root/trunk/omnipitr/lib/OmniPITR/Program/Backup/Slave.pm

Revision 214, 17.7 kB (checked in by depesz, 3 years ago)

1. Fix typo in changes.pod
2. perltidy over sources

Line 
1 package OmniPITR::Program::Backup::Slave;
2 use strict;
3 use warnings;
4
5 use base qw( OmniPITR::Program::Backup );
6
7 use File::Spec;
8 use File::Basename;
9 use English qw( -no_match_vars );
10 use File::Copy;
11 use File::Path;
12 use Getopt::Long;
13 use Carp;
14 use POSIX qw( strftime );
15 use Sys::Hostname;
16 use OmniPITR::Tools qw( run_command ext_for_compression );
17
18 =head1 make_data_archive()
19
20 Wraps all work necessary to make local .tar files (optionally compressed)
21 with content of PGDATA
22
23 =cut
24
25 sub make_data_archive {
26     my $self = shift;
27     $self->pause_xlog_removal();
28     $self->{ 'CONTROL' }->{ 'initial' } = $self->get_control_data();
29     $self->compress_pgdata();
30     return;
31 }
32
33 =head1 make_xlog_archive()
34
35 Wraps all work necessary to make local .tar files (optionally compressed)
36 with xlogs required to start PostgreSQL from backup.
37
38 =cut
39
40 sub make_xlog_archive {
41     my $self = shift;
42     $self->wait_for_checkpoint_location_change();
43     $self->compress_xlogs();
44     $self->unpause_xlog_removal();
45     return;
46 }
47
48 =head1 compress_xlogs()
49
50 Wrapper function which encapsulates all work required to compress xlog
51 segments that accumulated during backup of data directory.
52
53 =cut
54
55 sub compress_xlogs {
56     my $self = shift;
57
58     $self->make_dot_backup_file();
59     $self->uncompress_wal_archive_segments();
60
61     $self->log->time_start( 'Compressing xlogs' ) if $self->verbose;
62     $self->start_writers( 'xlog' );
63
64     my $source_transform_from = basename( $self->{ 'source' }->{ 'path' } );
65     $source_transform_from =~ s{^/*}{};
66     $source_transform_from =~ s{/*$}{};
67
68     my $dot_backup_transform_from = $self->{ 'temp-dir' };
69     $dot_backup_transform_from =~ s{^/*}{};
70     $dot_backup_transform_from =~ s{/*$}{};
71
72     my $transform_to = basename( $self->{ 'data-dir' } ) . '/pg_xlog';
73     my $transform_command = sprintf 's#^\(%s\|%s\)#%s#', $source_transform_from, $dot_backup_transform_from, $transform_to;
74
75     $self->tar_and_compress(
76         'work_dir'  => dirname( $self->{ 'source' }->{ 'path' } ),
77         'tar_dir'   => [ basename( $self->{ 'source' }->{ 'path' } ), File::Spec->catfile( $self->{ 'temp-dir' }, $self->{ 'dot_backup_filename' } ), ],
78         'transform' => $transform_command,
79     );
80
81     $self->log->time_finish( 'Compressing xlogs' ) if $self->verbose;
82
83     return;
84 }
85
86 =head1 uncompress_wal_archive_segments()
87
88 In case walarchive (--source option) is compressed, L<omnipitr-backup-slave>
89 needs to uncompress files to temp directory before making archive - so that
90 the archive will be easier to use.
91
92 This work is being done in this function.
93
94 =cut
95
96 sub uncompress_wal_archive_segments {
97     my $self = shift;
98     return if 'none' eq $self->{ 'source' }->{ 'compression' };
99
100     my $old_source = $self->{ 'source' }->{ 'path' };
101     my $new_source = File::Spec->catfile( $self->{ 'temp-dir' }, 'uncompresses_pg_xlogs' );
102     $self->{ 'source' }->{ 'path' } = $new_source;
103
104     mkpath( [ $new_source ], 0, oct( "755" ) );
105
106     opendir my $dir, $old_source or $self->log->fatal( 'Cannot open wal-archive (%s) : %s', $old_source, $OS_ERROR );
107     my $extension = ext_for_compression( $self->{ 'source' }->{ 'compression' } );
108     my @wal_segments = sort grep { -f File::Spec->catfile( $old_source, $_ ) && /\Q$extension\E\z/ } readdir( $dir );
109     close $dir;
110
111     $self->log->log( '%s wal segments have to be uncompressed', scalar @wal_segments );
112
113     for my $segment ( @wal_segments ) {
114         my $old_file = File::Spec->catfile( $old_source, $segment );
115         my $new_file = File::Spec->catfile( $new_source, $segment );
116         copy( $old_file, $new_file ) or $self->log->fatal( 'Cannot copy %s to %s: %s', $old_file, $new_file, $OS_ERROR );
117         $self->log->log( 'File copied: %s -> %s', $old_file, $new_file );
118         my @uncompress = ( $self->{ $self->{ 'source' }->{ 'compression' } . '-path' }, '-d', $new_file );
119         unshift @uncompress, $self->{ 'nice-path' } unless $self->{ 'not-nice' };
120         my $response = run_command( $self->{ 'temp-dir' }, @uncompress );
121         if ( $response->{ 'error_code' } ) {
122             $self->log->fatal( 'Error while uncompressing wal segment %s: %s', $new_file, $response );
123         }
124     }
125     return;
126 }
127
128 =head make_dot_backup_file()
129
130 Make I<SEGMENT>.I<OFFSET>.backup file that will be included in xlog archive.
131
132 This file contains vital information like start and end position of WAL
133 reply that is required to get consistent state.
134
135 =cut
136
137 sub make_dot_backup_file {
138     my $self = shift;
139
140     my $redo_location  = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's REDO location" };
141     my $final_location = $self->{ 'CONTROL' }->{ 'final' }->{ "Latest checkpoint location" };
142     my $timeline       = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's TimeLineID" };
143     my $offset         = $redo_location;
144     $offset =~ s#.*/##;
145     $offset =~ s/^.*?(.{0,6})$/$1/;
146
147     my $output_filename = sprintf '%s.%08s.backup', $self->convert_wal_location_and_timeline_to_filename( $redo_location, $timeline ), $offset;
148
149     my @content_lines = @{ $self->{ 'backup_file_data' } };
150     splice( @content_lines, 1, 0, sprintf 'STOP WAL LOCATION: %s (file %s)', $final_location, $self->convert_wal_location_and_timeline_to_filename( $final_location, $timeline ) );
151     splice( @content_lines, 4, 0, sprintf 'START TIME: %s', strftime( '%Y-%m-%d %H:%M:%S %Z', localtime time ) );
152
153     my $content = join( "\n", @content_lines ) . "\n";
154
155     my $filename = File::Spec->catfile( $self->{ 'temp-dir' }, $output_filename );
156     if ( open my $fh, '>', $filename ) {
157         print $fh $content;
158         close $fh;
159         $self->{ 'dot_backup_filename' } = $output_filename;
160         return;
161     }
162     $self->log->fatal( 'Cannot write .backup file file %s : %s', $output_filename, $OS_ERROR );
163 }
164
165 =head1 wait_for_checkpoint_location_change()
166
167 Just like the name suggests - this function periodically (every 5 seconds,
168 hardcoded, as there is not much sense in parametrizing it) checks
169 pg_controldata of PGDATA, and finishes if value in B<Latest checkpoint
170 location> will change.
171
172 =cut
173
174 sub wait_for_checkpoint_location_change {
175     my $self     = shift;
176     my $pre_wait = $self->get_control_data()->{ 'Latest checkpoint location' };
177     $self->log->log( 'Waiting for checkpoint' ) if $self->verbose;
178     while ( 1 ) {
179         sleep 5;
180         $self->{ 'CONTROL' }->{ 'final' } = $self->get_control_data();
181         last if $self->{ 'CONTROL' }->{ 'final' }->{ 'Latest checkpoint location' } ne $pre_wait;
182     }
183     $self->log->log( 'Checkpoint .' ) if $self->verbose;
184     return;
185 }
186
187 =head1 make_backup_label_temp_file()
188
189 Normal hot backup contains file named 'backup_label' in PGDATA archive.
190
191 Since this is not normal hot backup - PostgreSQL will not create this file,
192 and it has to be created separately by I<omnipitr-backup-slave>.
193
194 This file is created in temp directory (it is B<not> created in PGDATA), and
195 is included in tar in such a way that, on uncompressing, it will get to
196 unarchived PGDATA.
197
198 =cut
199
200 sub make_backup_label_temp_file {
201     my $self = shift;
202
203     my $redo_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's REDO location" };
204     my $last_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint location" };
205     my $timeline      = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's TimeLineID" };
206
207     my @content_lines = ();
208     push @content_lines, sprintf 'START WAL LOCATION: %s (file %s)', $redo_location, $self->convert_wal_location_and_timeline_to_filename( $redo_location, $timeline );
209     push @content_lines, sprintf 'CHECKPOINT LOCATION: %s', $last_location;
210     push @content_lines, sprintf 'START TIME: %s', strftime( '%Y-%m-%d %H:%M:%S %Z', localtime time );
211     push @content_lines, 'LABEL: OmniPITR_Slave_Hot_Backup';
212
213     $self->{ 'backup_file_data' } = \@content_lines;
214     my $content = join( "\n", @content_lines ) . "\n";
215
216     my $filename = File::Spec->catfile( $self->{ 'temp-dir' }, 'backup_label' );
217     if ( open my $fh, '>', $filename ) {
218         print $fh $content;
219         close $fh;
220         return;
221     }
222     $self->log->fatal( 'Cannot write backup_label file %s : %s', $filename, $OS_ERROR );
223 }
224
225 =head1 convert_wal_location_and_timeline_to_filename()
226
227 Helper function which converts WAL location and timeline number into
228 filename that given location will be in.
229
230 =cut
231
232 sub convert_wal_location_and_timeline_to_filename {
233     my $self = shift;
234     my ( $location, $timeline ) = @_;
235
236     my ( $series, $offset ) = split m{/}, $location;
237
238     $offset =~ s/.{0,6}$//;
239
240     my $location_filename = sprintf '%08s%08s%08s', $timeline, $series, $offset;
241
242     return $location_filename;
243 }
244
245 =head1 compress_pgdata()
246
247 Wrapper function which encapsulates all work required to compress data
248 directory.
249
250 =cut
251
252 sub compress_pgdata {
253     my $self = shift;
254
255     $self->make_backup_label_temp_file();
256
257     $self->log->time_start( 'Compressing $PGDATA' ) if $self->verbose;
258     $self->start_writers( 'data' );
259
260     my $transform_from = $self->{ 'temp-dir' };
261     $transform_from =~ s{^/*}{};
262     $transform_from =~ s{/*$}{};
263     my $transform_to = basename( $self->{ 'data-dir' } );
264     my $transform_command = sprintf 's#^%s/#%s/#', $transform_from, $transform_to;
265
266     my @excludes = qw( pg_log/* pg_xlog/0* pg_xlog/archive_status/* recovery.conf postmaster.pid );
267     for my $dir ( qw( pg_log pg_xlog ) ) {
268         push @excludes, $dir if -l File::Spec->catfile( $self->{ 'data-dir' }, $dir );
269     }
270
271     my ( $tablespaces, $transforms ) = $self->get_tablespaces_and_transforms();
272     push @{ $tablespaces }, basename( $self->{ 'data-dir' } ), File::Spec->catfile( $self->{ 'temp-dir' }, 'backup_label' );
273     push @{ $transforms }, $transform_command;
274
275     $self->tar_and_compress(
276         'work_dir'  => dirname( $self->{ 'data-dir' } ),
277         'tar_dir'   => $tablespaces,
278         'excludes'  => [ map { sprintf( '%s/%s', basename( $self->{ 'data-dir' } ), $_ ) } @excludes ],
279         'transform' => $transforms,
280     );
281
282     $self->log->time_finish( 'Compressing $PGDATA' ) if $self->verbose;
283     return;
284 }
285
286 =head1 pause_xlog_removal()
287
288 Creates trigger file that will pause removal of old segments by
289 I<omnipitr-restore>.
290
291 =cut
292
293 sub pause_xlog_removal {
294     my $self = shift;
295
296     if ( open my $fh, '>', $self->{ 'removal-pause-trigger' } ) {
297         print $fh $PROCESS_ID, "\n";
298         close $fh;
299         $self->{ 'removal-pause-trigger-created' } = 1;
300         return;
301     }
302     $self->log->fatal(
303         'Cannot create/write to removal pause trigger (%s) : %S',
304         $self->{ 'removal-pause-trigger' },
305         $OS_ERROR
306     );
307 }
308
309 =head1 unpause_xlog_removal()
310
311 Removed trigger file, effectively unpausing removal of old, obsolete log
312 segments in I<omnipitr-restore>.
313
314 =cut
315
316 sub unpause_xlog_removal {
317     my $self = shift;
318     unlink( $self->{ 'removal-pause-trigger' } );
319     delete $self->{ 'removal-pause-trigger-created' };
320     return;
321 }
322
323 =head1 DESTROY()
324
325 Destructor for object - removes created pause trigger;
326
327 =cut
328
329 sub DESTROY {
330     my $self = shift;
331     unlink( $self->{ 'removal-pause-trigger' } ) if $self->{ 'removal-pause-trigger-created' };
332     $self->SUPER::DESTROY();
333     return;
334 }
335
336 =head1 read_args()
337
338 Function which does all the parsing, and transformation of command line
339 arguments.
340
341 =cut
342
343 sub read_args {
344     my $self = shift;
345
346     my @argv_copy = @ARGV;
347
348     my %args = (
349         'temp-dir' => $ENV{ 'TMPDIR' } || '/tmp',
350         'gzip-path'          => 'gzip',
351         'bzip2-path'         => 'bzip2',
352         'lzma-path'          => 'lzma',
353         'tar-path'           => 'tar',
354         'nice-path'          => 'nice',
355         'rsync-path'         => 'rsync',
356         'pgcontroldata-path' => 'pg_controldata',
357         'filename-template'  => '__HOSTNAME__-__FILETYPE__-^Y-^m-^d.tar__CEXT__',
358     );
359
360     croak( 'Error while reading command line arguments. Please check documentation in doc/omnipitr-backup-slave.pod' )
361         unless GetOptions(
362         \%args,
363         'data-dir|D=s',
364         'source|s=s',
365         'dst-local|dl=s@',
366         'dst-remote|dr=s@',
367         'temp-dir|t=s',
368         'log|l=s',
369         'filename-template|f=s',
370         'removal-pause-trigger|p=s',
371         'pid-file',
372         'verbose|v',
373         'gzip-path|gp=s',
374         'bzip2-path|bp=s',
375         'lzma-path|lp=s',
376         'nice-path|np=s',
377         'tar-path|tp=s',
378         'rsync-path|rp=s',
379         'pgcontroldata-path|pp=s',
380         'not-nice|nn',
381         );
382
383     croak( '--log was not provided - cannot continue.' ) unless $args{ 'log' };
384     for my $key ( qw( log filename-template ) ) {
385         $args{ $key } =~ tr/^/%/;
386     }
387
388     for my $key ( grep { !/^dst-(?:local|remote)$/ } keys %args ) {
389         $self->{ $key } = $args{ $key };
390     }
391
392     for my $type ( qw( local remote ) ) {
393         my $D = [];
394         $self->{ 'destination' }->{ $type } = $D;
395
396         next unless defined $args{ 'dst-' . $type };
397
398         my %temp_for_uniq = ();
399         my @items = grep { !$temp_for_uniq{ $_ }++ } @{ $args{ 'dst-' . $type } };
400
401         for my $item ( @items ) {
402             my $current = { 'compression' => 'none', };
403             if ( $item =~ s/\A(gzip|bzip2|lzma)=// ) {
404                 $current->{ 'compression' } = $1;
405             }
406             $current->{ 'path' } = $item;
407             push @{ $D }, $current;
408         }
409     }
410
411     if ( $args{ 'source' } =~ s/\A(gzip|bzip2|lzma)=// ) {
412         $self->{ 'source' } = {
413             'compression' => $1,
414             'path'        => $args{ 'source' },
415         };
416     }
417     else {
418         $self->{ 'source' } = {
419             'compression' => 'none',
420             'path'        => $args{ 'source' },
421         };
422     }
423
424     $self->{ 'filename-template' } = strftime( $self->{ 'filename-template' }, localtime time() );
425     $self->{ 'filename-template' } =~ s/__HOSTNAME__/hostname()/ge;
426
427     # We do it here so it will actually work for reporing problems in validation
428     $self->{ 'log_template' } = $args{ 'log' };
429     $self->{ 'log' }          = OmniPITR::Log->new( $self->{ 'log_template' } );
430
431     $self->log->log( 'Called with parameters: %s', join( ' ', @argv_copy ) ) if $self->verbose;
432
433     return;
434 }
435
436 =head1 validate_args()
437
438 Does all necessary validation of given command line arguments.
439
440 One exception is for compression programs paths - technically, it could be
441 validated in here, but benefit would be pretty limited, and code to do so
442 relatively complex, as compression program path might, but doesn't have to
443 be actual file path - it might be just program name (without path), which is
444 the default.
445
446 =cut
447
448 sub validate_args {
449     my $self = shift;
450
451     $self->log->fatal( 'Data-dir was not provided!' ) unless defined $self->{ 'data-dir' };
452     $self->log->fatal( 'Provided data-dir (%s) does not exist!',   $self->{ 'data-dir' } ) unless -e $self->{ 'data-dir' };
453     $self->log->fatal( 'Provided data-dir (%s) is not directory!', $self->{ 'data-dir' } ) unless -d $self->{ 'data-dir' };
454     $self->log->fatal( 'Provided data-dir (%s) is not readable!',  $self->{ 'data-dir' } ) unless -r $self->{ 'data-dir' };
455
456     my $dst_count = scalar( @{ $self->{ 'destination' }->{ 'local' } } ) + scalar( @{ $self->{ 'destination' }->{ 'remote' } } );
457     $self->log->fatal( "No --dst-* has been provided!" ) if 0 == $dst_count;
458
459     $self->log->fatal( "Filename template does not contain __FILETYPE__ placeholder!" ) unless $self->{ 'filename-template' } =~ /__FILETYPE__/;
460     $self->log->fatal( "Filename template cannot contain / or \\ characters!" ) if $self->{ 'filename-template' } =~ m{[/\\]};
461
462     $self->log->fatal( 'Source of WAL files was not provided!' ) unless defined $self->{ 'source' }->{ 'path' };
463     $self->log->fatal( 'Provided source of wal files (%s) does not exist!',   $self->{ 'source' }->{ 'path' } ) unless -e $self->{ 'source' }->{ 'path' };
464     $self->log->fatal( 'Provided source of wal files (%s) is not directory!', $self->{ 'source' }->{ 'path' } ) unless -d $self->{ 'source' }->{ 'path' };
465     $self->log->fatal( 'Provided source of wal files (%s) is not readable!',  $self->{ 'source' }->{ 'path' } ) unless -r $self->{ 'source' }->{ 'path' };
466
467     $self->log->fatal( 'Temp-dir was not provided!' ) unless defined $self->{ 'temp-dir' };
468     $self->log->fatal( 'Provided temp-dir (%s) does not exist!',   $self->{ 'temp-dir' } ) unless -e $self->{ 'temp-dir' };
469     $self->log->fatal( 'Provided temp-dir (%s) is not directory!', $self->{ 'temp-dir' } ) unless -d $self->{ 'temp-dir' };
470     $self->log->fatal( 'Provided temp-dir (%s) is not writable!',  $self->{ 'temp-dir' } ) unless -w $self->{ 'temp-dir' };
471     $self->log->fatal( 'Provided temp-dir (%s) contains # character!', $self->{ 'temp-dir' } ) if $self->{ 'temp-dir' } =~ /#/;
472
473     $self->log->fatal( 'Removal pause trigger name was not provided!' ) unless defined $self->{ 'removal-pause-trigger' };
474     $self->log->fatal( 'Provided removal pause trigger file (%s) already exists!', $self->{ 'removal-pause-trigger' } ) if -e $self->{ 'removal-pause-trigger' };
475
476     $self->log->fatal( 'Directory for provided removal pause trigger (%s) does not exist!',   $self->{ 'removal-pause-trigger' } ) unless -e dirname( $self->{ 'removal-pause-trigger' } );
477     $self->log->fatal( 'Directory for provided removal pause trigger (%s) is not directory!', $self->{ 'removal-pause-trigger' } ) unless -d dirname( $self->{ 'removal-pause-trigger' } );
478     $self->log->fatal( 'Directory for provided removal pause trigger (%s) is not writable!',  $self->{ 'removal-pause-trigger' } ) unless -w dirname( $self->{ 'removal-pause-trigger' } );
479
480     return unless $self->{ 'destination' }->{ 'local' };
481
482     for my $d ( @{ $self->{ 'destination' }->{ 'local' } } ) {
483         my $dir = $d->{ 'path' };
484         $self->log->fatal( 'Choosen local destination dir (%s) does not exist. Cannot continue.',   $dir ) unless -e $dir;
485         $self->log->fatal( 'Choosen local destination dir (%s) is not directory. Cannot continue.', $dir ) unless -d $dir;
486         $self->log->fatal( 'Choosen local destination dir (%s) is not writable. Cannot continue.',  $dir ) unless -w $dir;
487     }
488
489     return;
490 }
491
492 1;
Note: See TracBrowser for help on using the browser.