root/trunk/omnipitr/lib/OmniPITR/Program/Backup/Slave.pm

Revision 179, 17.5 kB (checked in by depesz, 4 years ago)

add ability to disable nice usage

Line 
1 package OmniPITR::Program::Backup::Slave;
2 use strict;
3 use warnings;
4
5 use base qw( OmniPITR::Program::Backup );
6
7 use File::Spec;
8 use File::Basename;
9 use English qw( -no_match_vars );
10 use File::Copy;
11 use File::Path;
12 use Getopt::Long;
13 use Carp;
14 use POSIX qw( strftime );
15 use Sys::Hostname;
16 use OmniPITR::Tools qw( run_command ext_for_compression );
17
18 =head1 make_data_archive()
19
20 Wraps all work necessary to make local .tar files (optionally compressed)
21 with content of PGDATA
22
23 =cut
24
25 sub make_data_archive {
26     my $self = shift;
27     $self->pause_xlog_removal();
28     $self->{ 'CONTROL' }->{ 'initial' } = $self->get_control_data();
29     $self->compress_pgdata();
30     return;
31 }
32
33 =head1 make_xlog_archive()
34
35 Wraps all work necessary to make local .tar files (optionally compressed)
36 with xlogs required to start PostgreSQL from backup.
37
38 =cut
39
40 sub make_xlog_archive {
41     my $self = shift;
42     $self->wait_for_checkpoint_location_change();
43     $self->compress_xlogs();
44     $self->unpause_xlog_removal();
45     return;
46 }
47
48 =head1 compress_xlogs()
49
50 Wrapper function which encapsulates all work required to compress xlog
51 segments that accumulated during backup of data directory.
52
53 =cut
54
55 sub compress_xlogs {
56     my $self = shift;
57
58     $self->make_dot_backup_file();
59     $self->uncompress_wal_archive_segments();
60
61     $self->log->time_start( 'Compressing xlogs' ) if $self->verbose;
62     $self->start_writers( 'xlog' );
63
64     my $source_transform_from = basename( $self->{ 'source' }->{ 'path' } );
65     $source_transform_from =~ s{^/*}{};
66     $source_transform_from =~ s{/*$}{};
67
68     my $dot_backup_transform_from = $self->{ 'temp-dir' };
69     $dot_backup_transform_from =~ s{^/*}{};
70     $dot_backup_transform_from =~ s{/*$}{};
71
72     my $transform_to = basename( $self->{ 'data-dir' } ) . '/pg_xlog';
73     my $transform_command = sprintf 's#^\(%s\|%s\)#%s#', $source_transform_from, $dot_backup_transform_from, $transform_to;
74
75     $self->tar_and_compress(
76         'work_dir'  => dirname( $self->{ 'source' }->{ 'path' } ),
77         'tar_dir'   => [ basename( $self->{ 'source' }->{ 'path' } ), File::Spec->catfile( $self->{ 'temp-dir' }, $self->{ 'dot_backup_filename' } ), ],
78         'transform' => $transform_command,
79     );
80
81     $self->log->time_finish( 'Compressing xlogs' ) if $self->verbose;
82
83     return;
84 }
85
86 =head1 uncompress_wal_archive_segments()
87
88 In case walarchive (--source option) is compressed, L<omnipitr-backup-slave>
89 needs to uncompress files to temp directory before making archive - so that
90 the archive will be easier to use.
91
92 This work is being done in this function.
93
94 =cut
95
96 sub uncompress_wal_archive_segments {
97     my $self = shift;
98     return if 'none' eq $self->{ 'source' }->{ 'compression' };
99
100     my $old_source = $self->{ 'source' }->{ 'path' };
101     my $new_source = File::Spec->catfile( $self->{ 'temp-dir' }, 'uncompresses_pg_xlogs' );
102     $self->{ 'source' }->{ 'path' } = $new_source;
103
104     mkpath( [ $new_source ], 0, oct( "755" ) );
105
106     opendir my $dir, $old_source or $self->log->fatal( 'Cannot open wal-archive (%s) : %s', $old_source, $OS_ERROR );
107     my $extension = ext_for_compression( $self->{ 'source' }->{ 'compression' } );
108     my @wal_segments = sort grep { -f File::Spec->catfile( $old_source, $_ ) && /\Q$extension\E\z/ } readdir( $dir );
109     close $dir;
110
111     $self->log->log( '%s wal segments have to be uncompressed', scalar @wal_segments );
112
113     for my $segment ( @wal_segments ) {
114         my $old_file = File::Spec->catfile( $old_source, $segment );
115         my $new_file = File::Spec->catfile( $new_source, $segment );
116         copy( $old_file, $new_file ) or $self->log->fatal( 'Cannot copy %s to %s: %s', $old_file, $new_file, $OS_ERROR );
117         $self->log->log( 'File copied: %s -> %s', $old_file, $new_file );
118         my @uncompress = ( $self->{ $self->{ 'source' }->{ 'compression' } . '-path' }, '-d', $new_file );
119         unshift @uncompress, $self->{ 'nice-path' } unless $self->{ 'not-nice' };
120         my $response = run_command( $self->{ 'temp-dir' }, @uncompress );
121         if ( $response->{ 'error_code' } ) {
122             $self->log->fatal( 'Error while uncompressing wal segment %s: %s', $new_file, $response );
123         }
124     }
125     return;
126 }
127
128 =head make_dot_backup_file()
129
130 Make I<SEGMENT>.I<OFFSET>.backup file that will be included in xlog archive.
131
132 This file contains vital information like start and end position of WAL
133 reply that is required to get consistent state.
134
135 =cut
136
137 sub make_dot_backup_file {
138     my $self = shift;
139
140     my $redo_location  = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's REDO location" };
141     my $final_location = $self->{ 'CONTROL' }->{ 'final' }->{ "Latest checkpoint location" };
142     my $timeline       = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's TimeLineID" };
143     my $offset         = $redo_location;
144     $offset =~ s#.*/##;
145     $offset =~ s/^.*?(.{0,6})$/$1/;
146
147     my $output_filename = sprintf '%s.%08s.backup', $self->convert_wal_location_and_timeline_to_filename( $redo_location, $timeline ), $offset;
148
149     my @content_lines = @{ $self->{ 'backup_file_data' } };
150     splice( @content_lines, 1, 0, sprintf 'STOP WAL LOCATION: %s (file %s)', $final_location, $self->convert_wal_location_and_timeline_to_filename( $final_location, $timeline ) );
151     splice( @content_lines, 4, 0, sprintf 'START TIME: %s', strftime( '%Y-%m-%d %H:%M:%S %Z', localtime time ) );
152
153     my $content = join( "\n", @content_lines ) . "\n";
154
155     my $filename = File::Spec->catfile( $self->{ 'temp-dir' }, $output_filename );
156     if ( open my $fh, '>', $filename ) {
157         print $fh $content;
158         close $fh;
159         $self->{ 'dot_backup_filename' } = $output_filename;
160         return;
161     }
162     $self->log->fatal( 'Cannot write .backup file file %s : %s', $output_filename, $OS_ERROR );
163 }
164
165 =head1 wait_for_checkpoint_location_change()
166
167 Just like the name suggests - this function periodically (every 5 seconds,
168 hardcoded, as there is not much sense in parametrizing it) checks
169 pg_controldata of PGDATA, and finishes if value in B<Latest checkpoint
170 location> will change.
171
172 =cut
173
174 sub wait_for_checkpoint_location_change {
175     my $self     = shift;
176     my $pre_wait = $self->get_control_data()->{ 'Latest checkpoint location' };
177     $self->log->log( 'Waiting for checkpoint' ) if $self->verbose;
178     while ( 1 ) {
179         sleep 5;
180         $self->{ 'CONTROL' }->{ 'final' } = $self->get_control_data();
181         last if $self->{ 'CONTROL' }->{ 'final' }->{ 'Latest checkpoint location' } ne $pre_wait;
182     }
183     $self->log->log( 'Checkpoint .' ) if $self->verbose;
184     return;
185 }
186
187 =head1 make_backup_label_temp_file()
188
189 Normal hot backup contains file named 'backup_label' in PGDATA archive.
190
191 Since this is not normal hot backup - PostgreSQL will not create this file,
192 and it has to be created separately by I<omnipitr-backup-slave>.
193
194 This file is created in temp directory (it is B<not> created in PGDATA), and
195 is included in tar in such a way that, on uncompressing, it will get to
196 unarchived PGDATA.
197
198 =cut
199
200 sub make_backup_label_temp_file {
201     my $self = shift;
202
203     my $redo_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's REDO location" };
204     my $last_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint location" };
205     my $timeline      = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's TimeLineID" };
206
207     my @content_lines = ();
208     push @content_lines, sprintf 'START WAL LOCATION: %s (file %s)', $redo_location, $self->convert_wal_location_and_timeline_to_filename( $redo_location, $timeline );
209     push @content_lines, sprintf 'CHECKPOINT LOCATION: %s', $last_location;
210     push @content_lines, sprintf 'START TIME: %s', strftime( '%Y-%m-%d %H:%M:%S %Z', localtime time );
211     push @content_lines, 'LABEL: OmniPITR_Slave_Hot_Backup';
212
213     $self->{ 'backup_file_data' } = \@content_lines;
214     my $content = join( "\n", @content_lines ) . "\n";
215
216     my $filename = File::Spec->catfile( $self->{ 'temp-dir' }, 'backup_label' );
217     if ( open my $fh, '>', $filename ) {
218         print $fh $content;
219         close $fh;
220         return;
221     }
222     $self->log->fatal( 'Cannot write backup_label file %s : %s', $filename, $OS_ERROR );
223 }
224
225 =head1 convert_wal_location_and_timeline_to_filename()
226
227 Helper function which converts WAL location and timeline number into
228 filename that given location will be in.
229
230 =cut
231
232 sub convert_wal_location_and_timeline_to_filename {
233     my $self = shift;
234     my ( $location, $timeline ) = @_;
235
236     my ( $series, $offset ) = split m{/}, $location;
237
238     $offset =~ s/.{0,6}$//;
239
240     my $location_filename = sprintf '%08s%08s%08s', $timeline, $series, $offset;
241
242     return $location_filename;
243 }
244
245 =head1 compress_pgdata()
246
247 Wrapper function which encapsulates all work required to compress data
248 directory.
249
250 =cut
251
252 sub compress_pgdata {
253     my $self = shift;
254
255     $self->make_backup_label_temp_file();
256
257     $self->log->time_start( 'Compressing $PGDATA' ) if $self->verbose;
258     $self->start_writers( 'data' );
259
260     my $transform_from = $self->{ 'temp-dir' };
261     $transform_from =~ s{^/*}{};
262     $transform_from =~ s{/*$}{};
263     my $transform_to = basename( $self->{ 'data-dir' } );
264     my $transform_command = sprintf 's#^%s/#%s/#', $transform_from, $transform_to;
265
266     my @excludes = qw( pg_log/* pg_xlog/0* pg_xlog/archive_status/* recovery.conf postmaster.pid );
267     for my $dir ( qw( pg_log pg_xlog ) ) {
268         push @excludes, $dir if -l File::Spec->catfile( $self->{ 'data-dir' }, $dir );
269     }
270
271     $self->tar_and_compress(
272         'work_dir'  => dirname( $self->{ 'data-dir' } ),
273         'tar_dir'   => [ basename( $self->{ 'data-dir' } ), File::Spec->catfile( $self->{ 'temp-dir' }, 'backup_label' ) ],
274         'excludes'  => [ map { sprintf( '%s/%s', basename( $self->{ 'data-dir' } ), $_ ) } @excludes ],
275         'transform' => $transform_command,
276     );
277
278     $self->log->time_finish( 'Compressing $PGDATA' ) if $self->verbose;
279     return;
280 }
281
282 =head1 pause_xlog_removal()
283
284 Creates trigger file that will pause removal of old segments by
285 I<omnipitr-restore>.
286
287 =cut
288
289 sub pause_xlog_removal {
290     my $self = shift;
291
292     if ( open my $fh, '>', $self->{ 'removal-pause-trigger' } ) {
293         print $fh $PROCESS_ID, "\n";
294         close $fh;
295         $self->{ 'removal-pause-trigger-created' } = 1;
296         return;
297     }
298     $self->log->fatal(
299         'Cannot create/write to removal pause trigger (%s) : %S',
300         $self->{ 'removal-pause-trigger' },
301         $OS_ERROR
302     );
303 }
304
305 =head1 unpause_xlog_removal()
306
307 Removed trigger file, effectively unpausing removal of old, obsolete log
308 segments in I<omnipitr-restore>.
309
310 =cut
311
312 sub unpause_xlog_removal {
313     my $self = shift;
314     unlink( $self->{ 'removal-pause-trigger' } );
315     delete $self->{ 'removal-pause-trigger-created' };
316     return;
317 }
318
319 =head1 DESTROY()
320
321 Destructor for object - removes created pause trigger;
322
323 =cut
324
325 sub DESTROY {
326     my $self = shift;
327     unlink( $self->{ 'removal-pause-trigger' } ) if $self->{ 'removal-pause-trigger-created' };
328     $self->SUPER::DESTROY();
329     return;
330 }
331
332 =head1 read_args()
333
334 Function which does all the parsing, and transformation of command line
335 arguments.
336
337 =cut
338
339 sub read_args {
340     my $self = shift;
341
342     my @argv_copy = @ARGV;
343
344     my %args = (
345         'temp-dir' => $ENV{ 'TMPDIR' } || '/tmp',
346         'gzip-path'          => 'gzip',
347         'bzip2-path'         => 'bzip2',
348         'lzma-path'          => 'lzma',
349         'tar-path'           => 'tar',
350         'nice-path'          => 'nice',
351         'rsync-path'         => 'rsync',
352         'pgcontroldata-path' => 'pg_controldata',
353         'filename-template'  => '__HOSTNAME__-__FILETYPE__-^Y-^m-^d.tar__CEXT__',
354     );
355
356     croak( 'Error while reading command line arguments. Please check documentation in doc/omnipitr-backup-slave.pod' )
357         unless GetOptions(
358         \%args,
359         'data-dir|D=s',
360         'source|s=s',
361         'dst-local|dl=s@',
362         'dst-remote|dr=s@',
363         'temp-dir|t=s',
364         'log|l=s',
365         'filename-template|f=s',
366         'removal-pause-trigger|p=s',
367         'pid-file',
368         'verbose|v',
369         'gzip-path|gp=s',
370         'bzip2-path|bp=s',
371         'lzma-path|lp=s',
372         'nice-path|np=s',
373         'tar-path|tp=s',
374         'rsync-path|rp=s',
375         'pgcontroldata-path|pp=s',
376         'not-nice|nn',
377         );
378
379     croak( '--log was not provided - cannot continue.' ) unless $args{ 'log' };
380     for my $key ( qw( log filename-template ) ) {
381         $args{ $key } =~ tr/^/%/;
382     }
383
384     for my $key ( grep { !/^dst-(?:local|remote)$/ } keys %args ) {
385         $self->{ $key } = $args{ $key };
386     }
387
388     for my $type ( qw( local remote ) ) {
389         my $D = [];
390         $self->{ 'destination' }->{ $type } = $D;
391
392         next unless defined $args{ 'dst-' . $type };
393
394         my %temp_for_uniq = ();
395         my @items = grep { !$temp_for_uniq{ $_ }++ } @{ $args{ 'dst-' . $type } };
396
397         for my $item ( @items ) {
398             my $current = { 'compression' => 'none', };
399             if ( $item =~ s/\A(gzip|bzip2|lzma)=// ) {
400                 $current->{ 'compression' } = $1;
401             }
402             $current->{ 'path' } = $item;
403             push @{ $D }, $current;
404         }
405     }
406
407     if ( $args{ 'source' } =~ s/\A(gzip|bzip2|lzma)=// ) {
408         $self->{ 'source' } = {
409             'compression' => $1,
410             'path'        => $args{ 'source' },
411         };
412     }
413     else {
414         $self->{ 'source' } = {
415             'compression' => 'none',
416             'path'        => $args{ 'source' },
417         };
418     }
419
420     $self->{ 'filename-template' } = strftime( $self->{ 'filename-template' }, localtime time() );
421     $self->{ 'filename-template' } =~ s/__HOSTNAME__/hostname()/ge;
422
423     # We do it here so it will actually work for reporing problems in validation
424     $self->{ 'log_template' } = $args{ 'log' };
425     $self->{ 'log' }          = OmniPITR::Log->new( $self->{ 'log_template' } );
426
427     $self->log->log( 'Called with parameters: %s', join( ' ', @argv_copy ) ) if $self->verbose;
428
429     return;
430 }
431
432 =head1 validate_args()
433
434 Does all necessary validation of given command line arguments.
435
436 One exception is for compression programs paths - technically, it could be
437 validated in here, but benefit would be pretty limited, and code to do so
438 relatively complex, as compression program path might, but doesn't have to
439 be actual file path - it might be just program name (without path), which is
440 the default.
441
442 =cut
443
444 sub validate_args {
445     my $self = shift;
446
447     $self->log->fatal( 'Data-dir was not provided!' ) unless defined $self->{ 'data-dir' };
448     $self->log->fatal( 'Provided data-dir (%s) does not exist!',   $self->{ 'data-dir' } ) unless -e $self->{ 'data-dir' };
449     $self->log->fatal( 'Provided data-dir (%s) is not directory!', $self->{ 'data-dir' } ) unless -d $self->{ 'data-dir' };
450     $self->log->fatal( 'Provided data-dir (%s) is not readable!',  $self->{ 'data-dir' } ) unless -r $self->{ 'data-dir' };
451
452     my $dst_count = scalar( @{ $self->{ 'destination' }->{ 'local' } } ) + scalar( @{ $self->{ 'destination' }->{ 'remote' } } );
453     $self->log->fatal( "No --dst-* has been provided!" ) if 0 == $dst_count;
454
455     $self->log->fatal( "Filename template does not contain __FILETYPE__ placeholder!" ) unless $self->{ 'filename-template' } =~ /__FILETYPE__/;
456     $self->log->fatal( "Filename template cannot contain / or \\ characters!" ) if $self->{ 'filename-template' } =~ m{[/\\]};
457
458     $self->log->fatal( 'Source of WAL files was not provided!' ) unless defined $self->{ 'source' }->{ 'path' };
459     $self->log->fatal( 'Provided source of wal files (%s) does not exist!',   $self->{ 'source' }->{ 'path' } ) unless -e $self->{ 'source' }->{ 'path' };
460     $self->log->fatal( 'Provided source of wal files (%s) is not directory!', $self->{ 'source' }->{ 'path' } ) unless -d $self->{ 'source' }->{ 'path' };
461     $self->log->fatal( 'Provided source of wal files (%s) is not readable!',  $self->{ 'source' }->{ 'path' } ) unless -r $self->{ 'source' }->{ 'path' };
462
463     $self->log->fatal( 'Temp-dir was not provided!' ) unless defined $self->{ 'temp-dir' };
464     $self->log->fatal( 'Provided temp-dir (%s) does not exist!',   $self->{ 'temp-dir' } ) unless -e $self->{ 'temp-dir' };
465     $self->log->fatal( 'Provided temp-dir (%s) is not directory!', $self->{ 'temp-dir' } ) unless -d $self->{ 'temp-dir' };
466     $self->log->fatal( 'Provided temp-dir (%s) is not writable!',  $self->{ 'temp-dir' } ) unless -w $self->{ 'temp-dir' };
467     $self->log->fatal( 'Provided temp-dir (%s) contains # character!', $self->{ 'temp-dir' } ) if $self->{ 'temp-dir' } =~ /#/;
468
469     $self->log->fatal( 'Removal pause trigger name was not provided!' ) unless defined $self->{ 'removal-pause-trigger' };
470     $self->log->fatal( 'Provided removal pause trigger file (%s) already exists!', $self->{ 'removal-pause-trigger' } ) if -e $self->{ 'removal-pause-trigger' };
471
472     $self->log->fatal( 'Directory for provided removal pause trigger (%s) does not exist!',   $self->{ 'removal-pause-trigger' } ) unless -e dirname( $self->{ 'removal-pause-trigger' } );
473     $self->log->fatal( 'Directory for provided removal pause trigger (%s) is not directory!', $self->{ 'removal-pause-trigger' } ) unless -d dirname( $self->{ 'removal-pause-trigger' } );
474     $self->log->fatal( 'Directory for provided removal pause trigger (%s) is not writable!',  $self->{ 'removal-pause-trigger' } ) unless -w dirname( $self->{ 'removal-pause-trigger' } );
475
476     return unless $self->{ 'destination' }->{ 'local' };
477
478     for my $d ( @{ $self->{ 'destination' }->{ 'local' } } ) {
479         my $dir = $d->{ 'path' };
480         $self->log->fatal( 'Choosen local destination dir (%s) does not exist. Cannot continue.',   $dir ) unless -e $dir;
481         $self->log->fatal( 'Choosen local destination dir (%s) is not directory. Cannot continue.', $dir ) unless -d $dir;
482         $self->log->fatal( 'Choosen local destination dir (%s) is not writable. Cannot continue.',  $dir ) unless -w $dir;
483     }
484
485     return;
486 }
487
488 1;
Note: See TracBrowser for help on using the browser.