root/trunk/omnipitr/lib/OmniPITR/Program/Backup/Slave.pm

Revision 171, 27.5 kB (checked in by depesz, 4 years ago)

Alpha version of omnipitr-backup-slave. It passed simplest test, but it lacks documentation, and clearly code needs to be refactored - there are way too many copy/pasted functions.

Line 
1 package OmniPITR::Program::Backup::Slave;
2 use strict;
3 use warnings;
4
5 use base qw( OmniPITR::Program );
6
7 use File::Spec;
8 use File::Basename;
9 use Cwd;
10 use English qw( -no_match_vars );
11 use File::Copy;
12 use File::Path;
13 use Getopt::Long;
14 use Carp;
15 use POSIX qw( strftime );
16 use Sys::Hostname;
17 use OmniPITR::Tools qw( run_command ext_for_compression );
18
19 sub run {
20     my $self = shift;
21     $self->get_list_of_all_necessary_compressions();
22     $self->choose_base_local_destinations();
23
24     $self->pause_xlog_removal();
25
26     $self->get_initial_pg_control_data();
27
28     $self->compress_pgdata();
29
30     $self->wait_for_checkpoint_location_change();
31
32     $self->compress_xlogs();
33
34     $self->unpause_xlog_removal();
35
36     $self->deliver_to_all_destinations();
37 }
38
39 =head1 deliver_to_all_destinations()
40
41 Simple wrapper to have single point to call to deliver backups to all requested backups.
42
43 =cut
44
45 sub deliver_to_all_destinations {
46     my $self = shift;
47
48     $self->deliver_to_all_local_destinations();
49
50     $self->deliver_to_all_remote_destinations();
51
52     return;
53 }
54
55 =head1 deliver_to_all_local_destinations()
56
57 Copies backups to all local destinations which are not also base destinations for their respective compressions.
58
59 =cut
60
61 sub deliver_to_all_local_destinations {
62     my $self = shift;
63     return unless $self->{ 'destination' }->{ 'local' };
64     for my $dst ( @{ $self->{ 'destination' }->{ 'local' } } ) {
65         next if $dst->{ 'path' } eq $self->{ 'base' }->{ $dst->{ 'compression' } };
66
67         my $B = $self->{ 'base' }->{ $dst->{ 'compression' } };
68
69         for my $type ( qw( data xlog ) ) {
70
71             my $filename = $self->get_archive_filename( $type, $dst->{ 'compression' } );
72             my $source_filename = File::Spec->catfile( $B, $filename );
73             my $destination_filename = File::Spec->catfile( $dst->{ 'path' }, $filename );
74
75             my $time_msg = sprintf 'Copying %s to %s', $source_filename, $destination_filename;
76             $self->log->time_start( $time_msg ) if $self->verbose;
77
78             my $rc = copy( $source_filename, $destination_filename );
79
80             $self->log->time_finish( $time_msg ) if $self->verbose;
81
82             unless ( $rc ) {
83                 $self->log->error( 'Cannot copy %s to %s : %s', $source_filename, $destination_filename, $OS_ERROR );
84                 $self->{ 'had_errors' } = 1;
85             }
86
87         }
88     }
89     return;
90 }
91
92 =head1 deliver_to_all_remote_destinations()
93
94 Delivers backups to remote destinations using rsync program.
95
96 =cut
97
98 sub deliver_to_all_remote_destinations {
99     my $self = shift;
100     return unless $self->{ 'destination' }->{ 'remote' };
101     for my $dst ( @{ $self->{ 'destination' }->{ 'remote' } } ) {
102
103         my $B = $self->{ 'base' }->{ $dst->{ 'compression' } };
104
105         for my $type ( qw( data xlog ) ) {
106
107             my $filename = $self->get_archive_filename( $type, $dst->{ 'compression' } );
108             my $source_filename = File::Spec->catfile( $B, $filename );
109             my $destination_filename = $dst->{ 'path' };
110             $destination_filename =~ s{/*\z}{/};
111             $destination_filename .= $filename;
112
113             my $time_msg = sprintf 'Copying %s to %s', $source_filename, $destination_filename;
114             $self->log->time_start( $time_msg ) if $self->verbose;
115
116             my $response = run_command( $self->{ 'temp-dir' }, $self->{ 'rsync-path' }, $source_filename, $destination_filename );
117
118             $self->log->time_finish( $time_msg ) if $self->verbose;
119
120             if ( $response->{ 'error_code' } ) {
121                 $self->log->error( 'Cannot send archive %s to %s: %s', $source_filename, $destination_filename, $response );
122                 $self->{ 'had_errors' } = 1;
123             }
124         }
125     }
126     return;
127 }
128
129 sub compress_xlogs {
130     my $self = shift;
131
132     $self->make_dot_backup_file();
133     $self->uncompress_wal_archive_segments();
134
135     $self->log->time_start( 'Compressing xlogs' ) if $self->verbose;
136     $self->start_writers( 'xlog' );
137
138     my $source_transform_from = basename( $self->{ 'source' }->{ 'path' } );
139     $source_transform_from =~ s{^/*}{};
140     $source_transform_from =~ s{/*$}{};
141     my $source_transform_to = basename( $self->{ 'data-dir' } ) . '/pg_xlog';
142     my $source_transform_command = sprintf 's#^%s#%s#', $source_transform_from, $source_transform_to;
143
144     my $dot_backup_transform_from = File::Spec->catfile( $self->{ 'temp-dir' }, $self->{ 'dot_backup_filename' } );
145     $dot_backup_transform_from =~ s{^/*}{};
146     $dot_backup_transform_from =~ s{/*$}{};
147     my $dot_backup_transform_to = basename( $self->{ 'data-dir' } ) . '/pg_xlog/' . $self->{ 'dot_backup_filename' };
148     my $dot_backup_transform_command = sprintf 's#^%s#%s#', $dot_backup_transform_from, $dot_backup_transform_to;
149
150     $self->tar_and_compress(
151         'work_dir' => dirname( $self->{'source'}->{'path'} ),
152         'tar_dir'  => [ basename( $self->{ 'source' }->{ 'path' } ), File::Spec->catfile( $self->{ 'temp-dir' }, $self->{ 'dot_backup_filename' } ), ],
153         'transform' => [ $source_transform_command, $dot_backup_transform_command ],
154     );
155
156     $self->log->time_finish( 'Compressing xlogs' ) if $self->verbose;
157
158     return;
159 }
160
161 sub uncompress_wal_archive_segments {
162     my $self = shift;
163     return if 'none' eq $self->{ 'source' }->{ 'compression' };
164
165     my $old_source = $self->{ 'source' }->{ 'path' };
166     my $new_source = File::Spec->catfile( $self->{ 'temp-dir' }, 'uncompresses_pg_xlogs' );
167     $self->{ 'source' }->{ 'path' } = $new_source;
168
169     mkpath( [ $new_source ], 0, oct( "755" ) );
170
171     opendir my $dir, $old_source or $self->log->fatal( 'Cannot open wal-archive (%s) : %s', $old_source, $OS_ERROR );
172     my $extension = ext_for_compression( $self->{ 'source' }->{ 'compression' } );
173     my @wal_segments = sort grep { -f File::Spec->catfile( $old_source, $_ ) && /\Q$extension\E\z/ } readdir( $dir );
174     close $dir;
175
176     $self->log->log( '%s wal segments have to be uncompressed', scalar @wal_segments );
177
178     for my $segment ( @wal_segments ) {
179         my $old_file = File::Spec->catfile( $old_source, $segment );
180         my $new_file = File::Spec->catfile( $new_source, $segment );
181         copy( $old_file, $new_file ) or $self->log->fatal( 'Cannot copy %s to %s: %s', $old_file, $new_file, $OS_ERROR );
182         $self->log->log('File copied: %s -> %s', $old_file, $new_file);
183         my $response = run_command( $self->{ 'temp-dir' }, $self->{ 'nice-path' }, $self->{ $self->{'source'}->{'compression'} . '-path' }, '-d', $new_file );
184         if ( $response->{ 'error_code' } ) {
185             $self->log->fatal( 'Error while uncompressing wal segment %s: %s', $new_file, $response );
186         }
187     }
188     return;
189 }
190
191 sub make_dot_backup_file {
192     my $self = shift;
193
194     my $redo_location  = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's REDO location" };
195     my $final_location = $self->{ 'CONTROL' }->{ 'final' }->{ "Latest checkpoint location" };
196     my $timeline       = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's TimeLineID" };
197     my $offset         = $redo_location;
198     $offset =~ s#.*/##;
199     $offset =~ s/^.*?(.{0,6})$/$1/;
200
201     my $output_filename = sprintf '%s.%08s.backup', $self->convert_wal_location_and_timeline_to_filename( $redo_location, $timeline ), $offset;
202
203     my @content_lines = @{ $self->{ 'backup_file_data' } };
204     splice( @content_lines, 1, 0, sprintf 'STOP WAL LOCATION: %s (file %s)', $final_location, $self->convert_wal_location_and_timeline_to_filename( $final_location, $timeline ) );
205     splice( @content_lines, 4, 0, sprintf 'START TIME: %s', strftime( '%Y-%m-%d %H:%M:%S %Z', localtime time ) );
206
207     my $content = join( "\n", @content_lines ) . "\n";
208
209     my $filename = File::Spec->catfile( $self->{ 'temp-dir' }, $output_filename );
210     if ( open my $fh, '>', $filename ) {
211         print $fh $content;
212         close $fh;
213         $self->{ 'dot_backup_filename' } = $output_filename;
214         return;
215     }
216     $self->log->fatal( 'Cannot write .backup file file %s : %s', $output_filename, $OS_ERROR );
217 }
218
219 sub wait_for_checkpoint_location_change {
220     my $self     = shift;
221     my $pre_wait = $self->get_control_data()->{ 'Latest checkpoint location' };
222     $self->log->log( 'Waiting for checkpoint' ) if $self->verbose;
223     while ( 1 ) {
224         sleep 5;
225         $self->{ 'CONTROL' }->{ 'final' } = $self->get_control_data();
226         last if $self->{ 'CONTROL' }->{ 'final' }->{ 'Latest checkpoint location' } ne $pre_wait;
227     }
228     $self->log->log( 'Checkpoint .' ) if $self->verbose;
229     return;
230 }
231
232 sub make_backup_label_temp_file {
233     my $self = shift;
234
235     my $redo_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's REDO location" };
236     my $last_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint location" };
237     my $timeline      = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's TimeLineID" };
238
239     my @content_lines = ();
240     push @content_lines, sprintf 'START WAL LOCATION: %s (file %s)', $redo_location, $self->convert_wal_location_and_timeline_to_filename( $redo_location, $timeline );
241     push @content_lines, sprintf 'CHECKPOINT LOCATION: %s', $last_location;
242     push @content_lines, sprintf 'START TIME: %s', strftime( '%Y-%m-%d %H:%M:%S %Z', localtime time );
243     push @content_lines, 'LABEL: OmniPITR_Slave_Hot_Backup';
244
245     $self->{ 'backup_file_data' } = \@content_lines;
246     my $content = join( "\n", @content_lines ) . "\n";
247
248     my $filename = File::Spec->catfile( $self->{ 'temp-dir' }, 'backup_label' );
249     if ( open my $fh, '>', $filename ) {
250         print $fh $content;
251         close $fh;
252         return;
253     }
254     $self->log->fatal( 'Cannot write backup_label file %s : %s', $filename, $OS_ERROR );
255 }
256
257 sub convert_wal_location_and_timeline_to_filename {
258     my $self = shift;
259     my ( $location, $timeline ) = @_;
260
261     my ( $series, $offset ) = split m{/}, $location;
262
263     $offset =~ s/.{0,6}$//;
264
265     my $location_filename = sprintf '%08s%08s%08s', $timeline, $series, $offset;
266
267     return $location_filename;
268 }
269
270 =head1 get_archive_filename()
271
272 Helper function, which takes filetype and compression schema to use, and returns generated filename (based on filename-template command line option).
273
274 =cut
275
276 sub get_archive_filename {
277     my $self = shift;
278     my ( $type, $compression ) = @_;
279
280     my $ext = $compression eq 'none' ? '' : ext_for_compression( $compression );
281
282     my $filename = $self->{ 'filename-template' };
283     $filename =~ s/__FILETYPE__/$type/g;
284     $filename =~ s/__CEXT__/$ext/g;
285
286     return $filename;
287 }
288
289 =head1 start_writers()
290
291 Starts set of filehandles, which write to file, or to compression program, to create final archives.
292
293 Each compression schema gets its own filehandle, and printing data to it, will pass it to file directly or through compression program that has been chosen based on command line arguments.
294
295 =cut
296
297 sub start_writers {
298     my $self      = shift;
299     my $data_type = shift;
300
301     my %writers = ();
302
303     COMPRESSION:
304     while ( my ( $type, $dst_path ) = each %{ $self->{ 'base' } } ) {
305         my $filename = $self->get_archive_filename( $data_type, $type );
306
307         my $full_file_path = File::Spec->catfile( $dst_path, $filename );
308
309         if ( $type eq 'none' ) {
310             if ( open my $fh, '>', $full_file_path ) {
311                 $writers{ $type } = $fh;
312                 $self->log->log( "Starting \"none\" writer to $full_file_path" ) if $self->verbose;
313                 next COMPRESSION;
314             }
315             $self->clean_and_die( 'Cannot write to %s : %s', $full_file_path, $OS_ERROR );
316         }
317
318         my @command = map { quotemeta $_ } ( $self->{ 'nice-path' }, $self->{ $type . '-path' }, '--stdout', '-' );
319         push @command, ( '>', quotemeta( $full_file_path ) );
320
321         $self->log->log( "Starting \"%s\" writer to %s", $type, $full_file_path ) if $self->verbose;
322         if ( open my $fh, '|-', join( ' ', @command ) ) {
323             $writers{ $type } = $fh;
324             next COMPRESSION;
325         }
326         $self->clean_and_die( 'Cannot open command. Error: %s, Command: %s', $OS_ERROR, \@command );
327     }
328     $self->{ 'writers' } = \%writers;
329     return;
330 }
331
332 sub compress_pgdata {
333     my $self = shift;
334
335     $self->make_backup_label_temp_file();
336
337     $self->log->time_start( 'Compressing $PGDATA' ) if $self->verbose;
338     $self->start_writers( 'data' );
339
340     my $transform_from = $self->{ 'temp-dir' };
341     $transform_from =~ s{^/*}{};
342     $transform_from =~ s{/*$}{};
343     my $transform_to = basename( $self->{ 'data-dir' } );
344     my $transform_command = sprintf 's#^%s/#%s/#', $transform_from, $transform_to;
345
346     $self->tar_and_compress(
347         'work_dir'  => dirname( $self->{ 'data-dir' } ),
348         'tar_dir'   => [ basename( $self->{ 'data-dir' } ), File::Spec->catfile( $self->{ 'temp-dir' }, 'backup_label' ) ],
349         'excludes'  => [ map { sprintf( '%s/%s', basename( $self->{ 'data-dir' } ), $_ ) } qw( pg_log/* pg_xlog/0* pg_xlog/archive_status/* recovery.conf postmaster.pid ) ],
350         'transform' => [ $transform_command ],
351     );
352
353     $self->log->time_finish( 'Compressing $PGDATA' ) if $self->verbose;
354     return;
355 }
356
357 =head1 tar_and_compress()
358
359 Worker function which does all of the actual tar, and sending data to compression filehandles.
360
361 Takes hash (not hashref) as argument, and uses following keys from it:
362
363 =over
364
365 =item * tar_dir - arrayref with list of directories to compress
366
367 =item * work_dir - what should be current working directory when executing tar
368
369 =item * excludes - optional key, that (if exists) is treated as arrayref of shell globs (tar dir) of items to exclude from backup
370
371 =item * transform - optional key, that (if exists) is treated as arrayref of values for --transform option for tar
372
373 =back
374
375 If tar will print anything to STDERR it will be logged. Error status code is ignored, as it is expected that tar will generate errors (due to files modified while archiving).
376
377 =cut
378
379 sub tar_and_compress {
380     my $self = shift;
381     my %ARGS = @_;
382
383     $SIG{ 'PIPE' } = sub { $self->clean_and_die( 'Got SIGPIPE while tarring %s for %s', $ARGS{ 'tar_dir' }, $self->{ 'sigpipeinfo' } ); };
384
385     my @compression_command = ( $self->{ 'nice-path' }, $self->{ 'tar-path' }, 'cf', '-' );
386     if ( $ARGS{ 'excludes' } ) {
387         push @compression_command, map { '--exclude=' . $_ } @{ $ARGS{ 'excludes' } };
388     }
389     if ( $ARGS{ 'transform' } ) {
390         push @compression_command, map { '--transform=' . $_ } @{ $ARGS{ 'transform' } };
391     }
392     push @compression_command, @{ $ARGS{ 'tar_dir' } };
393
394     my $compression_str = join ' ', map { quotemeta $_ } @compression_command;
395
396     $self->prepare_temp_directory();
397     my $tar_stderr_filename = File::Spec->catfile( $self->{ 'temp-dir' }, 'tar.stderr' );
398     $compression_str .= ' 2> ' . quotemeta( $tar_stderr_filename );
399
400     my $previous_dir = getcwd;
401     chdir $ARGS{ 'work_dir' } if $ARGS{ 'work_dir' };
402
403     my $tar;
404     unless ( open $tar, '-|', $compression_str ) {
405         $self->clean_and_die( 'Cannot start tar (%s) : %s', $compression_str, $OS_ERROR );
406     }
407
408     chdir $previous_dir if $ARGS{ 'work_dir' };
409
410     my $buffer;
411     while ( my $len = sysread( $tar, $buffer, 8192 ) ) {
412         while ( my ( $type, $fh ) = each %{ $self->{ 'writers' } } ) {
413             $self->{ 'sigpipeinfo' } = $type;
414             my $written = syswrite( $fh, $buffer, $len );
415             next if $written == $len;
416             $self->clean_and_die( "Writting %u bytes to filehandle for <%s> compression wrote only %u bytes ?!", $len, $type, $written );
417         }
418     }
419     close $tar;
420
421     for my $fh ( values %{ $self->{ 'writers' } } ) {
422         close $fh;
423     }
424
425     delete $self->{ 'writers' };
426
427     my $stderr_output;
428     my $stderr;
429     unless ( open $stderr, '<', $tar_stderr_filename ) {
430         $self->log->log( 'Cannot open tar stderr file (%s) for reading: %s', $tar_stderr_filename );
431         return;
432     }
433     {
434         local $/;
435         $stderr_output = <$stderr>;
436     };
437     close $stderr;
438     return unless $stderr_output;
439     $self->log->log( 'Tar (%s) generated these output on stderr:', $compression_str );
440     $self->log->log( '==============================================' );
441     $self->log->log( '%s', $stderr_output );
442     $self->log->log( '==============================================' );
443     unlink $tar_stderr_filename;
444     return;
445 }
446
447 =head1 get_control_data()
448
449 Calls pg_controldata, and parses its output.
450
451 Verifies that output contains 2 critical pieces of information:
452
453 =over
454
455 =item * Latest checkpoint's REDO location
456
457 =item * Latest checkpoint's TimeLineID
458
459 =back
460
461 =cut
462
463 sub get_control_data {
464     my $self = shift;
465
466     $self->prepare_temp_directory();
467
468     my $response = run_command( $self->{ 'temp-dir' }, $self->{ 'pgcontroldata-path' }, $self->{ 'data-dir' } );
469     if ( $response->{ 'error_code' } ) {
470         $self->log->fatal( 'Error while getting pg_controldata for %s: %s', $self->{ 'data-dir' }, $response );
471     }
472
473     my $control_data = {};
474
475     my @lines = split( /\s*\n/, $response->{ 'stdout' } );
476     for my $line ( @lines ) {
477         unless ( $line =~ m{\A([^:]+):\s*(.*)\z} ) {
478             $self->log->fatal( 'Pg_controldata for %s contained unparseable line: [%s]', $self->{ 'data-dir' }, $line );
479         }
480         $control_data->{ $1 } = $2;
481     }
482
483     unless ( $control_data->{ "Latest checkpoint's REDO location" } ) {
484         $self->log->fatal( 'Pg_controldata for %s did not contain latest checkpoint redo location', $self->{ 'data-dir' } );
485     }
486     unless ( $control_data->{ "Latest checkpoint's TimeLineID" } ) {
487         $self->log->fatal( 'Pg_controldata for %s did not contain latest checkpoint timeline ID', $self->{ 'data-dir' } );
488     }
489
490     return $control_data;
491 }
492
493 sub get_initial_pg_control_data {
494     my $self = shift;
495
496     $self->{ 'CONTROL' }->{ 'initial' } = $self->get_control_data();
497
498     return;
499 }
500
501 sub pause_xlog_removal {
502     my $self = shift;
503
504     if ( open my $fh, '>', $self->{ 'removal-pause-trigger' } ) {
505         print $fh $PROCESS_ID, "\n";
506         close $fh;
507         $self->{ 'removal-pause-trigger-created' } = 1;
508         return;
509     }
510     $self->log->fatal(
511         'Cannot create/write to removal pause trigger (%s) : %S',
512         $self->{ 'removal-pause-trigger' },
513         $OS_ERROR
514     );
515 }
516
517 sub unpause_xlog_removal {
518     my $self = shift;
519     unlink( $self->{ 'removal-pause-trigger' } ) if $self->{ 'removal-pause-trigger-created' };
520     delete $self->{ 'removal-pause-trigger-created' };
521     return;
522 }
523
524 =head1 DESTROY()
525
526 Destroctor for object - removes temp directory on program exit.
527
528 =cut
529
530 sub DESTROY {
531     my $self = shift;
532     unlink( $self->{ 'removal-pause-trigger' } ) if $self->{ 'removal-pause-trigger-created' };
533     rmtree( [ $self->{ 'temp-dir-prepared' } ], 0 ) if $self->{ 'temp-dir-prepared' };
534     return;
535 }
536
537 =head1 prepare_temp_directory()
538
539 Helper function, which builds path for temp directory, and creates it.
540
541 Path is generated by using given temp-dir and 'omnipitr-backup-master' named.
542
543 For example, for temp-dir '/tmp' used temp directory would be /tmp/omnipitr-backup-master.
544
545 =cut
546
547 sub prepare_temp_directory {
548     my $self = shift;
549     return if $self->{ 'temp-dir-prepared' };
550     my $full_temp_dir = File::Spec->catfile( $self->{ 'temp-dir' }, basename( $PROGRAM_NAME ) );
551     mkpath( $full_temp_dir );
552     $self->{ 'temp-dir' }          = $full_temp_dir;
553     $self->{ 'temp-dir-prepared' } = $full_temp_dir;
554     return;
555 }
556
557 =head1 choose_base_local_destinations()
558
559 Chooses single local destination for every compression schema required by destinations specifications.
560
561 In case some compression schema exists only for remote destination, local temp directory is created in --temp-dir location.
562
563 =cut
564
565 sub choose_base_local_destinations {
566     my $self = shift;
567
568     my $base = { map { ( $_ => undef ) } @{ $self->{ 'compressions' } } };
569     $self->{ 'base' } = $base;
570
571     for my $dst ( @{ $self->{ 'destination' }->{ 'local' } } ) {
572         my $type = $dst->{ 'compression' };
573         next if defined $base->{ $type };
574         $base->{ $type } = $dst->{ 'path' };
575     }
576
577     my @unfilled = grep { !defined $base->{ $_ } } keys %{ $base };
578
579     return if 0 == scalar @unfilled;
580     $self->log->log( 'These compression(s) were given only for remote destinations. Usually this is not desired: %s', join( ', ', @unfilled ) );
581
582     $self->prepare_temp_directory();
583     for my $type ( @unfilled ) {
584         my $tmp_dir = File::Spec->catfile( $self->{ 'temp-dir' }, $type );
585         mkpath( $tmp_dir );
586         $base->{ $type } = $tmp_dir;
587     }
588
589     return;
590 }
591
592 =head1 get_list_of_all_necessary_compressions()
593
594 Scans list of destinations, and gathers list of all compressions that have to be made.
595
596 This is to be able to compress file only once even when having multiple destinations that require compressed format.
597
598 =cut
599
600 sub get_list_of_all_necessary_compressions {
601     my $self = shift;
602
603     my %compression = ();
604
605     for my $dst_type ( qw( local remote ) ) {
606         next unless my $dsts = $self->{ 'destination' }->{ $dst_type };
607         for my $destination ( @{ $dsts } ) {
608             $compression{ $destination->{ 'compression' } } = 1;
609         }
610     }
611     $self->{ 'compressions' } = [ keys %compression ];
612     return;
613 }
614
615 =head1 read_args()
616
617 Function which does all the parsing, and transformation of command line arguments.
618
619 =cut
620
621 sub read_args {
622     my $self = shift;
623
624     my @argv_copy = @ARGV;
625
626     my %args = (
627         'temp-dir' => $ENV{ 'TMPDIR' } || '/tmp',
628         'gzip-path'          => 'gzip',
629         'bzip2-path'         => 'bzip2',
630         'lzma-path'          => 'lzma',
631         'tar-path'           => 'tar',
632         'nice-path'          => 'nice',
633         'rsync-path'         => 'rsync',
634         'pgcontroldata-path' => 'pg_controldata',
635         'filename-template'  => '__HOSTNAME__-__FILETYPE__-^Y-^m-^d.tar__CEXT__',
636     );
637
638     croak( 'Error while reading command line arguments. Please check documentation in doc/omnipitr-backup-slave.pod' )
639         unless GetOptions(
640         \%args,
641         'data-dir|D=s',
642         'source|s=s',
643         'dst-local|dl=s@',
644         'dst-remote|dr=s@',
645         'temp-dir|t=s',
646         'log|l=s',
647         'filename-template|f=s',
648         'removal-pause-trigger|p=s',
649         'pid-file',
650         'verbose|v',
651         'gzip-path|gp=s',
652         'bzip2-path|bp=s',
653         'lzma-path|lp=s',
654         'nice-path|np=s',
655         'tar-path|tp=s',
656         'rsync-path|rp=s',
657         'pgcontroldata-path|pp=s',
658         );
659
660     croak( '--log was not provided - cannot continue.' ) unless $args{ 'log' };
661     for my $key ( qw( log filename-template ) ) {
662         $args{ $key } =~ tr/^/%/;
663     }
664
665     for my $key ( grep { !/^dst-(?:local|remote)$/ } keys %args ) {
666         $self->{ $key } = $args{ $key };
667     }
668
669     for my $type ( qw( local remote ) ) {
670         my $D = [];
671         $self->{ 'destination' }->{ $type } = $D;
672
673         next unless defined $args{ 'dst-' . $type };
674
675         my %temp_for_uniq = ();
676         my @items = grep { !$temp_for_uniq{ $_ }++ } @{ $args{ 'dst-' . $type } };
677
678         for my $item ( @items ) {
679             my $current = { 'compression' => 'none', };
680             if ( $item =~ s/\A(gzip|bzip2|lzma)=// ) {
681                 $current->{ 'compression' } = $1;
682             }
683             $current->{ 'path' } = $item;
684             push @{ $D }, $current;
685         }
686     }
687
688     if ( $args{ 'source' } =~ s/\A(gzip|bzip2|lzma)=// ) {
689         $self->{ 'source' } = {
690             'compression' => $1,
691             'path'        => $args{ 'source' },
692         };
693     }
694     else {
695         $self->{ 'source' } = {
696             'compression' => 'none',
697             'path'        => $args{ 'source' },
698         };
699     }
700
701     $self->{ 'filename-template' } = strftime( $self->{ 'filename-template' }, localtime time() );
702     $self->{ 'filename-template' } =~ s/__HOSTNAME__/hostname()/ge;
703
704     # We do it here so it will actually work for reporing problems in validation
705     $self->{ 'log_template' } = $args{ 'log' };
706     $self->{ 'log' }          = OmniPITR::Log->new( $self->{ 'log_template' } );
707
708     $self->log->log( 'Called with parameters: %s', join( ' ', @argv_copy ) ) if $self->verbose;
709
710     return;
711 }
712
713 =head1 validate_args()
714
715 Does all necessary validation of given command line arguments.
716
717 One exception is for compression programs paths - technically, it could be validated in here, but benefit would be pretty limited, and code to do so relatively complex, as compression program path
718 might, but doesn't have to be actual file path - it might be just program name (without path), which is the default.
719
720 =cut
721
722 sub validate_args {
723     my $self = shift;
724
725     $self->log->fatal( 'Data-dir was not provided!' ) unless defined $self->{ 'data-dir' };
726     $self->log->fatal( 'Provided data-dir (%s) does not exist!',   $self->{ 'data-dir' } ) unless -e $self->{ 'data-dir' };
727     $self->log->fatal( 'Provided data-dir (%s) is not directory!', $self->{ 'data-dir' } ) unless -d $self->{ 'data-dir' };
728     $self->log->fatal( 'Provided data-dir (%s) is not readable!',  $self->{ 'data-dir' } ) unless -r $self->{ 'data-dir' };
729
730     my $dst_count = scalar( @{ $self->{ 'destination' }->{ 'local' } } ) + scalar( @{ $self->{ 'destination' }->{ 'remote' } } );
731     $self->log->fatal( "No --dst-* has been provided!" ) if 0 == $dst_count;
732
733     $self->log->fatal( "Filename template does not contain __FILETYPE__ placeholder!" ) unless $self->{ 'filename-template' } =~ /__FILETYPE__/;
734     $self->log->fatal( "Filename template cannot contain / or \\ characters!" ) if $self->{ 'filename-template' } =~ m{[/\\]};
735
736     $self->log->fatal( 'Source of WAL files was not provided!' ) unless defined $self->{ 'source' };
737     $self->log->fatal( 'Provided source of wal files (%s) does not exist!',   $self->{ 'source' }->{ 'path' } ) unless -e $self->{ 'source' }->{ 'path' };
738     $self->log->fatal( 'Provided source of wal files (%s) is not directory!', $self->{ 'source' }->{ 'path' } ) unless -d $self->{ 'source' }->{ 'path' };
739     $self->log->fatal( 'Provided source of wal files (%s) is not readable!',  $self->{ 'source' }->{ 'path' } ) unless -r $self->{ 'source' }->{ 'path' };
740
741     $self->log->fatal( 'Temp-dir was not provided!' ) unless defined $self->{ 'temp-dir' };
742     $self->log->fatal( 'Provided temp-dir (%s) does not exist!',   $self->{ 'temp-dir' } ) unless -e $self->{ 'temp-dir' };
743     $self->log->fatal( 'Provided temp-dir (%s) is not directory!', $self->{ 'temp-dir' } ) unless -d $self->{ 'temp-dir' };
744     $self->log->fatal( 'Provided temp-dir (%s) is not writable!',  $self->{ 'temp-dir' } ) unless -w $self->{ 'temp-dir' };
745     $self->log->fatal( 'Provided temp-dir (%s) contains # character!', $self->{ 'temp-dir' } ) if $self->{ 'temp-dir' } =~ /#/;
746
747     $self->log->fatal( 'Removal pause trigger name was not provided!' ) unless defined $self->{ 'removal-pause-trigger' };
748     $self->log->fatal( 'Provided removal pause trigger file (%s) already exists!', $self->{ 'removal-pause-trigger' } ) if -e $self->{ 'removal-pause-trigger' };
749
750     $self->log->fatal( 'Directory for provided removal pause trigger (%s) does not exist!',   $self->{ 'removal-pause-trigger' } ) unless -e dirname( $self->{ 'removal-pause-trigger' } );
751     $self->log->fatal( 'Directory for provided removal pause trigger (%s) is not directory!', $self->{ 'removal-pause-trigger' } ) unless -d dirname( $self->{ 'removal-pause-trigger' } );
752     $self->log->fatal( 'Directory for provided removal pause trigger (%s) is not writable!',  $self->{ 'removal-pause-trigger' } ) unless -w dirname( $self->{ 'removal-pause-trigger' } );
753
754     return unless $self->{ 'destination' }->{ 'local' };
755
756     for my $d ( @{ $self->{ 'destination' }->{ 'local' } } ) {
757         my $dir = $d->{ 'path' };
758         $self->log->fatal( 'Choosen local destination dir (%s) does not exist. Cannot continue.',   $dir ) unless -e $dir;
759         $self->log->fatal( 'Choosen local destination dir (%s) is not directory. Cannot continue.', $dir ) unless -d $dir;
760         $self->log->fatal( 'Choosen local destination dir (%s) is not writable. Cannot continue.',  $dir ) unless -w $dir;
761     }
762
763     return;
764 }
765
766 1;
Note: See TracBrowser for help on using the browser.