root/trunk/omnipitr/lib/OmniPITR/Program/Backup/Master.pm

Revision 186, 13.7 kB (checked in by depesz, 4 years ago)

- fix problem with trailing / in datadir.

if datadir was provided with / at the end, it caused problem with finding out basename of datadir, which caused very misleading problem when making temporary directories.

Line 
1 package OmniPITR::Program::Backup::Master;
2 use strict;
3 use warnings;
4
5 use base qw( OmniPITR::Program::Backup );
6
7 use Carp;
8 use OmniPITR::Tools qw( run_command );
9 use English qw( -no_match_vars );
10 use File::Basename;
11 use Sys::Hostname;
12 use POSIX qw( strftime );
13 use File::Spec;
14 use File::Path qw( mkpath rmtree );
15 use File::Copy;
16 use Storable;
17 use Getopt::Long qw( :config no_ignore_case );
18
19 =head1 make_data_archive()
20
21 Wraps all work necessary to make local .tar files (optionally compressed)
22 with content of PGDATA
23
24 =cut
25
26 sub make_data_archive {
27     my $self = shift;
28     $self->start_pg_backup();
29     $self->compress_pgdata();
30     $self->stop_pg_backup();
31     return;
32 }
33
34 =head1 make_xlog_archive()
35
36 Wraps all work necessary to make local .tar files (optionally compressed)
37 with xlogs required to start PostgreSQL from backup.
38
39 =cut
40
41 sub make_xlog_archive {
42     my $self = shift;
43     $self->wait_for_final_xlog_and_remove_dst_backup();
44     $self->compress_xlogs();
45     return;
46 }
47
48 =head1 wait_for_file()
49
50 Helper function which waits for file to appear.
51
52 It will return only if the file appeared.
53
54 Return value is name of file.
55
56 =cut
57
58 sub wait_for_file {
59     my $self = shift;
60     my ( $dir, $filename_regexp ) = @_;
61
62     my $max_wait = 3600;    # It's 1 hour. There is no technical need to wait longer.
63     for my $i ( 0 .. $max_wait ) {
64         $self->log->log( 'Waiting for file matching %s in directory %s', $filename_regexp, $dir ) if 10 == $i;
65
66         opendir( my $dh, $dir ) or $self->log->fatal( 'Cannot open %s for scanning: %s', $dir, $OS_ERROR );
67         my @matching = grep { $_ =~ $filename_regexp } readdir $dh;
68         closedir $dh;
69
70         if ( 0 == scalar @matching ) {
71             sleep 1;
72             next;
73         }
74
75         my $reply_filename = shift @matching;
76         $self->log->log( 'File %s arrived after %u seconds.', $reply_filename, $i ) if $self->verbose;
77         return $reply_filename;
78     }
79
80     $self->log->fatal( 'Waited 1 hour for file matching %s, but it did not appear. Something is wrong. No sense in waiting longer.', $filename_regexp );
81
82     return;
83 }
84
85 =head1 wait_for_final_xlog_and_remove_dst_backup()
86
87 In PostgreSQL < 8.4 pg_stop_backup() finishes before .backup "wal segment"
88 is archived.
89
90 So we need to wait till it appears in backup xlog destination before we can
91 remove symlink.
92
93 =cut
94
95 sub wait_for_final_xlog_and_remove_dst_backup {
96     my $self = shift;
97
98     my $backup_file = $self->wait_for_file( $self->{ 'xlogs' }, $self->{ 'stop_backup_filename_re' } );
99
100     my $last_file = undef;
101
102     open my $fh, '<', File::Spec->catfile( $self->{ 'xlogs' }, $backup_file ) or $self->log->fatal( 'Cannot open backup file %s for reading: %s', $backup_file, $OS_ERROR );
103     while ( my $line = <$fh> ) {
104         next unless $line =~ m{\A STOP \s+ WAL \s+ LOCATION: .* file \s+ ( [0-9A-f]{24} ) }x;
105         $last_file = qr{\A$1\z};
106         last;
107     }
108     close $fh;
109
110     $self->log->fatal( '.backup file (%s) does not contain STOP WAL LOCATION line in recognizable format.', $backup_file ) unless $last_file;
111
112     $self->wait_for_file( $self->{ 'xlogs' }, $last_file );
113
114     unlink( $self->{ 'xlogs' } );
115 }
116
117 =head1 compress_xlogs()
118
119 Wrapper function which encapsulates all work required to compress xlog
120 segments that accumulated during backup of data directory.
121
122 =cut
123
124 sub compress_xlogs {
125     my $self = shift;
126     $self->log->time_start( 'Compressing xlogs' ) if $self->verbose;
127     $self->start_writers( 'xlog' );
128
129     $self->tar_and_compress(
130         'work_dir' => $self->{ 'xlogs' } . '.real',
131         'tar_dir'  => [ basename( $self->{ 'data-dir' } ) ],
132     );
133     $self->log->time_finish( 'Compressing xlogs' ) if $self->verbose;
134     rmtree( $self->{ 'xlogs' } . '.real', 0 );
135
136     return;
137 }
138
139 =head1 compress_pgdata()
140
141 Wrapper function which encapsulates all work required to compress data
142 directory.
143
144 =cut
145
146 sub compress_pgdata {
147     my $self = shift;
148     $self->log->time_start( 'Compressing $PGDATA' ) if $self->verbose;
149     $self->start_writers( 'data' );
150
151     my @excludes = qw( pg_log/* pg_xlog/0* pg_xlog/archive_status/* postmaster.pid );
152     for my $dir ( qw( pg_log pg_xlog ) ) {
153         push @excludes, $dir if -l File::Spec->catfile( $self->{ 'data-dir' }, $dir );
154     }
155
156     $self->tar_and_compress(
157         'work_dir' => dirname( $self->{ 'data-dir' } ),
158         'tar_dir'  => [ basename( $self->{ 'data-dir' } ) ],
159         'excludes' => \@excludes,
160     );
161
162     $self->log->time_finish( 'Compressing $PGDATA' ) if $self->verbose;
163     return;
164 }
165
166 =head1 stop_pg_backup()
167
168 Runs pg_stop_backup() PostgreSQL function, which is crucial in backup
169 process.
170
171 This happens after data directory compression, but before compression of
172 xlogs.
173
174 This function also removes temporary destination for xlogs (dst-backup for
175 omnipitr-archive).
176
177 =cut
178
179 sub stop_pg_backup {
180     my $self = shift;
181
182     $self->prepare_temp_directory();
183
184     my @command = ( @{ $self->{ 'psql' } }, "SELECT pg_stop_backup()" );
185
186     $self->log->time_start( 'pg_stop_backup()' ) if $self->verbose && $self->log;
187     my $status = run_command( $self->{ 'temp-dir' }, @command );
188     $self->log->time_finish( 'pg_stop_backup()' ) if $self->verbose && $self->log;
189
190     $self->log->fatal( 'Running pg_stop_backup() failed: %s', $status ) if $status->{ 'error_code' };
191
192     $status->{ 'stdout' } =~ s/\s*\z//;
193     $self->log->log( q{pg_stop_backup('omnipitr') returned %s.}, $status->{ 'stdout' } );
194
195     my $subdir = basename( $self->{ 'data-dir' } );
196     delete $self->{ 'pg_start_backup_done' };
197
198     return;
199 }
200
201 =head1 start_pg_backup()
202
203 Executes pg_start_backup() postgresql function, and (before it) creates
204 temporary destination for xlogs (dst-backup for omnipitr-archive).
205
206 =cut
207
208 sub start_pg_backup {
209     my $self = shift;
210
211     my $subdir = basename( $self->{ 'data-dir' } );
212     $self->log->fatal( 'Cannot create directory %s : %s', $self->{ 'xlogs' } . '.real',                 $OS_ERROR ) unless mkdir( $self->{ 'xlogs' } . '.real' );
213     $self->log->fatal( 'Cannot create directory %s : %s', $self->{ 'xlogs' } . ".real/$subdir",         $OS_ERROR ) unless mkdir( $self->{ 'xlogs' } . ".real/$subdir" );
214     $self->log->fatal( 'Cannot create directory %s : %s', $self->{ 'xlogs' } . ".real/$subdir/pg_xlog", $OS_ERROR ) unless mkdir( $self->{ 'xlogs' } . ".real/$subdir/pg_xlog" );
215     $self->log->fatal( 'Cannot symlink %s to %s: %s', $self->{ 'xlogs' } . ".real/$subdir/pg_xlog", $self->{ 'xlogs' }, $OS_ERROR )
216         unless symlink( $self->{ 'xlogs' } . ".real/$subdir/pg_xlog", $self->{ 'xlogs' } );
217
218     $self->prepare_temp_directory();
219
220     my @command = ( @{ $self->{ 'psql' } }, "SELECT pg_start_backup('omnipitr')" );
221
222     $self->log->time_start( 'pg_start_backup()' ) if $self->verbose;
223     my $status = run_command( $self->{ 'temp-dir' }, @command );
224     $self->log->time_finish( 'pg_start_backup()' ) if $self->verbose;
225
226     $self->log->fatal( 'Running pg_start_backup() failed: %s', $status ) if $status->{ 'error_code' };
227
228     $status->{ 'stdout' } =~ s/\s*\z//;
229     $self->log->log( q{pg_start_backup('omnipitr') returned %s.}, $status->{ 'stdout' } );
230     $self->log->fatal( 'Ouput from pg_start_backup is not parseable?!' ) unless $status->{ 'stdout' } =~ m{\A([0-9A-F]+)/([0-9A-F]{1,8})\z};
231
232     my ( $part_1, $part_2 ) = ( $1, $2 );
233     $part_2 =~ s/(.{1,6})\z//;
234     my $part_3 = $1;
235
236     my $expected_filename_suffix = sprintf '%08s%08s.%08s.backup', $part_1, $part_2, $part_3;
237     my $backup_filename_re = qr{\A[0-9A-F]{8}\Q$expected_filename_suffix\E\z};
238
239     $self->{ 'stop_backup_filename_re' } = $backup_filename_re;
240     delete $self->{ 'pg_start_backup_done' };
241
242     return;
243 }
244
245 =head1 DESTROY()
246
247 Destructor for object - removes created destination for omnipitr-archive,
248 and issues pg_stop_backup() to database.
249
250 =cut
251
252 sub DESTROY {
253     my $self = shift;
254     rmtree( [ $self->{ 'xlogs' } . '.real', $self->{ 'xlogs' } ], 0, );
255     $self->stop_pg_backup() if $self->{ 'pg_start_backup_done' };
256     $self->SUPER::DESTROY();
257     return;
258 }
259
260 =head1 read_args()
261
262 Function which does all the parsing, and transformation of command line
263 arguments.
264
265 =cut
266
267 sub read_args {
268     my $self = shift;
269
270     my @argv_copy = @ARGV;
271
272     my %args = (
273         'temp-dir' => $ENV{ 'TMPDIR' } || '/tmp',
274         'gzip-path'         => 'gzip',
275         'bzip2-path'        => 'bzip2',
276         'lzma-path'         => 'lzma',
277         'tar-path'          => 'tar',
278         'nice-path'         => 'nice',
279         'psql-path'         => 'psql',
280         'rsync-path'        => 'rsync',
281         'database'          => 'postgres',
282         'filename-template' => '__HOSTNAME__-__FILETYPE__-^Y-^m-^d.tar__CEXT__',
283     );
284
285     croak( 'Error while reading command line arguments. Please check documentation in doc/omnipitr-backup-master.pod' )
286         unless GetOptions(
287         \%args,
288         'data-dir|D=s',
289         'database|d=s',
290         'host|h=s',
291         'port|p=i',
292         'username|U=s',
293         'xlogs|x=s',
294         'dst-local|dl=s@',
295         'dst-remote|dr=s@',
296         'temp-dir|t=s',
297         'log|l=s',
298         'filename-template|f=s',
299         'pid-file',
300         'verbose|v',
301         'gzip-path|gp=s',
302         'bzip2-path|bp=s',
303         'lzma-path|lp=s',
304         'nice-path|np=s',
305         'psql-path|pp=s',
306         'tar-path|tp=s',
307         'rsync-path|rp=s',
308         'not-nice|nn',
309         );
310
311     croak( '--log was not provided - cannot continue.' ) unless $args{ 'log' };
312     for my $key ( qw( log filename-template ) ) {
313         $args{ $key } =~ tr/^/%/;
314     }
315
316     for my $key ( grep { !/^dst-(?:local|remote)$/ } keys %args ) {
317         $self->{ $key } = $args{ $key };
318     }
319
320     for my $type ( qw( local remote ) ) {
321         my $D = [];
322         $self->{ 'destination' }->{ $type } = $D;
323
324         next unless defined $args{ 'dst-' . $type };
325
326         my %temp_for_uniq = ();
327         my @items = grep { !$temp_for_uniq{ $_ }++ } @{ $args{ 'dst-' . $type } };
328
329         for my $item ( @items ) {
330             my $current = { 'compression' => 'none', };
331             if ( $item =~ s/\A(gzip|bzip2|lzma)=// ) {
332                 $current->{ 'compression' } = $1;
333             }
334             $current->{ 'path' } = $item;
335             push @{ $D }, $current;
336         }
337     }
338
339     $self->{ 'filename-template' } = strftime( $self->{ 'filename-template' }, localtime time() );
340     $self->{ 'filename-template' } =~ s/__HOSTNAME__/hostname()/ge;
341
342     # We do it here so it will actually work for reporing problems in validation
343     $self->{ 'log_template' } = $args{ 'log' };
344     $self->{ 'log' }          = OmniPITR::Log->new( $self->{ 'log_template' } );
345
346     $self->log->log( 'Called with parameters: %s', join( ' ', @argv_copy ) ) if $self->verbose;
347
348     my @psql = ();
349     push @psql, $self->{ 'psql-path' };
350     push @psql, '-qAtX';
351     push @psql, ( '-U', $self->{ 'username' } ) if $self->{ 'username' };
352     push @psql, ( '-d', $self->{ 'database' } ) if $self->{ 'database' };
353     push @psql, ( '-h', $self->{ 'host' } )     if $self->{ 'host' };
354     push @psql, ( '-p', $self->{ 'port' } )     if $self->{ 'port' };
355     push @psql, '-c';
356     $self->{ 'psql' } = \@psql;
357
358     return;
359 }
360
361 =head1 validate_args()
362
363 Does all necessary validation of given command line arguments.
364
365 One exception is for compression programs paths - technically, it could be
366 validated in here, but benefit would be pretty limited, and code to do so
367 relatively complex, as compression program path might, but doesn't have to
368 be actual file path - it might be just program name (without path), which is
369 the default.
370
371 =cut
372
373 sub validate_args {
374     my $self = shift;
375
376     $self->log->fatal( 'Data-dir was not provided!' ) unless defined $self->{ 'data-dir' };
377     $self->{'data-dir'} =~ s{/+$}{};
378     $self->log->fatal( 'Provided data-dir (%s) does not exist!',   $self->{ 'data-dir' } ) unless -e $self->{ 'data-dir' };
379     $self->log->fatal( 'Provided data-dir (%s) is not directory!', $self->{ 'data-dir' } ) unless -d $self->{ 'data-dir' };
380     $self->log->fatal( 'Provided data-dir (%s) is not readable!',  $self->{ 'data-dir' } ) unless -r $self->{ 'data-dir' };
381
382     my $dst_count = scalar( @{ $self->{ 'destination' }->{ 'local' } } ) + scalar( @{ $self->{ 'destination' }->{ 'remote' } } );
383     $self->log->fatal( "No --dst-* has been provided!" ) if 0 == $dst_count;
384
385     $self->log->fatal( "Filename template does not contain __FILETYPE__ placeholder!" ) unless $self->{ 'filename-template' } =~ /__FILETYPE__/;
386     $self->log->fatal( "Filename template cannot contain / or \\ characters!" ) if $self->{ 'filename-template' } =~ m{[/\\]};
387
388     $self->log->fatal( "Xlogs dir (--xlogs) was not given! Cannot work without it" ) unless defined $self->{ 'xlogs' };
389     $self->{ 'xlogs' } =~ s{/+$}{};
390     $self->log->fatal( "Xlogs dir (%s) already exists! It shouldn't.",           $self->{ 'xlogs' } ) if -e $self->{ 'xlogs' };
391     $self->log->fatal( "Xlogs side dir (%s.real) already exists! It shouldn't.", $self->{ 'xlogs' } ) if -e $self->{ 'xlogs' } . '.real';
392
393     my $xlog_parent = dirname( $self->{ 'xlogs' } );
394     $self->log->fatal( 'Xlogs dir (%s) parent (%s) does not exist. Cannot continue.',   $self->{ 'xlogs' }, $xlog_parent ) unless -e $xlog_parent;
395     $self->log->fatal( 'Xlogs dir (%s) parent (%s) is not directory. Cannot continue.', $self->{ 'xlogs' }, $xlog_parent ) unless -d $xlog_parent;
396     $self->log->fatal( 'Xlogs dir (%s) parent (%s) is not writable. Cannot continue.',  $self->{ 'xlogs' }, $xlog_parent ) unless -w $xlog_parent;
397
398     return unless $self->{ 'destination' }->{ 'local' };
399
400     for my $d ( @{ $self->{ 'destination' }->{ 'local' } } ) {
401         my $dir = $d->{ 'path' };
402         $self->log->fatal( 'Choosen local destination dir (%s) does not exist. Cannot continue.',   $dir ) unless -e $dir;
403         $self->log->fatal( 'Choosen local destination dir (%s) is not directory. Cannot continue.', $dir ) unless -d $dir;
404         $self->log->fatal( 'Choosen local destination dir (%s) is not writable. Cannot continue.',  $dir ) unless -w $dir;
405     }
406
407     return;
408 }
409
410 1;
Note: See TracBrowser for help on using the browser.