root/trunk/omnipitr/lib/OmniPITR/Program/Restore.pm

Revision 104, 13.6 kB (checked in by depesz, 4 years ago)

fix copy-paste mistake, add executable property

Line 
1 package OmniPITR::Program::Restore;
2 use strict;
3 use warnings;
4
5 use base qw( OmniPITR::Program );
6
7 use Carp;
8 use OmniPITR::Tools qw( :all );
9 use English qw( -no_match_vars );
10 use File::Basename;
11 use File::Spec;
12 use File::Path qw( make_path remove_tree );
13 use File::Copy;
14 use Storable;
15 use Getopt::Long;
16 use Cwd;
17
18 =head1 run()
19
20 Main function, called by actual script in bin/, wraps all work done by script with the sole exception of reading and validating command line arguments.
21
22 These tasks (reading and validating arguments) are in this module, but they are called from L<OmniPITR::Program::new()>
23
24 Name of called method should be self explanatory, and if you need further information - simply check doc for the method you have questions about.
25
26 =cut
27
28 sub run {
29     my $self = shift;
30
31     $SIG{ 'USR1' } = sub {
32         $self->{ 'finish' } = 'immediate';
33         return;
34     };
35
36     while ( 1 ) {
37         $self->try_to_restore_and_exit();
38         sleep 1;
39         next if $self->{ 'finish' };
40         $self->check_for_trigger_file();
41         next if $self->{ 'finish' };
42         $self->do_some_removal();
43     }
44 }
45
46 sub do_some_removal {
47     my $self = shift;
48
49     return unless $self->{ 'remove-unneeded' };
50
51     return if $self->{ 'removal-pause-trigger' } && -e $self->{ 'removal-pause-trigger' };
52
53     my $control_data = $self->get_control_data();
54     return unless $control_data;
55
56     my $last_important = $self->get_last_redo_segment( $control_data );
57     return unless $last_important;
58
59     my @to_be_removed = $self->get_list_of_segments_to_remove( $last_important );
60     return if 0 == scalar @to_be_removed;
61
62     for my $segment_name ( @to_be_removed ) {
63         return unless $self->handle_pre_removal_processing( $segment_name );
64
65         my $segment_file_name = File::Spec->catfile( $self->{ 'source' }->{ 'path' }, $segment_name );
66         $segment_file_name .= ext_for_compression( $self->{ 'source' }->{ 'compression' } ) if $self->{ 'source' }->{ 'compression' };
67         my $result = unlink $segment_file_name;
68         unless ( 1 == $result ) {
69             $self->log->error( 'Error while unlinking %s : %s', $segment_file_name, $OS_ERROR );
70             return;
71         }
72         $self->log->log( 'Segment %s (%s) removed, as it is too old (older than %s)', $segment_name, $segment_file_name, $last_important );
73     }
74     return;
75 }
76
77 sub handle_pre_removal_processing {
78     my $self         = shift;
79     my $segment_name = shift;
80     return unless $self->{ 'pre-removal-processing' };
81
82     $self->prepare_temp_directory();
83     my $xlog_dir  = File::Spec->catfile( $self->{ 'temp-dir' }, 'pg_xlog' );
84     my $xlog_file = File::Spec->catfile( $xlog_dir,             $segment_name );
85     make_path( $xlog_dir );
86
87     my $response = $self->copy_segment_to( $segment_name, $xlog_file );
88     if ( $response ) {
89         $self->log->error( 'Error while copying segment for pre removal processing for %s : %s', $segment_name, $response );
90         return;
91     }
92
93     my $previous_dir = gwtcwd();
94     chdir $self->{ 'temp-dir' };
95     my $full_command = $self->{ 'pre-removal-processing' } . " pg_xlog/$segment_name";
96     my $result = run_command( $self->{ 'tempdir' }, 'bash', '-c', $full_command );
97     chdir $previous_dir;
98
99     remove_tree( $xlog_dir );
100     return 1 unless $result->{ 'error_code' };
101
102     $self->log->error( 'Error while calling pre removal processing [%s] : %s', $full_command, Dumper( $result ) );
103
104     return;
105 }
106
107 sub get_list_of_segments_to_remove {
108     my $self           = shift;
109     my $last_important = shift;
110
111     my $extension = ext_for_compression( $self->{ 'source' }->{ 'compression' } ) if $self->{ 'source' }->{ 'compression' };
112     my $dir;
113
114     unless ( opendir( $dir, $self->{ 'source' } ) ) {
115         $self->log->error( 'Cannot open source directory (%s) for reading: %s', $self->{ 'source' }->{ 'path' }, $OS_ERROR );
116         return;
117     }
118     my @content = readdir $dir;
119     closedir $dir;
120
121     my @too_old = ();
122     for my $file ( @content ) {
123         $file =~ s/\Q$extension\E\z//;
124         next unless $file =~ m{\A[a-f0-9]{24}\z};
125         next unless $file lt $last_important;
126         push @too_old, $file;
127     }
128     return if 0 == scalar @too_old;
129
130     my @sorted = sort @too_old;
131     splice( @sorted, $self->{ 'remove-at-a-time' } );
132
133     return @sorted;
134 }
135
136 sub get_last_redo_segment {
137     my $self = shift;
138     my $CD   = shift;
139
140     my $segment  = $CD->{ "Latest checkpoint's REDO location" };
141     my $timeline = $CD->{ "Latest checkpoint's TimeLineID" };
142
143     my ( $series, $offset ) = split m{/}, $segment;
144
145     $offset =~ s/.{6}$//;
146
147     my $segment_filename = sprintf '%08s%08s%08s', $timeline, $series, $offset;
148
149     return $segment_filename;
150 }
151
152 sub get_control_data {
153     my $self = shift;
154
155     $self->prepare_temp_directory();
156
157     my $response = run_command( $self->{ 'temp-dir' }, $self->{ 'pgcontroldata-path' }, $self->{ 'data-dir' } );
158     if ( $response->{ 'error_code' } ) {
159         $self->log->error( 'Error while getting pg_controldata for %s: %s', $self->{ 'data-dir' }, Dumper( $response ) );
160         return;
161     }
162
163     my $control_data = {};
164
165     my @lines = split( /\s*\n/, $response->{ 'stdout' } );
166     for my $line ( @lines ) {
167         unless ( $line =~ m{\A([^:]+):\s*(.*)\z} ) {
168             $self->log->error( 'Pg_controldata for %s contained unparseable line: [%s]', $self->{ 'data-dir' }, $line );
169             $self->exit_with_status( 1 );
170         }
171         $control_data->{ $1 } = $2;
172     }
173
174     unless ( $control_data->{ "Latest checkpoint's REDO location" } ) {
175         $self->log->error( 'Pg_controldata for %s did not contain latest checkpoint redo location', $self->{ 'data-dir' } );
176         return;
177     }
178     unless ( $control_data->{ "Latest checkpoint's TimeLineID" } ) {
179         $self->log->error( 'Pg_controldata for %s did not contain latest checkpoint timeline ID', $self->{ 'data-dir' } );
180         return;
181     }
182
183     return $control_data;
184 }
185
186 sub try_to_restore_and_exit {
187     my $self = shift;
188
189     if ( $self->{ 'finish' } eq 'immediate' ) {
190         $self->log->error( 'Got immediate finish request. Dying.' );
191         $self->exit_with_status( 1 );
192     }
193
194     my $wanted_file = File::Spec->catfile( $self->{ 'source' }->{ 'path' }, $self->{ 'segment' } );
195     $wanted_file .= ext_for_compression( $self->{ 'source' }->{ 'compression' } ) if $self->{ 'source' }->{ 'compression' };
196
197     unless ( -e $wanted_file ) {
198         if ( $self->{ 'finish' } ) {
199             $self->log->error( 'Got finish request. Dying.' );
200             $self->exit_with_status( 1 );
201         }
202     }
203
204     if (   ( $self->{ 'recovery-delay' } )
205         && ( !$self->{ 'finish' } ) )
206     {
207         my @file_info  = stat( $wanted_file );
208         my $file_mtime = $file_info[ 9 ];
209         my $ok_since   = time - $self->{ 'recovery-delay' };
210         if ( $ok_since > $file_mtime ) {
211             if ( $self->{ 'verbose' } ) {
212                 unless ( $self->{ 'logged_delay' } ) {
213                     $self->log->log( 'Segment %s found, but it is too fresh (mtime = %u, accepted since %u)', $self->{ 'segment' }, $file_mtime, $ok_since );
214                     $self->{ 'logged_delay' } = 1;
215                 }
216             }
217             return;
218         }
219     }
220
221     my $full_destination = File::Spec->catfile( $self->{ 'data-dir' }, $self->{ 'segment_destination' } );
222
223     my $response = $self->copy_segment_to( $self->{ 'segment' }, $full_destination );
224
225     if ( $response ) {
226         $self->log->error( $response );
227         $self->exit_with_status( 1 );
228     }
229
230     $self->log->log( 'Segment %s restored', $self->{ 'segment' } );
231     $self->exit_with_status( 0 );
232 }
233
234 sub copy_segment_to {
235     my $self = shift;
236     my ( $segment_name, $destination ) = @_;
237
238     my $wanted_file = File::Spec->catfile( $self->{ 'source' }->{ 'path' }, $segment_name );
239     $wanted_file .= ext_for_compression( $self->{ 'source' }->{ 'compression' } ) if $self->{ 'source' }->{ 'compression' };
240
241     unless ( $self->{ 'source' }->{ 'compression' } ) {
242         if ( copy( $wanted_file, $destination ) ) {
243             return;
244         }
245         return sprintf( 'Copying %s to %s failed: %s', $wanted_file, $destination, $OS_ERROR );
246     }
247
248     my $compression = $self->{ 'source' }->{ 'compression' };
249     my $command = sprintf '%s --decompress --stdout %s > %s', quotemeta( $self->{ "$compression-path" } ), quotemeta( $wanted_file ), quotemeta( $destination );
250
251     $self->prepare_temp_directory();
252
253     my $response = run_command( $self->{ 'temp-dir' }, 'bash', '-c', $command );
254
255     return sprintf( 'Uncompressing %s to %s failed: %s', $wanted_file, $destination, Dumper( $response ) ) if $response->{ 'error_code' };
256     return;
257 }
258
259 sub check_for_trigger_file {
260     my $self = shift;
261
262     return unless $self->{ 'finish-trigger' };
263     return unless -e $self->{ 'finish-trigger' };
264
265     if ( open my $fh, '<', $self->{ 'finish-trigger' } ) {
266         local $INPUT_RECORD_SEPARATOR = undef;
267         my $content = <$fh>;
268         close $fh;
269
270         $self->{ 'finish' } = $content =~ m{\ANOW\n?\z} ? 'immediate' : 'smart';
271
272         $self->log->log( 'Finish trigger found, %s mode.', $self->{ 'finish' } );
273         return;
274     }
275     $self->log->fatal( 'Finish trigger (%s) exists, but cannot be open?! : %s', $self->{ 'finish-trigger' }, $OS_ERROR );
276 }
277
278 =head1 exit_with_status()
279
280 Exit function, doing cleanup (remove temp-dir), and exiting with given status.
281
282 =cut
283
284 sub exit_with_status {
285     my $self = shift;
286     my $code = shift;
287
288     remove_tree( $self->{ 'temp-dir' } ) if $self->{ 'temp-dir-prepared' };
289
290     exit( $code );
291 }
292
293 =head1 prepare_temp_directory()
294
295 Helper function, which builds path for temp directory, and creates it.
296
297 Path is generated by using given temp-dir and 'omnipitr-restore' name.
298
299 For example, for temp-dir '/tmp', actual, used temp directory would be /tmp/omnipitr-restore/.
300
301 =cut
302
303 sub prepare_temp_directory {
304     my $self = shift;
305     return if $self->{ 'temp-dir-prepared' };
306     my $full_temp_dir = File::Spec->catfile( $self->{ 'temp-dir' }, basename( $PROGRAM_NAME ) );
307     make_path( $full_temp_dir );
308     $self->{ 'temp-dir' }          = $full_temp_dir;
309     $self->{ 'temp-dir-prepared' } = 1;
310     return;
311 }
312
313 =head1 read_args()
314
315 Function which does all the parsing, and transformation of command line arguments.
316
317 It also verified base facts about passed WAL segment name, but all other validations, are being done in separate function: L<validate_args()>.
318
319 =cut
320
321 sub read_args {
322     my $self = shift;
323
324     my @argv_copy = @ARGV;
325
326     my %args = (
327         'bzip2-path'         => 'bzip2',
328         'data-dir'           => '.',
329         'gzip-path'          => 'gzip',
330         'lzma-path'          => 'lzma',
331         'pgcontroldata-path' => 'pg_controldata',
332         'remove-at-a-time'   => 3,
333         'temp-dir'           => $ENV{ 'TMPDIR' } || '/tmp',
334     );
335
336     croak( 'Error while reading command line arguments. Please check documentation in doc/omnipitr-archive.pod' )
337         unless GetOptions(
338         \%args,
339         'bzip2-path|bp=s',
340         'data-dir|D=s',
341         'finish-trigger|f=s',
342         'gzip-path|gp=s',
343         'log|l=s',
344         'lzma-path|lp=s',
345         'pgcontroldata-path|pp=s',
346         'pid-file=s',
347         'pre-removal-processing|h=s',
348         'remove-at-a-time|rt=i',
349         'recovery-delay|w=i',
350         'removal-pause-trigger|p=s',
351         'remove-unneeded|r=s',
352         'source|s=s',
353         'temp-dir|t=s',
354         'verbose|v',
355         );
356
357     croak( '--log was not provided - cannot continue.' ) unless $args{ 'log' };
358     $args{ 'log' } =~ tr/^/%/;
359
360     for my $key ( keys %args ) {
361         next if $key =~ m{ \A (?: source | log ) \z }x;    # Skip those, not needed in $self
362         $self->{ $key } = $args{ $key };
363     }
364
365     # We do it here so it will actually work for reporing problems in validation
366     $self->{ 'log_template' } = $args{ 'log' };
367     $self->{ 'log' }          = OmniPITR::Log->new( $self->{ 'log_template' } );
368
369     $self->log->fatal( 'Source path not provided!' ) unless $args{ 'source' };
370
371     if ( $args{ 'source' } =~ s/\A(gzip|bzip2|lzma)=// ) {
372         $self->{ 'source' }->{ 'compression' } = $1;
373     }
374     $self->{ 'source' }->{ 'path' } = $args{ 'source' };
375
376     # These could theoretically go into validation, but we need to check if we can get anything to put in segment* keys in $self
377     $self->log->fatal( 'WAL segment file name and/or destination have not been given' ) if 2 > scalar @ARGV;
378     $self->log->fatal( 'Too many arguments given.' ) if 2 < scalar @ARGV;
379
380     @{ $self }{ qw( segment segment_destination ) } = @ARGV;
381
382     $self->log->log( 'Called with parameters: %s', join( ' ', @argv_copy ) ) if $self->{ 'verbose' };
383
384     $self->{ 'finish' } = '';
385
386     return;
387 }
388
389 =head1 validate_args()
390
391 Does all necessary validation of given command line arguments.
392
393 One exception is for compression programs paths - technically, it could be validated in here, but benefit would be pretty limited, and code to do so relatively complex, as compression program path
394 might, but doesn't have to be actual file path - it might be just program name (without path), which is the default.
395
396 =cut
397
398 sub validate_args {
399     my $self = shift;
400
401     $self->log->fatal( 'Given data-dir (%s) is not valid', $self->{ 'data-dir' } ) unless -d $self->{ 'data-dir' } && -f File::Spec->catfile( $self->{ 'data-dir' }, 'PG_VERSION' );
402
403     $self->log->fatal( 'Given segment name is not valid (%s)', $self->{ 'segment' } ) unless $self->{ 'segment' } =~ m{\A[a-f0-9]{24}\z};
404
405     $self->log->fatal( 'Given source (%s) is not a directory', $self->{ 'source' }->{ 'path' } ) unless -d $self->{ 'source' }->{ 'path' };
406     $self->log->fatal( 'Given source (%s) is not readable',    $self->{ 'source' }->{ 'path' } ) unless -r $self->{ 'source' }->{ 'path' };
407     $self->log->fatal( 'Given source (%s) is not writable',    $self->{ 'source' }->{ 'path' } ) unless -w $self->{ 'source' }->{ 'path' };
408
409     return;
410 }
411
412 1;
Note: See TracBrowser for help on using the browser.