Changeset 233

Show
Ignore:
Timestamp:
04/23/11 18:05:36 (3 years ago)
Author:
depesz
Message:

alpha version

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • branches/omnipitr-with-slave-backups-calling-master/doc/omnipitr-backup-slave.pod

    r216 r233  
    1212 
    1313Where PostgreSQL datadir is located (path) 
     14 
     15=item --database (-d) 
     16 
     17Which database to connect to to issue required SQL queries. Defaults to 
     18template1. 
     19 
     20This is used only when --call-master is given. 
     21 
     22=item --host (-h) 
     23 
     24Which host to connect to when connecting to database to backup. Shouldn't really 
     25be changed in 99% of cases. Defaults to empty string - i.e. use UNIX sockets. 
     26 
     27This is used only when --call-master is given. 
     28 
     29=item --port (-P) 
     30 
     31Which port to connect to when connecting to database. Defaults to 5432. 
     32 
     33This is used only when --call-master is given. 
     34 
     35=item --username (-U) 
     36 
     37What username to use when connecting to database. Defaults to postgres. 
     38 
     39This is used only when --call-master is given. 
    1440 
    1541=item --source (-s) 
     
    6692the same as condigured removal-pause-trigger for L<omnipitr-restore>.  
    6793 
     94=item --call-master (-cm) 
     95 
     96If this option is given, L<omnipitr-backup-slave> will issue 
     97 
     98    SELECT pg_start_backup( '...' ); 
     99 
     100and 
     101 
     102    SELECT pg_stop_backup() 
     103 
     104on master database. 
     105 
     106Backups on PostgreSQL 9.0+, on slave, without --call-master can fail when later 
     107used. 
     108 
    68109=item --verbose (-v) 
    69110 
     
    97138 
    98139Full path to tar program - in case you can't set proper PATH environment 
     140variable. 
     141 
     142=item --psql-path (-pp) 
     143 
     144Full path to psql program - in case you can't set proper PATH environment 
    99145variable. 
    100146 
     
    359405(please see note above) 
    360406 
     407=item * If you're using L<omnipitr-backup-slave> on PostgreSQL 9.0+ you have to 
     408use --call-master. Otherwise created backup might fail when used as base for 
     409another replication slave that later on is promoted to standalone. 
     410 
    361411=back 
    362412 
  • branches/omnipitr-with-slave-backups-calling-master/lib/OmniPITR/Program/Backup.pm

    r222 r233  
    4949    my $self = shift; 
    5050    croak( "make_xlog_archive() method in OmniPITR::Program::Backup was not overridden!" ); 
     51} 
     52 
     53=head1 psql() 
     54 
     55Runs given query via psql (assuming options stored in $self->{'psql'}). 
     56 
     57In case of errors, it raises fatal error. 
     58 
     59Otherwise returns stdout of the psql. 
     60 
     61=cut 
     62 
     63sub psql { 
     64    my $self = shift; 
     65 
     66    my $query = shift; 
     67 
     68    $self->prepare_temp_directory(); 
     69 
     70    my @command = ( @{ $self->{ 'psql' } }, $query ); 
     71 
     72    $self->log->time_start( $query ) if $self->verbose; 
     73    my $status = run_command( $self->{ 'temp-dir' }, @command ); 
     74    $self->log->time_finish( $query ) if $self->verbose; 
     75 
     76    $self->log->fatal( 'Running [%s] via psql failed: %s', $query, $status ) if $status->{ 'error_code' }; 
     77 
     78    return $status->{'stdout'}; 
    5179} 
    5280 
  • branches/omnipitr-with-slave-backups-calling-master/lib/OmniPITR/Program/Backup/Master.pm

    r226 r233  
    147147    my $self = shift; 
    148148 
    149     $self->prepare_temp_directory(); 
    150  
    151     my @command = ( @{ $self->{ 'psql' } }, "SELECT pg_stop_backup()" ); 
    152  
    153     $self->log->time_start( 'pg_stop_backup()' ) if $self->verbose && $self->log; 
    154     my $status = run_command( $self->{ 'temp-dir' }, @command ); 
    155     $self->log->time_finish( 'pg_stop_backup()' ) if $self->verbose && $self->log; 
    156  
    157     $self->log->fatal( 'Running pg_stop_backup() failed: %s', $status ) if $status->{ 'error_code' }; 
    158  
    159     $status->{ 'stdout' } =~ s/\s*\z//; 
    160     $self->log->log( q{pg_stop_backup('omnipitr') returned %s.}, $status->{ 'stdout' } ); 
     149    my $stop_backup_output = $self->psql( 'SELECT pg_stop_backup()' ); 
     150 
     151    $stop_backup_output =~ s/\s*\z//; 
     152 
     153    $self->log->log( q{pg_stop_backup('omnipitr') returned %s.}, $stop_backup_output ); 
    161154 
    162155    my $subdir = basename( $self->{ 'data-dir' } ); 
     
    183176        unless symlink( $self->{ 'xlogs' } . ".real/$subdir/pg_xlog", $self->{ 'xlogs' } ); 
    184177 
    185     $self->prepare_temp_directory(); 
    186  
    187     my @command = ( @{ $self->{ 'psql' } }, "SELECT pg_start_backup('omnipitr')" ); 
    188  
    189     $self->log->time_start( 'pg_start_backup()' ) if $self->verbose; 
    190     my $status = run_command( $self->{ 'temp-dir' }, @command ); 
    191     $self->log->time_finish( 'pg_start_backup()' ) if $self->verbose; 
    192  
    193     $self->log->fatal( 'Running pg_start_backup() failed: %s', $status ) if $status->{ 'error_code' }; 
    194  
    195     $status->{ 'stdout' } =~ s/\s*\z//; 
    196     $self->log->log( q{pg_start_backup('omnipitr') returned %s.}, $status->{ 'stdout' } ); 
    197     $self->log->fatal( 'Ouput from pg_start_backup is not parseable?!' ) unless $status->{ 'stdout' } =~ m{\A([0-9A-F]+)/([0-9A-F]{1,8})\z}; 
     178    my $start_backup_output = $self->psql( "SELECT pg_start_backup('omnipitr')" ); 
     179 
     180    $start_backup_output =~ s/\s*\z//; 
     181    $self->log->log( q{pg_start_backup('omnipitr') returned %s.}, $start_backup_output ); 
     182    $self->log->fatal( 'Output from pg_start_backup is not parseable?!' ) unless $start_backup_output =~ m{\A([0-9A-F]+)/([0-9A-F]{1,8})\z}; 
    198183 
    199184    my ( $part_1, $part_2 ) = ( $1, $2 ); 
  • branches/omnipitr-with-slave-backups-calling-master/lib/OmniPITR/Program/Backup/Slave.pm

    r227 r233  
    2626    my $self = shift; 
    2727    $self->pause_xlog_removal(); 
    28     $self->{ 'CONTROL' }->{ 'initial' } = $self->get_control_data(); 
     28    $self->make_backup_label_temp_file(); 
    2929    $self->compress_pgdata(); 
    30     return; 
     30    $self->finish_pgdata_backup(); 
     31    return; 
     32
     33 
     34=head1 finish_pgdata_backup() 
     35 
     36Calls pg_stop_backup on master (if necessary), and waits for xlogs to be 
     37ready 
     38 
     39=cut 
     40 
     41sub finish_pgdata_backup { 
     42    my $self = shift; 
     43    return unless $self->{ 'call-master' }; 
     44 
     45    my $stop_backup_output = $self->psql( 'SELECT pg_stop_backup()' ); 
     46 
     47    $stop_backup_output =~ s/\s*\z//; 
     48    $self->log->log( q{pg_start_backup() returned %s.}, $stop_backup_output ); 
     49    $self->log->fatal( 'Output from pg_stop_backup is not parseable?!' ) unless $stop_backup_output =~ m{\A([0-9A-F]+)/([0-9A-F]{1,8})\z}; 
     50 
     51    my ( $part_1, $part_2 ) = ( $1, $2 ); 
     52    $part_2 =~ s/(.{1,6})\z//; 
     53    my $part_3 = $1; 
     54 
     55    my $expected_filename_suffix = sprintf '%08s%08s.%08s.backup', $part_1, $part_2, $part_3; 
     56 
     57    if ( 'none' ne $self->{ 'source' }->{ 'compression' } ) { 
     58        my $extension = ext_for_compression( $self->{ 'source' }->{ 'compression' } ); 
     59        $expected_filename_suffix .= $extension; 
     60    } 
     61 
     62    my $backup_filename_re = qr{\A[0-9A-F]{8}\Q$expected_filename_suffix\E\z}; 
     63 
     64    $self->{ 'stop_backup_filename_re' } = $backup_filename_re; 
     65 
     66    return; 
     67 
    3168} 
    3269 
     
    4077sub make_xlog_archive { 
    4178    my $self = shift; 
    42     $self->wait_for_checkpoint_location_change(); 
     79    $self->wait_for_xlog_archive_to_be_ready(); 
    4380    $self->compress_xlogs(); 
    4481    $self->unpause_xlog_removal(); 
     82    return; 
     83} 
     84 
     85=head1 wait_for_xlog_archive_to_be_ready() 
     86 
     87Waits till all necessary xlogs will be in archive, or (in case --call-master 
     88was not given) - for checkpoint on slave. 
     89 
     90=cut 
     91 
     92sub wait_for_xlog_archive_to_be_ready { 
     93    my $self = shift; 
     94    return $self->wait_for_checkpoint_location_change() unless $self->{ 'call-master' }; 
     95    $self->wait_for_file( $self->{ 'source' }->{ 'path' }, $self->{ 'stop_backup_filename_re' } ); 
    4596    return; 
    4697} 
     
    73124    my $transform_command = sprintf 's#^\(%s\|%s\)#%s#', $source_transform_from, $dot_backup_transform_from, $transform_to; 
    74125 
     126    my @stuff_to_compress = ( basename( $self->{ 'source' }->{ 'path' } ) ); 
     127    push @stuff_to_compress, File::Spec->catfile( $self->{ 'temp-dir' }, $self->{ 'dot_backup_filename' } ) if $self->{ 'dot_backup_filename' }; 
     128 
    75129    $self->tar_and_compress( 
    76130        'work_dir'  => dirname( $self->{ 'source' }->{ 'path' } ), 
    77         'tar_dir'   => [ basename( $self->{ 'source' }->{ 'path' } ), File::Spec->catfile( $self->{ 'temp-dir' }, $self->{ 'dot_backup_filename' } ), ]
     131        'tar_dir'   => \@stuff_to_compress
    78132        'transform' => $transform_command, 
    79133    ); 
     
    138192    my $self = shift; 
    139193 
     194    return if $self->{ 'call-master' }; 
     195 
    140196    my $redo_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's REDO location" }; 
    141197    my $timeline      = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's TimeLineID" }; 
     
    208264unarchived PGDATA. 
    209265 
     266If --call-master was given, it will run pg_start_backup() on master, and 
     267retrieve generated backup_label file. 
     268 
    210269=cut 
    211270 
     
    213272    my $self = shift; 
    214273 
    215     my $redo_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's REDO location" }; 
    216     my $last_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint location" }; 
    217     my $timeline      = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's TimeLineID" }; 
    218  
    219     my @content_lines = (); 
    220     push @content_lines, sprintf 'START WAL LOCATION: %s (file %s)', $redo_location, $self->convert_wal_location_and_timeline_to_filename( $redo_location, $timeline ); 
    221     push @content_lines, sprintf 'CHECKPOINT LOCATION: %s', $last_location; 
    222     push @content_lines, sprintf 'START TIME: %s', strftime( '%Y-%m-%d %H:%M:%S %Z', localtime time ); 
    223     push @content_lines, 'LABEL: OmniPITR_Slave_Hot_Backup'; 
    224  
    225     $self->{ 'backup_file_data' } = \@content_lines; 
    226     my $content = join( "\n", @content_lines ) . "\n"; 
     274    $self->{ 'CONTROL' }->{ 'initial' } = $self->get_control_data(); 
     275 
     276    if ( $self->{ 'call-master' } ) { 
     277        $self->get_backup_label_from_master(); 
     278    } 
     279    else { 
     280        my $redo_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's REDO location" }; 
     281        my $last_location = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint location" }; 
     282        my $timeline      = $self->{ 'CONTROL' }->{ 'initial' }->{ "Latest checkpoint's TimeLineID" }; 
     283 
     284        my @content_lines = (); 
     285        push @content_lines, sprintf 'START WAL LOCATION: %s (file %s)', $redo_location, $self->convert_wal_location_and_timeline_to_filename( $redo_location, $timeline ); 
     286        push @content_lines, sprintf 'CHECKPOINT LOCATION: %s', $last_location; 
     287        push @content_lines, sprintf 'START TIME: %s', strftime( '%Y-%m-%d %H:%M:%S %Z', localtime time ); 
     288        push @content_lines, 'LABEL: OmniPITR_Slave_Hot_Backup'; 
     289 
     290        $self->{ 'backup_file_data' } = \@content_lines; 
     291    } 
     292    my $content = join( "\n", @{ $self->{ 'backup_file_data' } } ) . "\n"; 
    227293 
    228294    my $filename = File::Spec->catfile( $self->{ 'temp-dir' }, 'backup_label' ); 
     
    235301} 
    236302 
     303=head1 get_backup_label_from_master() 
     304 
     305Wraps logic required to call pg_start_backup(), get response, and 
     306backup_label file content . 
     307 
     308=cut 
     309 
     310sub get_backup_label_from_master { 
     311    my $self = shift; 
     312 
     313    my $start_backup_output = $self->psql( "SELECT pg_start_backup('omnipitr_slave_backup_with_master_callback')" ); 
     314 
     315    $start_backup_output =~ s/\s*\z//; 
     316    $self->log->log( q{pg_start_backup('omnipitr') returned %s.}, $start_backup_output ); 
     317    $self->log->fatal( 'Output from pg_start_backup is not parseable?!' ) unless $start_backup_output =~ m{\A([0-9A-F]+)/([0-9A-F]{1,8})\z}; 
     318 
     319    my $backup_label_content = $self->psql( 
     320        "select pg_read_file( 'backup_label', 0, ( pg_stat_file( 'backup_label' ) ).size )", 
     321    ); 
     322 
     323    $self->{ 'backup_file_data' } = [ split( /\n/, $backup_label_content ) ]; 
     324 
     325    $self->wait_for_checkpoint_from_backup_label(); 
     326 
     327    return; 
     328} 
     329 
     330=head1 wait_for_checkpoint_from_backup_label() 
     331 
     332Waits till slave will do checkpoint in at least the same location as master 
     333did when pg_start_backup() was called. 
     334 
     335=cut 
     336 
     337sub wait_for_checkpoint_from_backup_label { 
     338    my $self = shift; 
     339 
     340    my @checkpoint_lines = grep { m{\ACHECKPOINT\s+LOCATION:\s+[a-f0-9]+/[0-9a-f]{8}\s*\z} } @{ $self->{ 'backup_file_data' } }; 
     341 
     342    $self->log->fatal( 'Cannot get checkpoint lines from: %s', $self->{ 'backup_file_data' } ) if 1 != scalar @checkpoint_lines; 
     343 
     344    my ( $major, $minor ) = $checkpoint_lines[ 0 ] =~ m{ \s+ ( [a-f0-9]+ ) / ( [a-f0-9]{8} ) \s* \z }xms; 
     345    $major = hex $major; 
     346    $minor = hex $minor; 
     347 
     348    $self->log->log( 'Waiting for checkpoint (based on backup_label from master)' ) if $self->verbose; 
     349    while ( 1 ) { 
     350        my $temp = $self->get_control_data(); 
     351 
     352        my ( $c_major, $c_minor ) = $temp->{ 'Latest checkpoint location' } =~ m{ \s+ ( [a-f0-9]+ ) / ( [a-f0-9]{8} ) \s* \z }xms; 
     353        $c_major = hex $c_major; 
     354        $c_minor = hex $c_minor; 
     355 
     356        last if $c_major > $major; 
     357        last if ( $c_major == $major ) && ( $c_minor >= $minor ); 
     358 
     359        sleep 5; 
     360    } 
     361    $self->log->log( 'Checkpoint .' ) if $self->verbose; 
     362    return; 
     363} 
     364 
    237365=head1 convert_wal_location_and_timeline_to_filename() 
    238366 
     
    264392sub compress_pgdata { 
    265393    my $self = shift; 
    266  
    267     $self->make_backup_label_temp_file(); 
    268394 
    269395    $self->log->time_start( 'Compressing $PGDATA' ) if $self->verbose; 
     
    365491        'tar-path'           => 'tar', 
    366492        'nice-path'          => 'nice', 
     493        'psql-path'          => 'psql', 
    367494        'rsync-path'         => 'rsync', 
    368495        'pgcontroldata-path' => 'pg_controldata', 
     
    373500        unless GetOptions( 
    374501        \%args, 
     502        'database|d=s', 
     503        'host|h=s', 
     504        'port|P=i', 
     505        'username|U=s', 
    375506        'data-dir|D=s', 
    376507        'source|s=s', 
     
    387518        'lzma-path|lp=s', 
    388519        'nice-path|np=s', 
     520        'psql-path|pp=s', 
    389521        'tar-path|tp=s', 
    390522        'rsync-path|rp=s', 
    391523        'pgcontroldata-path|pp=s', 
    392524        'not-nice|nn', 
     525        'call-master|cm', 
    393526        ); 
    394527 
     
    441574    $self->{ 'log' }          = OmniPITR::Log->new( $self->{ 'log_template' } ); 
    442575 
     576    my @psql = (); 
     577    push @psql, $self->{ 'psql-path' }; 
     578    push @psql, '-qAtX'; 
     579    push @psql, ( '-U', $self->{ 'username' } ) if $self->{ 'username' }; 
     580    push @psql, ( '-d', $self->{ 'database' } ) if $self->{ 'database' }; 
     581    push @psql, ( '-h', $self->{ 'host' } )     if $self->{ 'host' }; 
     582    push @psql, ( '-p', $self->{ 'port' } )     if $self->{ 'port' }; 
     583    push @psql, '-c'; 
     584    $self->{ 'psql' } = \@psql; 
     585 
    443586    $self->log->log( 'Called with parameters: %s', join( ' ', @argv_copy ) ) if $self->verbose; 
    444587