Changeset 131

Show
Ignore:
Timestamp:
04/21/10 21:28:01 (4 years ago)
Author:
depesz
Message:

work in progress. backup still doesn't work, but i want to commit to prevent code loss

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • trunk/omnipitr/doc/omnipitr-backup-master.pod

    r130 r131  
    3030 
    3131What username to use when connecting to database. Defaults to postgres. 
     32 
     33=item --xlogs (-x) 
     34 
     35Directory that (if doesn't exist) will be created. Then - will be used as source 
     36of xlogs to archive. Afterwards - it will be removed (if it didn't exist in the 
     37beginning). 
    3238 
    3339=item --dst-local (-dl) 
     
    5258Where to create temporary files (defaults to /tmp or I<$TMPDIR> environment 
    5359variable location) 
     60 
     61=item --state-dir (-s) 
     62 
     63Name of directory to use as state-directory to handle errors when sending wal 
     64segments to many locations. 
    5465 
    5566=item --log (-l) 
     
    100111variable. 
    101112 
     113=item --rsync-path (-rp) 
     114 
     115Full path to rsync program - in case you can't set proper PATH environment 
     116variable. 
     117 
     118=item --psql-path (-pp) 
     119 
     120Full path to psql program - in case you can't set proper PATH environment 
     121variable. 
     122 
    102123=back 
    103124 
  • trunk/omnipitr/lib/OmniPITR/Program.pm

    r128 r131  
    4040sub check_debug { 
    4141    my $self = shift; 
     42    return if 0 == scalar @ARGV; 
    4243    return unless '--debug' eq $ARGV[ 0 ]; 
    4344 
  • trunk/omnipitr/lib/OmniPITR/Program/Backup/Master.pm

    r86 r131  
    33use warnings; 
    44 
     5use base qw( OmniPITR::Program ); 
     6 
     7use Carp; 
     8use OmniPITR::Tools qw( :all ); 
     9use English qw( -no_match_vars ); 
     10use File::Basename; 
     11use Sys::Hostname; 
     12use POSIX qw( strftime ); 
     13use File::Spec; 
     14use File::Path qw( mkpath rmtree ); 
     15use File::Copy; 
     16use Storable; 
     17use Getopt::Long qw( :config no_ignore_case ); 
     18 
     19sub run { 
     20    my $self = shift; 
     21    $self->read_state(); 
     22    $self->get_list_of_all_necessary_compressions(); 
     23    $self->choose_base_local_destinations(); 
     24 
     25    return; 
     26} 
     27 
     28sub choose_base_local_destinations { 
     29    my $self = shift; 
     30 
     31    my $base = { map { ( $_ => undef ) } @{ $self->{ 'compressions' } } }; 
     32    $self->{ 'base' } = $base; 
     33 
     34    for my $dst ( @{ $self->{ 'destination' }->{ 'local' } } ) { 
     35        my $type = $dst->{ 'compression' }; 
     36        next if defined $base->{ $type }; 
     37        $base->{ $type } = $dst->{ 'path' }; 
     38    } 
     39 
     40    my @unfilled = grep { !defined $base->{ $_ } } keys %{ $base }; 
     41 
     42    return if 0 == scalar @unfilled; 
     43    $self->log->log( 'These compression(s) were given only for remote destinations. Usually this is not desired: %s', join( ', ', @unfilled ) ); 
     44 
     45    $self->prepare_temp_directory(); 
     46    for my $type ( @unfilled ) { 
     47        my $tmp_dir = File::Spec->catfile( $self->{ 'temp-dir' }, $type ); 
     48        mkpath( $tmp_dir ); 
     49        $base->{ $type } = $tmp_dir; 
     50    } 
     51    return; 
     52} 
     53 
     54sub DESTROY { 
     55    my $self = shift; 
     56    return unless $self->{ 'temp-dir-prepared' }; 
     57    rmtree( $self->{ 'temp-dir-prepared' } ); 
     58    return; 
     59} 
     60 
     61=head1 prepare_temp_directory() 
     62 
     63Helper function, which builds path for temp directory, and creates it. 
     64 
     65Path is generated by using given temp-dir, 'omnipitr-archive' name, and filename of segment. 
     66 
     67For example, for temp-dir '/tmp' and segment being pg_xlog/000000010000000000000003, actual, used temp directory would be /tmp/omnipitr-archive/000000010000000000000003/. 
     68 
     69=cut 
     70 
     71sub prepare_temp_directory { 
     72    my $self = shift; 
     73    my $full_temp_dir = File::Spec->catfile( $self->{ 'temp-dir' }, basename( $PROGRAM_NAME ) ); 
     74    mkpath( $full_temp_dir ); 
     75    $self->{ 'temp-dir' }          = $full_temp_dir; 
     76    $self->{ 'temp-dir-prepared' } = $full_temp_dir; 
     77    return; 
     78} 
     79 
     80=head1 read_state() 
     81 
     82Helper function to read state from state file. 
     83 
     84Name of state file is the same as filename of WAL segment being archived, but it is in state-dir. 
     85 
     86=cut 
     87 
     88sub read_state { 
     89    my $self = shift; 
     90    $self->{ 'state' } = {}; 
     91 
     92    return unless $self->{ 'state-dir' }; 
     93 
     94    $self->{ 'state-file' } = File::Spec->catfile( $self->{ 'state-dir' }, 'omnipitr-backup-master.state' ); 
     95    return unless -f $self->{ 'state-file' }; 
     96    $self->{ 'state' } = retrieve( $self->{ 'state-file' } ); 
     97    return; 
     98} 
     99 
     100=head1 save_state() 
     101 
     102Helper function to save state to state-file. 
     103 
     104=cut 
     105 
     106sub save_state { 
     107    my $self = shift; 
     108 
     109    return unless $self->{ 'state-file' }; 
     110 
     111    store( $self->{ 'state' }, $self->{ 'state-file' } ); 
     112 
     113    return; 
     114} 
     115 
     116=head1 get_list_of_all_necessary_compressions() 
     117 
     118Scans list of destinations, and gathers list of all compressions that have to be made. 
     119 
     120This is to be able to compress file only once even when having multiple destinations that require compressed format. 
     121 
     122=cut 
     123 
     124sub get_list_of_all_necessary_compressions { 
     125    my $self = shift; 
     126 
     127    my %compression = (); 
     128 
     129    for my $dst_type ( qw( local remote ) ) { 
     130        next unless my $dsts = $self->{ 'destination' }->{ $dst_type }; 
     131        for my $destination ( @{ $dsts } ) { 
     132            $compression{ $destination->{ 'compression' } } = 1; 
     133        } 
     134    } 
     135    $self->{ 'compressions' } = [ keys %compression ]; 
     136    return; 
     137} 
     138 
     139=head1 read_args() 
     140 
     141Function which does all the parsing, and transformation of command line arguments. 
     142 
     143It also verified base facts about passed WAL segment name, but all other validations, are being done in separate function: L<validate_args()>. 
     144 
     145=cut 
     146 
     147sub read_args { 
     148    my $self = shift; 
     149 
     150    my @argv_copy = @ARGV; 
     151 
     152    my %args = ( 
     153        'data-dir'          => '.', 
     154        'temp-dir'          => $ENV{ 'TMPDIR' } || '/tmp', 
     155        'gzip-path'         => 'gzip', 
     156        'bzip2-path'        => 'bzip2', 
     157        'lzma-path'         => 'lzma', 
     158        'tar-path'          => 'tar', 
     159        'nice-path'         => 'nice', 
     160        'psql-path'         => 'psql', 
     161        'rsync-path'        => 'rsync', 
     162        'database'          => 'postgres', 
     163        'filename-template' => '__HOSTNAME__-__FILETYPE__-^Y-^m-^d.tar__CEXT__', 
     164    ); 
     165 
     166    croak( 'Error while reading command line arguments. Please check documentation in doc/omnipitr-master-backup.pod' ) 
     167        unless GetOptions( 
     168        \%args, 
     169        'data-dir|D=s', 
     170        'database|d=s', 
     171        'host|h=s', 
     172        'port|p=i', 
     173        'username|U=s', 
     174        'xlogs|x=s', 
     175        'dst-local|dl=s@', 
     176        'dst-remote|dr=s@', 
     177        'temp-dir|t=s', 
     178        'log|l=s', 
     179        'filename-template|f=s', 
     180        'pid-file', 
     181        'verbose|v=s', 
     182        'gzip-path|gp=s', 
     183        'bzip2-path|bp=s', 
     184        'lzma-path|lp=s', 
     185        'nice-path|np=s', 
     186        'psql-path|pp=s', 
     187        'tar-path|tp=s', 
     188        'rsync-path|rp=s', 
     189        'state-dir|s=s', 
     190        ); 
     191 
     192    croak( '--log was not provided - cannot continue.' ) unless $args{ 'log' }; 
     193    for my $key ( qw( log filename-template ) ) { 
     194        $args{ $key } =~ tr/^/%/; 
     195    } 
     196 
     197    for my $key ( grep { !/^dst-(?:local|remote)$/ } keys %args ) { 
     198        $self->{ $key } = $args{ $key }; 
     199    } 
     200 
     201    for my $type ( qw( local remote ) ) { 
     202        my $D = []; 
     203        $self->{ 'destination' }->{ $type } = $D; 
     204 
     205        next unless defined $args{ 'dst-' . $type }; 
     206 
     207        my %temp_for_uniq = (); 
     208        my @items = grep { !$temp_for_uniq{ $_ }++ } @{ $args{ 'dst-' . $type } }; 
     209 
     210        for my $item ( @items ) { 
     211            my $current = { 'compression' => 'none', }; 
     212            if ( $item =~ s/\A(gzip|bzip2|lzma)=// ) { 
     213                $current->{ 'compression' } = $1; 
     214            } 
     215            $current->{ 'path' } = $item; 
     216            push @{ $D }, $current; 
     217        } 
     218    } 
     219 
     220    $self->{ 'filename-template' } = strftime( $self->{ 'filename-template' }, localtime time() ); 
     221    $self->{ 'filename-template' } =~ s/__HOSTNAME__/hostname()/ge; 
     222 
     223    # We do it here so it will actually work for reporing problems in validation 
     224    $self->{ 'log_template' } = $args{ 'log' }; 
     225    $self->{ 'log' }          = OmniPITR::Log->new( $self->{ 'log_template' } ); 
     226 
     227    $self->log->log( 'Called with parameters: %s', join( ' ', @argv_copy ) ) if $self->verbose; 
     228 
     229    return; 
     230} 
     231 
     232=head1 validate_args() 
     233 
     234Does all necessary validation of given command line arguments. 
     235 
     236One exception is for compression programs paths - technically, it could be validated in here, but benefit would be pretty limited, and code to do so relatively complex, as compression program path 
     237might, but doesn't have to be actual file path - it might be just program name (without path), which is the default. 
     238 
     239=cut 
     240 
     241sub validate_args { 
     242    my $self = shift; 
     243 
     244    my $dst_count = scalar( @{ $self->{ 'destination' }->{ 'local' } } ) + scalar( @{ $self->{ 'destination' }->{ 'remote' } } ); 
     245    $self->log->fatal( "No --dst-* has been provided!" ) if 0 == $dst_count; 
     246 
     247    if ( 1 < $dst_count ) { 
     248        $self->log->fatal( "More than 1 --dst-* has been provided, but no --state-dir!" ) if !$self->{ 'state-dir' }; 
     249        $self->log->fatal( "Given --state-dir (%s) does not exist",     $self->{ 'state-dir' } ) unless -e $self->{ 'state-dir' }; 
     250        $self->log->fatal( "Given --state-dir (%s) is not a directory", $self->{ 'state-dir' } ) unless -d $self->{ 'state-dir' }; 
     251        $self->log->fatal( "Given --state-dir (%s) is not writable",    $self->{ 'state-dir' } ) unless -w $self->{ 'state-dir' }; 
     252    } 
     253 
     254    $self->log->fatal( "Filename template does not contain __FILETYPE__ placeholder!" ) unless $self->{ 'filename-template' } =~ /__FILETYPE__/; 
     255    $self->log->fatal( "Filename template cannot contain / or \\ characters!" ) if $self->{ 'filename-template' } =~ m{[/\\]}; 
     256 
     257    return; 
     258} 
     259 
    52601; 
    6