Show
Ignore:
Timestamp:
01/06/12 22:16:56 (2 years ago)
Author:
depesz
Message:

Add support for -s and -t options (to show data for given check at given time)

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved
  • trunk/tools/system.monitoring/system_monitoring.pl

    r311 r321  
    1313use strict; 
    1414use warnings; 
     15use Carp; 
    1516use English qw( -no_match_vars ); 
    1617use Time::HiRes qw( time sleep ); 
     18use Time::Local qw( timelocal ); 
    1719use POSIX qw( strftime setsid ); 
    1820use Getopt::Long; 
     
    3840    $self->validate_config(); 
    3941 
     42    if ( $self->{ 'show_check' } ) { 
     43        $self->show_data(); 
     44        return; 
     45    } 
     46 
    4047    $self->daemonize(); 
    4148 
     
    4552    $self->main_loop(); 
    4653    return; 
     54} 
     55 
     56sub show_data { 
     57    my $self = shift; 
     58    croak( sprintf 'Requested check (%s) is not defined in config (%s)!', $self->{ 'show_check' }, $self->{ 'config_file' } ) unless $self->{ 'checks_hash' }->{ $self->{ 'show_check' } }; 
     59 
     60    my @files = $self->find_best_files_for_show(); 
     61 
     62    for my $use_file ( @files ) { 
     63 
     64        my $input; 
     65        if ( $use_file =~ m{\.gz\z} ) { 
     66            open $input, '-|', 'gzip --decompress --stdout ' . quotemeta( $use_file ) or croak( "Cannot gzip-decompress $use_file: $OS_ERROR" ); 
     67        } 
     68        elsif ( $use_file =~ m{\.bz2\z} ) { 
     69            open $input, '-|', 'bzip2 --decompress --stdout ' . quotemeta( $use_file ) or croak( "Cannot bzip2-decompress $use_file: $OS_ERROR" ); 
     70        } 
     71        else { 
     72            open $input, '<', $use_file or croak( "Cannot open $use_file: $OS_ERROR" ); 
     73        } 
     74 
     75        my $C = $self->{ 'checks_hash' }->{ $self->{ 'show_check' } }; 
     76 
     77        my $state           = 0; 
     78        my $last_time_str   = 0; 
     79        my $last_time_epoch = 0; 
     80        my $data_marker; 
     81 
     82        while ( <$input> ) { 
     83            croak( "Bad line in log: $_" ) unless m{^((\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)) \S+\t(\S+)}; 
     84            my ( $time_str, $line_code, @elements ) = ( $1, $8, $2, $3, $4, $5, $6, $7 ); 
     85            next if $line_code =~ m{\A(?:\?\?|:h)\z}; # ignore ?? and :h 
     86            $elements[ 1 ]--;    # month should be 0-11, and not 1-12 
     87 
     88            my $time_epoch = $time_str eq $last_time_str ? $last_time_epoch : timelocal( reverse @elements ); 
     89            $last_time_str   = $time_str; 
     90            $last_time_epoch = $time_epoch; 
     91 
     92            my $line_marker = $C->{ 'type' } eq 'periodic' ? $line_code : $time_str; 
     93 
     94            if ( $state == 0 ) { 
     95                next if $time_epoch < $self->{ 'show_time' }; 
     96                $state = 1; 
     97                print; 
     98                $data_marker = $line_marker; 
     99                next; 
     100            } 
     101            else { 
     102                return unless $data_marker eq $line_marker; 
     103                print; 
     104            } 
     105        } 
     106        close $input; 
     107    } 
     108 
     109    return; 
     110} 
     111 
     112sub find_best_files_for_show { 
     113    my $self = shift; 
     114 
     115    my @use_files = (); 
     116 
     117    my $use_ts = $self->{ 'show_time' }; 
     118 
     119    while ( $use_ts < time() ) { 
     120        my @t                = localtime( $use_ts ); 
     121        my $directory_prefix = strftime( '%Y/%m/%d', @t ); 
     122        my $full_directory   = File::Spec->catfile( $self->{ 'logdir' }, $directory_prefix ); 
     123        my $file_suffix      = strftime( '-%Y-%m-%d-%H.log', @t ); 
     124 
     125        my $file_name = $self->{ 'show_check' } . $file_suffix; 
     126        my $full_path = File::Spec->catfile( $full_directory, $file_name ); 
     127 
     128        for my $ext ( ( '', '.gz', '.bz2' ) ) { 
     129            next unless -f $full_path . $ext; 
     130            push @use_files, $full_path . $ext; 
     131            return @use_files if 2 == scalar @use_files; 
     132            last; 
     133        } 
     134        $use_ts += 3600; 
     135    } 
     136    croak( "There is no data for this check and this time." ) if 0 == scalar @use_files; 
     137    return @use_files; 
    47138} 
    48139 
     
    57148 
    58149    #<<<  do not let perltidy touch this 
    59     open( STDIN,  '<', '/dev/null' ) || die "can't read /dev/null: $!"
    60     open( STDOUT, '>', '/dev/null' ) || die "can't write to /dev/null: $!"
    61     defined( my $pid = fork() )      || die "can't fork: $!"
     150    open( STDIN,  '<', '/dev/null' ) || croak( "can't read /dev/null: $!" )
     151    open( STDOUT, '>', '/dev/null' ) || croak( "can't write to /dev/null: $!" )
     152    defined( my $pid = fork() )      || croak( "can't fork: $!" )
    62153    if ( $pid ) { 
    63154        # parent 
     
    66157    } 
    67158    $self->write_pidfile(); 
    68     ( setsid() != -1 )               || die "Can't start a new session: $!"
    69     open( STDERR, '>&', \*STDOUT )   || die "can't dup stdout: $!"
     159    ( setsid() != -1 )               || croak( "Can't start a new session: $!" )
     160    open( STDERR, '>&', \*STDOUT )   || croak( "can't dup stdout: $!" )
    70161    #>>> 
    71162    return; 
     
    84175    return unless $self->{ 'pidfile' }; 
    85176    my $pidfilename = $self->{ 'pidfile' }; 
    86     open my $pidfile, '>', $pidfilename or die "Cannot write to pidfile ($pidfilename): $OS_ERROR\n"
     177    open my $pidfile, '>', $pidfilename or croak( "Cannot write to pidfile ($pidfilename): $OS_ERROR\n" )
    87178    print $pidfile $PID . "\n"; 
    88179    close $pidfile; 
     
    95186    my $pidfilename = $self->{ 'pidfile' }; 
    96187    return unless -e $pidfilename; 
    97     die "Pidfile ($pidfilename) exists, but is not a file?!\n" unless -f $pidfilename; 
    98     open my $pidfile, '<', $pidfilename or die "Cannot read from pidfile ($pidfilename): $OS_ERROR\n"
     188    croak( "Pidfile ($pidfilename) exists, but is not a file?!\n" ) unless -f $pidfilename; 
     189    open my $pidfile, '<', $pidfilename or croak( "Cannot read from pidfile ($pidfilename): $OS_ERROR\n" )
    99190    my $old_pid_line = <$pidfile>; 
    100191    close $pidfile; 
    101192 
    102     die "Bad format of pidfile ($pidfilename)!\n" unless $old_pid_line =~ m{\A(\d+)\s*\z}; 
     193    croak( "Bad format of pidfile ($pidfilename)!\n" ) unless $old_pid_line =~ m{\A(\d+)\s*\z}; 
    103194    my $old_pid = $1; 
    104195    return if 0 == kill( 0, $old_pid ); 
    105     die "Old process ($old_pid) still exists!\n"
     196    croak( "Old process ($old_pid) still exists!\n" )
    106197} 
    107198 
     
    133224        last; 
    134225    } 
    135     die "Data from unknown input?! It shouldn't *ever* happen\n" unless $C; 
     226    croak( "Data from unknown input?! It shouldn't *ever* happen\n" ) unless $C; 
    136227 
    137228    my $read_data = ''; 
     
    205296    $mode = '<' if $command =~ s/\A\s*<\s*//; 
    206297 
    207     open my $fh, $mode, $command or die "Cannot open [$command] in mode [$mode]: $OS_ERROR\n"
     298    open my $fh, $mode, $command or croak( "Cannot open [$command] in mode [$mode]: $OS_ERROR\n" )
    208299    $self->{ 'select' }->add( $fh ); 
    209300    $C->{ 'input' } = $fh; 
     
    291382 
    292383        my $full_name = File::Spec->catfile( $full_directory, $C->{ 'name' } . $file_suffix ); 
    293         open my $fh, '>>', $full_name or die "Cannot write to $full_name: $OS_ERROR\n"
     384        open my $fh, '>>', $full_name or croak( "Cannot write to $full_name: $OS_ERROR\n" )
    294385        $C->{ 'fh' } = $fh; 
    295386 
     
    299390 
    300391            # File is empty 
    301             $C->{ 'buffer' } .= "\n" if $C->{ 'buffer' }; 
    302             $C->{ 'buffer' } .= $C->{ 'header' }; 
     392            my $tmp_job_id = $C->{ 'job_id' }; 
     393            $C->{ 'job_id' } = ':h'; 
     394            my $tmp_buffer = $C->{ 'buffer' }; 
     395            $C->{ 'buffer' } = $C->{ 'header' }; 
    303396            $self->print_log( $C ); 
     397            $C->{ 'job_id' } = $tmp_job_id; 
     398            $C->{ 'buffer' } = $tmp_buffer; 
    304399        } 
    305400    } 
     
    316411    my $self = shift; 
    317412 
    318     die "GLOBAL.logdir was not provided in config!\n" unless defined $self->{ 'logdir' }; 
    319     die "There are no checks to be run!\n"            unless defined $self->{ 'pre_checks' }; 
    320  
    321     die "Cannot chdir to " . $self->{ 'logdir' } . ": $OS_ERROR\n" unless chdir $self->{ 'logdir' }; 
     413    croak( "GLOBAL.logdir was not provided in config!\n" ) unless defined $self->{ 'logdir' }; 
     414    croak( "There are no checks to be run!\n" )            unless defined $self->{ 'pre_checks' }; 
     415 
     416    croak( "Cannot chdir to " . $self->{ 'logdir' } . ": $OS_ERROR\n" ) unless chdir $self->{ 'logdir' }; 
    322417 
    323418    my @checks = (); 
     
    326421        push @checks, $C; 
    327422 
    328         die "Bad type " . $C->{ 'type' } . " in check $check!\n" unless $C->{ 'type' } =~ m{\A(?:persistent|periodic)\z}; 
     423        croak( "Bad type " . $C->{ 'type' } . " in check $check!\n" ) unless $C->{ 'type' } =~ m{\A(?:persistent|periodic)\z}; 
    329424        next unless $C->{ 'type' } eq 'periodic'; 
    330425 
    331         die "Undefined interval for check $check!\n" unless defined $C->{ 'interval' }; 
    332         die "Bad interval (" . $C->{ 'interval' } . ") in check $check!\n" unless $C->{ 'interval' } =~ m{\A[1-9]\d*\z}; 
     426        croak( "Undefined interval for check $check!\n" ) unless defined $C->{ 'interval' }; 
     427        croak( "Bad interval (" . $C->{ 'interval' } . ") in check $check!\n" ) unless $C->{ 'interval' } =~ m{\A[1-9]\d*\z}; 
    333428 
    334429        $self->process_config_vars( $C ); 
     
    352447 
    353448    $self->{ 'checks' } = \@checks; 
     449    $self->{ 'checks_hash' } = { map { ( $_->{ 'name' } => $_ ) } @checks }; 
    354450    delete $self->{ 'pre_checks' }; 
    355451 
     
    375471    my $config_file_name = $self->{ 'config_file' }; 
    376472 
    377     open my $fh, '<', $config_file_name or die "Cannot open config file ($config_file_name) : $OS_ERROR\n"
     473    open my $fh, '<', $config_file_name or croak( "Cannot open config file ($config_file_name) : $OS_ERROR\n" )
    378474    while ( my $line = <$fh> ) { 
    379475        next if $line =~ m{^\s*#};     # comment 
     
    401497            next; 
    402498        } 
    403         die "Unknown line: [ $line ]\n"
     499        croak( "Unknown line: [ $line ]\n" )
    404500    } 
    405501    close $fh; 
     
    415511    my $self = shift; 
    416512 
    417     my $daemonize = undef; 
    418     exit( 1 ) unless GetOptions( 'daemonize|d' => \$daemonize ); 
    419     $self->{ 'daemonize' } = $daemonize; 
    420  
    421     die "You have to provide name of config file! Check: perldoc $PROGRAM_NAME\n" if 0 == scalar @ARGV; 
     513    my $daemonize  = undef; 
     514    my $show_check = undef; 
     515    my $show_time  = undef; 
     516    exit( 1 ) unless GetOptions( 
     517        'daemonize|d' => \$daemonize, 
     518        'show|s=s'    => \$show_check, 
     519        'time|t=s'    => \$show_time, 
     520    ); 
     521 
     522    if ( $show_time ) { 
     523        croak( "-t value has bad format (not YYYY-MM-DD HH:MI:SS)\n" ) unless $show_time =~ m{\A(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)\z}; 
     524        my @elements = ( $1, $2, $3, $4, $5, $6 ); 
     525        $elements[ 1 ]--;    # month is 0-11, and not 1-12! 
     526        $show_time = timelocal( reverse @elements ); 
     527    } 
     528    croak( "You cannot give -s without -t\n" ) if defined $show_check && !defined $show_time; 
     529    croak( "You cannot give -t without -s\n" ) if defined $show_time  && !defined $show_check; 
     530    $self->{ 'daemonize' }  = $daemonize; 
     531    $self->{ 'show_check' } = $show_check; 
     532    $self->{ 'show_time' }  = $show_time; 
     533 
     534    croak( "You have to provide name of config file! Check: perldoc $PROGRAM_NAME\n" ) if 0 == scalar @ARGV; 
    422535    $self->{ 'config_file' } = shift @ARGV; 
    423536 
     
    431544=head2 USAGE 
    432545 
    433 system_monitoring.pl [-d] <config_file> 
     546  system_monitoring.pl [-d] <config_file> 
     547 
     548  system_monitoring.pl -s check -t when <config_file> 
    434549 
    435550=head2 DESCRIPTION 
     
    448563All checks work in parallel, so there is no chance single check could lock 
    449564whole system_monitoring.pl. 
     565 
     566When run with -d option, it will daemonize itself and detach from terminal. 
     567 
     568When run with -s .. -t ... options, it will print values for given (-s) 
     569check for date being given (-t) or closest later. 
     570 
     571-t value (time to search for) has to be given in format: 
     572 
     573  YYYY-MM-DD HH:MI:SS 
     574 
     575For example: 
     576 
     577  2012-01-23 16:34:56 
    450578 
    451579=head2 Configuration file