root/trunk/tools/system_monitoring.pl

#!/usr/bin/perl -w

package main;
my $program = Monitoring->new();
$program->run();

exit;

package Monitoring;
use strict;
use English qw( -no_match_vars );
use Time::HiRes qw( time );
use POSIX qw( strftime );
use File::Spec;
use File::Path qw( mkpath );
use Data::Dumper;
use IO::Select;
use IO::Handle;

sub new {
    my $class = shift;
    return bless {}, $class;
}

sub run {
    my $self = shift;

    $self->read_config();
    $self->validate_config();

    $self->{ 'select' } = IO::Select->new();
    $self->start_persistent_processes();

    $self->main_loop();
    return;
}

sub main_loop {
    my $self = shift;
    while ( 1 ) {
        $self->{ 'current_time' } = time();
        $self->update_logger_filehandles();

        my $timeout = $self->calculate_timeout();
        my @ready   = $self->{ 'select' }->can_read( $timeout );
        for my $fh ( @ready ) {
            $self->handle_read( $fh );
        }
        $self->start_periodic_processes();
    }
}

sub handle_read {
    my $self = shift;
    my $fh   = shift;

    my $C;
    for my $check ( keys %{ $self->{ 'checks' } } ) {
        my $tmp = $self->{ 'checks' }->{ $check };
        next unless $tmp->{ 'input' };
        my $tmp_fh = $tmp->{ 'input' };
        next if "$tmp_fh" ne "$fh";    # Stringified reference to io handle
        $C = $tmp;
        last;
    }
    die "Data from unknown input?! It shouldn't *ever* happen\n" unless $C;

    my $read_data = '';
    while ( 1 ) {
        my $buffer;
        my $read_bytes = sysread( $fh, $buffer, 8192 );
        $read_data .= $buffer;
        last if 8192 > $read_bytes;
    }
    if ( '' eq $read_data ) {
        $self->{ 'select' }->remove( $fh );
        close $fh;
        delete $C->{ 'input' };
        return unless 'periodic' eq $C->{ 'type' };
        $C->{ 'next_call' } = $self->{ 'current_time' } + $C->{ 'interval' } if $self->{ 'current_time' } < $C->{ 'next_call' };
        return;
    }
    return if $C->{ 'ignore' };

    my $line_prefix = strftime( '%Y-%m-%d %H:%M:%S %Z | ', localtime( $self->{ 'current_time' } ) );
    $read_data =~ s/^/$line_prefix/gm;
    $read_data =~ s/([^\n])\z/$1\n/;
    print { $C->{ 'fh' } } $read_data;
    $C->{ 'fh' }->flush();
    return;
}

sub run_check {
    my $self    = shift;
    my $C       = shift;
    my $command = $C->{ 'exec' };

    my $mode = '-|';
    $mode = '<' if $command =~ s/\A\s*<\s*//;

    open my $fh, $mode, $command or die "Cannot open [$command] in mode [$mode]: $OS_ERROR\n";
    $self->{ 'select' }->add( $fh );
    $C->{ 'input' } = $fh;

    return;
}

sub start_periodic_processes {
    my $self = shift;
    for my $C ( values %{ $self->{ 'checks' } } ) {
        next unless 'periodic' eq $C->{ 'type' };
        next if defined $C->{ 'input' };
        next if ( defined $C->{ 'next_call' } ) && ( $C->{ 'next_call' } > $self->{ 'current_time' } );
        $self->run_check( $C );
        $C->{ 'next_call' } = $self->{ 'current_time' } + $C->{ 'interval' };
    }
    return;
}

sub start_persistent_processes {
    my $self = shift;
    for my $C ( values %{ $self->{ 'checks' } } ) {
        next unless 'persistent' eq $C->{ 'type' };
        $self->run_check( $C );
    }
    return;
}

sub calculate_timeout {
    my $self    = shift;
    my $nearest = undef;
    for my $check ( keys %{ $self->{ 'checks' } } ) {
        my $C = $self->{ 'checks' }->{ $check };
        next if 'persistent' eq $C->{ 'type' };
        next if defined $C->{ 'input' };
        return 0 unless defined $C->{ 'next_call' };
        if ( defined $nearest ) {
            $nearest = $C->{ 'next_call' } if $C->{ 'next_call' } < $nearest;
        }
        else {
            $nearest = $C->{ 'next_call' };
        }
    }
    $nearest = $self->{ 'current_time' } unless defined $nearest;
    my $sleep_time = $nearest - $self->{ 'current_time' };
    return $sleep_time < 0.5 ? 0.5 : $sleep_time;    # limit sleep time to 0.5s to avoid too aggresive calls.
}

sub update_logger_filehandles {
    my $self = shift;

    my $file_suffix = strftime( '-%Y-%m-%d-%H.log', localtime( $self->{ 'current_time' } ) );
    return if ( defined $self->{ 'previous-suffix' } ) && ( $self->{ 'previous-suffix' } eq $file_suffix );
    $self->{ 'previous-suffix' } = $file_suffix;

    my $directory_prefix = strftime( '%Y/%m/%d', localtime( $self->{ 'current_time' } ) );
    my $full_directory = File::Spec->catfile( $self->{ 'logdir' }, $directory_prefix );

    mkpath( [ $full_directory ], 0, oct( "750" ) ) unless -e $full_directory;

    for my $check ( keys %{ $self->{ 'checks' } } ) {
        my $C = $self->{ 'checks' }->{ $check };
        next if $C->{ 'ignore' };

        if ( $C->{ 'fh' } ) {
            close $C->{ 'fh' };
            delete $C->{ 'fh' };
        }

        my $full_name = File::Spec->catfile( $full_directory, $check . $file_suffix );
        open my $fh, '>>', $full_name or die "Cannot write to $full_name: $OS_ERROR\n";
        $C->{ 'fh' } = $fh;
    }

    return;
}

sub validate_config {
    my $self = shift;

    die "GLOBAL.logdir was not provided in config!\n" unless defined $self->{ 'logdir' };
    die "There are no checks to be run!\n"            unless defined $self->{ 'checks' };

    for my $check ( sort keys %{ $self->{ 'checks' } } ) {
        my $C = $self->{ 'checks' }->{ $check };
        $C->{ 'name' } = $check;

        die "Bad type " . $C->{ 'type' } . " in check $check!\n" unless $C->{ 'type' } =~ m{\A(?:persistent|periodic)\z};
        next unless $C->{ 'type' } eq 'periodic';

        die "Undefined interval for check $check!\n" unless defined $C->{ 'interval' };
        die "Bad interval (" . $C->{ 'interval' } . ") in check $check!\n" unless $C->{ 'interval' } =~ m{\A[1-9]\d*\z};
    }
    return;
}

sub read_config {
    my $self = shift;

    die "You have to provide name of config file! Check: perldoc $PROGRAM_NAME\n" if 0 == scalar @ARGV;
    my $config_file_name = shift @ARGV;

    open my $fh, '<', $config_file_name or die "Cannot open config file ($config_file_name) : $OS_ERROR\n";
    while ( my $line = <$fh> ) {
        next if $line =~ m{^\s*#};     # comment
        next if $line =~ m{^\s*\z};    # empty line
        $line =~ s{\A\s*}{};           # removing leading spaces
        $line =~ s{\s*\z}{};           # removing trailing spaces
        if ( $line =~ m{ \A GLOBAL\.logdir \s* = \s* (\S.*) \z }xmsi ) {
            $self->{ 'logdir' } = $1;
            next;
        }
        elsif ( $line =~ m{ \A check\.([A-Za-z0-9_]+)\.(type|exec|interval|ignore) \s* = \s* (\S.*) \z }xmsi ) {
            $self->{ 'checks' }->{ $1 }->{ $2 } = $3;
            next;
        }
        die "Unknown line: [ $line ]\n";
    }
    close $fh;
    return;
}

1;

=head1 system_monitoring.pl

=head2 USAGE

omnipitr-archive <config_file>

=head2 DESCRIPTION

system_monitoring.pl script is meant to provide single and solution for logging system data which change more often than it's practical for systems like cacti/nagios.

It is meant to be run on some low-privilege account, and gather the data, which are partitioned automatically by source, and time, and stored in simple text files.

After running, system_monitor.pl will check config, and if there are no errors - will start processing checks.

All checks work in parallel, so there is no chance single check could lock whole system_monitoring.pl.

=head2 Configuration file

Format of the configuration file is kept as simple as possible, to make this script very portable - which in this particular case means: no external (aside from core perl) dependencies.

Each line should be one of:

=over

=item * Comment (starts with #)

=item * Empty line (just white space characters)

=item * Setting

=back

Where setting line looks like:

    PARAM=value

with optional leading, trailing or around "=" whitespace.

Recognized parameters are:

=over

=item * GLOBAL.logdir - full path to log directory

=item * check.XXX.type - type of check with name XXX

=item * check.XXX.exec - what should be executed to get data for check XXX

=item * check.XXX.interval - how often to run check XXX

=item * check.XXX.ignore - should output be ignored?

=back

There are only two supported types:

=over

=item * persistent - which means given program is to be run in background, and whatever it will return should be logged. Such program "interval" will be ignored.

=item * periodic - which means that given program is to be run periodically as it will exit after returning data

=back

"exec" parameter is simply command line, to be run via shell, that will run the program.

If exec parameter starts with '<' character (with optional whitespace characters after), it is treated as filename to be read, and logged.

Due to the way it is internally processed - using "<" approach makes sense only for periodic checks - in case of permenent checks it would simply copy the file at start of system_monitoring.pl, and
ignore any changes to it afterwards. If you'd like to have something like 'tail -f' - use tail -f.

interval is time (in seconds) how often given program (of periodic type) should be run.

ignore is optional parameter which is checked using Perl boolean logic (any value other than empty string or 0 ar treated as true). Since system_monitoring doesn't let setting empty string as value
for option - it's best to not include ignore option for checks you want to log, and just add '...ignore=1' for those that you want to ignore.

If ignore is set, system_monitoring will not log output from such check.

This is helpful to build-in compression of older logs, using for example:

    check.cleanup.type=periodic
    check.cleanup.interval=300
    check.cleanup.exec=find /var/log/monitoring -type f -name '*.log' -mmin +120 -print0 | xargs -0 gzip
    check.cleanup.ignore=1

"XXX" (name of check) can consist only of upper and lower case letters, digits, and character _. That is it has to match regular expression:

    /\A[A-Za-z0-9_]+\z/

Output from all programs will be logged in files named:

    /logdir/YYYY/MM/DD/XXX-YYY-MM-DD-HH.log

where YYYY, MM, DD and HH are date and time parts of current (as of logging moment) time.

HH is 0 padded 24-hour style hour.

Example configuration:

    # Global configuration, log directory
    GLOBAL.logdir=/var/tmp/monitoring

    # Logging iostat output in 10 second intervals
    check.iostat.type=persistent
    check.iostat.exec=iostat -kx 10

    # Logging "ps auxwwn" every 30 seconds.
    check.ps.type=periodic
    check.ps.exec=ps auxwwn
    check.ps.interval=30

=head2 LICENSE

Copyright (c) 2010, OmniTI, Inc.

Permission to use, copy, modify, and distribute this software and its
documentation for any purpose, without fee, and without a written agreement
is hereby granted, provided that the above copyright notice and this
paragraph and the following two paragraphs appear in all copies.

IN NO EVENT SHALL OmniTI, Inc. BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
OmniTI, Inc. HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

OmniTI, Inc. SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS,
AND OmniTI, Inc. HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT,
UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

=head2 COPYRIGHT

The OmniPITR project is Copyright (c) 2010 OmniTI. All rights reserved.