root/trunk/pg_log/pg_log_monitor.pl

Revision 38, 12.6 kB (checked in by denish, 5 years ago)

pg_log monitor tool Version 0.1

Line 
1 #!/data/bin/perl
2
3 #Performs initial script configuration. Must be executed within BEGIN
4 #in order to dynamically set "use lib" value.
5 BEGIN{
6
7     @MAILER_RELAY_IPS = ('127.0.0.1');
8     $SCRIPT_DIR='/home/postgres/bin/pg_log';
9     $LIB_DIRECTORY='';
10     $SERVER_HOSTNAME='';
11     $TO_ADDRESS='';
12     $FROM_ADDRESS='';
13     $day = (localtime)[3];
14     $month = (localtime)[4];
15     $year = (localtime)[5];
16     $month += 1; $year += 1900;
17
18     $CURRENT_DATE = sprintf("%02d/%02d/%04d", $month, $day, $year);
19    
20     open(SC, '<', "$SCRIPT_DIR/settings.conf");
21         my @settings = <SC>;
22     close(SC);
23    
24     foreach my $line (@settings){
25         chomp($line);
26         my @setting = split(':', $line);       
27         if($setting[0] =~ m/^lib_directory/i){
28             $LIB_DIRECTORY = $setting[1];
29         }
30         elsif($setting[0] =~ m/^server_hostname/){
31             $SERVER_HOSTNAME = $setting[1];
32             chomp($SERVER_HOSTNAME);
33         }elsif($setting[0] =~ m/^from_address/){
34             $FROM_ADDRESS = $setting[1];
35             chomp($FROM_ADDRESS);
36         }elsif($setting[0] =~ m/^to_address/){
37             $TO_ADDRESS = $setting[1];
38             chomp($TO_ADDRESS);
39         }
40     }
41    
42 }
43
44 #Package/Class Includes
45 use lib $LIB_DIRECTORY;
46 use Time::Local;
47 use Switch;
48 use MIME::Lite;
49 use Socket;
50 use Sys::Hostname;
51 use Data::Dumper;
52
53 #Command Line Options
54 my $directory='';
55 my $timespan=1;
56 my $static_ignore=0;
57 my $dynamic_ignore=0;
58 my $start_with_yesterday=0;
59 my $cron=0;
60
61 #Grab CL Arguments
62 for(my $x = 0; $x <= $#ARGV; $x++){
63     switch($ARGV[$x]){         
64         case '-h' { display_help_text(); exit; }
65
66         case '-t' { $timespan=$ARGV[$x + 1]; }
67        
68         case '-d' { $directory=$ARGV[$x + 1]; }
69              
70         case '-i' { $static_ignore = 1; }
71              
72         case '-ix' { $dynamic_ignore = 1; }
73              
74         case '-c' { $cron = 1; }
75        
76         case '-y' { $start_with_yesterday = 1; }
77          }
78 }
79
80 #Check for required params
81 die "You must pass in a directory with the -d option." unless $directory;
82                
83 my @logfile_1_errors;
84 my @logfile_2_errors;
85 my @static_ignore_messages;
86 my @dynamic_ignore_regex;
87 my @unique_error_msgs;
88 my @seen_error_msgs;
89
90 #Load .dat configurations
91 @static_ignore_messages = load_static_ignore();
92 @dynamic_ignore_regex = load_dynamic_ignore();
93
94
95   #Main Application Logic
96
97    #Populate array of all logfile path names, sorted in descending order
98     opendir(FH, $directory);
99       my @file_list = grep(/\.log$/, readdir(FH));
100     closedir(FH);
101    
102     @file_list = sort {$b cmp $a} @file_list;
103
104    #Iterate through directory tree until timespan reached
105     for($count=0; $count < scalar(@file_list); $count++){   
106      
107       if($start_with_yesterday){
108          $start_with_yesterday=0;
109          next;
110       }
111      
112       $logfile_1_path = $directory."/".$file_list[$count];
113       $logfile_2_path = $directory."/".$file_list[$count + 1];
114    
115       @logfile_1_errors = load_errors($logfile_1_path);
116       @logfile_2_errors = load_errors($logfile_2_path);
117    
118       if($static_ignore){
119         @logfile_1_errors = remove_static_ignore(@logfile_1_errors);
120         @logfile_2_errors = remove_static_ignore(@logfile_2_errors);   
121       }
122    
123       if($dynamic_ignore){
124         @logfile_1_errors = remove_dynamic_ignore(@logfile_1_errors);
125         @logfile_2_errors = remove_dynamic_ignore(@logfile_2_errors);       
126       }
127  
128       my $new_ref; my $recurring_ref;
129       ($new_ref, $recurring_ref) = compare_logfiles( \@logfile_1_errors, \@logfile_2_errors);
130       @new = @$new_ref;
131       @recurring = @$recurring_ref;
132    
133       @new = remove_duplicates(@new);
134       @recurring = remove_duplicates(@recurring);
135    
136    
137       #Assemble Report for display or e-mail
138      
139       my $report = generate_output();
140      
141       if($cron){
142    
143           print "Sending e-mail. . .\n";
144          
145             my $addr = gethostbyname($name);
146                  
147             my $success='';
148             my $hostname = `hostname`;
149             my $report_date='';
150             ($report_date) = $logfile_1_path =~ m/(\d{4}-\d{2}-\d{2})\.log$/;
151             my $subject = "Postgres database pg_log errors on $SERVER_HOSTNAME for $report_date";
152             my $body = $report;
153             chomp($hostname);
154             foreach (@MAILER_RELAY_IPS){
155                 eval {
156                     my $msg = new MIME::Lite(  Type => 'text/plain',
157                                                 From => $FROM_ADDRESS,
158                                                 To => $TO_ADDRESS,
159                                                 Data => $body,
160                                                 Subject => $subject
161                     );
162
163                      $msg->send('smtp',$_, Debug=>0);
164                  };
165                  if (!$@) {
166                      $success = 1;
167                      last;
168                  }
169              }
170              if (!$success){
171                      die "E-mail attempt failed for pg_log_monitor.pl.\n";
172              }         
173    
174       }
175       else{       
176         print "\nCompleted report for $report_date.\n";
177         print "Action: [d]isplay error(s), [w]rite to file, or [s]kip: ";
178         $choice = <STDIN>;
179         chomp($choice);
180              
181             switch($choice){
182                 case 'd'  {
183                    print $report;
184                    print "\n\nPress enter to continue: ";
185                    <STDIN>;
186                 }
187
188                 case 'w'  {
189                     open(FH, '>>', 'new_pg_errors.txt');
190                         print FH $report;
191                     close(FH);
192                     print "\nAppeneded to file new_pg_errors.txt.\n\n";
193                     print "Press enter to continue: ";
194                     <STDIN>;                   
195                 }
196            }
197      }   
198
199      last unless ($count + 1) < $timespan;
200    
201     }#End main logic loop
202
203 ###########################
204 #### Support Functions ####
205 ###########################
206
207 sub load_errors(){
208
209 my $logpath = shift;
210 my @lines;   
211  
212     open(FH, '<', "$logpath");
213
214     while(my $line = <FH>){
215        chomp($line);
216        ($error_message) = $line =~ m/ERROR\:\s*(.+)/;
217        if($error_message){
218             $error_message =~ s/^\s+//;
219             $error_message =~ s/\s+$//;
220             push(@lines, $error_message);
221        }
222     }
223
224     close(FH);   
225    
226 return @lines;
227
228 }
229
230 sub load_static_ignore{
231
232 my @ignore_messages;
233
234     open(FH, '<', "$SCRIPT_DIR/static_ignore.dat");
235
236         while(my $line = <FH>){
237             chomp($line);
238             $line =~ s/^\s+//;
239             $line =~ s/\s+$//;
240             push(@ignore_messages, $line);   
241         }
242        
243     close(FH);
244
245 return @ignore_messages;
246
247 }
248
249 sub load_dynamic_ignore{
250
251 my @ignore_regex;
252
253     open(FH, '<', "$SCRIPT_DIR/dynamic_ignore.dat");
254
255         while(my $line = <FH>){
256             chomp($line);
257             $line =~ s/^\s+//;
258             $line =~ s/\s+$//;
259             push(@ignore_regex, $line);   
260         }
261        
262     close(FH);
263
264     foreach $line (@ignore_regex){
265        $line = replace_special_chars($line);
266        $line =~ s/::DYN::/\.\+/g;
267        $line = '^'.$line.'$';
268     }
269    
270 return @ignore_regex;
271
272 }
273
274
275 sub compare_logfiles(){
276
277 $ref_1 = $_[0];
278 $ref_2 = $_[1];
279
280 my @logfile_1 = @$ref_1;
281 my @logfile_2 = @$ref_2;
282
283 my @new_errors;
284 my @recurring_errors;
285 my $match=0;
286
287 foreach my $line1 (@logfile_1){
288
289     foreach my $line2 (@logfile_2){
290        $match = 1 unless ($line1 ne $line2);
291     }
292    
293     if($match){
294       push(@recurring_errors, $line1);
295     }
296     else{
297       push(@new_errors, $line1);
298     }
299    
300     $match = 0;
301    
302 }
303
304 return (\@new_errors, \@recurring_errors);
305    
306 }
307
308 sub remove_static_ignore{
309
310 my @stat = @_;
311 my @valid_lines;
312
313     my $static_error = 0;
314         foreach my $line_message (@stat){
315         foreach my $ignore_error (@static_ignore_messages){
316                 if($ignore_error eq $line_message){
317                 $static_error = 1;
318             }
319         }
320         push(@valid_lines, $line_message) unless $static_error;
321         $static_error = 0;
322     }
323
324 return @valid_lines;
325
326 }
327
328 sub remove_dynamic_ignore{
329
330 my @lines = @_;
331 my @valid_lines;
332
333     my $found_dyn_error = 0;
334         foreach my $line_message (@lines){
335         foreach my $ignore_regex (@dynamic_ignore_regex){
336                 if($line_message =~ m/$ignore_regex/){
337                 $found_dyn_error = 1;
338             }
339         }
340         push(@valid_lines, $line_message) unless $found_dyn_error;
341         $found_dyn_error = 0;
342     }
343
344 return @valid_lines;
345
346 }
347
348 sub remove_duplicates{
349
350 my @lines = @_;
351 my %hash   = map { $_ => 1 } @lines;
352 my @unique = keys %hash;
353    
354 return @unique;
355
356 }
357
358 sub generate_output{
359
360       ($report_date) = $logfile_1_path =~ m/(\d{4}-\d{2}-\d{2})\.log$/;
361          
362       $output  =  "\nReport For Date: $report_date\n";
363       $output .=  "Total New Unique Errors Found: ".@new;
364           if($static_ignore && $dynamic_ignore){
365               $output .= " (Minus Static & Dynamic Ignores)\n";
366           }
367           elsif($static_ignore){
368              $output .= " (Minus Static Ignores)\n";
369           }
370           elsif($dynamic_ignore){
371              $output .= " (Minus Dynamic Ignores)\n";
372           }
373           else{
374              $output .= "\n";
375           }     
376       $output .=  "Total Recurring Unique Errors: ".@recurring;
377        
378          if($static_ignore && $dynamic_ignore){
379              $output .= " (Minus Static & Dynamic Ignores)\n";
380          }
381          elsif($static_ignore){
382              $output .= " (Minus Static Ignores)\n";
383          }
384          elsif($dynamic_ignore){
385              $output .= " (Minus Dynamic Ignores)\n";
386          }
387          else{
388              $output .= "\n";
389          }   
390
391     if(scalar(@new) > 0){
392         $output .= "\nNew Errors: (including any duplicates)\n";
393         $output .= "--------------------------------------------------\n";   
394         my @full_new_lines;
395        
396         foreach $error (@new){
397        
398             $error = replace_special_chars($error);
399        
400             open(FH, '<', $logfile_1_path);
401
402             while(my $rawline = <FH>){
403             chomp($rawline);
404                 if($rawline =~ $error){
405                     push(@full_new_lines, $rawline);
406                 }
407            }
408            
409            close(FH);
410
411         }
412        
413         @full_new_lines = sort @full_new_lines;
414         foreach my $full_line (@full_new_lines){
415             $output .= $full_line."\n";
416         }
417        
418        
419     }
420
421     if(scalar(@recurring) > 0){
422         $output .= "\nRecurring Errors: (including any duplicates)\n";   
423         $output .= "--------------------------------------------------\n";
424         my @full_recurring_lines;
425      
426         foreach $error (@recurring){
427
428            open(FH, '<', $logfile_1_path);
429
430             while(my $rawline = <FH>){
431             chomp($rawline);
432             if($rawline =~ $error){
433                 push(@full_recurring_lines, $rawline);
434             }
435            }
436            
437            close(FH);
438
439         }
440        
441         @full_recurring_lines = sort @full_recurring_lines;
442         foreach my $full_line (@full_recurring_lines){
443             $output .= $full_line."\n";
444         }       
445        
446     }
447    
448     return $output;
449    
450 }
451
452 sub replace_special_chars(){
453
454 my $data_string = shift;
455
456 $data_string =~ s/([\$\.\"\'\(\)\|\*])/\\$1/g;
457
458 return $data_string;
459
460 }
461
462 sub display_settings(){
463     print "Initial Settings:\n";
464     print "-------------------------\n";
465     print "Lib Directory: $LIB_DIRECTORY\n";
466     print "Script Directory: $SCRIPT_DIR\n";
467     print "Server hostname: $SERVER_HOSTNAME\n";
468     print "To Address: $TO_ADDRESS\n";
469     print "From Address: $FROM_ADDRESS\n";
470     print "Current Date: $CURRENT_DATE\n";
471
472     print "\nCommand Line Arguments:\n";
473     print "-------------------------\n";
474     print "Logfile Directory: $directory\n";
475     print "Report Timespan: $timespan days\n";
476     print "Static Ignore: $static_ignore\n";
477     print "Dynamic Ignore: $dynamic_ignore\n";
478     print "Cron Switch: $cron\n";
479 }
480
481 sub display_help_text(){
482     print "Example Usage: pg_log_monitor.pl -d '/data/logs/pg_log' [option] [arg].\n\n";
483     print "Required Options:\n";
484     print "   -d: Accepts an absolute path to the log directory to be analyzed.\n";
485     print "Optional Flags:\n";
486     print "   -t: The timespan (in days) to analyze log files within. Default value is 2.\n";
487     print "   -i: Static ignore. Ignores static error messages stored in 'static_ignore.dat'.\n";
488     print "   -ix: Dynamic ignore. Ignores dynamic error message templates stored in 'dynamic_ignore.dat'.\n";
489     print "   -c: Run script in cron mode. Bypasses need for manual entry and e-mails errors to DBA team.\n";
490     print "   -y: Start checking from yesterday log file.\n";
491     print "   -h: Display help. Generates this message and exits script.\n";
492     print "\n\n";   
493 }
494
495 sub dump_array(){
496
497   my @array = @_;
498  
499   foreach $line (@array){
500  
501      print $line."\n";
502  
503   }
504
505 }
Note: See TracBrowser for help on using the browser.