root/src/jlog/jlog_sanity_check.pl

Revision cc981f5c55100675865b6960135d70148d8585af, 7.8 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 4 years ago)

pull in jlog @ 52

  • Property mode set to 100755
Line 
1 #!/opt/msys/3rdParty/bin/perl
2 use strict;
3 use warnings;
4 use Fcntl;
5
6 my $le = undef;
7 my $jlog_endian = shift;
8 if (defined $jlog_endian) {
9   if ($jlog_endian =~ /^(little|le)$/) {
10     $le = 1;
11   } elsif ($jlog_endian =~ /^(big|be)$/) {
12     $le = 0;
13   }
14 }
15
16 my $jlog = shift;
17 if (!defined $le or !defined $jlog) {
18   print "Usage: $0 <jlog endianness (be or le)> <path to jlog>\n";
19   exit 1;
20 }
21
22 sub unpack_32 {
23   if ($le) {
24     return unpack('V1', $_[0]);
25   } else {
26     return unpack('N1', $_[0]);
27   }
28 }
29 # we have to use 2 numbers to represent a 64-bit value so this works on 32-bit
30 sub unpack_64 {
31   if ($le) {
32     return unpack('V1', substr($_[0], 4, 4)), unpack('V1', substr($_[0], 0, 4));
33   } else {
34     return unpack('N1', substr($_[0], 0, 4)), unpack('N1', substr($_[0], 4, 4));
35   }
36 }
37
38 opendir(DIR, $jlog) or die "could not opendir $jlog: $!";
39 my $files = [ readdir(DIR) ];
40 closedir DIR;
41
42 my $metastore = grep /^metastore$/, @$files;
43 $files = [ grep !/^metastore$/, @$files ];
44 my $checkpoints = [ grep /^cp[.]/, @$files ];
45 $files = [ grep !/^cp[.]/, @$files ];
46 my $segments = [ sort { hex $a <=> hex $b } grep /^[0-9A-Fa-f]{8}$/, @$files ];
47 $files = [ grep !/^[0-9A-Fa-f]{8}$/, @$files ];
48 my $indexes = [ grep /^[0-9A-Fa-f]{8}.idx$/, @$files ];
49 $files = [ grep !/^[0-9A-Fa-f]{8}.idx$/, @$files ];
50
51 if (!$metastore) {
52   die "no metastore found\n";
53 }
54
55 $files = [ grep !/^[.]{1,2}$/, @$files ];
56 foreach (@$files) {
57   print "unexpected file found: $_ (skipping)\n";
58 }
59 undef $files;
60
61 if ((stat "$jlog/metastore")[7] != 12) {
62   die "metastore has invalid size\n";
63 }
64 my ($current_segment, $unit_limit, $safety);
65 sysopen(META, "$jlog/metastore", O_RDONLY)
66   or die "could not sysopen $jlog/metastore: $!";
67 my $data;
68 sysread(META, $data, 4) == 4 or die "metastore read error: $!";
69 $current_segment = unpack_32($data);
70 sysread(META, $data, 4) == 4 or die "metastore read error: $!";
71 $unit_limit = unpack_32($data);
72 sysread(META, $data, 4) == 4 or die "metastore read error: $!";
73 $safety = unpack_32($data);
74 close META;
75
76 my $oldest_cp_segment = 0xffffffff;
77 my $cpbyname = {};
78 foreach my $cp (@$checkpoints) {
79   my $cpname = $cp;
80   $cpname =~ s/^cp[.]//;
81   $cpname = pack('H*', $cpname);
82   if ((stat "$jlog/$cp")[7] != 8) {
83     print "checkpoint $cpname has invalid size\n";
84     next;
85   }
86   sysopen(CP, "$jlog/$cp", O_RDONLY) or die "could not sysopen $jlog/$cp: $!";
87   sysread(CP, $data, 4) == 4 or die "checkpoint $cpname: read error: $!";
88   my $segment = unpack_32($data);
89   sysread(CP, $data, 4) == 4 or die "checkpoint $cpname: read error: $!";
90   my $marker = unpack_32($data);
91   close CP;
92   if ($segment > $current_segment) {
93     print "checkpoint $cpname points to segment newer than current segment\n";
94     next;
95   }
96   $oldest_cp_segment = $segment if ($segment < $oldest_cp_segment);
97   $segment = sprintf "%08x", $segment;
98   $cpbyname->{$cpname} = {
99     segment => $segment,
100     marker => $marker,
101   };
102 }
103 if (!scalar keys %$cpbyname) {
104   die "no valid checkpoints\n";
105 }
106
107 my $lastnum = $oldest_cp_segment;
108 foreach my $seg (@$segments) {
109   my $num = hex $seg;
110   if ($num < $oldest_cp_segment) {
111     print "segment $seg is older than any checkpoint\n";
112   }
113   if ($num > $current_segment) {
114     print "segment $seg is newer than the current segment\n";
115   }
116   if ($num > $lastnum + 1) {
117     if ($num > $lastnum + 2) {
118       printf "segments %08x though %08x missing\n", $lastnum + 1, $num - 1;
119     } else {
120       printf "segment %08x missing\n", $lastnum + 1;
121     }
122   }
123   if ($num > $lastnum) {
124     $lastnum = $num;
125   }
126   my ($idx) = grep /$seg[.]idx/i, @$indexes;
127   $indexes = [ grep !/$seg[.]idx/i, @$indexes ];
128   $seg = [ $seg, $idx ];
129 }
130 if ($current_segment > $lastnum + 1) {
131   if ($current_segment > $lastnum + 2) {
132     printf "segments %08x though %08x missing\n",
133       $lastnum + 1, $current_segment - 1;
134   } else {
135     printf "segment %08x missing\n", $lastnum + 1;
136   }
137 }
138
139 foreach my $idx (@$indexes) {
140   print "index $idx doesn't correspond to any segment (skipping)\n";
141 }
142
143 foreach my $segdata (@$segments) {
144   my ($seg, $idx) = @$segdata;
145   my $last_marker = 0;
146   my $last_tv_sec = 0;
147   my $last_tv_usec = 0;
148   my $warned_timewarp = 0;
149   my $warned_toobig = 0;
150   my $data_off = 0;
151   my $idx_off = 0;
152   sysopen(SEG, "$jlog/$seg", O_RDONLY) or die "could not sysopen $jlog/$seg: $!";
153   my $data_len = (stat SEG)[7];
154   my $idx_len;
155   if ($idx) {
156     sysopen(IDX, "$jlog/$idx", O_RDONLY) or die "could not sysopen $jlog/$idx: $!";
157     $idx_len = (stat IDX)[7];
158   }
159
160   while ($data_off < $data_len) {
161     if (!$warned_toobig and ($data_off > $unit_limit)) {
162       print "segment $seg has message offset larger than unit limit\n";
163       $warned_toobig = 1;
164     }
165     if ($data_off + 16 > $data_len) {
166       print "segment $seg offset $data_off: not enough room for message header\n";
167       last;
168     }
169     sysread(SEG, $data, 16) == 16
170       or die "segment $seg offset $data_off: read error: $!";
171     my $reserved = unpack_32(substr $data, 0, 4);
172     my $tv_sec = unpack_32(substr $data, 4, 4);
173     my $tv_usec = unpack_32(substr $data, 8, 4);
174     my $mlen = unpack_32(substr $data, 12, 4);
175     if ($reserved) {
176       print "segment $seg offset $data_off: reserved field not 0\n";
177       last;
178     }
179     if (!$warned_timewarp) {
180       if ($tv_sec < $last_tv_sec or
181           ($tv_sec == $last_tv_sec and $tv_usec < $last_tv_usec))
182       {
183         print "segment $seg offset $data_off: time goes backwards\n";
184         $warned_timewarp = 1;
185       } else {
186         $last_tv_sec = $tv_sec;
187         $last_tv_usec = $tv_usec;
188       }
189     }
190     if ($data_off + $mlen > $data_len) {
191       print "segment $seg offset $data_off: not enough room for message body\n";
192       last;
193     }
194     sysread(SEG, $data, $mlen) == $mlen
195       or die "segment $seg offset $data_off + 16: read error: $!";
196     $last_marker++;
197
198     if ($idx) {
199       if ($idx_off == $idx_len) {
200         if ($current_segment > hex $seg) {
201           print "index $idx is incomplete (not an error)\n";
202         }
203         close IDX;
204         undef $idx;
205       } elsif ($idx_off + 8 > $idx_len) {
206         print "index $idx offset $idx_off: no room for next offset\n";
207         close IDX;
208         undef $idx;
209       } else {
210         sysread(IDX, $data, 8) == 8
211           or die "index $idx offset $idx_off: read error: $!";
212         my ($offh, $offl) = unpack_64($data);
213         if ($offh != 0 or $offl != $data_off) {
214           print "index $idx offset $idx_off: index points to wrong offset\n";
215           close IDX;
216           undef $idx;
217         } else {
218           $idx_off += 8;
219         }
220       }
221     }
222
223     $data_off += 16 + $mlen;
224   }
225   if ($data_off == $data_len) {
226     $segdata->[2] = $last_marker;
227   } else {
228     close IDX;
229     undef $idx;
230   }
231
232   close SEG;
233   if ($idx) {
234     if ($idx_off == $idx_len) {
235       if (hex $seg < $current_segment) {
236         print "index $idx not current or closed\n";
237       }
238     } elsif ($idx_off + 8 > $idx_len) {
239       print "index $idx offset $idx_off: no room for closing index\n";
240     } elsif ($idx_off + 8 < $idx_len) {
241       print "index $idx offset $idx_off: index too long\n";
242     } else {
243       sysread(IDX, $data, 8) == 8
244         or die "index $idx offset $idx_off: read error: $!";
245       my ($offh, $offl) = unpack_64($data);
246       if ($offh != 0 or $offl != 0) {
247         print "index $idx offset $idx_off: closing offset not 0\n";
248       }
249     }
250     close IDX;
251   }
252 }
253
254 foreach my $cp (keys %$cpbyname) {
255   if ($cpbyname->{$cp}{segment} ne '00000000' or
256       $cpbyname->{$cp}{marker} != 0)
257   {
258     my ($segdata) = grep { $_->[0] eq $cpbyname->{$cp}{segment} } @$segments;
259     if (!defined $segdata) {
260       if (hex $cpbyname->{$cp}{segment} != $current_segment) {
261         printf "checkpoint $cp points to nonexistent segment\n";
262       }
263     } elsif (!defined $segdata->[2]) {
264       print "checkpoint $cp points to a damaged segment\n";
265     } elsif ($cpbyname->{$cp}{marker} > $segdata->[2]) {
266       print "checkpoint $cp points past the end of segment\n";
267     }
268   }
269 }
Note: See TracBrowser for help on using the browser.