root/src/jlog/jlog_change_endian.pl

Revision cc981f5c55100675865b6960135d70148d8585af, 8.9 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 3 years ago)

pull in jlog @ 52

  • Property mode set to 100755
Line 
1 #!/opt/msys/3rdParty/bin/perl
2 use strict;
3 use warnings;
4 use Fcntl;
5 use File::Path;
6
7 my $src = shift;
8
9 my $fromle = undef;
10 my $jlog_endian = shift;
11 if (defined $jlog_endian) {
12   if ($jlog_endian =~ /^(tolittle|tole)$/) {
13     $fromle = 0;
14   } elsif ($jlog_endian =~ /^(tobig|tobe)$/) {
15     $fromle = 1;
16   }
17 }
18
19 my $dst = shift;
20
21 if (!defined $src or !defined $fromle or !defined $dst) {
22   print "Usage: $0 <source jlog> <endianness (tobe or tole)> <destination jlog>\n";
23   exit 1;
24 }
25
26 sub unpack_32 {
27   if ($fromle) {
28     return unpack('V1', $_[0]);
29   } else {
30     return unpack('N1', $_[0]);
31   }
32 }
33 sub repack_32 {
34   if ($fromle) {
35     return pack('N1', $_[0]);
36   } else {
37     return pack('V1', $_[0]);
38   }
39 }
40 # we have to use 2 numbers to represent a 64-bit value so this works on 32-bit
41 sub unpack_64 {
42   if ($fromle) {
43     return unpack('V1', substr($_[0], 4, 4)), unpack('V1', substr($_[0], 0, 4));
44   } else {
45     return unpack('N1', substr($_[0], 0, 4)), unpack('N1', substr($_[0], 4, 4));
46   }
47 }
48 sub repack_64 {
49   if ($fromle) {
50     return pack('N2', $_[0], $_[1]);
51   } else {
52     return pack('V2', $_[1], $_[0]);
53   }
54 }
55
56 mkdir($dst) or die "could not mkdir $dst: $!";
57 $SIG{__DIE__} = sub { rmtree $dst; };
58
59 opendir(DIR, $src) or die "could not opendir $src: $!";
60 my $files = [ readdir(DIR) ];
61 closedir DIR;
62
63 my $metastore = grep /^metastore$/, @$files;
64 $files = [ grep !/^metastore$/, @$files ];
65 my $checkpoints = [ grep /^cp[.]/, @$files ];
66 $files = [ grep !/^cp[.]/, @$files ];
67 my $segments = [ sort { hex $a <=> hex $b } grep /^[0-9A-Fa-f]{8}$/, @$files ];
68 $files = [ grep !/^[0-9A-Fa-f]{8}$/, @$files ];
69 my $indexes = [ grep /^[0-9A-Fa-f]{8}.idx$/, @$files ];
70 $files = [ grep !/^[0-9A-Fa-f]{8}.idx$/, @$files ];
71
72 if (!$metastore) {
73   die "no metastore found\n";
74 }
75
76 $files = [ grep !/^[.]{1,2}$/, @$files ];
77 foreach (@$files) {
78   print "unexpected file found: $_ (skipping)\n";
79 }
80 undef $files;
81
82 if ((stat "$src/metastore")[7] != 12) {
83   die "metastore has invalid size\n";
84 }
85 my ($current_segment, $unit_limit, $safety);
86 sysopen(META, "$src/metastore", O_RDONLY)
87   or die "could not sysopen $src/metastore: $!";
88 my $data;
89 sysread(META, $data, 4) == 4 or die "metastore read error: $!";
90 $current_segment = unpack_32($data);
91 sysread(META, $data, 4) == 4 or die "metastore read error: $!";
92 $unit_limit = unpack_32($data);
93 sysread(META, $data, 4) == 4 or die "metastore read error: $!";
94 $safety = unpack_32($data);
95 close META;
96
97 sysopen(META, "$dst/metastore", O_WRONLY|O_CREAT)
98   or die "could not sysopen $dst/metastore: $!";
99 $data = repack_32($current_segment);
100 syswrite(META, $data, 4) == 4 or die "metastore write error: $!";
101 $data = repack_32($unit_limit);
102 syswrite(META, $data, 4) == 4 or die "metastore write error: $!";
103 $data = repack_32($safety);
104 syswrite(META, $data, 4) == 4 or die "metastore write error: $!";
105 close META;
106
107 if (!scalar @$checkpoints) {
108   die "no checkpoints\n";
109 }
110 my $oldest_cp_segment = 0xffffffff;
111 my $cpbyname = {};
112 foreach my $cp (@$checkpoints) {
113   my $cpname = $cp;
114   $cpname =~ s/^cp[.]//;
115   $cpname = pack('H*', $cpname);
116   if ((stat "$src/$cp")[7] != 8) {
117     die "checkpoint $cpname has invalid size\n";
118   }
119   sysopen(CP, "$src/$cp", O_RDONLY) or die "could not sysopen $src/$cp: $!";
120   sysread(CP, $data, 4) == 4 or die "checkpoint $cpname: read error: $!";
121   my $segment = unpack_32($data);
122   sysread(CP, $data, 4) == 4 or die "checkpoint $cpname: read error: $!";
123   my $marker = unpack_32($data);
124   close CP;
125   if ($segment > $current_segment) {
126     die "checkpoint $cpname points to segment newer than current segment\n";
127   }
128   sysopen(CP, "$dst/$cp", O_WRONLY|O_CREAT) or die "could not sysopen $dst/$cp: $!";
129   $data = repack_32($segment);
130   syswrite(CP, $data, 4) == 4 or die "checkpoint $cpname: write error: $!";
131   $data = repack_32($marker);
132   syswrite(CP, $data, 4) == 4 or die "checkpoint $cpname: write error: $!";
133   close CP;
134   $oldest_cp_segment = $segment if ($segment < $oldest_cp_segment);
135   $segment = sprintf "%08x", $segment;
136   $cpbyname->{$cpname} = {
137     segment => $segment,
138     marker => $marker,
139   };
140 }
141
142 my $lastnum = $oldest_cp_segment;
143 foreach my $seg (@$segments) {
144   my $num = hex $seg;
145   if ($num < $oldest_cp_segment) {
146     die "segment $seg is older than any checkpoint\n";
147   }
148   if ($num > $current_segment) {
149     die "segment $seg is newer than the current segment\n";
150   }
151   if ($num > $lastnum + 1) {
152     if ($num > $lastnum + 2) {
153       printf "segments %08x though %08x missing\n", $lastnum + 1, $num - 1;
154     } else {
155       printf "segment %08x missing\n", $lastnum + 1;
156     }
157   }
158   if ($num > $lastnum) {
159     $lastnum = $num;
160   }
161   my ($idx) = grep /$seg[.]idx/i, @$indexes;
162   $indexes = [ grep !/$seg[.]idx/i, @$indexes ];
163   $seg = [ $seg, $idx ];
164 }
165 if ($current_segment > $lastnum + 1) {
166   if ($current_segment > $lastnum + 2) {
167     printf "segments %08x though %08x missing\n",
168       $lastnum + 1, $current_segment - 1;
169   } else {
170     printf "segment %08x missing\n", $lastnum + 1;
171   }
172 }
173
174 foreach my $idx (@$indexes) {
175   print "index $idx doesn't correspond to any segment (skipping)\n";
176 }
177
178 foreach my $segdata (@$segments) {
179   my ($seg, $idx) = @$segdata;
180   my $last_marker = 0;
181   my $warned_toobig = 0;
182   my $data_off = 0;
183   my $idx_off = 0;
184   sysopen(SEG, "$src/$seg", O_RDONLY) or die "could not sysopen $src/$seg: $!";
185   my $data_len = (stat SEG)[7];
186   sysopen(WSEG, "$dst/$seg", O_WRONLY|O_CREAT) or die "could not sysopen $dst/$seg: $!";
187   my $idx_len;
188   if ($idx) {
189     sysopen(IDX, "$src/$idx", O_RDONLY) or die "could not sysopen $src/$idx: $!";
190     $idx_len = (stat IDX)[7];
191     sysopen(WIDX, "$dst/$idx", O_WRONLY|O_CREAT) or die "could not sysopen $dst/$idx: $!";
192   }
193
194   while ($data_off < $data_len) {
195     if (!$warned_toobig and ($data_off > $unit_limit)) {
196       print "segment $seg has message offset larger than unit limit\n";
197       $warned_toobig = 1;
198     }
199     if ($data_off + 16 > $data_len) {
200       die "segment $seg offset $data_off: not enough room for message header\n";
201     }
202     sysread(SEG, $data, 16) == 16
203       or die "segment $seg offset $data_off: read error: $!";
204     my $reserved = unpack_32(substr $data, 0, 4);
205     my $tv_sec = unpack_32(substr $data, 4, 4);
206     my $tv_usec = unpack_32(substr $data, 8, 4);
207     my $mlen = unpack_32(substr $data, 12, 4);
208     if ($reserved) {
209       die "segment $seg offset $data_off: reserved field not 0\n";
210     }
211     if ($data_off + $mlen > $data_len) {
212       die "segment $seg offset $data_off: not enough room for message body\n";
213     }
214     $data = repack_32($reserved) . repack_32($tv_sec) .
215             repack_32($tv_usec) . repack_32($mlen);
216     syswrite(WSEG, $data, 16) == 16
217       or die "segment $seg offset $data_off: write error: $!";
218     sysread(SEG, $data, $mlen) == $mlen
219       or die "segment $seg offset $data_off + 16: read error: $!";
220     syswrite(WSEG, $data, $mlen) == $mlen
221       or die "segment $seg offset $data_off + 16: write error: $!";
222     $last_marker++;
223
224     if ($idx) {
225       if ($idx_off == $idx_len) {
226         if ($current_segment > hex $seg) {
227           print "index $idx is incomplete (not an error)\n";
228         }
229         close IDX;
230         close WIDX;
231         undef $idx;
232       } elsif ($idx_off + 8 > $idx_len) {
233         die "index $idx offset $idx_off: no room for next offset\n";
234       } else {
235         sysread(IDX, $data, 8) == 8
236           or die "index $idx offset $idx_off: read error: $!";
237         my ($offh, $offl) = unpack_64($data);
238         if ($offh != 0 or $offl != $data_off) {
239           die "index $idx offset $idx_off: index points to wrong offset\n";
240         } else {
241           $data = repack_64($offh, $offl);
242           syswrite(WIDX, $data, 8) == 8
243             or die "index $idx offset $idx_off: write error: $!";
244           $idx_off += 8;
245         }
246       }
247     }
248
249     $data_off += 16 + $mlen;
250   }
251   $segdata->[2] = $last_marker;
252
253   close SEG;
254   close WSEG;
255   if ($idx) {
256     if ($idx_off == $idx_len) {
257       if (hex $seg < $current_segment) {
258         print "index $idx not current or closed\n";
259       }
260     } elsif ($idx_off + 8 > $idx_len) {
261       die "index $idx offset $idx_off: no room for closing index\n";
262     } elsif ($idx_off + 8 < $idx_len) {
263       die "index $idx offset $idx_off: index too long\n";
264     } else {
265       sysread(IDX, $data, 8) == 8
266         or die "index $idx offset $idx_off: read error: $!";
267       my ($offh, $offl) = unpack_64($data);
268       if ($offh != 0 or $offl != 0) {
269         die "index $idx offset $idx_off: closing offset not 0\n";
270       }
271       $data = repack_64($offh, $offl);
272       syswrite(WIDX, $data, 8) == 8
273         or die "index $idx offset $idx_off: write error: $!";
274     }
275     close IDX;
276     close WIDX;
277   }
278 }
279
280 foreach my $cp (keys %$cpbyname) {
281   if ($cpbyname->{$cp}{segment} ne '00000000' or
282       $cpbyname->{$cp}{marker} != 0)
283   {
284     my ($segdata) = grep { $_->[0] eq $cpbyname->{$cp}{segment} } @$segments;
285     if (!defined $segdata) {
286       if (hex $cpbyname->{$cp}{segment} != $current_segment) {
287         die "checkpoint $cp points to nonexistent segment\n";
288       }
289     } elsif ($cpbyname->{$cp}{marker} > $segdata->[2]) {
290       die "checkpoint $cp points past the end of segment\n";
291     }
292   }
293 }
Note: See TracBrowser for help on using the browser.