root/zetaback

Revision 7937fdb5603ddb77ad61e8cfad23ce65e95d7e47, 19.8 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 11 years ago)

First draft of options docs, refs #2

  • Property mode set to 100755
Line 
1 z#!/usr/bin/perl
2
3 use strict;
4 use Getopt::Long;
5 use MIME::Base64;
6 use POSIX qw/strftime/;
7 use Pod::Usage;
8
9 use vars qw/%conf $version_string
10             $CONF $BLOCKSIZE $DEBUG $HOST $BACKUP
11             $RESTORE $RESTORE_HOST $RESTORE_ZFS $TIMESTAMP
12             $LIST $SUMMARY $SUMMARY_EXT
13             $EXPUNGE $NUETERED $ZFS
14             $VERSION $HELP/;
15 $version_string = '0.1';
16 $CONF = q^/etc/zetaback.conf^;
17 $BLOCKSIZE = 1024*64;
18
19 $conf{'default'}->{'time_format'} = "%Y-%m-%d %H:%M:%S";
20 $conf{'default'}->{'retention'} = 14 * 86400;
21
22 =pod
23
24 =head1 NAME
25
26 zetaback - perform backup, restore and retention policies for ZFS backups.
27
28 =head1 SYNOPSIS
29
30   zetaback -v
31
32   zetaback [-l | -s | -sx] [-c conf] [-d] [-h host] [-z zfs]
33
34   zetaback -b [-x] [-c conf] [-d] [-n] [-h host] [-z zfs]
35
36   zetaback -x [-b] [-c conf] [-d] [-n] [-h host] [-z zfs]
37
38   zetaback -r [-c conf] [-d] [-n] [-h host] [-z zfs] [-t timestamp]
39               [-rhost host] [-rzfs fs]
40
41 =cut
42
43 GetOptions(
44   "h=s"     => \$HOST,
45   "z=s"     => \$ZFS,
46   "c=s"     => \$CONF,
47   "b"       => \$BACKUP,
48   "l"       => \$LIST,
49   "s"       => \$SUMMARY,
50   "sx"      => \$SUMMARY_EXT,
51   "r"       => \$RESTORE,
52   "t=i"     => \$TIMESTAMP,
53   "rhost=s" => \$RESTORE_HOST,
54   "rzfs=s"  => \$RESTORE_ZFS,
55   "d"       => \$DEBUG,
56   "n"       => \$NUETERED,
57   "x"       => \$EXPUNGE,
58   "v"       => \$VERSION,
59 );
60
61 # actions allowed together 'x' and 'b' all others are exclusive:
62 my $actions = 0;
63 $actions++ if($BACKUP || $EXPUNGE);
64 $actions++ if($RESTORE);
65 $actions++ if($LIST);
66 $actions++ if($SUMMARY);
67 $actions++ if($SUMMARY_EXT);
68 $actions++ if($VERSION);
69 if($actions != 1) {
70   pod2usage({ -verbose => 0 });
71   exit -1;
72 }
73
74 =pod
75
76 =head1 DESCRIPTION
77
78 The B<zetaback> program is orchiestrates the backup (either full or
79 incremental) of remote ZFS filesysetms to a local store.  It handles
80 frequency requirements for both full and incemental backups as well
81 as retention policies.  In addition to backups, the B<zetaback> tool
82 allows for the restore of any backup to a specified host and zfs
83 filesystem.
84
85 =head1 OPTIONS
86
87 The non-optional action command line arguments define the invocation purpose
88 of B<zetaback>.  All other options are optional and refine the target
89 of the action specified.
90
91 =head2 Generic Options
92
93 The following options have the same meaning over several actions:
94
95 =over
96
97 =item -c <conf>
98
99 Use the specified file as the configuration file.  The default file, if
100 none is specified is /etc/zetaback.conf.
101
102 =item -d
103
104 Enable debugging output.
105
106 =item -n
107
108 Don't actually perform any remote commands or expunging.  This is useful with
109 the -d options to ascertain what would be done if the command was executed
110 without the -n option.
111
112 =item -t <timestamp>
113
114 Used during the restore process to Specify a point in time selection of a
115 backup image.  If omitted, the command becomes interactive.  This timestamp
116 is a UNIX timestamp abd is shown in output of -s and -sx actions.
117
118 =item -rhost <host>
119
120 Specifiy the remote host that is the target for a restore operation.  If
121 omitted the command becomes interactive.
122
123 =item -rzfs <zfs>
124
125 Specifiy the remote ZFS filesystem that is the target for a restore
126 operation.  If omitted the command becomes interactive.
127
128 =item -h <host>
129
130 Filters the operation to the host specified.  If <host> is the of the form
131 /patterm/, it matches 'pattern' as a perl regular expression against available
132 hosts.  If this option is omitted, no limit is enforced and all hosts are
133 used for the action.
134
135 =item -z <zfs>
136
137 Filters the operation to the zfs filesystem specified.  If <zfs> is of the
138 form /pattern/, it matches 'pattern' as a perl regular expression against
139 available zfs filesystems.  If this option is omitted, no filter is enforced
140 and all zfs filesystems are used for the action.
141
142 =back
143
144 =head2 Actions
145
146 =over
147
148 =item -v
149
150 Show the version.
151
152 =item -l
153
154 Show a brief listing of available backups.
155
156 =item -s
157
158 Like -l, -s will show a list of backups but provides also the most recent
159 backup information including timestamp type (full or incremental) and the
160 size on disk.
161
162 =item -sx
163
164 Shows a extended summary.  In addition to the output provided by the -s
165 action, the -sx action will show detail of each availble backup and note
166 the more recent full backup and incremental bakcup if such an incremental
167 backup exists more recent than the full backup.
168
169 =item -b
170
171 Performs a backup.  This option will investigate all eligible hosts, query
172 the available filesystems from the remote agent and determine if any such
173 filesystems require a new full or incremental backup to be taken.  This
174 option may be combined with the -x option (to clean up afterwards).
175
176 =item -x
177
178 Perform an expunge.  This option will determine which, if any, of the local
179 backups may be deleted given the retention policy specified in the
180 configuration.
181
182 =item -r
183
184 Perform a restore.  This option will operate on the specified backup and
185 restore it to the a zfs filesystem specified with -rzfs on the host specified
186 with the -rhost option.  The -h, -z and -t options may be used to filter
187 the source backup list.  If the filtered list contains more than one
188 source backup image, the command will act interactively.  If the -rhost
189 and -rzfs command are not specified, the command will act interactively.
190
191 =back
192
193 =cut
194
195 if($VERSION) {
196   print "zetaback: $version_string\n";
197   exit 0;
198 }
199
200 sub parse_config() {
201   local($/);
202   $/ = undef;
203   open(CONF, "<$CONF");
204   my $file = <CONF>;
205   while($file =~ m/^\s*(\S+)\s\s*{(.*?)}/gms) {
206     my $scope = $1;
207     my $filepart = $2;
208     $conf{$scope} ||= {};
209     foreach my $line (split /\n/, $filepart) {
210       if($line =~ /^\s*([^#]\S*)\s*=\s*(\S+)/) {
211         $conf{$scope}->{lc($1)} = $2;
212       }
213     }
214   }
215   close(CONF);
216 }
217 sub config_get($$) {
218   return $conf{$_[0]}->{$_[1]} || $conf{'default'}->{$_[1]};
219 }
220
221 sub dir_encode($) {
222   my $d = shift;
223   my $e = encode_base64($d, '');
224   $e =~ s/\//_/;
225   return $e;
226 }
227 sub dir_decode($) {
228   my $e = shift;
229   $e =~ s/_/\//;
230   return decode_base64($e);
231 }
232 sub pretty_size($) {
233   my $bytes = shift;
234   if($bytes > 1024*1024*1024) {
235     return sprintf("%0.2f Gb", $bytes / (1024*1024*1024));
236   }
237   if($bytes > 1024*1024) {
238     return sprintf("%0.2f Mb", $bytes / (1024*1024));
239   }
240   if($bytes > 1024) {
241     return sprintf("%0.2f Kb", $bytes / (1024));
242   }
243   return "$bytes b";
244 }
245 sub scan_for_backups($) {
246   my %info = ();
247   my $dir = shift;
248   $info{last_full} = $info{last_incremental} = $info{last_backup} = 0;
249   opendir(D, $dir) || return \%info;
250   foreach my $file (readdir(D)) {
251     if($file =~ /^(\d+)\.([^\.]+)\.full$/) {
252       my $whence = $1;
253       my $fs = dir_decode($2);
254       $info{$fs}->{full}->{$whence}->{'file'} = "$dir/$file";
255       $info{$fs}->{last_full} = $whence if($whence > $info{$fs}->{last_full});
256       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
257                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
258     }
259     elsif($file =~ /^(\d+).([^\.]+)\.incremental.(\d+)$/) {
260       my $whence = $1;
261       my $fs = dir_decode($2);
262       $info{$fs}->{incremental}->{$whence}->{'depends'} = $3;
263       $info{$fs}->{incremental}->{$whence}->{'file'} = "$dir/$file";
264       $info{$fs}->{last_incremental} = $whence if($whence > $info{$fs}->{last_incremental});
265       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
266                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
267     }
268   }
269   closedir(D);
270   return \%info;
271 }
272
273 parse_config();
274
275 sub zfs_remove_snap($$$) {
276   my ($host, $fs, $snap) = @_;
277   my $agent = config_get($host, 'agent');
278   return unless($snap);
279   print "Dropping $snap on $fs\n" if($DEBUG);
280   `ssh $host $agent -z $fs -d $snap`;
281 }
282
283 # Lots of args.. internally called.
284 sub zfs_do_backup($$$$$$) {
285   my ($host, $fs, $type, $point, $store, $dumpfile) = @_;
286   my $agent = config_get($host, 'agent');
287
288   # Do it. yeah.
289   open(LBACKUP, ">$store/.$dumpfile") || die "zfs_full_backup: cannot create dump\n";
290   eval {
291     open(RBACKUP, "ssh $host $agent -z $fs -$type $point |") || die "zfs_full_backup: cannot perform send\n";
292     my $buffer;
293     while(my $len = sysread(RBACKUP, $buffer, $BLOCKSIZE)) {
294       if(syswrite(LBACKUP, $buffer, $len) != $len) {
295         die "$!";
296       }
297     }
298     close(LBACKUP);
299     close(RBACKUP);
300     die "dump failed (zero bytes)\n" if(-z "$store/.$dumpfile");
301     rename("$store/.$dumpfile", "$store/$dumpfile") || die "cannot rename dump\n";
302   };
303   if($@) {
304     unlink("$store/.$dumpfile");
305     die "zfs_full_backup: failed $@";
306   }
307 }
308
309 sub zfs_full_backup($$$) {
310   my ($host, $fs, $store) = @_;
311
312   # Translate into a proper dumpfile nameA
313   my $point = time();
314   my $efs = dir_encode($fs);
315   my $dumpfile = "$point.$efs.full";
316
317   zfs_do_backup($host, $fs, 'f', $point, $store, $dumpfile);
318 }
319
320 sub zfs_incremental_backup($$$$) {
321   my ($host, $fs, $base, $store) = @_;
322   my $agent = config_get($host, 'agent');
323
324   # Translate into a proper dumpfile nameA
325   my $point = time();
326   my $efs = dir_encode($fs);
327   my $dumpfile = "$point.$efs.incremental.$base";
328
329   zfs_do_backup($host, $fs, 'i', $base, $store, $dumpfile);
330 }
331
332 sub perform_retention($$) {
333   my ($host, $store) = @_;
334   my $cutoff = time() - config_get($host, 'retention');
335   my $backup_info = scan_for_backups($store);
336  
337   foreach my $disk (sort keys %{$backup_info}) {
338     my $info = $backup_info->{$disk};
339     next unless(ref($info) eq 'HASH');
340     my %must_save;
341
342     # Get a list of all the full and incrementals, sorts newest to oldest
343     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
344     @backup_points = sort { $b <=> $a } @backup_points;
345
346     # We _cannot_ throw away _all_ out backups, so save the most recent no matter what
347     $must_save{$backup_points[0]} = 1;
348
349     # Walk the list for backups within our retention period.
350     foreach (@backup_points) {
351       if($_ >= $cutoff) {
352         $must_save{$_} = 1;
353       }
354       else {
355         # they are in decending order, once we miss, all will miss
356         last;
357       }
358     }
359
360     # Look for dependencies
361     foreach (@backup_points) {
362       if(exists($info->{incremental}->{$_})) {
363         print "   => $_ depends on $info->{incremental}->{$_}->{depends}\n" if($DEBUG);
364         $must_save{$info->{incremental}->{$_}} = 1
365       }
366     }
367     my @removals = grep { !exists($must_save{$_}) } @backup_points;
368     if($DEBUG) {
369       my $tf = config_get($host, 'time_format');
370       print "    => I can remove:\n";
371       foreach (@backup_points) {
372         print "      => ". strftime($tf, localtime($_));
373         print " [". (exists($info->{full}->{$_}) ? "full":"incremental") ."]";
374         print " XXX" if(!exists($must_save{$_}));
375         print "\n";
376       }
377     }
378     foreach (@removals) {
379       my $efs = dir_encode($disk);
380       my $filename;
381       if(exists($info->{full}->{$_})) {
382         $filename = "$store/$_.$efs.full";
383       }
384       elsif(exists($info->{incremental}->{$_})) {
385         $filename = "$store/$_.$efs.incremental.$info->{incremental}->{$_}->{depends}";
386       }
387       else {
388         print "ERROR: We tried to expunge $host $disk [$_], but couldn't find it.\n";
389       }
390       print "    => expunging $filename\n" if($DEBUG);
391       unless($NUETERED) {
392         unlink($filename) || print "ERROR: unlink $filename: $?\n";
393       }
394     }
395   }
396 }
397
398 sub choose($@) {
399   my($name, @list) = @_;
400   return $list[0] if(scalar(@list) == 1);
401   print "\n";
402   my $i = 1;
403   for (@list) {
404     printf " %3d) $_\n", $i++;
405   }
406   my $selection = 0;
407   while($selection !~ /^\d+$/ or
408         $selection < 1 or
409         $selection >= $i) {
410     print "$name: ";
411     chomp($selection = <>);
412   }
413   return $list[$selection - 1];
414 }
415
416 sub backup_chain($$) {
417   my ($info, $ts) = @_;
418   my @list;
419   push @list, $info->{full}->{$ts} if(exists($info->{full}->{$ts}));
420   if(exists($info->{incremental}->{$ts})) {
421     push @list, $info->{incremental}->{$ts};
422     push @list, backup_chain($info, $info->{incremental}->{$ts}->{depends});
423   }
424   return @list;
425 }
426
427 sub perform_restore() {
428   my %source;
429
430   foreach my $host (grep { $_ ne "default" } keys %conf) {
431     # If -h was specific, we will skip this host if the arg isn't
432     # an exact match or a pattern match
433     if($HOST &&
434        !(($HOST eq $host) ||
435          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
436       next;
437     }
438
439     my $store = config_get($host, 'store');
440     $store =~ s/%h/$host/g;;
441     mkdir $store if(! -d $store);
442
443     my $backup_info = scan_for_backups($store);
444     foreach my $disk (sort keys %{$backup_info}) {
445       my $info = $backup_info->{$disk};
446       next unless(ref($info) eq 'HASH');
447       next
448         if($ZFS &&      # if the pattern was specified it could
449            !($disk eq $ZFS ||        # be a specific match or a
450              ($ZFS =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
451       # We want to see this one
452       my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
453       my @source_points;
454       foreach (@backup_points) {
455         push @source_points, $_ if(!$TIMESTAMP || $TIMESTAMP == $_)
456       }
457       if(@source_points) {
458         $source{$host}->{$disk} = \@source_points;
459       }
460     }
461   }
462
463   if(! keys %source) {
464     print "No matching backups found\n";
465     return;
466   }
467
468   # Here goes the possibly interactive dialog
469   my $host = choose("Restore from host", keys %source);
470   my $disk = choose("Restore from ZFS", keys %{$source{$host}});
471   my $timestamp = choose("Restore as of timestamp", @{$source{$host}->{$disk}});
472   my $tf = config_get($host, 'time_format');
473
474   my $store = config_get($host, 'store');
475   $store =~ s/%h/$host/g;;
476   mkdir $store if(! -d $store);
477   my $backup_info = scan_for_backups($store);
478   my @backup_list = reverse backup_chain($backup_info->{$disk}, $timestamp);
479
480   if(!$RESTORE_HOST) {
481     print "Restore to host [$host]:";
482     chomp(my $input = <>);
483     $RESTORE_HOST = length($input) ? $input : $host;
484   }
485   if(!$RESTORE_ZFS) {
486     print "Restore to zfs [$disk]:";
487     chomp(my $input = <>);
488     $RESTORE_ZFS = length($input) ? $input : $disk;
489   }
490
491   # show intentions
492   print "Going to restore:\n";
493   print "\tfrom: $host\n";
494   print "\tfrom: $disk\n";
495   print "\t  at: $timestamp [" . strftime($tf, localtime($timestamp)) . "]\n";
496   print "\t  to: $RESTORE_HOST\n";
497   print "\t  to: $RESTORE_ZFS\n";
498   print "\n";
499
500   foreach(@backup_list) {
501     $_->{success} = zfs_restore_part($RESTORE_HOST, $RESTORE_ZFS, $_->{file});
502   }
503 }
504
505 sub zfs_restore_part($$$) {
506   my ($host, $fs, $file) = @_;
507   my $command;
508   if(exists($conf{$host})) {
509     my $agent = config_get($host, 'agent');
510     $command = "$agent -r -z $fs";
511   }
512   else {
513     $command = "/usr/sbin/zfs recv $fs";
514   }
515   print " => piping $file to $command\n" if($DEBUG);
516   unless($NUETERED) {
517     open(DUMP, "gzip -dfc $file |");
518     eval {
519       open(RECEIVER, "| ssh $host $command");
520       my $buffer;
521       while(my $len = sysread(DUMP, $buffer, $BLOCKSIZE)) {
522         if(syswrite(RECEIVER, $buffer, $len) != $len) {
523           die "$!";
524         }
525       }
526     };
527     close(DUMP);
528     close(RECEIVER);
529   }
530   return $?;
531 }
532
533 sub show_backups($$$) {
534   my ($host, $store, $diskpat) = @_;
535   my $backup_info = scan_for_backups($store);
536   my $tf = config_get($host, 'time_format');
537   foreach my $disk (sort keys %{$backup_info}) {
538     my $info = $backup_info->{$disk};
539     next unless(ref($info) eq 'HASH');
540     next
541       if($diskpat &&      # if the pattern was specified it could
542          !($disk eq $diskpat ||        # be a specific match or a
543            ($diskpat =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
544     # We want to see this one
545     print "$host:$disk\n";
546     next unless($SUMMARY || $SUMMARY_EXT);
547     if($SUMMARY_EXT) {
548       print "\tLast Full: ". ($info->{last_full} ? strftime($tf, localtime($info->{last_full})) : "Never") . "\n";
549       if($info->{last_full} < $info->{last_incremental}) {
550         print "\tLast Incr: ". strftime($tf, localtime($info->{last_incremental})). "\n";
551       }
552     }
553     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
554     @backup_points = sort { $a <=> $b } @backup_points;
555     unless ($SUMMARY_EXT) {
556       @backup_points = (pop @backup_points);
557     }
558     foreach (@backup_points) {
559       print "\t" . strftime($tf, localtime($_)) . " [$_] ";
560       if(exists($info->{full}->{$_})) {
561         my @st = stat($info->{full}->{$_}->{file});
562         print "FULL " . pretty_size($st[7]);
563       } else {
564         my @st = stat($info->{incremental}->{$_}->{file});
565         print "INCR from [$info->{incremental}->{$_}->{depends}] " . pretty_size($st[7]);
566       }
567       print "\n";
568     }
569     print "\n";
570   }
571 }
572
573 sub plan_and_run($$$) {
574   my ($host, $store, $diskpat) = @_;
575   print "Planning '$host'\n" if($DEBUG);
576   my $agent = config_get($host, 'agent');
577   open(ZFSLIST, "ssh $host $agent -l |") || next;
578   foreach my $diskline (<ZFSLIST>) {
579     chomp($diskline);
580     next unless($diskline =~ /^(\S+) \[([^\]]*)\]/);
581     my $diskname = $1;
582     my %snaps;
583     map { $snaps{$_} = 1 } (split(/,/, $2));
584
585     # If we are being selective (via -z) now is the time.
586     next
587       if($diskpat &&          # if the pattern was specified it could
588          !($diskname eq $diskpat ||        # be a specific match or a
589            ($diskpat =~ /^\/(.+)\/$/ && $diskname =~ /$1/))); # regex
590
591     print " => Scanning '$store' for old backups of '$diskname'.\n" if($DEBUG);
592     # Make directory on demand
593     my $backup_info = scan_for_backups($store);
594     # That gave us info on all backups, we just want this disk
595     $backup_info = $backup_info->{$diskname} || {};
596
597     # Should we do a backup?
598     my $backup_type = 'no';
599     if(time() > $backup_info->{last_backup} + config_get($host, 'backup_interval')) {
600       $backup_type = 'incremental';
601     }
602     if(time() > $backup_info->{last_full} + config_get($host, 'full_interval')) {
603       $backup_type = 'full';
604     }
605
606     # If we want an incremental, but have no full, then we need to upgrade to full
607     if($backup_type eq 'incremental') {
608       my $have_full_locally = 0;
609       # For each local full backup, see if the full backup still exists on the other end.
610       foreach (keys %{$backup_info->{'full'}}) {
611         $have_full_locally = 1 if(exists($snaps{'__zb_full_' . $_}));
612       }
613       $backup_type = 'full' unless($have_full_locally);
614     }
615
616     print " => doing $backup_type backup\n" if($DEBUG);
617     # We need to drop a __zb_base snap or a __zb_incr snap before we proceed
618     unless($NUETERED) {
619       if($backup_type eq 'full') {
620         eval { zfs_full_backup($host, $diskname, $store); };
621         if ($@) {
622           chomp(my $err = $@);
623           print " => failure $err\n";
624         }
625         else {
626           # Unless there was an error backing up, remove all the other full snaps
627           foreach (keys %snaps) {
628             zfs_remove_snap($host, $diskname, $_) if(/^__zb_full_(\d+)/)
629           }
630         }
631       }
632       if($backup_type eq 'incremental') {
633         zfs_remove_snap($host, $diskname, '__zb_incr') if($snaps{'__zb_incr'});
634         # Find the newest full from which to do an incremental (NOTE: reverse numeric sort)
635         my @fulls = sort { $b <=> $a } (keys %{$backup_info->{'full'}});
636         zfs_incremental_backup($host, $diskname, $fulls[0], $store);
637       }
638     }
639   }
640   close(ZFSLIST);
641 }
642
643 if($RESTORE) {
644   perform_restore();
645 }
646 else {
647   foreach my $host (grep { $_ ne "default" } keys %conf) {
648     # If -h was specific, we will skip this host if the arg isn't
649     # an exact match or a pattern match
650     if($HOST &&
651        !(($HOST eq $host) ||
652          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
653       next;
654     }
655  
656     my $store = config_get($host, 'store');
657     $store =~ s/%h/$host/g;;
658     mkdir $store if(! -d $store);
659  
660     if($LIST || $SUMMARY || $SUMMARY_EXT) {
661       show_backups($host, $store, $ZFS);
662     }
663     if($BACKUP) {
664       plan_and_run($host, $store, $ZFS);
665     }
666     if($EXPUNGE) {
667       perform_retention($host, $store);
668     }
669   }
670 }
671
672 exit 0;
673
674 =pod
675
676 =head1 FILES
677
678 =over
679
680 =item /etc/zetaback.conf
681
682 The main zetaback configuration file.  The location of the file can be
683 specified on the command line with the -c flag.
684
685 =back
686
687 =head1 SEE ALSO
688
689 zetaback.conf(5), zetaback_agent(1), zetback_agent.conf(5)
690
691 =cut
Note: See TracBrowser for help on using the browser.