root/zetaback.in

Revision 2f7771043693427156e4040c47f807cd956cb1a0, 26.1 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 7 years ago)

reference the license from each source file

  • Property mode set to 100755
Line 
1 #!/usr/bin/perl
2
3 # Copyright (c) 2007 OmniTI Computer Consulting, Inc. All rights reserved.
4 # For information on licensing see:
5 #   https://labs.omniti.com/zetaback/trunk/LICENSE
6
7 use strict;
8 use Getopt::Long;
9 use MIME::Base64;
10 use POSIX qw/strftime/;
11 use Fcntl qw/:flock/;
12 use IO::File;
13 use Pod::Usage;
14
15 use vars qw/%conf %locks $version_string
16             $PREFIX $CONF $BLOCKSIZE $DEBUG $HOST $BACKUP
17             $RESTORE $RESTORE_HOST $RESTORE_ZFS $TIMESTAMP
18             $LIST $SUMMARY $SUMMARY_EXT
19             $EXPUNGE $NEUTERED $ZFS
20             $VERSION $HELP/;
21 $version_string = '0.1';
22 $PREFIX = q^__PREFIX__^;
23 $CONF = qq^$PREFIX/etc/zetaback.conf^;
24 $BLOCKSIZE = 1024*64;
25
26 $conf{'default'}->{'time_format'} = "%Y-%m-%d %H:%M:%S";
27 $conf{'default'}->{'retention'} = 14 * 86400;
28
29 =pod
30
31 =head1 NAME
32
33 zetaback - perform backup, restore and retention policies for ZFS backups.
34
35 =head1 SYNOPSIS
36
37   zetaback -v
38
39   zetaback [-l | -s | -sx] [-c conf] [-d] [-h host] [-z zfs]
40
41   zetaback -b [-x] [-c conf] [-d] [-n] [-h host] [-z zfs]
42
43   zetaback -x [-b] [-c conf] [-d] [-n] [-h host] [-z zfs]
44
45   zetaback -r [-c conf] [-d] [-n] [-h host] [-z zfs] [-t timestamp]
46               [-rhost host] [-rzfs fs]
47
48 =cut
49
50 GetOptions(
51   "h=s"     => \$HOST,
52   "z=s"     => \$ZFS,
53   "c=s"     => \$CONF,
54   "b"       => \$BACKUP,
55   "l"       => \$LIST,
56   "s"       => \$SUMMARY,
57   "sx"      => \$SUMMARY_EXT,
58   "r"       => \$RESTORE,
59   "t=i"     => \$TIMESTAMP,
60   "rhost=s" => \$RESTORE_HOST,
61   "rzfs=s"  => \$RESTORE_ZFS,
62   "d"       => \$DEBUG,
63   "n"       => \$NEUTERED,
64   "x"       => \$EXPUNGE,
65   "v"       => \$VERSION,
66 );
67
68 # actions allowed together 'x' and 'b' all others are exclusive:
69 my $actions = 0;
70 $actions++ if($BACKUP || $EXPUNGE);
71 $actions++ if($RESTORE);
72 $actions++ if($LIST);
73 $actions++ if($SUMMARY);
74 $actions++ if($SUMMARY_EXT);
75 $actions++ if($VERSION);
76 if($actions != 1) {
77   pod2usage({ -verbose => 0 });
78   exit -1;
79 }
80
81 =pod
82
83 =head1 DESCRIPTION
84
85 The B<zetaback> program is orchestrates the backup (either full or
86 incremental) of remote ZFS filesysetms to a local store.  It handles
87 frequency requirements for both full and incemental backups as well
88 as retention policies.  In addition to backups, the B<zetaback> tool
89 allows for the restore of any backup to a specified host and zfs
90 filesystem.
91
92 =head1 OPTIONS
93
94 The non-optional action command line arguments define the invocation purpose
95 of B<zetaback>.  All other options are optional and refine the target
96 of the action specified.
97
98 =head2 Generic Options
99
100 The following options have the same meaning over several actions:
101
102 =over
103
104 =item -c <conf>
105
106 Use the specified file as the configuration file.  The default file, if
107 none is specified is /etc/zetaback.conf.  The prefix for this file may
108 also be specified in the Makefile.
109
110 =item -d
111
112 Enable debugging output.
113
114 =item -n
115
116 Don't actually perform any remote commands or expunging.  This is useful with
117 the -d options to ascertain what would be done if the command was actually
118 executed.
119
120 =item -t <timestamp>
121
122 Used during the restore process to specify a backup image from the desired
123 point in time.  If omitted, the command becomes interactive.  This timestamp
124 is a UNIX timestamp and is shown in output of -s and -sx actions.
125
126 =item -rhost <host>
127
128 Specify the remote host that is the target for a restore operation.  If
129 omitted the command becomes interactive.
130
131 =item -rzfs <zfs>
132
133 Specify the remote ZFS filesystem that is the target for a restore
134 operation.  If omitted the command becomes interactive.
135
136 =item -h <host>
137
138 Filters the operation to the host specified.  If <host> is of the form
139 /pattern/, it matches 'pattern' as a perl regular expression against available
140 hosts.  If this option is omitted, no limit is enforced and all hosts are
141 used for the action.
142
143 =item -z <zfs>
144
145 Filters the operation to the zfs filesystem specified.  If <zfs> is of the
146 form /pattern/, it matches 'pattern' as a perl regular expression against
147 available zfs filesystems.  If this option is omitted, no filter is enforced
148 and all zfs filesystems are used for the action.
149
150 =back
151
152 =head2 Actions
153
154 =over
155
156 =item -v
157
158 Show the version.
159
160 =item -l
161
162 Show a brief listing of available backups.
163
164 =item -s
165
166 Like -l, -s will show a list of backups but provides also the most recent
167 backup information including timestamp type (full or incremental) and the
168 size on disk.
169
170 =item -sx
171
172 Shows a extended summary.  In addition to the output provided by the -s
173 action, the -sx action will show detail for each availble backup.  For
174 full backups, the detail will include any more recent full backups, if
175 they exist.  For incremental backups, the detail will include any
176 incremental backups that are more recent than the last full backup.
177
178 =item -b
179
180 Performs a backup.  This option will investigate all eligible hosts, query
181 the available filesystems from the remote agent and determine if any such
182 filesystems require a new full or incremental backup to be taken.  This
183 option may be combined with the -x option (to clean up afterwards).
184
185 =item -x
186
187 Perform an expunge.  This option will determine which, if any, of the local
188 backups may be deleted given the retention policy specified in the
189 configuration.
190
191 =item -r
192
193 Perform a restore.  This option will operate on the specified backup and
194 restore it to the zfs filesystem specified with -rzfs on the host specified
195 with the -rhost option.  The -h, -z and -t options may be used to filter
196 the source backup list.  If the filtered list contains more than one
197 source backup image, the command will act interactively.  If the -rhost
198 and -rzfs command are not specified, the command will act interactively.
199
200 =back
201
202 =cut
203
204 if($VERSION) {
205   print "zetaback: $version_string\n";
206   exit 0;
207 }
208
209 =pod
210
211 =head1 CONFIGURATION
212
213 The zetaback configuration file consists of a default stanza, containing
214 settings that can be overridden on a per-host basis.  A stanza begins
215 either with the string 'default', or a fully-qualified hostname, with
216 settings enclosed in braces ({}).  Single-line comments begin with a hash
217 ('#'), and whitespace is ignored, so feel free to indent for better
218 readability.  Every host to be backed up must have a host stanza in the
219 configuration file.
220
221 =head2 Settings
222
223 The following settings are valid in both the default and host scopes:
224
225 =over
226
227 =item store
228
229 The base directory under which to keep backups.  An interpolated variable
230 '%h' can be used, which expands to the hostname.  There is no default for
231 this setting.
232
233 =item agent
234
235 The location of the zetaback_agent binary on the host.  There is no default
236 for this setting.
237
238 =item time_format
239
240 All timestamps within zetaback are in Unix timestamp format.  This setting
241 provides a string for formatting all timestamps on output.  The sequences
242 available are identical to those in strftime(3).  If not specified, the
243 default is '%Y-%m-%d %H:%M:%S'.
244
245 =item backup_interval
246
247 The frequency (in seconds) at which to perform incremental backups.  An
248 incremental backup will be performed if the current time is more than
249 backup_interval since the last incremental backup.  If there is no full backup
250 for a particular filesystem, then a full backup is performed.  There is no
251 default for this setting.
252
253 =item full_interval
254
255 The frequency (in seconds) at which to perform full backups.  A full backup will
256 be performed if the current time is more than full_interval since the last full
257 backup.
258
259 =item retention
260
261 The retention time (in seconds) for backups.  Defaults to (14 * 86400), or two
262 weeks.
263
264 =back
265
266 =head1 CONFIGURATION EXAMPLES
267
268 =head2 Uniform hosts
269
270 This config results in backups stored in /var/spool/zfs_backups, with a
271 subdirectory for each host.  Incremental backups will be performed
272 approximately once per day, assuming zetaback is run hourly.  Full backups
273 will be done once per week.  Time format and retention are default.
274
275   default {
276     store = /var/spool/zfs_backups/%h
277     agent = /usr/local/bin/zetaback_agent
278     backup_interval = 83000
279     full_interval = 604800
280   }
281
282   host1 {}
283
284   host2 {}
285
286 =head2 Non-uniform hosts
287
288 Here, host1's and host2's agents are found in different places, and host2's
289 backups should be stored in a different path.
290
291   default {
292     store = /var/spool/zfs_backups/%h
293     agent = /usr/local/bin/zetaback_agent
294     backup_interval = 83000
295     full_interval = 604800
296   }
297
298   host1 {
299     agent = /opt/local/bin/zetaback_agent
300   }
301
302   host2 {
303     store = /var/spool/alt_backups/%h
304     agent = /www/bin/zetaback_agent
305   }
306
307 =cut
308
309 # Make the parser more formal:
310 # config => stanza*
311 # stanza => string { kvp* }
312 # kvp    => string = string
313 my $str_re = qr/(?:"(?:\\\\|\\"|[^"])*"|\S+)/;
314 my $kvp_re = qr/($str_re)\s*=\s*($str_re)/;
315 my $stanza_re = qr/($str_re)\s*\{((?:\s*$kvp_re)*)\s*\}/;
316
317 sub parse_config() {
318   local($/);
319   $/ = undef;
320   open(CONF, "<$CONF");
321   my $file = <CONF>;
322   # Rip comments
323   $file =~ s/^\s*#.*$//mg;
324   while($file =~ m/$stanza_re/gm) {
325     my $scope = $1;
326     my $filepart = $2;
327     $scope =~ s/^"(.*)"$/$1/;
328     $conf{$scope} ||= {};
329     while($filepart =~ m/$kvp_re/gm) {
330       my $key = $1;
331       my $value = $2;
332       $key =~ s/^"(.*)"$/$1/;
333       $value =~ s/^"(.*)"$/$1/;
334       $conf{$scope}->{lc($key)} = $value;
335     }
336   }
337   close(CONF);
338 }
339 sub config_get($$) {
340   return $conf{$_[0]}->{$_[1]} || $conf{'default'}->{$_[1]};
341 }
342
343 sub dir_encode($) {
344   my $d = shift;
345   my $e = encode_base64($d, '');
346   $e =~ s/\//_/;
347   return $e;
348 }
349 sub dir_decode($) {
350   my $e = shift;
351   $e =~ s/_/\//;
352   return decode_base64($e);
353 }
354 sub pretty_size($) {
355   my $bytes = shift;
356   if($bytes > 1024*1024*1024) {
357     return sprintf("%0.2f Gb", $bytes / (1024*1024*1024));
358   }
359   if($bytes > 1024*1024) {
360     return sprintf("%0.2f Mb", $bytes / (1024*1024));
361   }
362   if($bytes > 1024) {
363     return sprintf("%0.2f Kb", $bytes / (1024));
364   }
365   return "$bytes b";
366 }
367 sub lock($;$$) {
368   my ($host, $file, $nowait) = @_;
369   print "Acquiring lock for $host:$file\n" if($DEBUG);
370   $file ||= 'master.lock';
371   my $store = config_get($host, 'store');
372   $store =~ s/%h/$host/g;
373   return 1 if(exists($locks{"$host:$file"}));
374   open(LOCK, "+>>$store/$file") || return 0;
375   unless(flock(LOCK, LOCK_EX | ($nowait ? LOCK_NB : 0))) {
376     close(LOCK);
377     return 0;
378   }
379   $locks{"$host:$file"} = \*LOCK;
380   return 1;
381 }
382 sub unlock($;$$) {
383   my ($host, $file, $remove) = @_;
384   print "Releasing lock for $host:$file\n" if($DEBUG);
385   $file ||= 'master.lock';
386   my $store = config_get($host, 'store');
387   $store =~ s/%h/$host/g;
388   return 0 unless(exists($locks{"$host:$file"}));
389   *UNLOCK = $locks{$file};
390   unlink("$store/$file") if($remove);
391   flock(UNLOCK, LOCK_UN);
392   close(UNLOCK);
393   return 1;
394 }
395 sub scan_for_backups($) {
396   my %info = ();
397   my $dir = shift;
398   $info{last_full} = $info{last_incremental} = $info{last_backup} = 0;
399   opendir(D, $dir) || return \%info;
400   foreach my $file (readdir(D)) {
401     if($file =~ /^(\d+)\.([^\.]+)\.full$/) {
402       my $whence = $1;
403       my $fs = dir_decode($2);
404       $info{$fs}->{full}->{$whence}->{'file'} = "$dir/$file";
405       $info{$fs}->{last_full} = $whence if($whence > $info{$fs}->{last_full});
406       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
407                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
408     }
409     elsif($file =~ /^(\d+).([^\.]+)\.incremental.(\d+)$/) {
410       my $whence = $1;
411       my $fs = dir_decode($2);
412       $info{$fs}->{incremental}->{$whence}->{'depends'} = $3;
413       $info{$fs}->{incremental}->{$whence}->{'file'} = "$dir/$file";
414       $info{$fs}->{last_incremental} = $whence if($whence > $info{$fs}->{last_incremental});
415       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
416                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
417     }
418   }
419   closedir(D);
420   return \%info;
421 }
422
423 parse_config();
424
425 sub zetaback_log($$;@) {
426   my ($host, $mess, @args) = @_;
427   my $tf = config_get($host, 'time_format');
428   my $file = config_get($host, 'logfile');
429   my $fileh;
430   if(defined($file)) {
431     $fileh = IO::File->new(">>$file");
432   }
433   $fileh ||= IO::File->new(">&STDERR");
434   printf $fileh "%s: $mess", strftime($tf, localtime(time)), @args;
435   $fileh->close();
436 }
437
438 sub zfs_remove_snap($$$) {
439   my ($host, $fs, $snap) = @_;
440   my $agent = config_get($host, 'agent');
441   return unless($snap);
442   print "Dropping $snap on $fs\n" if($DEBUG);
443   `ssh $host $agent -z $fs -d $snap`;
444 }
445
446 # Lots of args.. internally called.
447 sub zfs_do_backup($$$$$$) {
448   my ($host, $fs, $type, $point, $store, $dumpfile) = @_;
449   my $agent = config_get($host, 'agent');
450
451   # Do it. yeah.
452   open(LBACKUP, ">$store/.$dumpfile") || die "zfs_full_backup: cannot create dump\n";
453   eval {
454     if(my $pid = fork()) {
455       close(LBACKUP);
456       waitpid($pid, 0);
457       die "error: $?" if($?);
458     }
459     else {
460       my @cmd = ('ssh', $host, $agent, '-z', $fs, "-$type", $point);
461       open STDIN, "/dev/null" || exit(-1);
462       open STDOUT, ">&LBACKUP" || exit(-1);
463       exec { $cmd[0] } @cmd;
464       print STDERR "$cmd[0] failed: $?\n";
465       exit($?);
466     }
467     die "dump failed (zero bytes)\n" if(-z "$store/.$dumpfile");
468     rename("$store/.$dumpfile", "$store/$dumpfile") || die "cannot rename dump\n";
469   };
470   if($@) {
471     unlink("$store/.$dumpfile");
472     chomp(my $error = $@);
473     $error =~ s/[\r\n]+/ /gsm;
474     zetaback_log($host, "FAILED[$error] $host:$fs $type\n");
475     die "zfs_full_backup: failed $@";
476   }
477   my @st = stat("$store/$dumpfile");
478   my $size = pretty_size($st[7]);
479   zetaback_log($host, "SUCCESS[$size] $host:$fs $type\n");
480 }
481
482 sub zfs_full_backup($$$) {
483   my ($host, $fs, $store) = @_;
484
485   # Translate into a proper dumpfile nameA
486   my $point = time();
487   my $efs = dir_encode($fs);
488   my $dumpfile = "$point.$efs.full";
489
490   zfs_do_backup($host, $fs, 'f', $point, $store, $dumpfile);
491 }
492
493 sub zfs_incremental_backup($$$$) {
494   my ($host, $fs, $base, $store) = @_;
495   my $agent = config_get($host, 'agent');
496
497   # Translate into a proper dumpfile nameA
498   my $point = time();
499   my $efs = dir_encode($fs);
500   my $dumpfile = "$point.$efs.incremental.$base";
501
502   zfs_do_backup($host, $fs, 'i', $base, $store, $dumpfile);
503 }
504
505 sub perform_retention($$) {
506   my ($host, $store) = @_;
507   my $cutoff = time() - config_get($host, 'retention');
508   my $backup_info = scan_for_backups($store);
509  
510   foreach my $disk (sort keys %{$backup_info}) {
511     my $info = $backup_info->{$disk};
512     next unless(ref($info) eq 'HASH');
513     my %must_save;
514
515     # Get a list of all the full and incrementals, sorts newest to oldest
516     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
517     @backup_points = sort { $b <=> $a } @backup_points;
518
519     # We _cannot_ throw away _all_ out backups, so save the most recent no matter what
520     $must_save{$backup_points[0]} = 1;
521
522     # Walk the list for backups within our retention period.
523     foreach (@backup_points) {
524       if($_ >= $cutoff) {
525         $must_save{$_} = 1;
526       }
527       else {
528         # they are in decending order, once we miss, all will miss
529         last;
530       }
531     }
532
533     # Look for dependencies
534     foreach (@backup_points) {
535       if(exists($info->{incremental}->{$_})) {
536         print "   => $_ depends on $info->{incremental}->{$_}->{depends}\n" if($DEBUG);
537         $must_save{$info->{incremental}->{$_}} = 1
538       }
539     }
540     my @removals = grep { !exists($must_save{$_}) } @backup_points;
541     if($DEBUG) {
542       my $tf = config_get($host, 'time_format');
543       print "    => I can remove:\n";
544       foreach (@backup_points) {
545         print "      => ". strftime($tf, localtime($_));
546         print " [". (exists($info->{full}->{$_}) ? "full":"incremental") ."]";
547         print " XXX" if(!exists($must_save{$_}));
548         print "\n";
549       }
550     }
551     foreach (@removals) {
552       my $efs = dir_encode($disk);
553       my $filename;
554       if(exists($info->{full}->{$_})) {
555         $filename = "$store/$_.$efs.full";
556       }
557       elsif(exists($info->{incremental}->{$_})) {
558         $filename = "$store/$_.$efs.incremental.$info->{incremental}->{$_}->{depends}";
559       }
560       else {
561         print "ERROR: We tried to expunge $host $disk [$_], but couldn't find it.\n";
562       }
563       print "    => expunging $filename\n" if($DEBUG);
564       unless($NEUTERED) {
565         unlink($filename) || print "ERROR: unlink $filename: $?\n";
566       }
567     }
568   }
569 }
570
571 sub __default_sort($$) { return $_[0] cmp $_[1]; }
572    
573 sub choose($$;$) {
574   my($name, $obj, $sort) = @_;
575   $sort ||= \&__default_sort;;
576   my @list;
577   my $hash;
578   if(ref $obj eq 'ARRAY') {
579     @list = sort { $sort->($a,$b); } (@$obj);
580     map { $hash->{$_} = $_; } @list;
581   }
582   elsif(ref $obj eq 'HASH') {
583     @list = sort { $sort->($a,$b); } (keys %$obj);
584     $hash = $obj;
585   }
586   else {
587     die "choose passed bad object: " . ref($obj) . "\n";
588   }
589   return $list[0] if(scalar(@list) == 1);
590   print "\n";
591   my $i = 1;
592   for (@list) {
593     printf " %3d) $hash->{$_}\n", $i++;
594   }
595   my $selection = 0;
596   while($selection !~ /^\d+$/ or
597         $selection < 1 or
598         $selection >= $i) {
599     print "$name: ";
600     chomp($selection = <>);
601   }
602   return $list[$selection - 1];
603 }
604
605 sub backup_chain($$) {
606   my ($info, $ts) = @_;
607   my @list;
608   push @list, $info->{full}->{$ts} if(exists($info->{full}->{$ts}));
609   if(exists($info->{incremental}->{$ts})) {
610     push @list, $info->{incremental}->{$ts};
611     push @list, backup_chain($info, $info->{incremental}->{$ts}->{depends});
612   }
613   return @list;
614 }
615
616 sub perform_restore() {
617   my %source;
618
619   foreach my $host (grep { $_ ne "default" } keys %conf) {
620     # If -h was specific, we will skip this host if the arg isn't
621     # an exact match or a pattern match
622     if($HOST &&
623        !(($HOST eq $host) ||
624          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
625       next;
626     }
627
628     my $store = config_get($host, 'store');
629     $store =~ s/%h/$host/g;;
630     mkdir $store if(! -d $store);
631
632     my $backup_info = scan_for_backups($store);
633     foreach my $disk (sort keys %{$backup_info}) {
634       my $info = $backup_info->{$disk};
635       next unless(ref($info) eq 'HASH');
636       next
637         if($ZFS &&      # if the pattern was specified it could
638            !($disk eq $ZFS ||        # be a specific match or a
639              ($ZFS =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
640       # We want to see this one
641       my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
642       my @source_points;
643       foreach (@backup_points) {
644         push @source_points, $_ if(!$TIMESTAMP || $TIMESTAMP == $_)
645       }
646       if(@source_points) {
647         $source{$host}->{$disk} = \@source_points;
648       }
649     }
650   }
651
652   if(! keys %source) {
653     print "No matching backups found\n";
654     return;
655   }
656
657   # Here goes the possibly interactive dialog
658   my $host = choose("Restore from host",  [keys %source]);
659   my $disk = choose("Restore from ZFS", [keys %{$source{$host}}]);
660  
661   # Times are special.  We build a human readable form and use a numerical
662   # sort function instead of the default lexical one.
663   my %times;
664   my $tf = config_get($host, 'time_format');
665   map { $times{$_} = strftime($tf, localtime($_)); } @{$source{$host}->{$disk}};
666   my $timestamp = choose("Restore as of timestamp", \%times,
667                          sub { $_[0] <=> $_[1]; });
668
669   my $store = config_get($host, 'store');
670   $store =~ s/%h/$host/g;;
671   mkdir $store if(! -d $store);
672   my $backup_info = scan_for_backups($store);
673   my @backup_list = reverse backup_chain($backup_info->{$disk}, $timestamp);
674
675   if(!$RESTORE_HOST) {
676     print "Restore to host [$host]:";
677     chomp(my $input = <>);
678     $RESTORE_HOST = length($input) ? $input : $host;
679   }
680   if(!$RESTORE_ZFS) {
681     print "Restore to zfs [$disk]:";
682     chomp(my $input = <>);
683     $RESTORE_ZFS = length($input) ? $input : $disk;
684   }
685
686   # show intentions
687   print "Going to restore:\n";
688   print "\tfrom: $host\n";
689   print "\tfrom: $disk\n";
690   print "\t  at: $timestamp [" . strftime($tf, localtime($timestamp)) . "]\n";
691   print "\t  to: $RESTORE_HOST\n";
692   print "\t  to: $RESTORE_ZFS\n";
693   print "\n";
694
695   foreach(@backup_list) {
696     $_->{success} = zfs_restore_part($RESTORE_HOST, $RESTORE_ZFS, $_->{file}, $_->{depends});
697   }
698 }
699
700 sub zfs_restore_part($$$;$) {
701   my ($host, $fs, $file, $dep) = @_;
702   my $command;
703   if(exists($conf{$host})) {
704     my $agent = config_get($host, 'agent');
705     $command = "$agent -r -z $fs";
706     $command .= " -b $dep" if($dep);
707   }
708   else {
709     $command = "/usr/sbin/zfs recv $fs";
710   }
711   print " => piping $file to $command\n" if($DEBUG);
712   if($NEUTERED) {
713     print "gzip -dfc $file | ssh $host $command\n" if ($DEBUG);
714   }
715   else {
716     open(DUMP, "gzip -dfc $file |");
717     eval {
718       open(RECEIVER, "| ssh $host $command");
719       my $buffer;
720       while(my $len = sysread(DUMP, $buffer, $BLOCKSIZE)) {
721         if(syswrite(RECEIVER, $buffer, $len) != $len) {
722           die "$!";
723         }
724       }
725     };
726     close(DUMP);
727     close(RECEIVER);
728   }
729   return $?;
730 }
731
732 sub show_backups($$$) {
733   my ($host, $store, $diskpat) = @_;
734   my $backup_info = scan_for_backups($store);
735   my $tf = config_get($host, 'time_format');
736   foreach my $disk (sort keys %{$backup_info}) {
737     my $info = $backup_info->{$disk};
738     next unless(ref($info) eq 'HASH');
739     next
740       if($diskpat &&      # if the pattern was specified it could
741          !($disk eq $diskpat ||        # be a specific match or a
742            ($diskpat =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
743     # We want to see this one
744     print "$host:$disk\n";
745     next unless($SUMMARY || $SUMMARY_EXT);
746     if($SUMMARY_EXT) {
747       print "\tLast Full: ". ($info->{last_full} ? strftime($tf, localtime($info->{last_full})) : "Never") . "\n";
748       if($info->{last_full} < $info->{last_incremental}) {
749         print "\tLast Incr: ". strftime($tf, localtime($info->{last_incremental})). "\n";
750       }
751     }
752     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
753     @backup_points = sort { $a <=> $b } @backup_points;
754     unless ($SUMMARY_EXT) {
755       @backup_points = (pop @backup_points);
756     }
757     foreach (@backup_points) {
758       print "\t" . strftime($tf, localtime($_)) . " [$_] ";
759       if(exists($info->{full}->{$_})) {
760         my @st = stat($info->{full}->{$_}->{file});
761         print "FULL " . pretty_size($st[7]);
762       } else {
763         my @st = stat($info->{incremental}->{$_}->{file});
764         print "INCR from [$info->{incremental}->{$_}->{depends}] " . pretty_size($st[7]);
765       }
766       print "\n";
767     }
768     print "\n";
769   }
770 }
771
772 sub plan_and_run($$$) {
773   my ($host, $store, $diskpat) = @_;
774   print "Planning '$host'\n" if($DEBUG);
775   my $agent = config_get($host, 'agent');
776   my $took_action = 1;
777   while($took_action) {
778     $took_action = 0;
779     my @disklist;
780
781     # We need a lock for the listing.
782     return unless(lock($host, ".list"));
783     open(ZFSLIST, "ssh $host $agent -l |") || next;
784     @disklist = grep { chomp } (<ZFSLIST>);
785     close(ZFSLIST);
786
787     foreach my $diskline (@disklist) {
788       chomp($diskline);
789       next unless($diskline =~ /^(\S+) \[([^\]]*)\]/);
790       my $diskname = $1;
791       my %snaps;
792       map { $snaps{$_} = 1 } (split(/,/, $2));
793  
794       # If we are being selective (via -z) now is the time.
795       next
796         if($diskpat &&          # if the pattern was specified it could
797            !($diskname eq $diskpat ||        # be a specific match or a
798              ($diskpat =~ /^\/(.+)\/$/ && $diskname =~ /$1/))); # regex
799  
800       print " => Scanning '$store' for old backups of '$diskname'.\n" if($DEBUG);
801
802       # Make directory on demand
803       my $backup_info = scan_for_backups($store);
804       # That gave us info on all backups, we just want this disk
805       $backup_info = $backup_info->{$diskname} || {};
806  
807       # Should we do a backup?
808       my $backup_type = 'no';
809       if(time() > $backup_info->{last_backup} + config_get($host, 'backup_interval')) {
810         $backup_type = 'incremental';
811       }
812       if(time() > $backup_info->{last_full} + config_get($host, 'full_interval')) {
813         $backup_type = 'full';
814       }
815  
816       # If we want an incremental, but have no full, then we need to upgrade to full
817       if($backup_type eq 'incremental') {
818         my $have_full_locally = 0;
819         # For each local full backup, see if the full backup still exists on the other end.
820         foreach (keys %{$backup_info->{'full'}}) {
821           $have_full_locally = 1 if(exists($snaps{'__zb_full_' . $_}));
822         }
823         $backup_type = 'full' unless($have_full_locally);
824       }
825  
826       print " => doing $backup_type backup\n" if($DEBUG);
827       # We need to drop a __zb_base snap or a __zb_incr snap before we proceed
828       unless($NEUTERED || $backup_type eq 'no') {
829         # attempt to lock this action, if it fails, skip -- someone else is working it.
830         next unless(lock($host, dir_encode($diskname), 1));
831         unlock($host, '.list');
832
833         if($backup_type eq 'full') {
834           eval { zfs_full_backup($host, $diskname, $store); };
835           if ($@) {
836             chomp(my $err = $@);
837             print " => failure $err\n";
838           }
839           else {
840             # Unless there was an error backing up, remove all the other full snaps
841             foreach (keys %snaps) {
842               zfs_remove_snap($host, $diskname, $_) if(/^__zb_full_(\d+)/)
843             }
844           }
845           $took_action = 1;
846         }
847         if($backup_type eq 'incremental') {
848           zfs_remove_snap($host, $diskname, '__zb_incr') if($snaps{'__zb_incr'});
849           # Find the newest full from which to do an incremental (NOTE: reverse numeric sort)
850           my @fulls = sort { $b <=> $a } (keys %{$backup_info->{'full'}});
851           zfs_incremental_backup($host, $diskname, $fulls[0], $store);
852           $took_action = 1;
853         }
854         unlock($host, dir_encode($diskname), 1);
855       }
856       last if($took_action);
857     }
858     unlock($host, '.list');
859   }
860 }
861
862 if($RESTORE) {
863   perform_restore();
864 }
865 else {
866   foreach my $host (grep { $_ ne "default" } keys %conf) {
867     # If -h was specific, we will skip this host if the arg isn't
868     # an exact match or a pattern match
869     if($HOST &&
870        !(($HOST eq $host) ||
871          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
872       next;
873     }
874  
875     my $store = config_get($host, 'store');
876     $store =~ s/%h/$host/g;;
877     mkdir $store if(! -d $store);
878  
879     if($LIST || $SUMMARY || $SUMMARY_EXT) {
880       show_backups($host, $store, $ZFS);
881     }
882     if($BACKUP) {
883       plan_and_run($host, $store, $ZFS);
884     }
885     if($EXPUNGE) {
886       perform_retention($host, $store);
887     }
888   }
889 }
890
891 exit 0;
892
893 =pod
894
895 =head1 FILES
896
897 =over
898
899 =item /etc/zetaback.conf
900
901 The main zetaback configuration file.  The location of the file can be
902 specified on the command line with the -c flag.  The prefix of this
903 file may also be specified in the Makefile.
904
905 =back
906
907 =head1 SEE ALSO
908
909 zetaback_agent(1)
910
911 =cut
Note: See TracBrowser for help on using the browser.