root/zetaback.in

Revision 84122018b81afc540eefcf4990e05f69553f86bf, 34.6 kB (checked in by Mark Harrison <mark@omniti.com>, 5 years ago)

Fix viewing of backups, and change the storage method to use actual
filesystems.

  • Property mode set to 100755
Line 
1 #!/usr/bin/perl
2
3 # Copyright (c) 2007 OmniTI Computer Consulting, Inc. All rights reserved.
4 # For information on licensing see:
5 #   https://labs.omniti.com/zetaback/trunk/LICENSE
6
7 use strict;
8 use Getopt::Long;
9 use MIME::Base64;
10 use POSIX qw/strftime/;
11 use Fcntl qw/:flock/;
12 use File::Copy;
13 use IO::File;
14 use Pod::Usage;
15
16 use vars qw/%conf %locks $version_string
17             $PREFIX $CONF $BLOCKSIZE $DEBUG $HOST $BACKUP
18             $RESTORE $RESTORE_HOST $RESTORE_ZFS $TIMESTAMP
19             $LIST $SUMMARY $SUMMARY_EXT $SUMMARY_VIOLATORS
20             $FORCE_FULL $FORCE_INC $EXPUNGE $NEUTERED $ZFS
21             $SHOW_FILENAMES $ARCHIVE $VERSION $HELP/;
22 $version_string = q$URL$;
23 $version_string =~ s#/branches/#/b#;
24 $version_string =~ s#^.*/([^/]+)/[^/]+$#$1#;
25 $PREFIX = q^__PREFIX__^;
26 $CONF = qq^$PREFIX/etc/zetaback.conf^;
27 $BLOCKSIZE = 1024*64;
28
29 $conf{'default'}->{'time_format'} = "%Y-%m-%d %H:%M:%S";
30 $conf{'default'}->{'retention'} = 14 * 86400;
31 $conf{'default'}->{'compressionlevel'} = 1;
32 $conf{'default'}->{'dataset_backup'} = 0;
33
34 =pod
35
36 =head1 NAME
37
38 zetaback - perform backup, restore and retention policies for ZFS backups.
39
40 =head1 SYNOPSIS
41
42   zetaback -v
43
44   zetaback [-l | -s | -sx | -sv] [--files] [-c conf] [-d] [-h host] [-z zfs]
45
46   zetaback -a [-c conf] [-d] [-h host] [-z zfs]
47
48   zetaback -b [-ff] [-fi] [-x] [-c conf] [-d] [-n] [-h host] [-z zfs]
49
50   zetaback -x [-b] [-c conf] [-d] [-n] [-h host] [-z zfs]
51
52   zetaback -r [-c conf] [-d] [-n] [-h host] [-z zfs] [-t timestamp]
53               [-rhost host] [-rzfs fs]
54
55 =cut
56
57 GetOptions(
58   "h=s"     => \$HOST,
59   "z=s"     => \$ZFS,
60   "c=s"     => \$CONF,
61   "a"       => \$ARCHIVE,
62   "b"       => \$BACKUP,
63   "l"       => \$LIST,
64   "s"       => \$SUMMARY,
65   "sx"      => \$SUMMARY_EXT,
66   "sv"      => \$SUMMARY_VIOLATORS,
67   "r"       => \$RESTORE,
68   "t=i"     => \$TIMESTAMP,
69   "rhost=s" => \$RESTORE_HOST,
70   "rzfs=s"  => \$RESTORE_ZFS,
71   "d"       => \$DEBUG,
72   "n"       => \$NEUTERED,
73   "x"       => \$EXPUNGE,
74   "v"       => \$VERSION,
75   "ff"      => \$FORCE_FULL,
76   "fi"      => \$FORCE_INC,
77   "files"   => \$SHOW_FILENAMES,
78 );
79
80 # actions allowed together 'x' and 'b' all others are exclusive:
81 my $actions = 0;
82 $actions++ if($ARCHIVE);
83 $actions++ if($BACKUP || $EXPUNGE);
84 $actions++ if($RESTORE);
85 $actions++ if($LIST);
86 $actions++ if($SUMMARY);
87 $actions++ if($SUMMARY_EXT);
88 $actions++ if($SUMMARY_VIOLATORS);
89 $actions++ if($VERSION);
90 $actions++ if($BACKUP && $FORCE_FULL && $FORCE_INC);
91 if($actions != 1) {
92   pod2usage({ -verbose => 0 });
93   exit -1;
94 }
95
96 =pod
97
98 =head1 DESCRIPTION
99
100 The B<zetaback> program orchestrates the backup (either full or
101 incremental) of remote ZFS filesystems to a local store.  It handles
102 frequency requirements for both full and incemental backups as well
103 as retention policies.  In addition to backups, the B<zetaback> tool
104 allows for the restore of any backup to a specified host and zfs
105 filesystem.
106
107 =head1 OPTIONS
108
109 The non-optional action command line arguments define the invocation purpose
110 of B<zetaback>.  All other arguments are optional and refine the target
111 of the action specified.
112
113 =head2 Generic Options
114
115 The following arguments have the same meaning over several actions:
116
117 =over
118
119 =item -c <conf>
120
121 Use the specified file as the configuration file.  The default file, if
122 none is specified is /usr/local/etc/zetaback.conf.  The prefix of this
123 file may also be specified as an argument to the configure script.
124
125 =item -d
126
127 Enable debugging output.
128
129 =item -n
130
131 Don't actually perform any remote commands or expunging.  This is useful with
132 the -d argument to ascertain what would be done if the command was actually
133 executed.
134
135 =item -t <timestamp>
136
137 Used during the restore process to specify a backup image from the desired
138 point in time.  If omitted, the command becomes interactive.  This timestamp
139 is a UNIX timestamp and is shown in the output of the -s and -sx actions.
140
141 =item -rhost <host>
142
143 Specify the remote host that is the target for a restore operation.  If
144 omitted the command becomes interactive.
145
146 =item -rzfs <zfs>
147
148 Specify the remote ZFS filesystem that is the target for a restore
149 operation.  If omitted the command becomes interactive.
150
151 =item -h <host>
152
153 Filters the operation to the host specified.  If <host> is of the form
154 /pattern/, it matches 'pattern' as a perl regular expression against available
155 hosts.  If omitted, no limit is enforced and all hosts are used for the action.
156
157 =item -z <zfs>
158
159 Filters the operation to the zfs filesystem specified.  If <zfs> is of the
160 form /pattern/, it matches 'pattern' as a perl regular expression against
161 available zfs filesystems.  If omitted, no filter is enforced and all zfs
162 filesystems are used for the action.
163
164 =back
165
166 =head2 Actions
167
168 =over
169
170 =item -v
171
172 Show the version.
173
174 =item -l
175
176 Show a brief listing of available backups.
177
178 =item -s
179
180 Like -l, -s will show a list of backups but provides additional information
181 about the backups including timestamp, type (full or incremental) and the
182 size on disk.
183
184 =item -sx
185
186 Shows an extended summary.  In addition to the output provided by the -s
187 action, the -sx action will show detail for each availble backup.  For
188 full backups, the detail will include any more recent full backups, if
189 they exist.  For incremental backups, the detail will include any
190 incremental backups that are more recent than the last full backup.
191
192 =item -sv
193
194 Display all backups in the current store that violate the configured
195 backup policy. This is where the most recent full backup is older than
196 full_interval seconds ago, or the most recent incremental backup is older
197 than backup_interval seconds ago.
198
199 =item --files
200
201 Display the on-disk file corresponding to each backup named in the output.
202 This is useful with the -sv flag to name violating files.  Often times,
203 violators are filesystems that have been removed on the host machines and
204 zetaback can no longer back them up.  Be very careful if you choose to
205 automate the removal of such backups as filesystems that would be backed up
206 by the next regular zetaback run will often show up as violators.
207
208 =item -a
209
210 Performs an archive.  This option will look at all eligible backup points
211 (as restricted by -z and -h) and move those to the configured archive
212 directory.  The recommended use is to first issue -sx --files then
213 carefully review available backup points and prune those that are
214 unneeded.  Then invoke with -a to move only the remaining "desired"
215 backup points into the archives.  Archived backups do not appear in any
216 listings or in the list of policy violators generated by the -sv option.
217 In effect, they are no longer "visible" to zetaback.
218
219 =item -b
220
221 Performs a backup.  This option will investigate all eligible hosts, query
222 the available filesystems from the remote agent and determine if any such
223 filesystems require a new full or incremental backup to be taken.  This
224 option may be combined with the -x option (to clean up afterwards.)
225
226 =item -ff
227
228 Forces a full backup to be taken on each filesystem encountered.  This is
229 used in combination with -b.  It is recommended to use this option only when
230 targeting specific filesystems (via the -h and -z options.)  Forcing a full
231 backup across all machines will cause staggered backups to coalesce and
232 could cause performance issues.
233
234 =item -fi
235
236 Forces an incremental backup to be taken on each filesystem encountered. 
237 This is used in combination with -b.  It is recommended to use this option
238 only when targeting specific filesystems (via the -h and -z options.)  Forcing
239 an incremental backup across all machines will cause staggered backups
240 to coalesce and could cause performance issues.
241
242 =item -x
243
244 Perform an expunge.  This option will determine which, if any, of the local
245 backups may be deleted given the retention policy specified in the
246 configuration.
247
248 =item -r
249
250 Perform a restore.  This option will operate on the specified backup and
251 restore it to the ZFS filesystem specified with -rzfs on the host specified
252 with the -rhost option.  The -h, -z and -t options may be used to filter
253 the source backup list.  If the filtered list contains more than one
254 source backup image, the command will act interactively.  If the -rhost
255 and -rzfs command are not specified, the command will act interactively.
256
257 =back
258
259 =cut
260
261 if($VERSION) {
262   print "zetaback: $version_string\n";
263   exit 0;
264 }
265
266 =pod
267
268 =head1 CONFIGURATION
269
270 The zetaback configuration file consists of a default stanza, containing
271 settings that can be overridden on a per-host basis.  A stanza begins
272 either with the string 'default', or a fully-qualified hostname, with
273 settings enclosed in braces ({}).  Single-line comments begin with a hash
274 ('#'), and whitespace is ignored, so feel free to indent for better
275 readability.  Every host to be backed up must have a host stanza in the
276 configuration file.
277
278 =head2 Settings
279
280 The following settings are valid in both the default and host scopes:
281
282 =over
283
284 =item store
285
286 The base directory under which to keep backups.  An interpolated variable
287 '%h' can be used, which expands to the hostname.  There is no default for
288 this setting.
289
290 =item archive
291
292 The base directory under which archives are stored.  The format is the same
293 as the store setting.  This is the destination to which files are relocated
294 when issuing an archive action (-a).
295
296 =item agent
297
298 The location of the zetaback_agent binary on the host.  There is no default
299 for this setting.
300
301 =item time_format
302
303 All timestamps within zetaback are in UNIX timestamp format.  This setting
304 provides a string for formatting all timestamps on output.  The sequences
305 available are identical to those in strftime(3).  If not specified, the
306 default is '%Y-%m-%d %H:%M:%S'.
307
308 =item backup_interval
309
310 The frequency (in seconds) at which to perform incremental backups.  An
311 incremental backup will be performed if the current time is more than
312 backup_interval since the last incremental backup.  If there is no full backup
313 for a particular filesystem, then a full backup is performed.  There is no
314 default for this setting.
315
316 =item full_interval
317
318 The frequency (in seconds) at which to perform full backups.  A full backup will
319 be performed if the current time is more than full_interval since the last full
320 backup.
321
322 =item retention
323
324 The retention time (in seconds) for backups.  Defaults to (14 * 86400), or two
325 weeks.
326
327 =item compressionlevel
328
329 Compress files using gzip at the specified compression level. 0 means no
330 compression. Accepted values are 1-9. Defaults to 1 (fastest/minimal
331 compression.)
332
333 =item ssh_config
334
335 Full path to an alternate ssh client config.  This is useful for specifying a
336 less secure but faster cipher for some hosts, or using a different private
337 key.  There is no default for this setting.
338
339 =item dataset_backup
340
341 By default zetaback backs zfs filesystems up to files. This option lets you
342 specify that the backup go be stored as a zfs dataset on the backup host.
343
344 =back
345
346 =head1 CONFIGURATION EXAMPLES
347
348 =head2 Uniform hosts
349
350 This config results in backups stored in /var/spool/zfs_backups, with a
351 subdirectory for each host.  Incremental backups will be performed
352 approximately once per day, assuming zetaback is run hourly.  Full backups
353 will be done once per week.  Time format and retention are default.
354
355   default {
356     store = /var/spool/zfs_backups/%h
357     agent = /usr/local/bin/zetaback_agent
358     backup_interval = 83000
359     full_interval = 604800
360   }
361
362   host1 {}
363
364   host2 {}
365
366 =head2 Non-uniform hosts
367
368 Here, host1's and host2's agents are found in different places, and host2's
369 backups should be stored in a different path.
370
371   default {
372     store = /var/spool/zfs_backups/%h
373     agent = /usr/local/bin/zetaback_agent
374     backup_interval = 83000
375     full_interval = 604800
376   }
377
378   host1 {
379     agent = /opt/local/bin/zetaback_agent
380   }
381
382   host2 {
383     store = /var/spool/alt_backups/%h
384     agent = /www/bin/zetaback_agent
385   }
386
387 =cut
388
389 # Make the parser more formal:
390 # config => stanza*
391 # stanza => string { kvp* }
392 # kvp    => string = string
393 my $str_re = qr/(?:"(?:\\\\|\\"|[^"])*"|\S+)/;
394 my $kvp_re = qr/($str_re)\s*=\s*($str_re)/;
395 my $stanza_re = qr/($str_re)\s*\{((?:\s*$kvp_re)*)\s*\}/;
396
397 sub parse_config() {
398   local($/);
399   $/ = undef;
400   open(CONF, "<$CONF") || die "Unable to open config file: $CONF";
401   my $file = <CONF>;
402   # Rip comments
403   $file =~ s/^\s*#.*$//mg;
404   while($file =~ m/$stanza_re/gm) {
405     my $scope = $1;
406     my $filepart = $2;
407     $scope =~ s/^"(.*)"$/$1/;
408     $conf{$scope} ||= {};
409     while($filepart =~ m/$kvp_re/gm) {
410       my $key = $1;
411       my $value = $2;
412       $key =~ s/^"(.*)"$/$1/;
413       $value =~ s/^"(.*)"$/$1/;
414       $conf{$scope}->{lc($key)} = $value;
415     }
416   }
417   close(CONF);
418 }
419 sub config_get($$) {
420   return $conf{$_[0]}->{$_[1]} || $conf{'default'}->{$_[1]};
421 }
422
423 sub dir_encode($) {
424   my $d = shift;
425   my $e = encode_base64($d, '');
426   $e =~ s/\//_/;
427   return $e;
428 }
429 sub dir_decode($) {
430   my $e = shift;
431   $e =~ s/_/\//;
432   return decode_base64($e);
433 }
434 sub pretty_size($) {
435   my $bytes = shift;
436   if($bytes > 1024*1024*1024) {
437     return sprintf("%0.2f Gb", $bytes / (1024*1024*1024));
438   }
439   if($bytes > 1024*1024) {
440     return sprintf("%0.2f Mb", $bytes / (1024*1024));
441   }
442   if($bytes > 1024) {
443     return sprintf("%0.2f Kb", $bytes / (1024));
444   }
445   return "$bytes b";
446 }
447 sub lock($;$$) {
448   my ($host, $file, $nowait) = @_;
449   print "Acquiring lock for $host:$file\n" if($DEBUG);
450   $file ||= 'master.lock';
451   my $store = config_get($host, 'store');
452   $store =~ s/%h/$host/g;
453   return 1 if(exists($locks{"$host:$file"}));
454   open(LOCK, "+>>$store/$file") || return 0;
455   unless(flock(LOCK, LOCK_EX | ($nowait ? LOCK_NB : 0))) {
456     close(LOCK);
457     return 0;
458   }
459   $locks{"$host:$file"} = \*LOCK;
460   return 1;
461 }
462 sub unlock($;$$) {
463   my ($host, $file, $remove) = @_;
464   print "Releasing lock for $host:$file\n" if($DEBUG);
465   $file ||= 'master.lock';
466   my $store = config_get($host, 'store');
467   $store =~ s/%h/$host/g;
468   return 0 unless(exists($locks{"$host:$file"}));
469   *UNLOCK = $locks{$file};
470   unlink("$store/$file") if($remove);
471   flock(UNLOCK, LOCK_UN);
472   close(UNLOCK);
473   return 1;
474 }
475 sub scan_for_backups($) {
476   my %info = ();
477   my $dir = shift;
478   $info{last_full} = $info{last_incremental} = $info{last_backup} = 0;
479   # Look for standard file based backups first
480   opendir(D, $dir) || return \%info;
481   foreach my $file (readdir(D)) {
482     if($file =~ /^(\d+)\.([^\.]+)\.full$/) {
483       my $whence = $1;
484       my $fs = dir_decode($2);
485       $info{$fs}->{full}->{$whence}->{'file'} = "$dir/$file";
486       $info{$fs}->{last_full} = $whence if($whence > $info{$fs}->{last_full});
487       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
488                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
489     }
490     elsif($file =~ /^(\d+).([^\.]+)\.incremental.(\d+)$/) {
491       my $whence = $1;
492       my $fs = dir_decode($2);
493       $info{$fs}->{incremental}->{$whence}->{'depends'} = $3;
494       $info{$fs}->{incremental}->{$whence}->{'file'} = "$dir/$file";
495       $info{$fs}->{last_incremental} = $whence if($whence > $info{$fs}->{last_incremental});
496       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
497                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
498     }
499   }
500   closedir(D);
501   # Now look for zfs based backups
502   my $storefs;
503   eval {
504     $storefs = get_fs_from_mountpoint($dir);
505   };
506   return \%info if ($@);
507   my $rv = open(ZFSLIST, "__ZFS__ list -H -r -t snapshot $storefs |");
508   return \%info unless $rv;
509   while (<ZFSLIST>) {
510       my @F = split(' ');
511       my ($fs, $snap) = split('@', $F[0]);
512       $fs =~ s/$storefs\/?//;
513       my ($whence) = ($snap =~ /__zb_dset_(\d+)/);
514       next unless $whence;
515       $info{$fs}->{full}->{$whence}->{'snapshot'} = $snap;
516       # TODO - the info may need to be different here - no last_full or
517       # last_incremental
518       $info{$fs}->{last_full} = $whence if($whence > $info{$fs}->{last_full});
519       $info{$fs}->{last_backup} = $whence if ($whence >
520           $info{$fs}->{last_backup});
521   }
522   close(ZFSLIST);
523   die "Unable to determine zfs filesystem for $dir" unless $storefs;
524
525   return \%info;
526 }
527
528 parse_config();
529
530 sub zetaback_log($$;@) {
531   my ($host, $mess, @args) = @_;
532   my $tf = config_get($host, 'time_format');
533   my $file = config_get($host, 'logfile');
534   my $fileh;
535   if(defined($file)) {
536     $fileh = IO::File->new(">>$file");
537   }
538   $fileh ||= IO::File->new(">&STDERR");
539   printf $fileh "%s: $mess", strftime($tf, localtime(time)), @args;
540   $fileh->close();
541 }
542
543 sub zfs_remove_snap($$$) {
544   my ($host, $fs, $snap) = @_;
545   my $agent = config_get($host, 'agent');
546   my $ssh_config = config_get($host, 'ssh_config');
547   $ssh_config = "-F $ssh_config" if($ssh_config);
548   print "Using custom ssh config file: $ssh_config\n" if($DEBUG);
549   return unless($snap);
550   print "Dropping $snap on $fs\n" if($DEBUG);
551   `ssh $ssh_config $host $agent -z $fs -d $snap`;
552 }
553
554 # Lots of args.. internally called.
555 sub zfs_do_backup($$$$$$) {
556   my ($host, $fs, $type, $point, $store, $dumpname) = @_;
557   my $agent = config_get($host, 'agent');
558   my $ssh_config = config_get($host, 'ssh_config');
559   $ssh_config = "-F $ssh_config" if($ssh_config);
560   print "Using custom ssh config file: $ssh_config\n" if($DEBUG);
561
562
563   # compression is meaningless for dataset backups
564   if ($type ne "d") {
565     my $cl = config_get($host, 'compressionlevel');
566     if ($cl >= 1 && $cl <= 9) {
567         open(LBACKUP, "|gzip -$cl >$store/.$dumpname") ||
568         die "zfs_full_backup: cannot create dump\n";
569     } else {
570         open(LBACKUP, ">$store/.$dumpname") ||
571         die "zfs_full_backup: cannot create dump\n";
572     }
573   }
574   # Do it. yeah.
575   eval {
576     if(my $pid = fork()) {
577       close(LBACKUP) unless ($type eq "d");
578       waitpid($pid, 0);
579       die "error: $?" if($?);
580     }
581     else {
582       my @cmd = ('ssh', split(/ /, $ssh_config), $host, $agent, '-z', $fs, "-$type", $point);
583       open STDIN, "/dev/null" || exit(-1);
584       if ($type eq "d") {
585         my $storefs = get_fs_from_mountpoint($store);
586         open STDOUT, "|__ZFS__ recv $storefs/$dumpname"
587       } else {
588         open STDOUT, ">&LBACKUP" || exit(-1);
589       }
590       exec { $cmd[0] } @cmd;
591       print STDERR "$cmd[0] failed: $?\n";
592       exit($?);
593     }
594     if ($type ne "d") {
595         die "dump failed (zero bytes)\n" if(-z "$store/.$dumpname");
596         rename("$store/.$dumpname", "$store/$dumpname") || die "cannot rename dump\n";
597     }
598   };
599   if($@) {
600     unlink("$store/.$dumpname");
601     chomp(my $error = $@);
602     $error =~ s/[\r\n]+/ /gsm;
603     zetaback_log($host, "FAILED[$error] $host:$fs $type\n");
604     die "zfs_full_backup: failed $@";
605   }
606   my $size;
607   if ($type ne "d") {
608     my @st = stat("$store/$dumpname");
609     $size = pretty_size($st[7]);
610   } else {
611     $size = `__ZFS__ get -Ho value used $dumpname`;
612   }
613   zetaback_log($host, "SUCCESS[$size] $host:$fs $type\n");
614 }
615
616 sub zfs_full_backup($$$) {
617   my ($host, $fs, $store) = @_;
618
619   # Translate into a proper dumpname
620   my $point = time();
621   my $efs = dir_encode($fs);
622   my $dumpname = "$point.$efs.full";
623
624   zfs_do_backup($host, $fs, 'f', $point, $store, $dumpname);
625 }
626
627 sub zfs_incremental_backup($$$$) {
628   my ($host, $fs, $base, $store) = @_;
629   my $agent = config_get($host, 'agent');
630
631   # Translate into a proper dumpname
632   my $point = time();
633   my $efs = dir_encode($fs);
634   my $dumpname = "$point.$efs.incremental.$base";
635
636   zfs_do_backup($host, $fs, 'i', $base, $store, $dumpname);
637 }
638
639 sub zfs_dataset_backup($$$$) {
640   my ($host, $fs, $base, $store) = @_;
641   my $agent = config_get($host, 'agent');
642
643   my $point = time();
644   my $dumpname = "$host/$fs\@$point";
645
646   zfs_do_backup($host, $fs, 'd', $base, $store, $dumpname);
647 }
648
649 sub perform_retention($$) {
650   my ($host, $store) = @_;
651   my $cutoff = time() - config_get($host, 'retention');
652   my $backup_info = scan_for_backups($store);
653  
654   foreach my $disk (sort keys %{$backup_info}) {
655     my $info = $backup_info->{$disk};
656     next unless(ref($info) eq 'HASH');
657     my %must_save;
658
659     # Get a list of all the full and incrementals, sorts newest to oldest
660     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
661     @backup_points = sort { $b <=> $a } @backup_points;
662
663     # We _cannot_ throw away _all_ our backups,
664     # so save the most recent incremental and full no matter what
665     $must_save{$backup_points[0]} = 1;
666     my @fulls = grep { exists($info->{full}->{$_}) } @backup_points;
667     $must_save{$fulls[0]} = 1;
668
669     # Walk the list for backups within our retention period.
670     foreach (@backup_points) {
671       if($_ >= $cutoff) {
672         $must_save{$_} = 1;
673       }
674       else {
675         # they are in decending order, once we miss, all will miss
676         last;
677       }
678     }
679
680     # Look for dependencies
681     foreach (@backup_points) {
682       if(exists($info->{incremental}->{$_})) {
683         print "   => $_ depends on $info->{incremental}->{$_}->{depends}\n" if($DEBUG);
684         $must_save{$info->{incremental}->{$_}->{depends}} = 1
685       }
686     }
687     my @removals = grep { !exists($must_save{$_}) } @backup_points;
688     if($DEBUG) {
689       my $tf = config_get($host, 'time_format');
690       print "    => I can remove:\n";
691       foreach (@backup_points) {
692         print "      => ". strftime($tf, localtime($_));
693         print " [". (exists($info->{full}->{$_}) ? "full":"incremental") ."]";
694         print " XXX" if(!exists($must_save{$_}));
695         print "\n";
696       }
697     }
698     foreach (@removals) {
699       my $efs = dir_encode($disk);
700       my $filename;
701       if(exists($info->{full}->{$_})) {
702         $filename = "$store/$_.$efs.full";
703       }
704       elsif(exists($info->{incremental}->{$_})) {
705         $filename = "$store/$_.$efs.incremental.$info->{incremental}->{$_}->{depends}";
706       }
707       else {
708         print "ERROR: We tried to expunge $host $disk [$_], but couldn't find it.\n";
709       }
710       print "    => expunging $filename\n" if($DEBUG);
711       unless($NEUTERED) {
712         unlink($filename) || print "ERROR: unlink $filename: $?\n";
713       }
714     }
715   }
716 }
717
718 sub __default_sort($$) { return $_[0] cmp $_[1]; }
719    
720 sub choose($$;$) {
721   my($name, $obj, $sort) = @_;
722   $sort ||= \&__default_sort;;
723   my @list;
724   my $hash;
725   if(ref $obj eq 'ARRAY') {
726     @list = sort { $sort->($a,$b); } (@$obj);
727     map { $hash->{$_} = $_; } @list;
728   }
729   elsif(ref $obj eq 'HASH') {
730     @list = sort { $sort->($a,$b); } (keys %$obj);
731     $hash = $obj;
732   }
733   else {
734     die "choose passed bad object: " . ref($obj) . "\n";
735   }
736   return $list[0] if(scalar(@list) == 1);
737   print "\n";
738   my $i = 1;
739   for (@list) {
740     printf " %3d) $hash->{$_}\n", $i++;
741   }
742   my $selection = 0;
743   while($selection !~ /^\d+$/ or
744         $selection < 1 or
745         $selection >= $i) {
746     print "$name: ";
747     chomp($selection = <>);
748   }
749   return $list[$selection - 1];
750 }
751
752 sub backup_chain($$) {
753   my ($info, $ts) = @_;
754   my @list;
755   push @list, $info->{full}->{$ts} if(exists($info->{full}->{$ts}));
756   if(exists($info->{incremental}->{$ts})) {
757     push @list, $info->{incremental}->{$ts};
758     push @list, backup_chain($info, $info->{incremental}->{$ts}->{depends});
759   }
760   return @list;
761 }
762
763 sub get_fs_from_mountpoint($) {
764     my ($mountpoint) = @_;
765     my $fs;
766     my $rv = open(ZFSLIST, "__ZFS__ list -H |");
767     die "Unable to determine zfs filesystem for $mountpoint" unless $rv;
768     while (<ZFSLIST>) {
769         my @F = split(' ');
770         if ($F[-1] eq $mountpoint) {
771             $fs = $F[0];
772             last;
773         }
774     }
775     close(ZFSLIST);
776     die "Unable to determine zfs filesystem for $mountpoint" unless $fs;
777     return $fs;
778 }
779
780 sub perform_restore() {
781   my %source;
782
783   foreach my $host (grep { $_ ne "default" } keys %conf) {
784     # If -h was specific, we will skip this host if the arg isn't
785     # an exact match or a pattern match
786     if($HOST &&
787        !(($HOST eq $host) ||
788          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
789       next;
790     }
791
792     my $store = config_get($host, 'store');
793     $store =~ s/%h/$host/g;;
794     mkdir $store if(! -d $store);
795
796     my $backup_info = scan_for_backups($store);
797     foreach my $disk (sort keys %{$backup_info}) {
798       my $info = $backup_info->{$disk};
799       next unless(ref($info) eq 'HASH');
800       next
801         if($ZFS &&      # if the pattern was specified it could
802            !($disk eq $ZFS ||        # be a specific match or a
803              ($ZFS =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
804       # We want to see this one
805       my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
806       my @source_points;
807       foreach (@backup_points) {
808         push @source_points, $_ if(!$TIMESTAMP || $TIMESTAMP == $_)
809       }
810       if(@source_points) {
811         $source{$host}->{$disk} = \@source_points;
812       }
813     }
814   }
815
816   if(! keys %source) {
817     print "No matching backups found\n";
818     return;
819   }
820
821   # Here goes the possibly interactive dialog
822   my $host = choose("Restore from host",  [keys %source]);
823   my $disk = choose("Restore from ZFS", [keys %{$source{$host}}]);
824  
825   # Times are special.  We build a human readable form and use a numerical
826   # sort function instead of the default lexical one.
827   my %times;
828   my $tf = config_get($host, 'time_format');
829   map { $times{$_} = strftime($tf, localtime($_)); } @{$source{$host}->{$disk}};
830   my $timestamp = choose("Restore as of timestamp", \%times,
831                          sub { $_[0] <=> $_[1]; });
832
833   my $store = config_get($host, 'store');
834   $store =~ s/%h/$host/g;;
835   mkdir $store if(! -d $store);
836   my $backup_info = scan_for_backups($store);
837   my @backup_list = reverse backup_chain($backup_info->{$disk}, $timestamp);
838
839   if(!$RESTORE_HOST) {
840     print "Restore to host [$host]:";
841     chomp(my $input = <>);
842     $RESTORE_HOST = length($input) ? $input : $host;
843   }
844   if(!$RESTORE_ZFS) {
845     print "Restore to zfs [$disk]:";
846     chomp(my $input = <>);
847     $RESTORE_ZFS = length($input) ? $input : $disk;
848   }
849
850   # show intentions
851   print "Going to restore:\n";
852   print "\tfrom: $host\n";
853   print "\tfrom: $disk\n";
854   print "\t  at: $timestamp [" . strftime($tf, localtime($timestamp)) . "]\n";
855   print "\t  to: $RESTORE_HOST\n";
856   print "\t  to: $RESTORE_ZFS\n";
857   print "\n";
858
859   foreach(@backup_list) {
860     $_->{success} = zfs_restore_part($RESTORE_HOST, $RESTORE_ZFS, $_->{file}, $_->{depends});
861   }
862 }
863
864 sub zfs_restore_part($$$;$) {
865   my ($host, $fs, $file, $dep) = @_;
866   my $ssh_config = config_get($host, 'ssh_config');
867   $ssh_config = "-F $ssh_config" if($ssh_config);
868   print "Using custom ssh config file: $ssh_config\n" if($DEBUG);
869   my $command;
870   if(exists($conf{$host})) {
871     my $agent = config_get($host, 'agent');
872     $command = "$agent -r -z $fs";
873     $command .= " -b $dep" if($dep);
874   }
875   else {
876     $command = "__ZFS__ recv $fs";
877   }
878   print " => piping $file to $command\n" if($DEBUG);
879   if($NEUTERED) {
880     print "gzip -dfc $file | ssh $ssh_config $host $command\n" if ($DEBUG);
881   }
882   else {
883     open(DUMP, "gzip -dfc $file |");
884     eval {
885       open(RECEIVER, "| ssh $ssh_config $host $command");
886       my $buffer;
887       while(my $len = sysread(DUMP, $buffer, $BLOCKSIZE)) {
888         if(syswrite(RECEIVER, $buffer, $len) != $len) {
889           die "$!";
890         }
891       }
892     };
893     close(DUMP);
894     close(RECEIVER);
895   }
896   return $?;
897 }
898
899 sub pretty_print_backup($$$) {
900   my ($info, $host, $point) = @_;
901   my $tf = config_get($host, 'time_format');
902   print "\t" . strftime($tf, localtime($point)) . " [$point] ";
903   if(exists($info->{full}->{$point})) {
904     my @st = stat($info->{full}->{$point}->{file});
905     print "FULL " . pretty_size($st[7]);
906     print "\n\tfile: $info->{full}->{$point}->{file}" if($SHOW_FILENAMES);
907   } else {
908     my @st = stat($info->{incremental}->{$point}->{file});
909     print "INCR from [$info->{incremental}->{$point}->{depends}] " . pretty_size($st[7]);
910     print "\n\tfile: $info->{incremental}->{$point}->{file}" if($SHOW_FILENAMES);
911   }
912   print "\n";
913 }
914
915 sub show_backups($$$) {
916   my ($host, $store, $diskpat) = @_;
917   my $backup_info = scan_for_backups($store);
918   my $tf = config_get($host, 'time_format');
919   my @files;
920   foreach my $disk (sort keys %{$backup_info}) {
921     my $info = $backup_info->{$disk};
922     next unless(ref($info) eq 'HASH');
923     next
924       if($diskpat &&      # if the pattern was specified it could
925          !($disk eq $diskpat ||        # be a specific match or a
926            ($diskpat =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
927
928     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
929     @backup_points = sort { $a <=> $b } @backup_points;
930     @backup_points = (pop @backup_points) unless ($ARCHIVE || $SUMMARY_EXT || $SUMMARY_VIOLATORS);
931
932     # Quick short-circuit in the case of retention violation checks
933     if($SUMMARY_VIOLATORS) {
934       if(time() > $info->{last_full} + config_get($host, 'full_interval') ||
935          time() > $info->{last_backup} + config_get($host, 'backup_interval')) {
936         print "$host:$disk\n";
937         pretty_print_backup($info, $host, $info->{last_full});
938         # Only print the last backup if it isn't the same as the last full
939         if ($info->{last_full} != $info->{last_backup}) {
940             pretty_print_backup($info, $host, $info->{last_backup});
941         }
942       }
943       next;
944     }
945
946     # We want to see this one
947     print "$host:$disk\n";
948     next unless($SUMMARY || $SUMMARY_EXT || $ARCHIVE);
949     if($SUMMARY_EXT) {
950       print "\tLast Full: ". ($info->{last_full} ? strftime($tf, localtime($info->{last_full})) : "Never") . "\n";
951       if($info->{last_full} < $info->{last_incremental}) {
952         print "\tLast Incr: ". strftime($tf, localtime($info->{last_incremental})). "\n";
953       }
954     }
955     foreach (@backup_points) {
956       pretty_print_backup($info, $host, $_);
957       push @files, exists($info->{full}->{$_}) ? $info->{full}->{$_}->{file} : $info->{incremental}->{$_}->{file};
958     }
959     print "\n";
960   }
961   if($ARCHIVE && scalar(@files)) {
962     my $archive = config_get($host, 'archive');
963     $archive =~ s/%h/$host/g;
964     if(! -d $archive) {
965       mkdir $archive || die "Cannot mkdir($archive)\n";
966     }
967     print "\nAre you sure you would like to archive ".scalar(@files)." file(s)? ";
968     while(($_ = <>) !~ /(?:y|n|yes|no)$/i) {
969       print "Are you sure you would like to archive ".scalar(@files)." file(s)? ";
970     }
971     if(/^y/i) {
972       foreach my $file (@files) {
973         (my $afile = $file) =~ s/^$store/$archive/;
974         move($file, $afile) || print "Error archiving $file: $!\n";
975       }
976     }
977   }
978 }
979
980 sub plan_and_run($$$) {
981   my ($host, $store, $diskpat) = @_;
982   my $ssh_config = config_get($host, 'ssh_config');
983   $ssh_config = "-F $ssh_config" if($ssh_config);
984   my %suppress;
985   print "Planning '$host'\n" if($DEBUG);
986   my $agent = config_get($host, 'agent');
987   my $took_action = 1;
988   while($took_action) {
989     $took_action = 0;
990     my @disklist;
991
992     # We need a lock for the listing.
993     return unless(lock($host, ".list"));
994
995     # Get list of zfs filesystems from the agent
996     open(SILENT, ">&", \*STDERR);
997     close(STDERR);
998     my $rv = open(ZFSLIST, "ssh $ssh_config $host $agent -l |");
999     open(STDERR, ">&", \*SILENT);
1000     close(SILENT);
1001     next unless $rv;
1002     @disklist = grep { chomp } (<ZFSLIST>);
1003     close(ZFSLIST);
1004
1005     foreach my $diskline (@disklist) {
1006       chomp($diskline);
1007       next unless($diskline =~ /^(\S+) \[([^\]]*)\]/);
1008       my $diskname = $1;
1009       my %snaps;
1010       map { $snaps{$_} = 1 } (split(/,/, $2));
1011  
1012       # We've just done this.
1013       next if($suppress{"$host:$diskname"});
1014       # If we are being selective (via -z) now is the time.
1015       next
1016         if($diskpat &&          # if the pattern was specified it could
1017            !($diskname eq $diskpat ||        # be a specific match or a
1018              ($diskpat =~ /^\/(.+)\/$/ && $diskname =~ /$1/))); # regex
1019  
1020       print " => Scanning '$store' for old backups of '$diskname'.\n" if($DEBUG);
1021
1022       # Make directory on demand
1023       my $backup_info = scan_for_backups($store);
1024       # That gave us info on all backups, we just want this disk
1025       $backup_info = $backup_info->{$diskname} || {};
1026  
1027       # Should we do a backup?
1028       my $backup_type = 'no';
1029       if(time() > $backup_info->{last_backup} + config_get($host, 'backup_interval')) {
1030         $backup_type = 'incremental';
1031       }
1032       if(time() > $backup_info->{last_full} + config_get($host, 'full_interval')) {
1033         $backup_type = 'full';
1034       }
1035  
1036       # If we want an incremental, but have no full, then we need to upgrade to full
1037       if($backup_type eq 'incremental') {
1038         my $have_full_locally = 0;
1039         # For each local full backup, see if the full backup still exists on the other end.
1040         foreach (keys %{$backup_info->{'full'}}) {
1041           $have_full_locally = 1 if(exists($snaps{'__zb_full_' . $_}));
1042         }
1043         $backup_type = 'full' unless($have_full_locally);
1044       }
1045       $backup_type = 'full' if($FORCE_FULL);
1046       $backup_type = 'incremental' if($FORCE_INC);
1047
1048       print " => doing $backup_type backup\n" if($DEBUG);
1049       # We need to drop a __zb_base snap or a __zb_incr snap before we proceed
1050       unless($NEUTERED || $backup_type eq 'no') {
1051         # attempt to lock this action, if it fails, skip -- someone else is working it.
1052         next unless(lock($host, dir_encode($diskname), 1));
1053         unlock($host, '.list');
1054
1055         if($backup_type eq 'full') {
1056           eval { zfs_full_backup($host, $diskname, $store); };
1057           if ($@) {
1058             chomp(my $err = $@);
1059             print " => failure $err\n";
1060           }
1061           else {
1062             # Unless there was an error backing up, remove all the other full snaps
1063             foreach (keys %snaps) {
1064               zfs_remove_snap($host, $diskname, $_) if(/^__zb_full_(\d+)/)
1065             }
1066           }
1067           $took_action = 1;
1068         }
1069         if($backup_type eq 'incremental') {
1070           eval {
1071             zfs_remove_snap($host, $diskname, '__zb_incr') if($snaps{'__zb_incr'});
1072             # Find the newest full from which to do an incremental (NOTE: reverse numeric sort)
1073             my @fulls = sort { $b <=> $a } (keys %{$backup_info->{'full'}});
1074             zfs_incremental_backup($host, $diskname, $fulls[0], $store);
1075           };
1076           if ($@) {
1077             chomp(my $err = $@);
1078             print " => failure $err\n";
1079           }
1080           else {
1081             $took_action = 1;
1082           }
1083         }
1084         unlock($host, dir_encode($diskname), 1);
1085       }
1086       $suppress{"$host:$diskname"} = 1;
1087       last if($took_action);
1088     }
1089     unlock($host, '.list');
1090   }
1091 }
1092
1093 if($RESTORE) {
1094   perform_restore();
1095 }
1096 else {
1097   foreach my $host (grep { $_ ne "default" } keys %conf) {
1098     # If -h was specific, we will skip this host if the arg isn't
1099     # an exact match or a pattern match
1100     if($HOST &&
1101        !(($HOST eq $host) ||
1102          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
1103       next;
1104     }
1105  
1106     my $store = config_get($host, 'store');
1107     $store =~ s/%h/$host/g;;
1108     mkdir $store if(! -d $store);
1109  
1110     if($LIST || $SUMMARY || $SUMMARY_EXT || $SUMMARY_VIOLATORS || $ARCHIVE) {
1111       show_backups($host, $store, $ZFS);
1112     }
1113     if($BACKUP) {
1114       plan_and_run($host, $store, $ZFS);
1115     }
1116     if($EXPUNGE) {
1117       perform_retention($host, $store);
1118     }
1119   }
1120 }
1121
1122 exit 0;
1123
1124 =pod
1125
1126 =head1 FILES
1127
1128 =over
1129
1130 =item zetaback.conf
1131
1132 The main zetaback configuration file.  The location of the file can be
1133 specified on the command line with the -c flag.  The prefix of this
1134 file may also be specified as an argument to the configure script.
1135
1136 =back
1137
1138 =head1 SEE ALSO
1139
1140 zetaback_agent(1)
1141
1142 =cut
Note: See TracBrowser for help on using the browser.