root/zetaback.in

Revision d373c5760c878cf5f7b7588cae226eee627307a1, 54.4 kB (checked in by Mark Harrison <mark@omniti.com>, 4 years ago)

Documentation on multi-restore mode

  • Property mode set to 100755
Line 
1 #!/usr/bin/perl
2 # vim: sts=2 sw=2 ts=8 et
3
4 # Copyright (c) 2007 OmniTI Computer Consulting, Inc. All rights reserved.
5 # For information on licensing see:
6 #   https://labs.omniti.com/zetaback/trunk/LICENSE
7
8 use strict;
9 use Getopt::Long;
10 use MIME::Base64;
11 use POSIX qw/strftime/;
12 use Fcntl qw/:flock/;
13 use File::Path qw/mkpath/;
14 use File::Copy;
15 use IO::File;
16 use Pod::Usage;
17
18 use vars qw/%conf %locks $version_string $process_lock
19             $PREFIX $CONF $BLOCKSIZE $DEBUG $HOST $BACKUP
20             $RESTORE $RESTORE_HOST $RESTORE_ZFS $TIMESTAMP
21             $LIST $SUMMARY $SUMMARY_EXT $SUMMARY_VIOLATORS
22             $SUMMARY_VIOLATORS_VERBOSE $FORCE_FULL $FORCE_INC
23             $EXPUNGE $NEUTERED $ZFS $SHOW_FILENAMES $ARCHIVE
24             $VERSION $HELP/;
25 $version_string = '1.0.6';
26 $PREFIX = q^__PREFIX__^;
27 $CONF = qq^$PREFIX/etc/zetaback.conf^;
28 $BLOCKSIZE = 1024*64;
29
30 $conf{'default'}->{'time_format'} = "%Y-%m-%d %H:%M:%S";
31 $conf{'default'}->{'retention'} = 14 * 86400;
32 $conf{'default'}->{'compressionlevel'} = 1;
33 $conf{'default'}->{'dataset_backup'} = 0;
34 $conf{'default'}->{'violator_grace_period'} = 21600;
35
36 =pod
37
38 =head1 NAME
39
40 zetaback - perform backup, restore and retention policies for ZFS backups.
41
42 =head1 SYNOPSIS
43
44   zetaback -v
45
46   zetaback [-l|-s|-sx|-sv|-svv] [--files] [-c conf] [-d] [-h host] [-z zfs]
47
48   zetaback -a [-c conf] [-d] [-h host] [-z zfs]
49
50   zetaback -b [-ff] [-fi] [-x] [-c conf] [-d] [-n] [-h host] [-z zfs]
51
52   zetaback -x [-b] [-c conf] [-d] [-n] [-h host] [-z zfs]
53
54   zetaback -r [-c conf] [-d] [-n] [-h host] [-z zfs] [-t timestamp]
55               [-rhost host] [-rzfs fs]
56
57 =cut
58
59 GetOptions(
60   "h=s"     => \$HOST,
61   "z=s"     => \$ZFS,
62   "c=s"     => \$CONF,
63   "a"       => \$ARCHIVE,
64   "b"       => \$BACKUP,
65   "l"       => \$LIST,
66   "s"       => \$SUMMARY,
67   "sx"      => \$SUMMARY_EXT,
68   "sv"      => \$SUMMARY_VIOLATORS,
69   "svv"     => \$SUMMARY_VIOLATORS_VERBOSE,
70   "r"       => \$RESTORE,
71   "t=i"     => \$TIMESTAMP,
72   "rhost=s" => \$RESTORE_HOST,
73   "rzfs=s"  => \$RESTORE_ZFS,
74   "d"       => \$DEBUG,
75   "n"       => \$NEUTERED,
76   "x"       => \$EXPUNGE,
77   "v"       => \$VERSION,
78   "ff"      => \$FORCE_FULL,
79   "fi"      => \$FORCE_INC,
80   "files"   => \$SHOW_FILENAMES,
81 );
82
83 # actions allowed together 'x' and 'b' all others are exclusive:
84 my $actions = 0;
85 $actions++ if($ARCHIVE);
86 $actions++ if($BACKUP || $EXPUNGE);
87 $actions++ if($RESTORE);
88 $actions++ if($LIST);
89 $actions++ if($SUMMARY);
90 $actions++ if($SUMMARY_EXT);
91 $actions++ if($SUMMARY_VIOLATORS);
92 $actions++ if($SUMMARY_VIOLATORS_VERBOSE);
93 $actions++ if($VERSION);
94 $actions++ if($BACKUP && $FORCE_FULL && $FORCE_INC);
95 if($actions != 1) {
96   pod2usage({ -verbose => 0 });
97   exit -1;
98 }
99
100 =pod
101
102 =head1 DESCRIPTION
103
104 The B<zetaback> program orchestrates the backup (either full or
105 incremental) of remote ZFS filesystems to a local store.  It handles
106 frequency requirements for both full and incemental backups as well
107 as retention policies.  In addition to backups, the B<zetaback> tool
108 allows for the restore of any backup to a specified host and zfs
109 filesystem.
110
111 =head1 OPTIONS
112
113 The non-optional action command line arguments define the invocation purpose
114 of B<zetaback>.  All other arguments are optional and refine the target
115 of the action specified.
116
117 =head2 Generic Options
118
119 The following arguments have the same meaning over several actions:
120
121 =over
122
123 =item -c <conf>
124
125 Use the specified file as the configuration file.  The default file, if
126 none is specified is /usr/local/etc/zetaback.conf.  The prefix of this
127 file may also be specified as an argument to the configure script.
128
129 =item -d
130
131 Enable debugging output.
132
133 =item -n
134
135 Don't actually perform any remote commands or expunging.  This is useful with
136 the -d argument to ascertain what would be done if the command was actually
137 executed.
138
139 =item -t <timestamp>
140
141 Used during the restore process to specify a backup image from the desired
142 point in time.  If omitted, the command becomes interactive.  This timestamp
143 is a UNIX timestamp and is shown in the output of the -s and -sx actions.
144
145 =item -rhost <host>
146
147 Specify the remote host that is the target for a restore operation.  If
148 omitted the command becomes interactive.
149
150 =item -rzfs <zfs>
151
152 Specify the remote ZFS filesystem that is the target for a restore
153 operation.  If omitted the command becomes interactive.
154
155 =item -h <host>
156
157 Filters the operation to the host specified.  If <host> is of the form
158 /pattern/, it matches 'pattern' as a perl regular expression against available
159 hosts.  If omitted, no limit is enforced and all hosts are used for the action.
160
161 =item -z <zfs>
162
163 Filters the operation to the zfs filesystem specified.  If <zfs> is of the
164 form /pattern/, it matches 'pattern' as a perl regular expression against
165 available zfs filesystems.  If omitted, no filter is enforced and all zfs
166 filesystems are used for the action.
167
168 =back
169
170 =head2 Actions
171
172 =over
173
174 =item -v
175
176 Show the version.
177
178 =item -l
179
180 Show a brief listing of available backups.
181
182 =item -s
183
184 Like -l, -s will show a list of backups but provides additional information
185 about the backups including timestamp, type (full or incremental) and the
186 size on disk.
187
188 =item -sx
189
190 Shows an extended summary.  In addition to the output provided by the -s
191 action, the -sx action will show detail for each availble backup.  For
192 full backups, the detail will include any more recent full backups, if
193 they exist.  For incremental backups, the detail will include any
194 incremental backups that are more recent than the last full backup.
195
196 =item -sv
197
198 Display all backups in the current store that violate the configured
199 backup policy. This is where the most recent full backup is older than
200 full_interval seconds ago, or the most recent incremental backup is older
201 than backup_interval seconds ago.
202
203 If, at the time of the most recent backup, a filesystem no longer exists on
204 the server (because it was deleted), then backups of this filesystem are not
205 included in the list of violators. To include these filesystems, use the -svv
206 option instead.
207
208 =item -svv
209
210 The violators summary will exclude backups of filesystems that are no longer
211 on the server in the list of violators. Use this option to include those
212 filesystems.
213
214 =item --files
215
216 Display the on-disk file corresponding to each backup named in the output.
217 This is useful with the -sv flag to name violating files.  Often times,
218 violators are filesystems that have been removed on the host machines and
219 zetaback can no longer back them up.  Be very careful if you choose to
220 automate the removal of such backups as filesystems that would be backed up
221 by the next regular zetaback run will often show up as violators.
222
223 =item -a
224
225 Performs an archive.  This option will look at all eligible backup points
226 (as restricted by -z and -h) and move those to the configured archive
227 directory.  The recommended use is to first issue -sx --files then
228 carefully review available backup points and prune those that are
229 unneeded.  Then invoke with -a to move only the remaining "desired"
230 backup points into the archives.  Archived backups do not appear in any
231 listings or in the list of policy violators generated by the -sv option.
232 In effect, they are no longer "visible" to zetaback.
233
234 =item -b
235
236 Performs a backup.  This option will investigate all eligible hosts, query
237 the available filesystems from the remote agent and determine if any such
238 filesystems require a new full or incremental backup to be taken.  This
239 option may be combined with the -x option (to clean up afterwards.)
240
241 =item -ff
242
243 Forces a full backup to be taken on each filesystem encountered.  This is
244 used in combination with -b.  It is recommended to use this option only when
245 targeting specific filesystems (via the -h and -z options.)  Forcing a full
246 backup across all machines will cause staggered backups to coalesce and
247 could cause performance issues.
248
249 =item -fi
250
251 Forces an incremental backup to be taken on each filesystem encountered. 
252 This is used in combination with -b.  It is recommended to use this option
253 only when targeting specific filesystems (via the -h and -z options.)  Forcing
254 an incremental backup across all machines will cause staggered backups
255 to coalesce and could cause performance issues.
256
257 =item -x
258
259 Perform an expunge.  This option will determine which, if any, of the local
260 backups may be deleted given the retention policy specified in the
261 configuration.
262
263 =item -r
264
265 Perform a restore.  This option will operate on the specified backup and
266 restore it to the ZFS filesystem specified with -rzfs on the host specified
267 with the -rhost option.  The -h, -z and -t options may be used to filter
268 the source backup list.  If the filtered list contains more than one
269 source backup image, the command will act interactively.  If the -rhost
270 and -rzfs command are not specified, the command will act interactively.
271
272 When running interactively, you can choose multiple filesystems from the list
273 using ranges. For example 1-4,5,10-11. If you do this, zetaback will enter
274 multi-restore mode. In this mode it will automatically select the most recent
275 backup, and restore filessytems in bulk.
276
277 In multi-restore mode, you have the option to specify a base filesystem to
278 restore to. This filesystem will be added as a prefix to the original
279 filesystem name, so if you picked a prefix of data/restore, and one of the
280 filesystems you are restoring is called data/set/myfilesystem, then the
281 filesystem will be restored to data/restore/data/set/myfilesystem.
282
283 Note that, just like in regular restore mode, zetaback won't create
284 intermediate filesystems for you when restoring, and these should either exist
285 beforehand, or you should make sure you pick a set of filesystems that will
286 restore the entire tree for you, for example, you should restore data as well
287 as data/set before restoring data/set/foo.
288
289 =back
290
291 =cut
292
293 if($VERSION) {
294   print "zetaback: $version_string\n";
295   exit 0;
296 }
297
298 =pod
299
300 =head1 CONFIGURATION
301
302 The zetaback configuration file consists of a default stanza, containing
303 settings that can be overridden on a per-host basis.  A stanza begins
304 either with the string 'default', or a fully-qualified hostname, with
305 settings enclosed in braces ({}).  Single-line comments begin with a hash
306 ('#'), and whitespace is ignored, so feel free to indent for better
307 readability.  Every host to be backed up must have a host stanza in the
308 configuration file.
309
310 =head2 Storage Classes
311
312 In addition to the default and host stanzas, the configuration file can also
313 contain 'class' stanzas. Classes allow you to override settings on a
314 per-filesystem basis rather than a per-host basis. A class stanza begins with
315 the name of the class, and has a setting 'type = class'. For example:
316
317   myclass {
318     type = class
319     store = /path/to/alternate/store
320   }
321
322 To add a filesystem to a class, set a zfs user property on the relevant
323 filesystem. This must be done on the server that runs the zetaback agent, and
324 not the zetaback server itself.
325
326   zfs set com.omniti.labs.zetaback:class=myclass pool/fs
327
328 Note that user properties (and therefore classes) are are only available on
329 Solaris 10 8/07 and newer, and on Solaris Express build 48 and newer. Only the
330 server running the agent needs to have user property support, not the zetaback
331 server itself.
332
333 The following settings can be included in a class stanza. All other settings
334 will be ignored, and their default (or per host) settings used instead:
335
336 =over
337
338 =item *
339
340 store
341
342 =item *
343
344 full_interval
345
346 =item *
347
348 backup_interval
349
350 =item *
351
352 retention
353
354 =item *
355
356 dataset_backup
357
358 =item *
359
360 violator_grace_period
361
362 =back
363
364 =head2 Settings
365
366 The following settings are valid in both the default and host scopes:
367
368 =over
369
370 =item store
371
372 The base directory under which to keep backups.  An interpolated variable
373 '%h' can be used, which expands to the hostname.  There is no default for
374 this setting.
375
376 =item archive
377
378 The base directory under which archives are stored.  The format is the same
379 as the store setting.  This is the destination to which files are relocated
380 when issuing an archive action (-a).
381
382 =item agent
383
384 The location of the zetaback_agent binary on the host.  There is no default
385 for this setting.
386
387 =item time_format
388
389 All timestamps within zetaback are in UNIX timestamp format.  This setting
390 provides a string for formatting all timestamps on output.  The sequences
391 available are identical to those in strftime(3).  If not specified, the
392 default is '%Y-%m-%d %H:%M:%S'.
393
394 =item backup_interval
395
396 The frequency (in seconds) at which to perform incremental backups.  An
397 incremental backup will be performed if the current time is more than
398 backup_interval since the last incremental backup.  If there is no full backup
399 for a particular filesystem, then a full backup is performed.  There is no
400 default for this setting.
401
402 =item full_interval
403
404 The frequency (in seconds) at which to perform full backups.  A full backup will
405 be performed if the current time is more than full_interval since the last full
406 backup.
407
408 =item retention
409
410 The retention time (in seconds) for backups.  This can be a simple number, in
411 which case all backups older than this will be expunged.
412
413 The retention specification can also be more complex, and consist of pairs of
414 values separated by a comma. The first value is a time period in seconds, and
415 the second value is how many backups should be retained within that period.
416 For example:
417
418 retention = 3600,4;86400,11
419
420 This will keep up to 4 backups for the first hour, and an additional 11
421 backups over 24 hours. The times do not stack. In other words, the 11 backups
422 would be kept during the period from 1 hour old to 24 hours old, or one every
423 2 hours.
424
425 Any backups older than the largest time given are deleted. In the above
426 example, all backups older than 24 hours are deleted.
427
428 If a second number is not specified, then all backups are kept within that
429 period.
430
431 Note: Full backups are never deleted if they are depended upon by an
432 incremental. In addition, the most recent backup is never deleted, regardless
433 of how old it is.
434
435 This value defaults to (14 * 86400), or two weeks.
436
437 =item compressionlevel
438
439 Compress files using gzip at the specified compression level. 0 means no
440 compression. Accepted values are 1-9. Defaults to 1 (fastest/minimal
441 compression.)
442
443 =item ssh_config
444
445 Full path to an alternate ssh client config.  This is useful for specifying a
446 less secure but faster cipher for some hosts, or using a different private
447 key.  There is no default for this setting.
448
449 =item dataset_backup
450
451 By default zetaback backs zfs filesystems up to files. This option lets you
452 specify that the backup go be stored as a zfs dataset on the backup host.
453
454 =item offline
455
456 Setting this option to 1 for a host will mark it as being 'offline'. Hosts
457 that are marked offline will not be backed up, will not have any old backups
458 expunged and will not be included in the list of policy violators. However,
459 the host will still be shown when listing backups and archiving.
460
461 =item violator_grace_period
462
463 This setting controls the grace period used when deciding if a backup has
464 violated its backup window. It is used to prevent false positives in the case
465 where a filesystem is still being backed up. For example, if it is 25 hours
466 since the last daily backup, but the daily backup is in progress, the grace
467 period will mean that it is not shown in the violators list.
468
469 Like all intervals, this period is in seconds. The default is 21600 seconds (6
470 hours).
471
472 =back
473
474 =head2 Global Settings
475
476 The following settings are only valid in the default scope:
477
478 =over
479
480 =item process_limit
481
482 This setting limits the number of concurrent zetaback processes that can run
483 at one time. Zetaback already has locks on hosts and datasets to prevent
484 conflicting backups, and this allows you to have multiple zetaback instances
485 running in the event a backup takes some time to complete, while still keeping
486 a limit on the resources used. If this configuration entry is missing, then no
487 limiting will occur.
488
489 =back
490
491 =head1 CONFIGURATION EXAMPLES
492
493 =head2 Uniform hosts
494
495 This config results in backups stored in /var/spool/zfs_backups, with a
496 subdirectory for each host.  Incremental backups will be performed
497 approximately once per day, assuming zetaback is run hourly.  Full backups
498 will be done once per week.  Time format and retention are default.
499
500   default {
501     store = /var/spool/zfs_backups/%h
502     agent = /usr/local/bin/zetaback_agent
503     backup_interval = 83000
504     full_interval = 604800
505   }
506
507   host1 {}
508
509   host2 {}
510
511 =head2 Non-uniform hosts
512
513 Here, host1's and host2's agents are found in different places, and host2's
514 backups should be stored in a different path.
515
516   default {
517     store = /var/spool/zfs_backups/%h
518     agent = /usr/local/bin/zetaback_agent
519     backup_interval = 83000
520     full_interval = 604800
521   }
522
523   host1 {
524     agent = /opt/local/bin/zetaback_agent
525   }
526
527   host2 {
528     store = /var/spool/alt_backups/%h
529     agent = /www/bin/zetaback_agent
530   }
531
532 =cut
533
534 # Make the parser more formal:
535 # config => stanza*
536 # stanza => string { kvp* }
537 # kvp    => string = string
538 my $str_re = qr/(?:"(?:\\\\|\\"|[^"])*"|\S+)/;
539 my $kvp_re = qr/($str_re)\s*=\s*($str_re)/;
540 my $stanza_re = qr/($str_re)\s*\{((?:\s*$kvp_re)*)\s*\}/;
541
542 sub parse_config() {
543   local($/);
544   $/ = undef;
545   open(CONF, "<$CONF") || die "Unable to open config file: $CONF";
546   my $file = <CONF>;
547   # Rip comments
548   $file =~ s/^\s*#.*$//mg;
549   while($file =~ m/$stanza_re/gm) {
550     my $scope = $1;
551     my $filepart = $2;
552     $scope =~ s/^"(.*)"$/$1/;
553     $conf{$scope} ||= {};
554     while($filepart =~ m/$kvp_re/gm) {
555       my $key = $1;
556       my $value = $2;
557       $key =~ s/^"(.*)"$/$1/;
558       $value =~ s/^"(.*)"$/$1/;
559       $conf{$scope}->{lc($key)} = $value;
560     }
561   }
562   close(CONF);
563 }
564 sub config_get($$;$) {
565   # Params: host, key, class
566   # Order of precedence: class, host, default
567   if ($_[2]) {
568     return $conf{$_[2]}->{$_[1]} || $conf{$_[0]}->{$_[1]} ||
569         $conf{'default'}->{$_[1]};
570   } else {
571     return $conf{$_[0]}->{$_[1]} || $conf{'default'}->{$_[1]};
572   }
573 }
574
575 sub get_store($;$) {
576   my ($host, $class) = @_;
577   my $store = config_get($host, 'store', $class);
578   $store =~ s/%h/$host/g;;
579   return $store;
580 }
581
582 sub get_classes() {
583   my @classes = (""); # The default/blank class is always present
584   foreach my $key (keys %conf) {
585     if ($conf{$key}->{'type'} eq 'class') {
586       push @classes, $key;
587     }
588   }
589   return @classes;
590 }
591
592 sub fs_encode($) {
593   my $d = shift;
594   my @parts = split('@', $d);
595   my $e = encode_base64($parts[0], '');
596   $e =~ s/\//_/g;
597   $e =~ s/=/-/g;
598   $e =~ s/\+/\./g;
599   if (exists $parts[1]) {
600     $e .= "\@$parts[1]";
601   }
602   return $e;
603 }
604 sub fs_decode($) {
605   my $e = shift;
606   $e =~ s/_/\//g;
607   $e =~ s/-/=/g;
608   $e =~ s/\./\+/g;
609   return decode_base64($e);
610 }
611 sub dir_encode($) {
612   my $d = shift;
613   my $e = encode_base64($d, '');
614   $e =~ s/\//_/;
615   return $e;
616 }
617 sub dir_decode($) {
618   my $e = shift;
619   $e =~ s/_/\//;
620   return decode_base64($e);
621 }
622 sub pretty_size($) {
623   my $bytes = shift;
624   if($bytes > 1024*1024*1024) {
625     return sprintf("%0.2f Gb", $bytes / (1024*1024*1024));
626   }
627   if($bytes > 1024*1024) {
628     return sprintf("%0.2f Mb", $bytes / (1024*1024));
629   }
630   if($bytes > 1024) {
631     return sprintf("%0.2f Kb", $bytes / (1024));
632   }
633   return "$bytes b";
634 }
635 sub lock($;$$) {
636   my ($host, $file, $nowait) = @_;
637   print "Acquiring lock for $host:$file\n" if($DEBUG);
638   $file ||= 'master.lock';
639   my $store = get_store($host); # Don't take classes into account - not needed
640   mkpath($store) if(! -d $store);
641   return 1 if(exists($locks{"$host:$file"}));
642   open(LOCK, "+>>$store/$file") || return 0;
643   unless(flock(LOCK, LOCK_EX | ($nowait ? LOCK_NB : 0))) {
644     close(LOCK);
645     return 0;
646   }
647   $locks{"$host:$file"} = \*LOCK;
648   return 1;
649 }
650 sub unlock($;$$) {
651   my ($host, $file, $remove) = @_;
652   print "Releasing lock for $host:$file\n" if($DEBUG);
653   $file ||= 'master.lock';
654   my $store = get_store($host); # Don't take classes into account - not needed
655   mkpath($store) if(! -d $store);
656   return 0 unless(exists($locks{"$host:$file"}));
657   *UNLOCK = $locks{$file};
658   unlink("$store/$file") if($remove);
659   flock(UNLOCK, LOCK_UN);
660   close(UNLOCK);
661   return 1;
662 }
663 sub limit_running_processes() {
664     my $max = $conf{'default'}->{'process_limit'};
665     return unless defined($max);
666     print "Aquiring process lock\n" if $DEBUG;
667     for (my $i=0; $i < $max; $i++) {
668         my $file = "/tmp/.zetaback_$i.lock";
669         print "$file\n" if $DEBUG;
670         open ($process_lock, "+>>$file") || next;
671         if (flock($process_lock, LOCK_EX | LOCK_NB)) {
672             print "Process lock succeeded: $file\n" if $DEBUG;
673             return 1;
674         } else {
675             close($process_lock);
676         }
677     }
678     print "Too many zetaback processes running. Exiting...\n" if $DEBUG;
679     exit 0;
680 }
681 sub scan_for_backups($) {
682   my %info = ();
683   my $dir = shift;
684   $info{last_full} = $info{last_incremental} = $info{last_backup} = 0;
685   # Look for standard file based backups first
686   opendir(D, $dir) || return \%info;
687   foreach my $file (readdir(D)) {
688     if($file =~ /^(\d+)\.([^\.]+)\.full$/) {
689       my $whence = $1;
690       my $fs = dir_decode($2);
691       $info{$fs}->{full}->{$whence}->{'file'} = "$dir/$file";
692       $info{$fs}->{last_full} = $whence if($whence > $info{$fs}->{last_full});
693       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
694                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
695     }
696     elsif($file =~ /^(\d+).([^\.]+)\.incremental.(\d+)$/) {
697       my $whence = $1;
698       my $fs = dir_decode($2);
699       $info{$fs}->{incremental}->{$whence}->{'depends'} = $3;
700       $info{$fs}->{incremental}->{$whence}->{'file'} = "$dir/$file";
701       $info{$fs}->{last_incremental} = $whence if($whence > $info{$fs}->{last_incremental});
702       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
703                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
704     }
705   }
706   closedir(D);
707   # Now look for zfs based backups
708   my $storefs;
709   eval {
710     $storefs = get_fs_from_mountpoint($dir);
711   };
712   return \%info if ($@);
713   my $rv = open(ZFSLIST, "__ZFS__ list -H -r -t snapshot $storefs |");
714   return \%info unless $rv;
715   while (<ZFSLIST>) {
716       my @F = split(' ');
717       my ($rawfs, $snap) = split('@', $F[0]);
718       my ($whence) = ($snap =~ /(\d+)/);
719       next unless $whence;
720       my @fsparts = split('/', $rawfs);
721       my $fs = fs_decode($fsparts[-1]);
722       # Treat a dataset backup as a full backup from the point of view of the
723       # backup lists
724       $info{$fs}->{full}->{$whence}->{'snapshot'} = $snap;
725       $info{$fs}->{full}->{$whence}->{'dataset'} = "$rawfs\@$snap";
726       # Note - this field isn't set for file backups - we probably should do
727       # this
728       $info{$fs}->{full}->{$whence}->{'pretty_size'} = "$F[1]";
729       $info{$fs}->{last_full} = $whence if ($whence >
730           $info{$fs}->{last_full});
731       $info{$fs}->{last_backup} = $whence if ($whence >
732           $info{$fs}->{last_backup});
733   }
734   close(ZFSLIST);
735
736   return \%info;
737 }
738
739 parse_config();
740
741 sub zetaback_log($$;@) {
742   my ($host, $mess, @args) = @_;
743   my $tf = config_get($host, 'time_format');
744   my $file = config_get($host, 'logfile');
745   my $fileh;
746   if(defined($file)) {
747     $fileh = IO::File->new(">>$file");
748   }
749   $fileh ||= IO::File->new(">&STDERR");
750   printf $fileh "%s: $mess", strftime($tf, localtime(time)), @args;
751   $fileh->close();
752 }
753
754 sub zfs_remove_snap($$$) {
755   my ($host, $fs, $snap) = @_;
756   my $agent = config_get($host, 'agent');
757   my $ssh_config = config_get($host, 'ssh_config');
758   $ssh_config = "-F $ssh_config" if($ssh_config);
759   print "Using custom ssh config file: $ssh_config\n" if($DEBUG);
760   return unless($snap);
761   print "Dropping $snap on $fs\n" if($DEBUG);
762   `ssh $ssh_config $host $agent -z $fs -d $snap`;
763 }
764
765 # Lots of args.. internally called.
766 sub zfs_do_backup($$$$$$;$) {
767   my ($host, $fs, $type, $point, $store, $dumpname, $base) = @_;
768   my ($storefs, $encodedname);
769   my $agent = config_get($host, 'agent');
770   my $ssh_config = config_get($host, 'ssh_config');
771   $ssh_config = "-F $ssh_config" if($ssh_config);
772   print "Using custom ssh config file: $ssh_config\n" if($DEBUG);
773
774   # compression is meaningless for dataset backups
775   if ($type ne "s") {
776     my $cl = config_get($host, 'compressionlevel');
777     if ($cl >= 1 && $cl <= 9) {
778         open(LBACKUP, "|gzip -$cl >$store/.$dumpname") ||
779         die "zfs_do_backup $host:$fs $type: cannot create dump\n";
780     } else {
781         open(LBACKUP, ">$store/.$dumpname") ||
782         die "zfs_do_backup $host:$fs $type: cannot create dump\n";
783     }
784   } else {
785     # Dataset backup - pipe received filesystem to zfs recv
786     eval {
787       $storefs = get_fs_from_mountpoint($store);
788     };
789     if ($@) {
790       # The zfs filesystem doesn't exist, so we have to work out what it
791       # would be
792       my $basestore = $store;
793       $basestore =~ s/\/?%h//g;
794       $storefs = get_fs_from_mountpoint($basestore);
795       $storefs="$storefs/$host";
796     }
797     $encodedname = fs_encode($dumpname);
798     print STDERR "Receiving to zfs filesystem $storefs/$encodedname\n"
799       if($DEBUG);
800     zfs_create_intermediate_filesystems("$storefs/$encodedname");
801     open(LBACKUP, "|__ZFS__ recv $storefs/$encodedname");
802   }
803   # Do it. yeah.
804   eval {
805     if(my $pid = fork()) {
806       close(LBACKUP);
807       waitpid($pid, 0);
808       die "error: $?" if($?);
809     }
810     else {
811       my @cmd = ('ssh', split(/ /, $ssh_config), $host, $agent, '-z', $fs);
812       if ($type eq "i" || ($type eq "s" && $base)) {
813         push @cmd, ("-i", $base);
814       }
815       if ($type eq "f" || $type eq "s") {
816         push @cmd, ("-$type", $point);
817       }
818       open STDIN, "/dev/null" || exit(-1);
819       open STDOUT, ">&LBACKUP" || exit(-1);
820       print STDERR "   => @cmd\n" if($DEBUG);
821       unless (exec { $cmd[0] } @cmd) {
822         print STDERR "$cmd[0] failed: $!\n";
823         exit(1);
824       }
825     }
826     if ($type ne "s") {
827       die "dump failed (zero bytes)\n" if(-z "$store/.$dumpname");
828       rename("$store/.$dumpname", "$store/$dumpname") || die "cannot rename dump\n";
829     } else {
830       # Check everything is ok
831       `__ZFS__ list $storefs/$encodedname`;
832       die "dump failed (received snapshot $storefs/$encodedname does not exist)\n"
833         if $?;
834     }
835   };
836   if($@) {
837     if ($type ne "s") {
838         unlink("$store/.$dumpname");
839     }
840     chomp(my $error = $@);
841     $error =~ s/[\r\n]+/ /gsm;
842     zetaback_log($host, "FAILED[$error] $host:$fs $type\n");
843     die "zfs_do_backup $host:$fs $type: $error";
844   }
845   my $size;
846   if ($type ne "s") {
847     my @st = stat("$store/$dumpname");
848     $size = pretty_size($st[7]);
849   } else {
850     $size = `__ZFS__ get -Ho value used $storefs/$encodedname`;
851     chomp $size;
852   }
853   zetaback_log($host, "SUCCESS[$size] $host:$fs $type\n");
854 }
855
856 sub zfs_create_intermediate_filesystems($) {
857   my ($fs) = @_;
858   my $idx=0;
859   while (($idx = index($fs, '/', $idx+1)) != -1) {
860       my $fspart = substr($fs, 0, $idx);
861       `__ZFS__ list $fspart 2>&1`;
862       if ($?) {
863         print STDERR "Creating intermediate zfs filesystem: $fspart\n"
864           if $DEBUG;
865         `__ZFS__ create $fspart`;
866       }
867   }
868 }
869
870 sub zfs_full_backup($$$) {
871   my ($host, $fs, $store) = @_;
872
873   # Translate into a proper dumpname
874   my $point = time();
875   my $efs = dir_encode($fs);
876   my $dumpname = "$point.$efs.full";
877
878   zfs_do_backup($host, $fs, 'f', $point, $store, $dumpname);
879 }
880
881 sub zfs_incremental_backup($$$$) {
882   my ($host, $fs, $base, $store) = @_;
883   my $agent = config_get($host, 'agent');
884
885   # Translate into a proper dumpname
886   my $point = time();
887   my $efs = dir_encode($fs);
888   my $dumpname = "$point.$efs.incremental.$base";
889
890   zfs_do_backup($host, $fs, 'i', $point, $store, $dumpname, $base);
891 }
892
893 sub zfs_dataset_backup($$$$) {
894   my ($host, $fs, $base, $store) = @_;
895   my $agent = config_get($host, 'agent');
896
897   my $point = time();
898   my $dumpname = "$fs\@$point";
899
900   zfs_do_backup($host, $fs, 's', $point, $store, $dumpname, $base);
901 }
902
903 sub perform_retention($) {
904   my ($host) = @_;
905   my $now = time();
906
907   if ($DEBUG) {
908     print "Performing retention for $host\n";
909   }
910
911   foreach my $class (get_classes()) {
912     if ($DEBUG) {
913       if ($class) {
914         print "=> Class: $class\n" if $class;
915       } else {
916         print "=> Class: (none)\n";
917       }
918     }
919     my $retention = config_get($host, 'retention', $class);
920     my $store = get_store($host, $class);
921     my $backup_info = scan_for_backups($store);
922     foreach my $disk (sort keys %{$backup_info}) {
923       my $info = $backup_info->{$disk};
924       next unless(ref($info) eq 'HASH');
925       my %must_save;
926
927       if ($DEBUG) {
928         print "   $disk\n";
929       }
930
931       # Get a list of all the full and incrementals, sorts newest to oldest
932       my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
933       @backup_points = sort { $b <=> $a } @backup_points;
934
935       # We _cannot_ throw away _all_ our backups,
936       # so save the most recent incremental and full no matter what
937       push(@{$must_save{$backup_points[0]}}, "most recent backup");
938       my @fulls = grep { exists($info->{full}->{$_}) } @backup_points;
939       push(@{$must_save{$fulls[0]}}, "most recent full");
940
941       # Process retention policy
942       my @parts = split(/;/, $retention);
943       my %retention_map;
944       foreach (@parts) {
945         my ($period, $amount) = split(/,/);
946         if (!defined($amount)) {
947           $amount = -1;
948         }
949         $retention_map{$period} = $amount;
950       }
951       my @periods = sort { $a <=> $b } keys(%retention_map);
952       my %backup_bins;
953       foreach(@periods) {
954         $backup_bins{$_} = ();
955       }
956       my $cutoff = $now - $periods[0];
957       # Sort backups into time period sections
958       foreach (@backup_points) {
959         # @backup_points is in descending order (newest first)
960         while ($_ <= $cutoff) {
961           # Move to the next largest bin if the current backup is not in the
962           # current bin. However, if there is no larger bin, then don't
963           shift(@periods);
964           if (@periods) {
965             $cutoff = $now - $periods[0];
966           } else {
967             last;
968           }
969         }
970         # Throw away all backups older than the largest time period specified
971         if (!@periods) {
972           last;
973         }
974         push(@{$backup_bins{$periods[0]}}, $_);
975       }
976       foreach (keys(%backup_bins)) {
977         my $keep = $retention_map{$_}; # How many backups to keep
978         if ($backup_bins{$_}) {
979           my @backups = @{$backup_bins{$_}};
980           my $total = @backups;  # How many backups we have
981           # If we didn't specify how many to keep, keep them all
982           if ($keep == -1) { $keep = $total };
983           # If we have less backups than we should keep, keep them all
984           if ($total < $keep) { $keep = $total };
985           for (my $i = 1; $i <= $keep; $i++) {
986             my $idx = int(($i * $total) / $keep) - 1;
987             push(@{$must_save{$backups[$idx]}}, "retention policy - $_");
988           }
989         }
990       }
991       if ($DEBUG) {
992         print "    => Backup bins:\n";
993         foreach my $a (keys(%backup_bins)) {
994           print "      => $a\n";
995           foreach my $i (@{$backup_bins{$a}}) {
996             my $trans = $now - $i;
997             print "         => $i ($trans seconds old)";
998             if (exists($must_save{$i})) { print " => keep" };
999             print "\n";
1000           }
1001         }
1002       }
1003
1004       # Look for dependencies
1005       foreach (@backup_points) {
1006         if(exists($info->{incremental}->{$_})) {
1007           print "   => $_ depends on $info->{incremental}->{$_}->{depends}\n" if($DEBUG);
1008           if (exists($must_save{$_})) {
1009             push(@{$must_save{$info->{incremental}->{$_}->{depends}}},
1010               "dependency");
1011           }
1012         }
1013       }
1014
1015       my @removals = grep { !exists($must_save{$_}) } @backup_points;
1016       if($DEBUG) {
1017         my $tf = config_get($host, 'time_format');
1018         print "    => Candidates for removal:\n";
1019         foreach (@backup_points) {
1020           print "      => ". strftime($tf, localtime($_));
1021           print " ($_)";
1022           print " [". (exists($info->{full}->{$_}) ? "full":"incremental") ."]";
1023           if (exists($must_save{$_})) {
1024             my $reason = join(", ", @{$must_save{$_}});
1025             print " => keep ($reason)";
1026           } else {
1027             print " => remove";
1028           }
1029           print "\n";
1030         }
1031       }
1032       foreach (@removals) {
1033         my $efs = dir_encode($disk);
1034         my $filename;
1035         my $dataset;
1036         if(exists($info->{full}->{$_}->{file})) {
1037           $filename = $info->{full}->{$_}->{file};
1038         } elsif(exists($info->{incremental}->{$_}->{file})) {
1039           $filename = $info->{incremental}->{$_}->{file};
1040         } elsif(exists($info->{full}->{$_}->{dataset})) {
1041           $dataset = $info->{full}->{$_}->{dataset};
1042         } elsif(exists($info->{incremental}->{$_}->{dataset})) {
1043           $dataset = $info->{incremental}->{$_}->{dataset};
1044         } else {
1045           print "ERROR: We tried to expunge $host $disk [$_], but couldn't find it.\n";
1046         }
1047         print "    => expunging ${filename}${dataset}\n" if($DEBUG);
1048         unless($NEUTERED) {
1049           if ($filename) {
1050             unlink($filename) || print "ERROR: unlink $filename: $?\n";
1051           } elsif ($dataset) {
1052             `__ZFS__ destroy $dataset`;
1053             if ($?) {
1054               print "ERROR: zfs destroy $dataset: $?\n";
1055             }
1056           }
1057         }
1058       }
1059     }
1060   }
1061 }
1062
1063 sub __default_sort($$) { return $_[0] cmp $_[1]; }
1064    
1065 sub choose($$;$$) {
1066   my($name, $obj, $many, $sort) = @_;
1067   $sort ||= \&__default_sort;;
1068   my @list;
1069   my $hash;
1070   if(ref $obj eq 'ARRAY') {
1071     @list = sort { $sort->($a,$b); } (@$obj);
1072     map { $hash->{$_} = $_; } @list;
1073   }
1074   elsif(ref $obj eq 'HASH') {
1075     @list = sort { $sort->($a,$b); } (keys %$obj);
1076     $hash = $obj;
1077   }
1078   else {
1079     die "choose passed bad object: " . ref($obj) . "\n";
1080   }
1081   return \@list if(scalar(@list) == 1) && $many;
1082   return $list[0] if(scalar(@list) == 1) && !$many;
1083   print "\n";
1084   my $i = 1;
1085   for (@list) {
1086     printf " %3d) $hash->{$_}\n", $i++;
1087   }
1088   if ($many) {
1089     my @selection;
1090     my $range;
1091     while(1) {
1092       print "$name: ";
1093       chomp($range = <>);
1094       next if ($range !~ /^[\d,-]+$/);
1095       my @parts = split(',', $range);
1096       foreach my $part (@parts) {
1097           my ($from, $to) = ($part =~ /(\d+)(?:-(\d+))?/);
1098           if ($from < 1 || $to > scalar(@list)) {
1099               print "Invalid range: $from-$to\n";
1100               @selection = ();
1101               last;
1102           }
1103           if ($to) {
1104             push @selection, @list[$from - 1 .. $to - 1];
1105           } else {
1106             push @selection, @list[$from - 1];
1107           }
1108       }
1109       if (@selection) {
1110           last;
1111       }
1112     }
1113     return \@selection;
1114   } else {
1115     my $selection = 0;
1116     while($selection !~ /^\d+$/ or
1117           $selection < 1 or
1118           $selection >= $i) {
1119       print "$name: ";
1120       chomp($selection = <>);
1121     }
1122     return $list[$selection - 1];
1123   }
1124 }
1125
1126 sub backup_chain($$) {
1127   my ($info, $ts) = @_;
1128   my @list;
1129   push @list, $info->{full}->{$ts} if(exists($info->{full}->{$ts}));
1130   if(exists($info->{incremental}->{$ts})) {
1131     push @list, $info->{incremental}->{$ts};
1132     push @list, backup_chain($info, $info->{incremental}->{$ts}->{depends});
1133   }
1134   return @list;
1135 }
1136
1137 sub get_fs_from_mountpoint($) {
1138     my ($mountpoint) = @_;
1139     my $fs;
1140     my $rv = open(ZFSLIST, "__ZFS__ list -t filesystem -H |");
1141     die "Unable to determine zfs filesystem for $mountpoint" unless $rv;
1142     while (<ZFSLIST>) {
1143         my @F = split(' ');
1144         if ($F[-1] eq $mountpoint) {
1145             $fs = $F[0];
1146             last;
1147         }
1148     }
1149     close(ZFSLIST);
1150     die "Unable to determine zfs filesystem for $mountpoint" unless $fs;
1151     return $fs;
1152 }
1153
1154 sub perform_restore() {
1155   my (%source, %classmap);
1156
1157   foreach my $host (grep { $_ ne "default" && $conf{$_}->{"type"} ne "class"}
1158       keys %conf) {
1159     # If -h was specific, we will skip this host if the arg isn't
1160     # an exact match or a pattern match
1161     if($HOST &&
1162        !(($HOST eq $host) ||
1163          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
1164       next;
1165     }
1166
1167     foreach my $class (get_classes()) {
1168       if ($DEBUG) {
1169         if ($class) {
1170           print "=> Class: $class\n" if $class;
1171         } else {
1172           print "=> Class: (none)\n";
1173         }
1174       }
1175       my $store = get_store($host, $class);
1176       my $backup_info = scan_for_backups($store);
1177       foreach my $disk (sort keys %{$backup_info}) {
1178         my $info = $backup_info->{$disk};
1179         next unless(ref($info) eq 'HASH');
1180         next
1181           if($ZFS &&      # if the pattern was specified it could
1182             !($disk eq $ZFS ||        # be a specific match or a
1183               ($ZFS =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
1184         # We want to see this one
1185         my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
1186         my @source_points;
1187         foreach (@backup_points) {
1188           push @source_points, $_ if(!$TIMESTAMP || $TIMESTAMP == $_)
1189         }
1190         if(@source_points) {
1191           $source{$host}->{$disk} = \@source_points;
1192           $classmap{$host}->{$disk} = $class;
1193         }
1194       }
1195     }
1196   }
1197
1198   if(! keys %source) {
1199     print "No matching backups found\n";
1200     return;
1201   }
1202
1203   # Here goes the possibly interactive dialog
1204   my $host = choose("Restore from host",  [keys %source]);
1205   my $disks = choose("Restore from ZFS", [keys %{$source{$host}}], 1);
1206
1207   if (scalar(@$disks) > 1) {
1208     # We selected multiple backups, only the latest backup of each should be
1209     # used
1210     print "Multiple filesystems selected, choosing latest backup for each\n";
1211     my $backup_list = {};
1212     foreach my $disk (@$disks) {
1213       my $store = get_store($host, $classmap{$host}->{$disk});
1214       my $backup_info = scan_for_backups($store);
1215       $backup_list->{$disk} = [ reverse backup_chain($backup_info->{$disk},
1216           $backup_info->{$disk}->{last_backup}) ];
1217     }
1218
1219     if(!$RESTORE_HOST) {
1220       print "Restore to host [$host]:";
1221       chomp(my $input = <>);
1222       $RESTORE_HOST = length($input) ? $input : $host;
1223     }
1224     if(!$RESTORE_ZFS) {
1225       print "Restore at base zfs (filesystem must exist) []:";
1226       chomp(my $input = <>);
1227       $RESTORE_ZFS = $input;
1228     }
1229
1230     # show intentions
1231     print "Going to restore:\n";
1232     print "\tfrom: $host\n";
1233     foreach my $disk (@$disks) {
1234       print "\tfrom: $disk\n";
1235     }
1236     print "\t  to: $RESTORE_HOST\n";
1237     print "\t  at base zfs: $RESTORE_ZFS\n";
1238     print "\n";
1239
1240     foreach my $disk (@$disks) {
1241       print "Restoring: $disk\n";
1242       foreach(@{$backup_list->{$disk}}) {
1243         my $restore_dataset = $disk;
1244         if ($RESTORE_ZFS) {
1245           $restore_dataset = "$RESTORE_ZFS/$restore_dataset";
1246         }
1247         $_->{success} = zfs_restore_part($RESTORE_HOST, $restore_dataset, $_->{file}, $_->{dataset}, $_->{depends});
1248       }
1249     }
1250   } else {
1251     my $disk = $disks->[0];
1252     # Times are special.  We build a human readable form and use a numerical
1253     # sort function instead of the default lexical one.
1254     my %times;
1255     my $tf = config_get($host, 'time_format');
1256     map { $times{$_} = strftime($tf, localtime($_)); } @{$source{$host}->{$disk}};
1257     my $timestamp = choose("Restore as of timestamp", \%times, 0,
1258                             sub { $_[0] <=> $_[1]; });
1259
1260     my $store = get_store($host, $classmap{$host}->{$disk});
1261     my $backup_info = scan_for_backups($store);
1262     my @backup_list = reverse backup_chain($backup_info->{$disk}, $timestamp);
1263
1264     if(!$RESTORE_HOST) {
1265       print "Restore to host [$host]:";
1266       chomp(my $input = <>);
1267       $RESTORE_HOST = length($input) ? $input : $host;
1268     }
1269     if(!$RESTORE_ZFS) {
1270       print "Restore to zfs [$disk]:";
1271       chomp(my $input = <>);
1272       $RESTORE_ZFS = length($input) ? $input : $disk;
1273     }
1274
1275     # show intentions
1276     print "Going to restore:\n";
1277     print "\tfrom: $host\n";
1278     print "\tfrom: $disk\n";
1279     print "\t  at: $timestamp [" . strftime($tf, localtime($timestamp)) . "]\n";
1280     print "\t  to: $RESTORE_HOST\n";
1281     print "\t  to: $RESTORE_ZFS\n";
1282     print "\n";
1283
1284     foreach(@backup_list) {
1285       $_->{success} = zfs_restore_part($RESTORE_HOST, $RESTORE_ZFS, $_->{file}, $_->{dataset}, $_->{depends});
1286     }
1287   }
1288
1289 }
1290
1291 sub zfs_restore_part($$$$;$) {
1292   my ($host, $fs, $file, $dataset, $dep) = @_;
1293   unless ($file || $dataset) {
1294     print STDERR "=> No dataset or filename given to restore. Bailing out.";
1295     return 1;
1296   }
1297   my $ssh_config = config_get($host, 'ssh_config');
1298   $ssh_config = "-F $ssh_config" if($ssh_config);
1299   print "Using custom ssh config file: $ssh_config\n" if($DEBUG);
1300   my $command;
1301   if(exists($conf{$host})) {
1302     my $agent = config_get($host, 'agent');
1303     $command = "$agent -r -z $fs";
1304     $command .= " -b $dep" if($dep);
1305   }
1306   else {
1307     $command = "__ZFS__ recv $fs";
1308   }
1309   if ($file) {
1310     print " => piping $file to $command\n" if($DEBUG);
1311     print "gzip -dfc $file | ssh $ssh_config $host $command\n" if ($DEBUG && $NEUTERED);
1312   } elsif ($dataset) {
1313     print " => piping $dataset to $command using zfs send\n" if ($DEBUG);
1314     print "zfs send $dataset | ssh $ssh_config $host $command\n" if ($DEBUG && $NEUTERED);
1315   }
1316   unless($NEUTERED) {
1317     if ($file) {
1318       open(DUMP, "gzip -dfc $file |");
1319     } elsif ($dataset) {
1320       open(DUMP, "__ZFS__ send $dataset |");
1321     }
1322     eval {
1323       open(RECEIVER, "| ssh $ssh_config $host $command");
1324       my $buffer;
1325       while(my $len = sysread(DUMP, $buffer, $BLOCKSIZE)) {
1326         if(syswrite(RECEIVER, $buffer, $len) != $len) {
1327           die "$!";
1328         }
1329       }
1330     };
1331     close(DUMP);
1332     close(RECEIVER);
1333   }
1334   return $?;
1335 }
1336
1337 sub pretty_print_backup($$$) {
1338   my ($info, $host, $point) = @_;
1339   my $tf = config_get($host, 'time_format');
1340   print "\t" . strftime($tf, localtime($point)) . " [$point] ";
1341   if(exists($info->{full}->{$point})) {
1342     if ($info->{full}->{$point}->{file}) {
1343       my @st = stat($info->{full}->{$point}->{file});
1344       print "FULL " . pretty_size($st[7]);
1345       print "\n\tfile: $info->{full}->{$point}->{file}" if($SHOW_FILENAMES);
1346     } elsif ($info->{full}->{$point}->{dataset}) {
1347       print "FULL $info->{full}->{$point}->{pretty_size}";
1348       print "\n\tdataset: $info->{full}->{$point}->{dataset}"
1349         if($SHOW_FILENAMES);
1350     }
1351   } else {
1352     my @st = stat($info->{incremental}->{$point}->{file});
1353     print "INCR from [$info->{incremental}->{$point}->{depends}] " . pretty_size($st[7]);
1354     print "\n\tfile: $info->{incremental}->{$point}->{file}" if($SHOW_FILENAMES);
1355   }
1356   print "\n";
1357 }
1358
1359 sub show_backups($$) {
1360   my ($host, $diskpat) = @_;
1361   my (@files, @datasets, %classmap);
1362   my $tf = config_get($host, 'time_format');
1363   foreach my $class (get_classes()) {
1364     if ($DEBUG) {
1365       if ($class) {
1366         print "=> Class: $class\n" if $class;
1367       } else {
1368         print "=> Class: (none)\n";
1369       }
1370     }
1371     my $store = get_store($host, $class);
1372     my $backup_info = scan_for_backups($store);
1373     foreach my $disk (sort keys %{$backup_info}) {
1374       my $info = $backup_info->{$disk};
1375       next unless(ref($info) eq 'HASH');
1376       next
1377         if($diskpat &&      # if the pattern was specified it could
1378           !($disk eq $diskpat ||        # be a specific match or a
1379             ($diskpat =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
1380
1381       my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
1382       @backup_points = sort { $a <=> $b } @backup_points;
1383       @backup_points = (pop @backup_points) unless ($ARCHIVE || $SUMMARY_EXT);
1384
1385       # We want to see this one
1386       print "$host:$disk\n";
1387       next unless($SUMMARY || $SUMMARY_EXT || $ARCHIVE);
1388       if($SUMMARY_EXT) {
1389         print "\tLast Full: ". ($info->{last_full} ? strftime($tf, localtime($info->{last_full})) : "Never") . "\n";
1390         if($info->{last_full} < $info->{last_incremental}) {
1391           print "\tLast Incr: ". strftime($tf, localtime($info->{last_incremental})). "\n";
1392         }
1393       }
1394       foreach (@backup_points) {
1395         pretty_print_backup($info, $host, $_);
1396         if(exists($info->{full}->{$_}->{file})) {
1397           push @files, $info->{full}->{$_}->{file};
1398           $classmap{$info->{full}->{$_}->{file}} = $class;
1399         } elsif(exists($info->{incremental}->{$_}->{file})) {
1400           push @files, $info->{incremental}->{$_}->{file};
1401           $classmap{$info->{incremental}->{$_}->{file}} = $class;
1402         } elsif(exists($info->{full}->{$_}->{dataset})) {
1403           push @datasets, $info->{full}->{$_}->{dataset};
1404           $classmap{$info->{full}->{$_}->{dataset}} = $class;
1405         }
1406       }
1407       print "\n";
1408     }
1409   }
1410   if($ARCHIVE && (scalar(@files) || scalar(@datasets))) {
1411     print "\nAre you sure you would like to archive ".scalar(@files).
1412       " file(s) and ".scalar(@datasets)." dataset(s)? ";
1413     while(($_ = <>) !~ /(?:y|n|yes|no)$/i) {
1414       print "\nAre you sure you would like to archive ".scalar(@files).
1415         " file(s) and ".scalar(@datasets)." dataset(s)? ";
1416     }
1417     if(/^y/i) {
1418       if (@files) {
1419         my $archive = config_get($host, 'archive');
1420         $archive =~ s/%h/$host/g;
1421         if(! -d $archive) {
1422           mkdir $archive || die "Cannot mkdir($archive)\n";
1423         }
1424         foreach my $file (@files) {
1425           my $store = get_store($host, $classmap{$file});
1426           (my $afile = $file) =~ s/^$store/$archive/;
1427           move($file, $afile) || print "Error archiving $file: $!\n";
1428         }
1429       }
1430       if (@datasets) {
1431         my $archive = config_get($host, 'archive');
1432         (my $basearchive = $archive) =~ s/\/?%h//g;
1433         my $basearchivefs;
1434         eval {
1435           $basearchivefs = get_fs_from_mountpoint($basearchive);
1436         };
1437         die "Unable to find archive filesystem. The archive directory must be the root of a zfs filesystem to archive datasets." if $@;
1438         my $archivefs = "$basearchivefs/$host";
1439         `__ZFS__ create $archivefs`; # We don't care if this fails
1440         my %seen = ();
1441         foreach my $dataset (@datasets) {
1442           my $store = get_store($host, $classmap{$dataset});
1443           my $storefs = get_fs_from_mountpoint($store);
1444           $dataset =~ s/@.*$//; # Only rename filesystems, not snapshots
1445           next if $seen{$dataset}++; # Only rename a filesystem once
1446           (my $adataset = $dataset) =~ s/^$storefs/$archivefs/;
1447           `__ZFS__ rename $dataset $adataset`;
1448           if ($?) {
1449             print "Error archiving $dataset\n";
1450           }
1451         }
1452       }
1453     }
1454   }
1455 }
1456
1457 sub show_violators($$) {
1458   my ($host, $diskpat) = @_;
1459   my $host_store = get_store($host);
1460   my $filesystems = {};
1461   if (open (my $fh, "$host_store/.fslist")) {
1462     while (<$fh>) {
1463       chomp;
1464       $filesystems->{$_} = 1;
1465     }
1466     close($fh);
1467   } elsif ($DEBUG) {
1468     print "=> $host_store/.fslist not present, skipping missing FS detection\n";
1469   }
1470   foreach my $class (get_classes()) {
1471     if ($DEBUG) {
1472       if ($class) {
1473         print "=> Class: $class\n" if $class;
1474       } else {
1475         print "=> Class: (none)\n";
1476       }
1477     }
1478     my $store = get_store($host, $class);
1479     my $backup_info = scan_for_backups($store);
1480     foreach my $disk (sort keys %{$backup_info}) {
1481       my $info = $backup_info->{$disk};
1482       next unless(ref($info) eq 'HASH');
1483       next if (
1484         $diskpat &&      # if the pattern was specified it could
1485         !(
1486           $disk eq $diskpat ||                        # be a specific match
1487           ($diskpat =~ /^\/(.+)\/$/ && $disk =~ /$1/) # or a regex
1488         )
1489       ); # regex
1490       # Backups for filesystems that no longer exist aren't violators
1491       if (!$SUMMARY_VIOLATORS_VERBOSE && %{$filesystems} &&
1492         !defined $filesystems->{$disk}) {
1493         print "=> $disk doesn't exist on server, not marking as violator\n"
1494           if ($DEBUG);
1495         next;
1496       }
1497
1498
1499       my @violators = ();
1500
1501       # No recent full
1502       if (time() > $info->{last_full} +
1503           config_get($host, 'full_interval', $class) +
1504           config_get($host, 'violator_grace_period', $class)) {
1505         push @violators, {
1506           "host" => $host, "disk" => $disk,
1507           "reason" => "No recent full backup",
1508           "backup" => "full"
1509         }
1510       }
1511
1512       # No recent incremental
1513       if (time() > $info->{last_backup} +
1514           config_get($host, 'backup_interval', $class) +
1515           config_get($host, 'violator_grace_period', $class)) {
1516         push @violators, {
1517           "host" => $host, "disk" => $disk,
1518           "reason" => "No recent incremental backup",
1519           "backup" => "backup"
1520         }
1521       }
1522
1523       for my $v (@violators) {
1524         print "$v->{host}:$v->{disk} - $v->{reason}\n";
1525         pretty_print_backup($info, $host, $info->{"last_$v->{'backup'}"});
1526       }
1527     }
1528   }
1529 }
1530
1531 sub plan_and_run($$) {
1532   my ($host, $diskpat) = @_;
1533   my $store;
1534   my $ssh_config = config_get($host, 'ssh_config');
1535   $ssh_config = "-F $ssh_config" if($ssh_config);
1536   my %suppress;
1537   print "Planning '$host'\n" if($DEBUG);
1538   my $agent = config_get($host, 'agent');
1539   my $took_action = 1;
1540   while($took_action) {
1541     $took_action = 0;
1542     my @disklist;
1543
1544     # We need a lock for the listing.
1545     return unless(lock($host, ".list"));
1546
1547     # Get list of zfs filesystems from the agent
1548     open(SILENT, ">&", \*STDERR);
1549     close(STDERR);
1550     my $rv = open(ZFSLIST, "ssh $ssh_config $host $agent -l |");
1551     open(STDERR, ">&", \*SILENT);
1552     close(SILENT);
1553     next unless $rv;
1554     @disklist = grep { chomp } (<ZFSLIST>);
1555     close(ZFSLIST);
1556     # Write the filesystem list out to a file for use by the violators list
1557     my $store = get_store($host); # Don't take classes into account - not needed
1558     mkpath($store) if(! -d $store);
1559     open(my $fh, ">$store/.fslist");
1560     foreach my $diskline (@disklist) {
1561       # Get only the filesystem and not the snapshots/classes
1562       (my $filesystem = $diskline) =~ s/ \[.*//;
1563       print $fh "$filesystem\n";
1564     }
1565     close($fh);
1566     if ($DEBUG) {
1567       print " => Filesystems for $host (zetaback_agent -l output)\n";
1568       foreach my $diskline (@disklist) {
1569         print "    $diskline\n";
1570       }
1571     }
1572
1573     foreach my $diskline (@disklist) {
1574       chomp($diskline);
1575       next unless($diskline =~ /^(\S+) \[([^\]]*)\](?: {([^}]*)})?/);
1576       my $diskname = $1;
1577       my %snaps;
1578       map { $snaps{$_} = 1 } (split(/,/, $2));
1579       my $class = $3;
1580  
1581       # We've just done this.
1582       next if($suppress{"$host:$diskname"});
1583       # If we are being selective (via -z) now is the time.
1584       next
1585         if($diskpat &&          # if the pattern was specified it could
1586            !($diskname eq $diskpat ||        # be a specific match or a
1587              ($diskpat =~ /^\/(.+)\/$/ && $diskname =~ /$1/))); # regex
1588  
1589       $store = get_store($host, $class);
1590       if ($DEBUG) {
1591         if ($class) {
1592             print STDERR "=> Class is $class\n";
1593         } else {
1594             print STDERR "=> No/default class\n";
1595         }
1596       }
1597       print " => Scanning '$store' for old backups of '$diskname'.\n" if($DEBUG);
1598
1599       # Make directory on demand
1600       mkpath($store) if(! -d $store);
1601       my $backup_info = scan_for_backups($store);
1602       # That gave us info on all backups, we just want this disk
1603       $backup_info = $backup_info->{$diskname} || {};
1604  
1605       # Should we do a backup?
1606       my $backup_type = 'no';
1607       if(time() > $backup_info->{last_backup} + config_get($host,
1608           'backup_interval', $class)) {
1609         $backup_type = 'incremental';
1610       }
1611       if(time() > $backup_info->{last_full} + config_get($host,
1612           'full_interval', $class)) {
1613         $backup_type = 'full';
1614       }
1615       # If we want an incremental, but have no full, then we need to upgrade to full
1616       if($backup_type eq 'incremental') {
1617         my $have_full_locally = 0;
1618         # For each local full backup, see if the full backup still exists on the other end.
1619         foreach (keys %{$backup_info->{'full'}}) {
1620           $have_full_locally = 1 if(exists($snaps{'__zb_full_' . $_}));
1621         }
1622         $backup_type = 'full' unless($have_full_locally);
1623       }
1624       $backup_type = 'full' if($FORCE_FULL);
1625       $backup_type = 'incremental' if($FORCE_INC);
1626       $backup_type = 'dataset' if(config_get($host, 'dataset_backup', $class)
1627         eq 1 && $backup_type ne 'no');
1628
1629       print " => doing $backup_type backup\n" if($DEBUG);
1630       # We need to drop a __zb_base snap or a __zb_incr snap before we proceed
1631       unless($NEUTERED || $backup_type eq 'no') {
1632         # attempt to lock this action, if it fails, skip -- someone else is working it.
1633         next unless(lock($host, dir_encode($diskname), 1));
1634         unlock($host, '.list');
1635
1636         if($backup_type eq 'full') {
1637           eval { zfs_full_backup($host, $diskname, $store); };
1638           if ($@) {
1639             chomp(my $err = $@);
1640             print " => failure $err\n";
1641           }
1642           else {
1643             # Unless there was an error backing up, remove all the other full snaps
1644             foreach (keys %snaps) {
1645               zfs_remove_snap($host, $diskname, $_) if(/^__zb_full_(\d+)/)
1646             }
1647           }
1648           $took_action = 1;
1649         }
1650         if($backup_type eq 'incremental') {
1651           eval {
1652             zfs_remove_snap($host, $diskname, '__zb_incr') if($snaps{'__zb_incr'});
1653             # Find the newest full from which to do an incremental (NOTE: reverse numeric sort)
1654             my @fulls = sort { $b <=> $a } (keys %{$backup_info->{'full'}});
1655             zfs_incremental_backup($host, $diskname, $fulls[0], $store);
1656           };
1657           if ($@) {
1658             chomp(my $err = $@);
1659             print " => failure $err\n";
1660           }
1661           else {
1662             $took_action = 1;
1663           }
1664         }
1665         if($backup_type eq 'dataset') {
1666           my @backups = sort { $b <=> $a } (keys %{$backup_info->{'full'}});
1667           eval { zfs_dataset_backup($host, $diskname, $backups[0], $store); };
1668           if ($@) {
1669             chomp(my $err = $@);
1670             print " => failure $err\n";
1671           }
1672           else {
1673             # Unless there was an error backing up, remove all the other dset snaps
1674             foreach (keys %snaps) {
1675               zfs_remove_snap($host, $diskname, $_) if(/^__zb_dset_(\d+)/)
1676             }
1677           }
1678           $took_action = 1;
1679         }
1680         unlock($host, dir_encode($diskname), 1);
1681       }
1682       $suppress{"$host:$diskname"} = 1;
1683       last if($took_action);
1684     }
1685     unlock($host, '.list');
1686   }
1687 }
1688
1689 ## Start of main program
1690 limit_running_processes;
1691
1692 if($RESTORE) {
1693   perform_restore();
1694 }
1695 else {
1696   foreach my $host (grep { $_ ne "default" && $conf{$_}->{"type"} ne "class"}
1697       keys %conf) {
1698     # If -h was specific, we will skip this host if the arg isn't
1699     # an exact match or a pattern match
1700     if($HOST &&
1701        !(($HOST eq $host) ||
1702          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
1703       next;
1704     }
1705
1706     # Skip if the host is marked as 'offline' and we are not listing backups
1707     if (config_get($host, 'offline') == 1 &&
1708         !$LIST && !$SUMMARY && !$SUMMARY_EXT && !$ARCHIVE) {
1709       next;
1710     }
1711
1712     if($LIST || $SUMMARY || $SUMMARY_EXT || $ARCHIVE) {
1713       show_backups($host, $ZFS);
1714     }
1715     if ($SUMMARY_VIOLATORS || $SUMMARY_VIOLATORS_VERBOSE) {
1716       show_violators($host, $ZFS);
1717     }
1718     if($BACKUP) {
1719       plan_and_run($host, $ZFS);
1720     }
1721     if($EXPUNGE) {
1722       perform_retention($host);
1723     }
1724   }
1725 }
1726
1727 exit 0;
1728
1729 =pod
1730
1731 =head1 FILES
1732
1733 =over
1734
1735 =item zetaback.conf
1736
1737 The main zetaback configuration file.  The location of the file can be
1738 specified on the command line with the -c flag.  The prefix of this
1739 file may also be specified as an argument to the configure script.
1740
1741 =back
1742
1743 =head1 SEE ALSO
1744
1745 zetaback_agent(1)
1746
1747 =cut
Note: See TracBrowser for help on using the browser.