root/zetaback

Revision 52c8f50e2f7095ebfcaf92b28a3e9761cc4aefce, 14.9 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 7 years ago)

Major progress towards restore. Not using agent (which needs to be smart enough to use send/recv or backup/restore appropriately). No snapshot destination support. refs #1

  • Property mode set to 100755
Line 
1 #!/usr/bin/perl
2
3 use strict;
4 use Getopt::Long;
5 use MIME::Base64;
6 use POSIX qw/strftime/;
7 use Data::Dumper;
8
9 use vars qw/$CONF %conf $BLOCKSIZE $DEBUG $HOST $BACKUP
10             $RESTORE $RESTORE_HOST $RESTORE_ZFS $TIMESTAMP
11             $LIST $SUMMARY $SUMMARY_EXT
12             $EXPUNGE $NUETERED $ZFS/;
13 $CONF = q^/var/spool/zfs_backups/zetaback.conf^;
14 $BLOCKSIZE = 1024*64;
15
16 $conf{'default'}->{'time_format'} = "%Y-%m-%d %H:%M:%S";
17 $conf{'default'}->{'retention'} = 14 * 86400;
18
19 GetOptions(
20   "h=s"     => \$HOST,
21   "z=s"     => \$ZFS,
22   "c=s"     => \$CONF,
23   "b"       => \$BACKUP,
24   "l"       => \$LIST,
25   "s"       => \$SUMMARY,
26   "sx"      => \$SUMMARY_EXT,
27   "r"       => \$RESTORE,
28   "t=i"     => \$TIMESTAMP,
29   "rhost=s" => \$RESTORE_HOST,
30   "rfs=s"   => \$RESTORE_ZFS,
31   "d"       => \$DEBUG,
32   "n"       => \$NUETERED,
33   "x"       => \$EXPUNGE,
34 );
35
36 sub parse_config() {
37   local($/);
38   $/ = undef;
39   open(CONF, "<$CONF");
40   my $file = <CONF>;
41   while($file =~ m/^\s*(\S+)\s\s*{(.*?)}/gms) {
42     my $scope = $1;
43     my $filepart = $2;
44     $conf{$scope} ||= {};
45     foreach my $line (split /\n/, $filepart) {
46       if($line =~ /^\s*([^#]\S*)\s*=\s*(\S+)/) {
47         $conf{$scope}->{lc($1)} = $2;
48       }
49     }
50   }
51   close(CONF);
52 }
53 sub config_get($$) {
54   return $conf{$_[0]}->{$_[1]} || $conf{'default'}->{$_[1]};
55 }
56
57 sub dir_encode($) {
58   my $d = shift;
59   my $e = encode_base64($d, '');
60   $e =~ s/\//_/;
61   return $e;
62 }
63 sub dir_decode($) {
64   my $e = shift;
65   $e =~ s/_/\//;
66   return decode_base64($e);
67 }
68 sub pretty_size($) {
69   my $bytes = shift;
70   if($bytes > 1024*1024*1024) {
71     return sprintf("%0.2f Gb", $bytes / (1024*1024*1024));
72   }
73   if($bytes > 1024*1024) {
74     return sprintf("%0.2f Mb", $bytes / (1024*1024));
75   }
76   if($bytes > 1024) {
77     return sprintf("%0.2f Kb", $bytes / (1024));
78   }
79   return "$bytes b";
80 }
81 sub scan_for_backups($) {
82   my %info = ();
83   my $dir = shift;
84   $info{last_full} = $info{last_incremental} = $info{last_backup} = 0;
85   opendir(D, $dir) || return \%info;
86   foreach my $file (readdir(D)) {
87     if($file =~ /^(\d+)\.([^\.]+)\.full$/) {
88       my $whence = $1;
89       my $fs = dir_decode($2);
90       $info{$fs}->{full}->{$whence}->{'file'} = "$dir/$file";
91       $info{$fs}->{last_full} = $whence if($whence > $info{$fs}->{last_full});
92       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
93                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
94     }
95     elsif($file =~ /^(\d+).([^\.]+)\.incremental.(\d+)$/) {
96       my $whence = $1;
97       my $fs = dir_decode($2);
98       $info{$fs}->{incremental}->{$whence}->{'depends'} = $3;
99       $info{$fs}->{incremental}->{$whence}->{'file'} = "$dir/$file";
100       $info{$fs}->{last_incremental} = $whence if($whence > $info{$fs}->{last_incremental});
101       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
102                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
103     }
104   }
105   closedir(D);
106   return \%info;
107 }
108
109 parse_config();
110
111 sub zfs_remove_snap($$$) {
112   my ($host, $fs, $snap) = @_;
113   my $agent = config_get($host, 'agent');
114   return unless($snap);
115   print "Dropping $snap on $fs\n" if($DEBUG);
116   `ssh $host $agent -z $fs -d $snap`;
117 }
118
119 # Lots of args.. internally called.
120 sub zfs_do_backup($$$$$$) {
121   my ($host, $fs, $type, $point, $store, $dumpfile) = @_;
122   my $agent = config_get($host, 'agent');
123
124   # Do it. yeah.
125   open(LBACKUP, ">$store/.$dumpfile") || die "zfs_full_backup: cannot create dump\n";
126   eval {
127     open(RBACKUP, "ssh $host $agent -z $fs -$type $point |") || die "zfs_full_backup: cannot perform send\n";
128     my $buffer;
129     while(my $len = sysread(RBACKUP, $buffer, $BLOCKSIZE)) {
130       if(syswrite(LBACKUP, $buffer, $len) != $len) {
131         die "$!";
132       }
133     }
134     close(LBACKUP);
135     close(RBACKUP);
136     die "dump failed (zero bytes)\n" if(-z "$store/.$dumpfile");
137     rename("$store/.$dumpfile", "$store/$dumpfile") || die "cannot rename dump\n";
138   };
139   if($@) {
140     unlink("$store/.$dumpfile");
141     die "zfs_full_backup: failed $@";
142   }
143 }
144
145 sub zfs_full_backup($$$) {
146   my ($host, $fs, $store) = @_;
147
148   # Translate into a proper dumpfile nameA
149   my $point = time();
150   my $efs = dir_encode($fs);
151   my $dumpfile = "$point.$efs.full";
152
153   zfs_do_backup($host, $fs, 'f', $point, $store, $dumpfile);
154 }
155
156 sub zfs_incremental_backup($$$$) {
157   my ($host, $fs, $base, $store) = @_;
158   my $agent = config_get($host, 'agent');
159
160   # Translate into a proper dumpfile nameA
161   my $point = time();
162   my $efs = dir_encode($fs);
163   my $dumpfile = "$point.$efs.incremental.$base";
164
165   zfs_do_backup($host, $fs, 'i', $base, $store, $dumpfile);
166 }
167
168 sub perform_retention($$) {
169   my ($host, $store) = @_;
170   my $cutoff = time() - config_get($host, 'retention');
171   my $backup_info = scan_for_backups($store);
172  
173   foreach my $disk (sort keys %{$backup_info}) {
174     my $info = $backup_info->{$disk};
175     next unless(ref($info) eq 'HASH');
176     my %must_save;
177
178     # Get a list of all the full and incrementals, sorts newest to oldest
179     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
180     @backup_points = sort { $b <=> $a } @backup_points;
181
182     # We _cannot_ throw away _all_ out backups, so save the most recent no matter what
183     $must_save{$backup_points[0]} = 1;
184
185     # Walk the list for backups within our retention period.
186     foreach (@backup_points) {
187       if($_ >= $cutoff) {
188         $must_save{$_} = 1;
189       }
190       else {
191         # they are in decending order, once we miss, all will miss
192         last;
193       }
194     }
195
196     # Look for dependencies
197     foreach (@backup_points) {
198       if(exists($info->{incremental}->{$_})) {
199         print "   => $_ depends on $info->{incremental}->{$_}->{depends}\n" if($DEBUG);
200         $must_save{$info->{incremental}->{$_}} = 1
201       }
202     }
203     my @removals = grep { !exists($must_save{$_}) } @backup_points;
204     if($DEBUG) {
205       my $tf = config_get($host, 'time_format');
206       print "    => I can remove:\n";
207       foreach (@backup_points) {
208         print "      => ". strftime($tf, localtime($_));
209         print " [". (exists($info->{full}->{$_}) ? "full":"incremental") ."]";
210         print " XXX" if(!exists($must_save{$_}));
211         print "\n";
212       }
213     }
214     foreach (@removals) {
215       my $efs = dir_encode($disk);
216       my $filename;
217       if(exists($info->{full}->{$_})) {
218         $filename = "$store/$_.$efs.full";
219       }
220       elsif(exists($info->{incremental}->{$_})) {
221         $filename = "$store/$_.$efs.incremental.$info->{incremental}->{$_}->{depends}";
222       }
223       else {
224         print "ERROR: We tried to expunge $host $disk [$_], but couldn't find it.\n";
225       }
226       print "    => expunging $filename\n" if($DEBUG);
227       unless($NUETERED) {
228         unlink($filename) || print "ERROR: unlink $filename: $?\n";
229       }
230     }
231   }
232 }
233
234 sub choose($@) {
235   my($name, @list) = @_;
236   return $list[0] if(scalar(@list) == 1);
237   print "\n";
238   my $i = 1;
239   for (@list) {
240     printf " %3d) $_\n", $i++;
241   }
242   my $selection = 0;
243   while($selection !~ /^\d+$/ or
244         $selection < 1 or
245         $selection >= $i) {
246     print "$name: ";
247     chomp($selection = <>);
248   }
249   return $list[$selection - 1];
250 }
251
252 sub backup_chain($$) {
253   my ($info, $ts) = @_;
254   my @list;
255   push @list, $info->{full}->{$ts} if(exists($info->{full}->{$ts}));
256   if(exists($info->{incremental}->{$ts})) {
257     push @list, $info->{incremental}->{$ts};
258     push @list, backup_chain($info, $info->{incremental}->{$ts}->{depends});
259   }
260   return @list;
261 }
262
263 sub perform_restore() {
264   my %source;
265
266   foreach my $host (grep { $_ ne "default" } keys %conf) {
267     # If -h was specific, we will skip this host if the arg isn't
268     # an exact match or a pattern match
269     if($HOST &&
270        !(($HOST eq $host) ||
271          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
272       next;
273     }
274
275     my $store = config_get($host, 'store');
276     $store =~ s/%h/$host/g;;
277     mkdir $store if(! -d $store);
278
279     my $backup_info = scan_for_backups($store);
280     foreach my $disk (sort keys %{$backup_info}) {
281       my $info = $backup_info->{$disk};
282       next unless(ref($info) eq 'HASH');
283       next
284         if($ZFS &&      # if the pattern was specified it could
285            !($disk eq $ZFS ||        # be a specific match or a
286              ($ZFS =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
287       # We want to see this one
288       my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
289       my @source_points;
290       foreach (@backup_points) {
291         push @source_points, $_ if(!$TIMESTAMP || $TIMESTAMP == $_)
292       }
293       if(@source_points) {
294         $source{$host}->{$disk} = \@source_points;
295       }
296     }
297   }
298
299   if(! keys %source) {
300     print "No matching backups found\n";
301     return;
302   }
303
304   # Here goes the possibly interactive dialog
305   my $host = choose("Restore from host", keys %source);
306   my $disk = choose("Restore from ZFS", keys %{$source{$host}});
307   my $timestamp = choose("Restore as of timestamp", @{$source{$host}->{$disk}});
308   my $tf = config_get($host, 'time_format');
309
310   my $store = config_get($host, 'store');
311   $store =~ s/%h/$host/g;;
312   mkdir $store if(! -d $store);
313   my $backup_info = scan_for_backups($store);
314   my @backup_list = reverse backup_chain($backup_info->{$disk}, $timestamp);
315
316   if(!$RESTORE_HOST) {
317     print "Restore to host [$host]:";
318     chomp(my $input = <>);
319     $RESTORE_HOST = length($input) ? $input : $host;
320   }
321   if(!$RESTORE_ZFS) {
322     print "Restore to zfs [$disk]:";
323     chomp(my $input = <>);
324     $RESTORE_ZFS = length($input) ? $input : $disk;
325   }
326
327   # show intentions
328   print "Going to restore:\n";
329   print "\tfrom: $host\n";
330   print "\tfrom: $disk\n";
331   print "\t  at: $timestamp [" . strftime($tf, localtime($timestamp)) . "]\n";
332   print "\t  to: $RESTORE_HOST\n";
333   print "\t  to: $RESTORE_ZFS\n";
334   print "\n";
335
336   foreach(@backup_list) {
337     $_->{success} = zfs_restore_part($RESTORE_HOST, $RESTORE_ZFS, $_->{file});
338   }
339 }
340
341 sub zfs_restore_part($$$) {
342   my ($host, $fs, $file) = @_;
343   open(DUMP, "gzip -dfc $file |");
344   eval {
345     open(RECEIVER, "| ssh $host /sbin/zfs recv $fs");
346     my $buffer;
347     while(my $len = sysread(DUMP, $buffer, $BLOCKSIZE)) {
348       if(syswrite(RECEIVER, $buffer, $len) != $len) {
349         die "$!";
350       }
351     }
352   };
353   close(DUMP);
354   close(RECEIVER);
355   return $?;
356 }
357
358 sub show_backups($$$) {
359   my ($host, $store, $diskpat) = @_;
360   my $backup_info = scan_for_backups($store);
361   my $tf = config_get($host, 'time_format');
362   foreach my $disk (sort keys %{$backup_info}) {
363     my $info = $backup_info->{$disk};
364     next unless(ref($info) eq 'HASH');
365     next
366       if($diskpat &&      # if the pattern was specified it could
367          !($disk eq $diskpat ||        # be a specific match or a
368            ($diskpat =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
369     # We want to see this one
370     print "$host:$disk\n";
371     next unless($SUMMARY || $SUMMARY_EXT);
372     if($SUMMARY_EXT) {
373       print "\tLast Full: ". ($info->{last_full} ? strftime($tf, localtime($info->{last_full})) : "Never") . "\n";
374       if($info->{last_full} < $info->{last_incremental}) {
375         print "\tLast Incr: ". strftime($tf, localtime($info->{last_incremental})). "\n";
376       }
377     }
378     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
379     @backup_points = sort { $a <=> $b } @backup_points;
380     unless ($SUMMARY_EXT) {
381       @backup_points = (pop @backup_points);
382     }
383     foreach (@backup_points) {
384       print "\t" . strftime($tf, localtime($_)) . " [$_] ";
385       if(exists($info->{full}->{$_})) {
386         my @st = stat($info->{full}->{$_}->{file});
387         print "FULL " . pretty_size($st[7]);
388       } else {
389         my @st = stat($info->{incremental}->{$_}->{file});
390         print "INCR from [$info->{incremental}->{$_}->{depends}] " . pretty_size($st[7]);
391       }
392       print "\n";
393     }
394     print "\n";
395   }
396 }
397
398 sub plan_and_run($$$) {
399   my ($host, $store, $diskpat) = @_;
400   print "Planning '$host'\n" if($DEBUG);
401   my $agent = config_get($host, 'agent');
402   open(ZFSLIST, "ssh $host $agent -l |") || next;
403   foreach my $diskline (<ZFSLIST>) {
404     chomp($diskline);
405     next unless($diskline =~ /^(\S+) \[([^\]]*)\]/);
406     my $diskname = $1;
407     my %snaps;
408     map { $snaps{$_} = 1 } (split(/,/, $2));
409
410     # If we are being selective (via -z) now is the time.
411     next
412       if($diskpat &&          # if the pattern was specified it could
413          !($diskname eq $diskpat ||        # be a specific match or a
414            ($diskpat =~ /^\/(.+)\/$/ && $diskname =~ /$1/))); # regex
415
416     print " => Scanning '$store' for old backups of '$diskname'.\n" if($DEBUG);
417     # Make directory on demand
418     my $backup_info = scan_for_backups($store);
419     # That gave us info on all backups, we just want this disk
420     $backup_info = $backup_info->{$diskname} || {};
421
422     # Should we do a backup?
423     my $backup_type = 'no';
424     if(time() > $backup_info->{last_backup} + config_get($host, 'backup_interval')) {
425       $backup_type = 'incremental';
426     }
427     if(time() > $backup_info->{last_full} + config_get($host, 'full_interval')) {
428       $backup_type = 'full';
429     }
430
431     # If we want an incremental, but have no full, then we need to upgrade to full
432     if($backup_type eq 'incremental') {
433       my $have_full_locally = 0;
434       # For each local full backup, see if the full backup still exists on the other end.
435       foreach (keys %{$backup_info->{'full'}}) {
436         $have_full_locally = 1 if(exists($snaps{'__zb_full_' . $_}));
437       }
438       $backup_type = 'full' unless($have_full_locally);
439     }
440
441     print " => doing $backup_type backup\n" if($DEBUG);
442     # We need to drop a __zb_base snap or a __zb_incr snap before we proceed
443     unless($NUETERED) {
444       if($backup_type eq 'full') {
445         eval { zfs_full_backup($host, $diskname, $store); };
446         if ($@) {
447           chomp(my $err = $@);
448           print " => failure $err\n";
449         }
450         else {
451           # Unless there was an error backing up, remove all the other full snaps
452           foreach (keys %snaps) {
453             zfs_remove_snap($host, $diskname, $_) if(/^__zb_full_(\d+)/)
454           }
455         }
456       }
457       if($backup_type eq 'incremental') {
458         zfs_remove_snap($host, $diskname, '__zb_incr') if($snaps{'__zb_incr'});
459         # Find the newest full from which to do an incremental (NOTE: reverse numeric sort)
460         my @fulls = sort { $b <=> $a } (keys %{$backup_info->{'full'}});
461         zfs_incremental_backup($host, $diskname, $fulls[0], $store);
462       }
463     }
464   }
465   close(ZFSLIST);
466 }
467
468 if($RESTORE) {
469   perform_restore();
470 }
471 else {
472   foreach my $host (grep { $_ ne "default" } keys %conf) {
473     # If -h was specific, we will skip this host if the arg isn't
474     # an exact match or a pattern match
475     if($HOST &&
476        !(($HOST eq $host) ||
477          ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
478       next;
479     }
480  
481     my $store = config_get($host, 'store');
482     $store =~ s/%h/$host/g;;
483     mkdir $store if(! -d $store);
484  
485     if($BACKUP) {
486       plan_and_run($host, $store, $ZFS);
487     }
488     if($LIST || $SUMMARY || $SUMMARY_EXT) {
489       show_backups($host, $store, $ZFS);
490     }
491     if($EXPUNGE) {
492       perform_retention($host, $store);
493     }
494   }
495 }
Note: See TracBrowser for help on using the browser.