root/zetaback

Revision e57012c0617601ca20d6bb6087d68e6f1124ae0b, 11.0 kB (checked in by Theo Schlossnagle <jesus@omniti.com>, 7 years ago)

initial export/import from people/jesus/soltools/zetaback

  • Property mode set to 100755
Line 
1 #!/usr/bin/perl
2
3 use strict;
4 use Getopt::Long;
5 use MIME::Base64;
6 use POSIX qw/strftime/;
7 use Data::Dumper;
8
9 use vars qw/$CONF %conf $BLOCKSIZE $DEBUG $HOST $BACKUP $SUMMARY $RESTORE
10             $EXPUNGE $NUETERED $ZFS/;
11 $CONF = q^/var/spool/zfs_backups/zetaback.conf^;
12 $BLOCKSIZE = 1024*64;
13
14 $conf{'default'}->{'time_format'} = "%Y-%m-%d %H:%M:%S";
15 $conf{'default'}->{'retention'} = 14 * 86400;
16
17 GetOptions(
18   "h=s" => \$HOST,
19   "z=s" => \$ZFS,
20   "c=s" => \$CONF,
21   "b"   => \$BACKUP,
22   "s"   => \$SUMMARY,
23   "r"   => \$RESTORE,
24   "d"   => \$DEBUG,
25   "n"   => \$NUETERED,
26   "x"   => \$EXPUNGE,
27 );
28
29 sub parse_config() {
30   local($/);
31   $/ = undef;
32   open(CONF, "<$CONF");
33   my $file = <CONF>;
34   while($file =~ m/^\s*(\S+)\s\s*{(.*?)}/gms) {
35     my $scope = $1;
36     my $filepart = $2;
37     $conf{$scope} ||= {};
38     foreach my $line (split /\n/, $filepart) {
39       if($line =~ /^\s*([^#]\S*)\s*=\s*(\S+)/) {
40         $conf{$scope}->{lc($1)} = $2;
41       }
42     }
43   }
44   close(CONF);
45 }
46 sub config_get($$) {
47   return $conf{$_[0]}->{$_[1]} || $conf{'default'}->{$_[1]};
48 }
49
50 sub dir_encode($) {
51   my $d = shift;
52   my $e = encode_base64($d, '');
53   $e =~ s/\//_/;
54   return $e;
55 }
56 sub dir_decode($) {
57   my $e = shift;
58   $e =~ s/_/\//;
59   return decode_base64($e);
60 }
61 sub pretty_size($) {
62   my $bytes = shift;
63   if($bytes > 1024*1024*1024) {
64     return sprintf("%0.2f Gb", $bytes / (1024*1024*1024));
65   }
66   if($bytes > 1024*1024) {
67     return sprintf("%0.2f Mb", $bytes / (1024*1024));
68   }
69   if($bytes > 1024) {
70     return sprintf("%0.2f Kb", $bytes / (1024));
71   }
72   return "$bytes b";
73 }
74 sub scan_for_backups($) {
75   my %info = ();
76   my $dir = shift;
77   $info{last_full} = $info{last_incremental} = $info{last_backup} = 0;
78   opendir(D, $dir) || return \%info;
79   foreach my $file (readdir(D)) {
80     if($file =~ /^(\d+)\.([^\.]+)\.full$/) {
81       my $whence = $1;
82       my $fs = dir_decode($2);
83       $info{$fs}->{full}->{$whence}->{'file'} = "$dir/$file";
84       $info{$fs}->{last_full} = $whence if($whence > $info{$fs}->{last_full});
85       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
86                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
87     }
88     elsif($file =~ /^(\d+).([^\.]+)\.incremental.(\d+)$/) {
89       my $whence = $1;
90       my $fs = dir_decode($2);
91       $info{$fs}->{incremental}->{$whence}->{'depends'} = $3;
92       $info{$fs}->{incremental}->{$whence}->{'file'} = "$dir/$file";
93       $info{$fs}->{last_incremental} = $whence if($whence > $info{$fs}->{last_incremental});
94       $info{$fs}->{last_backup} = $info{$fs}->{last_incremental} > $info{$fs}->{last_full} ?
95                                      $info{$fs}->{last_incremental} : $info{$fs}->{last_full};
96     }
97   }
98   closedir(D);
99   return \%info;
100 }
101
102 parse_config();
103
104 sub zfs_remove_snap($$$) {
105   my ($host, $fs, $snap) = @_;
106   my $agent = config_get($host, 'agent');
107   return unless($snap);
108   print "Dropping $snap on $fs\n" if($DEBUG);
109   `ssh $host $agent -z $fs -d $snap`;
110 }
111
112 # Lots of args.. internally called.
113 sub zfs_do_backup($$$$$$) {
114   my ($host, $fs, $type, $point, $store, $dumpfile) = @_;
115   my $agent = config_get($host, 'agent');
116
117   # Do it. yeah.
118   open(LBACKUP, ">$store/.$dumpfile") || die "zfs_full_backup: cannot create dump\n";
119   eval {
120     open(RBACKUP, "ssh $host $agent -z $fs -$type $point |") || die "zfs_full_backup: cannot perform send\n";
121     my $buffer;
122     while(my $len = sysread(RBACKUP, $buffer, $BLOCKSIZE)) {
123       if(syswrite(LBACKUP, $buffer, $len) != $len) {
124         die "$!";
125       }
126     }
127     close(LBACKUP);
128     close(RBACKUP);
129     die "dump failed (zero bytes)\n" if(-z "$store/.$dumpfile");
130     rename("$store/.$dumpfile", "$store/$dumpfile") || die "cannot rename dump\n";
131   };
132   if($@) {
133     unlink("$store/.$dumpfile");
134     die "zfs_full_backup: failed $@";
135   }
136 }
137
138 sub zfs_full_backup($$$) {
139   my ($host, $fs, $store) = @_;
140
141   # Translate into a proper dumpfile nameA
142   my $point = time();
143   my $efs = dir_encode($fs);
144   my $dumpfile = "$point.$efs.full";
145
146   zfs_do_backup($host, $fs, 'f', $point, $store, $dumpfile);
147 }
148
149 sub zfs_incremental_backup($$$$) {
150   my ($host, $fs, $base, $store) = @_;
151   my $agent = config_get($host, 'agent');
152
153   # Translate into a proper dumpfile nameA
154   my $point = time();
155   my $efs = dir_encode($fs);
156   my $dumpfile = "$point.$efs.incremental.$base";
157
158   zfs_do_backup($host, $fs, 'i', $base, $store, $dumpfile);
159 }
160
161 sub perform_retention($$) {
162   my ($host, $store) = @_;
163   my $cutoff = time() - config_get($host, 'retention');
164   my $backup_info = scan_for_backups($store);
165  
166   foreach my $disk (sort keys %{$backup_info}) {
167     my $info = $backup_info->{$disk};
168     next unless(ref($info) eq 'HASH');
169     my %must_save;
170
171     # Get a list of all the full and incrementals, sorts newest to oldest
172     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
173     @backup_points = sort { $b <=> $a } @backup_points;
174
175     # We _cannot_ throw away _all_ out backups, so save the most recent no matter what
176     $must_save{$backup_points[0]} = 1;
177
178     # Walk the list for backups within our retention period.
179     foreach (@backup_points) {
180       if($_ >= $cutoff) {
181         $must_save{$_} = 1;
182       }
183       else {
184         # they are in decending order, once we miss, all will miss
185         last;
186       }
187     }
188
189     # Look for dependencies
190     foreach (@backup_points) {
191       if(exists($info->{incremental}->{$_})) {
192         print "   => $_ depends on $info->{incremental}->{$_}->{depends}\n" if($DEBUG);
193         $must_save{$info->{incremental}->{$_}} = 1
194       }
195     }
196     my @removals = grep { !exists($must_save{$_}) } @backup_points;
197     if($DEBUG) {
198       my $tf = config_get($host, 'time_format');
199       print "    => I can remove:\n";
200       foreach (@backup_points) {
201         print "      => ". strftime($tf, localtime($_));
202         print " [". (exists($info->{full}->{$_}) ? "full":"incremental") ."]";
203         print " XXX" if(!exists($must_save{$_}));
204         print "\n";
205       }
206     }
207     foreach (@removals) {
208       my $efs = dir_encode($disk);
209       my $filename;
210       if(exists($info->{full}->{$_})) {
211         $filename = "$store/$_.$efs.full";
212       }
213       elsif(exists($info->{incremental}->{$_})) {
214         $filename = "$store/$_.$efs.incremental.$info->{incremental}->{$_}->{depends}";
215       }
216       else {
217         print "ERROR: We tried to expunge $host $disk [$_], but couldn't find it.\n";
218       }
219       print "    => expunging $filename\n" if($DEBUG);
220       unless($NUETERED) {
221         unlink($filename) || print "ERROR: unlink $filename: $?\n";
222       }
223     }
224   }
225 }
226
227 sub show_backups($$$) {
228   my ($host, $store, $diskpat) = @_;
229   my $backup_info = scan_for_backups($store);
230   my $tf = config_get($host, 'time_format');
231   foreach my $disk (sort keys %{$backup_info}) {
232     my $info = $backup_info->{$disk};
233     next unless(ref($info) eq 'HASH');
234     next
235       if($diskpat &&      # if the pattern was specified it could
236          !($disk eq $diskpat ||        # be a specific match or a
237            ($diskpat =~ /^\/(.+)\/$/ && $disk =~ /$1/))); # regex
238     # We want to see this one
239     print "$disk:\n";
240     print "\tLast Full: ". ($info->{last_full} ? strftime($tf, localtime($info->{last_full})) : "Never") . "\n";
241     if($info->{last_full} < $info->{last_incremental}) {
242       print "\tLast Incr: ". strftime($tf, localtime($info->{last_incremental})). "\n";
243     }
244     my @backup_points = (keys %{$info->{full}}, keys %{$info->{incremental}});
245     @backup_points = sort { $a <=> $b } @backup_points;
246     print "\tBackups available:\n";
247     foreach (@backup_points) {
248       print "\t\t" . strftime($tf, localtime($_)) . " [$_] ";
249       if(exists($info->{full}->{$_})) {
250         my @st = stat($info->{full}->{$_}->{file});
251         print "FULL " . pretty_size($st[7]);
252       } else {
253         my @st = stat($info->{incremental}->{$_}->{file});
254         print "INCR from [$info->{incremental}->{$_}->{depends}] " . pretty_size($st[7]);
255       }
256       print "\n";
257     }
258     print "\n";
259   }
260 }
261
262 sub plan_and_run($$$) {
263   my ($host, $store, $diskpat) = @_;
264   print "Planning '$host'\n" if($DEBUG);
265   my $agent = config_get($host, 'agent');
266   open(ZFSLIST, "ssh $host $agent -l |") || next;
267   foreach my $diskline (<ZFSLIST>) {
268     chomp($diskline);
269     next unless($diskline =~ /^(\S+) \[([^\]]*)\]/);
270     my $diskname = $1;
271     my %snaps;
272     map { $snaps{$_} = 1 } (split(/,/, $2));
273
274     # If we are being selective (via -z) now is the time.
275     next
276       if($diskpat &&          # if the pattern was specified it could
277          !($diskname eq $diskpat ||        # be a specific match or a
278            ($diskpat =~ /^\/(.+)\/$/ && $diskname =~ /$1/))); # regex
279
280     print " => Scanning '$store' for old backups of '$diskname'.\n" if($DEBUG);
281     # Make directory on demand
282     my $backup_info = scan_for_backups($store);
283     # That gave us info on all backups, we just want this disk
284     $backup_info = $backup_info->{$diskname} || {};
285
286     # Should we do a backup?
287     my $backup_type = 'no';
288     if(time() > $backup_info->{last_backup} + config_get($host, 'backup_interval')) {
289       $backup_type = 'incremental';
290     }
291     if(time() > $backup_info->{last_full} + config_get($host, 'full_interval')) {
292       $backup_type = 'full';
293     }
294
295     # If we want an incremental, but have no full, then we need to upgrade to full
296     if($backup_type eq 'incremental') {
297       my $have_full_locally = 0;
298       # For each local full backup, see if the full backup still exists on the other end.
299       foreach (keys %{$backup_info->{'full'}}) {
300         $have_full_locally = 1 if(exists($snaps{'__zb_full_' . $_}));
301       }
302       $backup_type = 'full' unless($have_full_locally);
303     }
304
305     print " => doing $backup_type backup\n" if($DEBUG);
306     # We need to drop a __zb_base snap or a __zb_incr snap before we proceed
307     unless($NUETERED) {
308       if($backup_type eq 'full') {
309         eval { zfs_full_backup($host, $diskname, $store); };
310         if ($@) {
311           chomp(my $err = $@);
312           print " => failure $err\n";
313         }
314         else {
315           # Unless there was an error backing up, remove all the other full snaps
316           foreach (keys %snaps) {
317             zfs_remove_snap($host, $diskname, $_) if(/^__zb_full_(\d+)/)
318           }
319         }
320       }
321       if($backup_type eq 'incremental') {
322         zfs_remove_snap($host, $diskname, '__zb_incr') if($snaps{'__zb_incr'});
323         # Find the newest full from which to do an incremental (NOTE: reverse numeric sort)
324         my @fulls = sort { $b <=> $a } (keys %{$backup_info->{'full'}});
325         zfs_incremental_backup($host, $diskname, $fulls[0], $store);
326       }
327     }
328   }
329   close(ZFSLIST);
330 }
331
332 foreach my $host (grep { $_ ne "default" } keys %conf) {
333   # If -h was specific, we will skip this host if the arg isn't
334   # an exact match or a pattern match
335   if($HOST &&
336      !(($HOST eq $host) ||
337        ($HOST =~ /^\/(.*)\/$/ && $host =~ /$1/))) {
338     next;
339   }
340
341   my $store = config_get($host, 'store');
342   $store =~ s/%h/$host/g;;
343   mkdir $store if(! -d $store);
344
345   if($BACKUP) {
346     plan_and_run($host, $store, $ZFS);
347   }
348   if($SUMMARY) {
349     show_backups($host, $store, $ZFS);
350   }
351   if($EXPUNGE) {
352     perform_retention($host, $store);
353   }
354 }
355
Note: See TracBrowser for help on using the browser.