Merge pull request #636 from jimklimov/autoCreation-and-not

Extend `--autoCreation` effect (or default lack thereof) to newly appearing sub-datasets

.github/workflows/spelling/expect.txt

@@ -379,6 +379,7 @@ newestbe
 nf
 nh
 noaction
+noauto
 nobase
 nodelay
 nodestroy

CHANGES

@@ -10,6 +10,7 @@ znapzend (0.21.3) unstable; urgency=medium
   * Fixed CI recipes and contents for spell-checker
   * Added rc-script and integration documentation for FreeBSD and similar platforms
   * Converted configure.ac and numerous Makefile.am to avoid GNU Make syntax in favor of portability: tested with Solaris/illumos Sun make and with FreeBSD make
+  * Extended `--autoCreation` effect (or lack thereof) to newly appearing sub-datasets; added a `--noautoCreation` option to help override configuration file settings (where used)
 
  -- Jim Klimov <[email protected]>  Tue, 12 Mar 2024 13:42:28 +0100
 

bin/znapzend

@@ -16,7 +16,7 @@ sub main {
     my $opts = {};
     GetOptions($opts, qw(help|h man debug|d noaction|n nodestroy features=s),
         qw(sudo pfexec rootExec=s daemonize pidfile=s logto=s loglevel=s),
-        qw(runonce:s connectTimeout=s timeWarp=i nodelay autoCreation version),
+        qw(runonce:s connectTimeout=s timeWarp=i nodelay autoCreation noautoCreation version),
         qw(forcedSnapshotSuffix=s forbidDestRollback),
         qw(skipIntermediates|i sendIntermediates|I),
             # Note: the intended usage is either via feature as done
@@ -85,6 +85,15 @@ sub main {
         $opts->{forbidDestRollback} = 0;
     }
 
+    if (defined($opts->{noautoCreation})) {
+        $opts->{autoCreation} = 0;
+        delete $opts->{noautoCreation};
+    } else {
+        if (defined($opts->{autoCreation})) {
+            $opts->{autoCreation} = 1;
+        }
+    }
+
     if (defined($opts->{sinceForced}) && ($opts->{sinceForced} eq '')) {
         delete $opts->{sinceForced};
     }
@@ -202,6 +211,8 @@ B<znapzend> [I<options>...]
  --connectTimeout=x     sets the ConnectTimeout for ssh commands
  --autoCreation         automatically create dataset on destination if it does
                         not exist
+ --noautoCreation       avoid automatically creating a dataset on destination
+                        if it does not exist (default)
  --timeWarp=x           shift znapzend's sense of NOW into the future
                         by x seconds
  --skipOnPreSnapCmdFail skip snapshots if the pre-snap-command fails
@@ -610,6 +621,12 @@ sets the ssh connection timeout (in seconds)
 
 Automatically create a dataset on a destination host if it's not there yet.
 
+=item B<--noautoCreation>
+
+Avoid automatically creating a dataset on a destination host if it's not
+there yet. This is the default behavior; the option is available to help
+explicitly override a setting inherited from a configuration file, etc.
+
 =item B<--timeWarp>=x
 
 Shift ZnapZend's sense of time into the future by x seconds.

lib/ZnapZend.pm

@@ -431,6 +431,8 @@ my $refreshBackupPlans = sub {
                             . "' does not exist or is offline. will be rechecked every run..."
                             . ( $self->autoCreation ? "" : " Consider running znapzend --autoCreation" ) );
                 };
+
+                $self->zLog->debug('refreshBackupPlans(): detected dst_' . $key . '_valid status for ' . $backupSet->{"dst_$key"} . ': ' . $backupSet->{"dst_$key" . '_valid'}) if ($self->debug);
             }
             $backupSet->{"dst$key" . 'PlanHash'}
                 = $self->zTime->backupPlanToHash($backupSet->{"dst_$key" . '_plan'});
@@ -567,7 +569,10 @@ my $sendRecvCleanup = sub {
                 if ($self->autoCreation && !$self->sendRaw) {
                     my ($zpool) = $backupSet->{"dst_$key"} =~ /(^[^\/]+)\//;
 
-                    # check if we can access destination zpool, if so create parent dataset
+                    # check if we can access destination zpool, if so -
+                    # create parent dataset (e.g. this backupSet root;
+                    # note that if we recurse into children that may be
+                    # absent, they are treated separately below)
                     $self->zZfs->dataSetExists($zpool) && do {
                         $self->zLog->info("creating destination dataset '" . $backupSet->{"dst_$key"} . "'...");
 
@@ -581,13 +586,15 @@ my $sendRecvCleanup = sub {
                 }
                 ( $backupSet->{"dst_$key" . '_valid'} || ($self->sendRaw && $self->autoCreation) ) or do {
                     my $errmsg = "destination '" . $backupSet->{"dst_$key"}
-                        . "' does not exist or is offline. ignoring it for this round...";
+                        . "' does not exist or is offline; ignoring it for this round...";
                     $self->zLog->warn($errmsg);
                     push (@sendFailed, $errmsg);
                     $thisSendFailed = 1;
                     next;
                 };
             };
+
+            $self->zLog->debug('sendRecvCleanup(): detected dst_' . $key . '_valid status for ' . $backupSet->{"dst_$key"} . ': ' . $backupSet->{"dst_$key" . '_valid'}) if ($self->debug);
         }
 
         #sending loop through all subdatasets
@@ -607,6 +614,19 @@ my $sendRecvCleanup = sub {
                 next;
             }
 
+            # Time to check if the target sub-dataset exists
+            # at all (unless we would auto-create one anyway).
+            if (!$self->autoCreation && !$self->sendRaw && !$self->zZfs->dataSetExists($dstDataSet)) {
+                my $errmsg = "sub-destination '" . $dstDataSet
+                    . "' does not exist or is offline; ignoring it for this round... Consider "
+                    . ( $self->autoCreation || $self->sendRaw ? "" : "running znapzend --autoCreation or " )
+                    . "disabling this dataset from znapzend handling.";
+                $self->zLog->warn($errmsg);
+                push (@sendFailed, $errmsg);
+                $thisSendFailed = 1;
+                next;
+            }
+
             {   # scoping
                 local $@;
                 eval {
@@ -615,8 +635,12 @@ my $sendRecvCleanup = sub {
                         'since=="' . $self->since . '"'.
                         ', skipIntermediates=="' . $self->skipIntermediates . '"' .
                         ', forbidDestRollback=="' . $self->forbidDestRollback . '"' .
+                        ', autoCreation=="' . $self->autoCreation . '"' .
+                        ', sendRaw=="' . $self->sendRaw . '"' .
+                        ', valid=="' . ( $backupSet->{"dst_$key" . '_valid'} ? "true" : "false" ) . '"' .
                         ', justCreated=="' . ( $backupSet->{"dst_$key" . '_justCreated'} ? "true" : "false" ) . '"'
                         ) if $self->debug;
+
                     if ($self->since) {
                         # Make sure that if we use the "--sinceForced=X" or
                         # "--since=X" option, this named snapshot exists (or

lib/ZnapZend/Config.pm

@@ -204,6 +204,7 @@ my $checkBackupSets = sub {
         for my $dst (grep { /^dst_[^_]+$/ } keys %$backupSet){
             #store backup destination validity. will be checked where used
             $backupSet->{$dst . '_valid'} = $self->zfs->dataSetExists($backupSet->{$dst});
+            $self->zLog->debug('checkBackupSets(): detected ' . $dst . '_valid status for ' . $backupSet->{$dst} . ': ' . $backupSet->{$dst . '_valid'}) if ($self->debug);
 
             #if a backup destination is given, we also need a plan
             $backupSet->{$dst . '_plan'} or die "ERROR: no backup plan given for destination\n";

man/znapzend.1

@@ -176,6 +176,8 @@ znapzend \- znapzend daemon
 \& \-\-connectTimeout=x     sets the ConnectTimeout for ssh commands
 \& \-\-autoCreation         automatically create dataset on destination if it does
 \&                        not exist
+\& \-\-noautoCreation       avoid automatically creating a dataset on destination
+\&                        if it does not exist (default)
 \& \-\-timeWarp=x           shift znapzend\*(Aqs sense of NOW into the future
 \&                        by x seconds
 \& \-\-skipOnPreSnapCmdFail skip snapshots if the pre\-snap\-command fails
@@ -569,6 +571,11 @@ sets the ssh connection timeout (in seconds)
 .IP "\fB\-\-autoCreation\fR" 4
 .IX Item "--autoCreation"
 Automatically create a dataset on a destination host if it's not there yet.
+.IP "\fB\-\-noautoCreation\fR" 4
+.IX Item "--noautoCreation"
+Avoid automatically creating a dataset on a destination host if it's not
+there yet. This is the default behavior; the option is available to help
+explicitly override a setting inherited from a configuration file, etc.
 .IP "\fB\-\-timeWarp\fR=x" 4
 .IX Item "--timeWarp=x"
 Shift ZnapZend's sense of time into the future by x seconds.