Fix for issue #80: enabling archive-copy causes failing differential & incremental backups

README.md

@@ -227,6 +227,7 @@ The following recovery types are supported:
 - `xid` - recover to the transaction id specified in `--target`.
 - `time` - recover to the time specified in `--target`.
 - `preserve` - preserve the existing `recovery.conf` file.
+- `none` - no `recovery.conf` file is written so PostgreSQL will attempt to achieve consistency using WAL segments present in `pg_xlog`.  Provide the required WAL segments or use the `archive-copy` setting to include them with the backup.
 
 ```
 required: n
@@ -610,6 +611,8 @@ example: archive-check=n
 ##### `archive-copy` key
 
 Store WAL segments required to make the backup consistent in the backup's pg_xlog path.  This slightly paranoid option protects against corruption or premature expiration in the WAL segment archive.  PITR won't be possible without the WAL segment archive and this option also consumes more space.
+
+Even though WAL segments will be restored with the backup, PostgreSQL will ignore them if a `recovery.conf` file exists and instead use `archive_command` to fetch WAL segments.  Specifying `type=none` when restoring will not create `recovery.conf` and force PostgreSQL to use the WAL segments in pg_xlog.  This will get the database to a consistent state.
 ```
 required: n
 default: n
@@ -731,7 +734,7 @@ example: db-path=/data/db
 
 ### v0.75: IN DEVELOPMENT: enterprise features: monitoring, throttling, retention period
 
-*
+* Fixed an issue where archive-copy would fail on an incr/diff backup when hardlink=n.  In this case the pg_xlog path does not already exist and must be created.
 
 ### v0.65: Improved resume and restore logging, compact restores
 

doc/doc.xml

@@ -219,6 +219,7 @@ Run a <id>full</id> backup on the <id>db</id> stanza.  <param>--type</param> can
 								<li><id>xid</id> - recover to the transaction id specified in <param>--target</param>.</li>
 								<li><id>time</id> - recover to the time specified in <param>--target</param>.</li>
 								<li><id>preserve</id> - preserve the existing <file>recovery.conf</file> file.</li>
+								<li><id>none</id> - no <file>recovery.conf</file> file is written so <postgres/> will attempt to achieve consistency using WAL segments present in <path>pg_xlog</path>.  Provide the required WAL segments or use the <setting>archive-copy</setting> setting to include them with the backup.</li>
 							</ul></text>
 						<example>xid</example>
 					</option>
@@ -562,7 +563,9 @@ Run a <id>full</id> backup on the <id>db</id> stanza.  <param>--type</param> can
 
 					<!-- CONFIG - BACKUP SECTION - ARCHIVE-COPY -->
 					<config-key id="archive-copy">
-						<text>Store WAL segments required to make the backup consistent in the backup's pg_xlog path.  This slightly paranoid option protects against corruption or premature expiration in the WAL segment archive.  PITR won't be possible without the WAL segment archive and this option also consumes more space.</text>
+						<text>Store WAL segments required to make the backup consistent in the backup's pg_xlog path.  This slightly paranoid option protects against corruption or premature expiration in the WAL segment archive.  PITR won't be possible without the WAL segment archive and this option also consumes more space.
+
+							Even though WAL segments will be restored with the backup, <postgres/> will ignore them if a <file>recovery.conf</file> file exists and instead use <setting>archive_command</setting> to fetch WAL segments.  Specifying <setting>type=none</setting> when restoring will not create <file>recovery.conf</file> and force <postgres/> to use the WAL segments in pg_xlog.  This will get the database to a consistent state.</text>
 
 						<example>y</example>
 					</config-key>
@@ -684,7 +687,7 @@ Run a <id>full</id> backup on the <id>db</id> stanza.  <param>--type</param> can
 			<release-version version="0.75" title="IN DEVELOPMENT: enterprise features: monitoring, throttling, retention period">
 				<release-feature-bullet-list>
 					<release-feature>
-						<text></text>
+						<text>Fixed an issue where archive-copy would fail on an incr/diff backup when hardlink=n.  In this case the pg_xlog path does not already exist and must be created.</text>
 					</release-feature>
 				</release-feature-bullet-list>
 			</release-version>

lib/BackRest/Backup.pm

@@ -646,14 +646,9 @@ sub backup
     $oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_VERSION, undef, version_get());
 
     # Build the backup manifest
-    my %oTablespaceMap;
+    my $oTablespaceMap = $bNoStartStop ? undef : $oDb->tablespace_map_get();
 
-    if (!$bNoStartStop)
+    $oBackupManifest->build($oFile, $strDbClusterPath, $oLastManifest, $bNoStartStop, $oTablespaceMap);
-    {
-        $oDb->tablespace_map_get(\%oTablespaceMap);
-    }
-
-    $oBackupManifest->build($oFile, $strDbClusterPath, $oLastManifest, $bNoStartStop, \%oTablespaceMap);
     &log(TEST, TEST_MANIFEST_BUILD);
 
     # Check if an aborted backup exists for this stanza
@@ -794,12 +789,13 @@ sub backup
 
                 # Copy the log file from the archive repo to the backup
                 my $strDestinationFile = "base/pg_xlog/${strArchive}" . ($bCompress ? ".$oFile->{strCompressExtension}" : '');
+                my $bArchiveCompressed = $strArchiveFile =~ "^.*\.$oFile->{strCompressExtension}\$";
 
                 my ($bCopyResult, $strCopyChecksum, $lCopySize) =
                     $oFile->copy(PATH_BACKUP_ARCHIVE, $strArchiveFile,
                                  PATH_BACKUP_TMP, $strDestinationFile,
-                                 $strArchiveFile =~ "^.*\.$oFile->{strCompressExtension}\$",
+                                 $bArchiveCompressed, $bCompress,
-                                 $bCompress, undef, $lModificationTime);
+                                 undef, $lModificationTime, undef, true);
 
                 # Add the archive file to the manifest so it can be part of the restore and checked in validation
                 my $strPathSection = 'base:path';

lib/BackRest/Db.pm

@@ -124,10 +124,13 @@ sub psql_execute
 sub tablespace_map_get
 {
     my $self = shift;
-    my $oHashRef = shift;
+
+    my $oHashRef = {};
 
     data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute(
                     'copy (select oid, spcname from pg_tablespace) to stdout'), "\t");
+
+    return $oHashRef;
 }
 
 ####################################################################################################################################

lib/BackRest/File.pm

@@ -504,7 +504,7 @@ sub move
 
                 if (!$self->exists(PATH_ABSOLUTE, dirname($strPathOpDestination)))
                 {
-                    $strError = "${strPathOpDestination} does not exist";
+                    $strError = dirname($strPathOpDestination) . " destination path does not exist";
                     $iErrorCode = COMMAND_ERR_FILE_MISSING;
                 }
 
@@ -1408,7 +1408,7 @@ sub copy
 
                 if (!$self->exists(PATH_ABSOLUTE, dirname($strDestinationTmpOp)))
                 {
-                    $strError = dirname($strDestinationTmpOp) . ' does not exist';
+                    $strError = dirname($strDestinationTmpOp) . ' destination path does not exist';
                     $iErrorCode = COMMAND_ERR_FILE_MISSING;
                 }
 

lib/BackRest/Manifest.pm

@@ -551,7 +551,7 @@ sub build
         }
 
         # If bNoStartStop then build the tablespace map from pg_tblspc path
-        if ($bNoStartStop)
+        if ($bNoStartStop && !defined($oTablespaceMapRef))
         {
             $oTablespaceMapRef = {};
 

test/lib/BackRestTest/BackupTest.pm

@@ -1319,7 +1319,7 @@ sub BackRestTestBackup_RestoreCompare
     }
 
     # Generate the tablespace map for real backups
-    my %oTablespaceMap;
+    my $oTablespaceMap = undef;
     # ${$oTablespaceMapRef}{oid}{$strName}{name} = $strName;
 
     if (!$bSynthetic && defined(${$oExpectedManifestRef}{'backup:tablespace'}))
@@ -1327,10 +1327,8 @@ sub BackRestTestBackup_RestoreCompare
         foreach my $strTablespaceName (keys(${$oExpectedManifestRef}{'backup:tablespace'}))
         {
             my $strTablespaceOid = ${$oExpectedManifestRef}{'backup:tablespace'}{$strTablespaceName}{link};
-            #
-            # confess "GOT HERE - $strTablespaceOid, $strTablespaceName";
 
-            $oTablespaceMap{oid}{$strTablespaceOid}{name} = $strTablespaceName;
+            $$oTablespaceMap{oid}{$strTablespaceOid}{name} = $strTablespaceName;
         }
     }
 
@@ -1338,7 +1336,7 @@ sub BackRestTestBackup_RestoreCompare
     my $oActualManifest = new BackRest::Manifest("${strTestPath}/actual.manifest", false);
 
     my $oTablespaceMapRef = undef;
-    $oActualManifest->build($oFile, ${$oExpectedManifestRef}{'backup:path'}{'base'}, $oLastManifest, $bSynthetic, \%oTablespaceMap);
+    $oActualManifest->build($oFile, ${$oExpectedManifestRef}{'backup:path'}{'base'}, $oLastManifest, true, $oTablespaceMap);
 
     # Generate checksums for all files if required
     # Also fudge size if this is a synthetic test - sizes may change during backup.
@@ -2306,7 +2304,7 @@ sub BackRestTestBackup_Test
                                             $bRemote ? BACKUP : undef,         # remote
                                             $bCompress,                        # compress
                                             undef,                             # checksum
-                                            $bRemote ? undef : true,           # hardlink
+                                            $bRemote ? undef : false,          # hardlink
                                             $iThreadMax,                       # thread-max
                                             $bArchiveAsync,                    # archive-async
                                             undef);                            # compress-async
@@ -2318,7 +2316,7 @@ sub BackRestTestBackup_Test
                                                 $bRemote ? DB : undef,         # remote
                                                 $bCompress,                    # compress
                                                 undef,                         # checksum
-                                                true,                          # hardlink
+                                                false,                         # hardlink
                                                 $iThreadMax,                   # thread-max
                                                 undef,                         # archive-async
                                                 undef);                        # compress-async

test/lib/BackRestTest/CommonTest.pm

@@ -941,6 +941,8 @@ sub BackRestTestCommon_ConfigCreate
         {
             $oParamHash{'global:backup'}{'hardlink'} = 'y';
         }
+
+        $oParamHash{'global:backup'}{'archive-copy'} = 'y';
     }
 
     if (defined($bCompress) && !$bCompress)