Fixed a bug where internal symlinks were not being created for tablespaces in the repository.

This issue was only apparent when trying to bring up clusters in-place with filesystem snapshots and did not affect normal backup and restore.

doc/xml/release.xml

@@ -135,6 +135,12 @@
     <release-list>
         <release date="XXXX-XX-XX" version="1.12dev" title="UNDER DEVELOPMENT">
             <release-core-list>
+                <release-bug-list>
+                    <release-item>
+                        <p>Fixed a bug where internal symlinks were not being created for tablespaces in the repository.  This issue was only apparent when trying to bring up clusters in-place with filesystem snapshots and did not affect normal backup and restore.</p>
+                    </release-item>
+                </release-bug-list>
+
                 <release-refactor-list>
                     <release-item>
                         <release-item-contributor-list>

lib/pgBackRest/Backup.pm

@@ -191,7 +191,7 @@ sub tmpClean
         my $strDelete = $oFileLocal->pathGet(PATH_BACKUP_TMP, $strFile);
 
         # If a path then delete it, all the files should have already been deleted since we are going in reverse order
-        if (-d $strDelete)
+        if (!-X $strDelete && -d $strDelete)
         {
             logDebugMisc($strOperation, "remove path ${strDelete}");
 
@@ -267,13 +267,24 @@ sub processManifest
     my $lFileTotal = 0;
     my $lSizeTotal = 0;
 
-    # If this is a full backup or hard-linked then create all paths
+    # If this is a full backup or hard-linked then create all paths and tablespace links
     if ($bHardLink || $strType eq BACKUP_TYPE_FULL)
     {
+        # Create paths
         foreach my $strPath ($oBackupManifest->keys(MANIFEST_SECTION_TARGET_PATH))
         {
             $oFileMaster->pathCreate(PATH_BACKUP_TMP, $strPath);
         }
+
+        # Create tablespace links
+        for my $strTarget ($oBackupManifest->keys(MANIFEST_SECTION_BACKUP_TARGET))
+        {
+            if ($oBackupManifest->isTargetTablespace($strTarget))
+            {
+                $oFileMaster->linkCreate(
+                    PATH_BACKUP_TMP, $strTarget, PATH_BACKUP_TMP, MANIFEST_TARGET_PGDATA . "/${strTarget}", false, true);
+            }
+        }
     }
 
     # Iterate all files in the manifest

lib/pgBackRest/File.pm

@@ -447,13 +447,27 @@ sub linkCreate
         # Generate relative path if requested
         if ($bRelative)
         {
-            my $iCommonLen = commonPrefix($strSource, $strDestination);
+            # Determine how much of the paths are common
+            my @strySource = split('/', $strSource);
+            my @stryDestination = split('/', $strDestination);
 
-            if ($iCommonLen != 0)
+            while (defined($strySource[0]) && defined($stryDestination[0]) && $strySource[0] eq $stryDestination[0])
             {
-                $strSource = ('../' x substr($strDestination, $iCommonLen) =~ tr/\///) . substr($strSource, $iCommonLen);
+                shift(@strySource);
+                shift(@stryDestination);
             }
 
+            # Add relative path sections
+            $strSource = '';
+
+            for (my $iIndex = 0; $iIndex < @stryDestination - 1; $iIndex++)
+            {
+                $strSource .= '../';
+            }
+
+            # Add path to source
+            $strSource .= join('/', @strySource);
+
             logDebugMisc
             (
                 $strOperation, 'apply relative path',

test/expect/backup-synthetic-002.log

@@ -1018,6 +1018,7 @@ P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <f
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1, strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1/[TS_PATH-1], strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1/[TS_PATH-1]/16384, strPathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/1, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/1, strSourcePathType = backup:tmp
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/global/pg_control to [BACKUP-FULL-2]
 P00  DEBUG:     File->linkCreate(): bHard = true, bPathCreate = true, bRelative = false, strDestinationFile = pg_data/global/pg_control, strDestinationPathType = backup:tmp, strSourceFile = [BACKUP-FULL-2]/pg_data/global/pg_control, strSourcePathType = backup:cluster
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/postgresql.conf to [BACKUP-FULL-2]
@@ -1306,8 +1307,9 @@ P00  DEBUG:     File->manifestRecurse(): iDepth = 1, oManifestHashRef = [hash],
 P00  DEBUG:     File->manifestRecurse(): iDepth = 2, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1, strPathOp = [TEST_PATH]/db-master/repo/temp/db.tmp, strPathType = backup:tmp
 P00  DEBUG:     File->manifestRecurse(): iDepth = 3, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1/[TS_PATH-1], strPathOp = [TEST_PATH]/db-master/repo/temp/db.tmp, strPathType = backup:tmp
 P00  DEBUG:     File->manifestRecurse(): iDepth = 4, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1/[TS_PATH-1]/16384, strPathOp = [TEST_PATH]/db-master/repo/temp/db.tmp, strPathType = backup:tmp
-P00  DEBUG:     Backup->fileNotInManifest=>: stryFile = (pg_data/PG_VERSION, pg_data/base/1/12000, pg_data/base/1/PG_VERSION, pg_data/base/16384/17000, pg_data/base/16384/PG_VERSION, pg_data/base/32768/33000, pg_data/base/32768/PG_VERSION, pg_data/global/pg_control, pg_data/pg_stat/global.stat, pg_data/postgresql.conf)
+P00  DEBUG:     Backup->fileNotInManifest=>: stryFile = (pg_data/PG_VERSION, pg_data/base/1/12000, pg_data/base/1/PG_VERSION, pg_data/base/16384/17000, pg_data/base/16384/PG_VERSION, pg_data/base/32768/33000, pg_data/base/32768/PG_VERSION, pg_data/global/pg_control, pg_data/pg_stat/global.stat, pg_data/pg_tblspc/1, pg_data/postgresql.conf)
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/postgresql.conf
+P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/pg_tblspc/1
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/pg_stat/global.stat
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/global/pg_control
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/base/32768/PG_VERSION
@@ -1338,6 +1340,8 @@ P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <f
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2, strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2/[TS_PATH-1], strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2/[TS_PATH-1]/32768, strPathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/1, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/1, strSourcePathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/2, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/2, strSourcePathType = backup:tmp
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/global/pg_control to [BACKUP-FULL-2]
 P00  DEBUG:     File->linkCreate(): bHard = true, bPathCreate = true, bRelative = false, strDestinationFile = pg_data/global/pg_control, strDestinationPathType = backup:tmp, strSourceFile = [BACKUP-FULL-2]/pg_data/global/pg_control, strSourcePathType = backup:cluster
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/postgresql.conf to [BACKUP-FULL-2]

test/expect/backup-synthetic-004.log

@@ -1016,6 +1016,7 @@ P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <f
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1, strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1/[TS_PATH-1], strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1/[TS_PATH-1]/16384, strPathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/1, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/1, strSourcePathType = backup:tmp
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/global/pg_control to [BACKUP-FULL-2]
 P00  DEBUG:     File->linkCreate(): bHard = true, bPathCreate = true, bRelative = false, strDestinationFile = pg_data/global/pg_control, strDestinationPathType = backup:tmp, strSourceFile = [BACKUP-FULL-2]/pg_data/global/pg_control, strSourcePathType = backup:cluster
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/postgresql.conf to [BACKUP-FULL-2]
@@ -1303,8 +1304,9 @@ P00  DEBUG:     File->manifestRecurse(): iDepth = 1, oManifestHashRef = [hash],
 P00  DEBUG:     File->manifestRecurse(): iDepth = 2, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1, strPathOp = [TEST_PATH]/db-master/repo/temp/db.tmp, strPathType = backup:tmp
 P00  DEBUG:     File->manifestRecurse(): iDepth = 3, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1/[TS_PATH-1], strPathOp = [TEST_PATH]/db-master/repo/temp/db.tmp, strPathType = backup:tmp
 P00  DEBUG:     File->manifestRecurse(): iDepth = 4, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1/[TS_PATH-1]/16384, strPathOp = [TEST_PATH]/db-master/repo/temp/db.tmp, strPathType = backup:tmp
-P00  DEBUG:     Backup->fileNotInManifest=>: stryFile = (pg_data/PG_VERSION.gz, pg_data/base/1/12000.gz, pg_data/base/1/PG_VERSION.gz, pg_data/base/16384/17000.gz, pg_data/base/16384/PG_VERSION.gz, pg_data/base/32768/33000.gz, pg_data/base/32768/PG_VERSION.gz, pg_data/global/pg_control.gz, pg_data/pg_stat/global.stat.gz, pg_data/postgresql.conf.gz)
+P00  DEBUG:     Backup->fileNotInManifest=>: stryFile = (pg_data/PG_VERSION.gz, pg_data/base/1/12000.gz, pg_data/base/1/PG_VERSION.gz, pg_data/base/16384/17000.gz, pg_data/base/16384/PG_VERSION.gz, pg_data/base/32768/33000.gz, pg_data/base/32768/PG_VERSION.gz, pg_data/global/pg_control.gz, pg_data/pg_stat/global.stat.gz, pg_data/pg_tblspc/1, pg_data/postgresql.conf.gz)
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/postgresql.conf.gz
+P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/pg_tblspc/1
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/pg_stat/global.stat.gz
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/global/pg_control.gz
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/db-master/repo/temp/db.tmp/pg_data/base/32768/PG_VERSION.gz
@@ -1335,6 +1337,8 @@ P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <f
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2, strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2/[TS_PATH-1], strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2/[TS_PATH-1]/32768, strPathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/1, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/1, strSourcePathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/2, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/2, strSourcePathType = backup:tmp
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/global/pg_control to [BACKUP-FULL-2]
 P00  DEBUG:     File->linkCreate(): bHard = true, bPathCreate = true, bRelative = false, strDestinationFile = pg_data/global/pg_control, strDestinationPathType = backup:tmp, strSourceFile = [BACKUP-FULL-2]/pg_data/global/pg_control, strSourcePathType = backup:cluster
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/postgresql.conf to [BACKUP-FULL-2]

test/expect/backup-synthetic-006.log

@@ -1041,6 +1041,7 @@ P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <f
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1, strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1/[TS_PATH-1], strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1/[TS_PATH-1]/16384, strPathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/1, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/1, strSourcePathType = backup:tmp
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/global/pg_control to [BACKUP-FULL-2]
 P00  DEBUG:     File->linkCreate(): bHard = true, bPathCreate = true, bRelative = false, strDestinationFile = pg_data/global/pg_control, strDestinationPathType = backup:tmp, strSourceFile = [BACKUP-FULL-2]/pg_data/global/pg_control, strSourcePathType = backup:cluster
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/postgresql.conf to [BACKUP-FULL-2]
@@ -1339,8 +1340,9 @@ P00  DEBUG:     File->manifestRecurse(): iDepth = 1, oManifestHashRef = [hash],
 P00  DEBUG:     File->manifestRecurse(): iDepth = 2, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1, strPathOp = [TEST_PATH]/backup/repo/temp/db.tmp, strPathType = backup:tmp
 P00  DEBUG:     File->manifestRecurse(): iDepth = 3, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1/[TS_PATH-1], strPathOp = [TEST_PATH]/backup/repo/temp/db.tmp, strPathType = backup:tmp
 P00  DEBUG:     File->manifestRecurse(): iDepth = 4, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1/[TS_PATH-1]/16384, strPathOp = [TEST_PATH]/backup/repo/temp/db.tmp, strPathType = backup:tmp
-P00  DEBUG:     Backup->fileNotInManifest=>: stryFile = (pg_data/PG_VERSION, pg_data/base/1/12000, pg_data/base/1/PG_VERSION, pg_data/base/16384/17000, pg_data/base/16384/PG_VERSION, pg_data/base/32768/33000, pg_data/base/32768/PG_VERSION, pg_data/global/pg_control, pg_data/pg_stat/global.stat, pg_data/postgresql.conf)
+P00  DEBUG:     Backup->fileNotInManifest=>: stryFile = (pg_data/PG_VERSION, pg_data/base/1/12000, pg_data/base/1/PG_VERSION, pg_data/base/16384/17000, pg_data/base/16384/PG_VERSION, pg_data/base/32768/33000, pg_data/base/32768/PG_VERSION, pg_data/global/pg_control, pg_data/pg_stat/global.stat, pg_data/pg_tblspc/1, pg_data/postgresql.conf)
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/postgresql.conf
+P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/pg_tblspc/1
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/pg_stat/global.stat
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/global/pg_control
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/base/32768/PG_VERSION
@@ -1371,6 +1373,8 @@ P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <f
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2, strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2/[TS_PATH-1], strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2/[TS_PATH-1]/32768, strPathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/1, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/1, strSourcePathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/2, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/2, strSourcePathType = backup:tmp
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/global/pg_control to [BACKUP-FULL-2]
 P00  DEBUG:     File->linkCreate(): bHard = true, bPathCreate = true, bRelative = false, strDestinationFile = pg_data/global/pg_control, strDestinationPathType = backup:tmp, strSourceFile = [BACKUP-FULL-2]/pg_data/global/pg_control, strSourcePathType = backup:cluster
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/postgresql.conf to [BACKUP-FULL-2]

test/expect/backup-synthetic-008.log

@@ -1037,6 +1037,7 @@ P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <f
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1, strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1/[TS_PATH-1], strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/1/[TS_PATH-1]/16384, strPathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/1, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/1, strSourcePathType = backup:tmp
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/global/pg_control to [BACKUP-FULL-2]
 P00  DEBUG:     File->linkCreate(): bHard = true, bPathCreate = true, bRelative = false, strDestinationFile = pg_data/global/pg_control, strDestinationPathType = backup:tmp, strSourceFile = [BACKUP-FULL-2]/pg_data/global/pg_control, strSourcePathType = backup:cluster
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/postgresql.conf to [BACKUP-FULL-2]
@@ -1333,8 +1334,9 @@ P00  DEBUG:     File->manifestRecurse(): iDepth = 1, oManifestHashRef = [hash],
 P00  DEBUG:     File->manifestRecurse(): iDepth = 2, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1, strPathOp = [TEST_PATH]/backup/repo/temp/db.tmp, strPathType = backup:tmp
 P00  DEBUG:     File->manifestRecurse(): iDepth = 3, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1/[TS_PATH-1], strPathOp = [TEST_PATH]/backup/repo/temp/db.tmp, strPathType = backup:tmp
 P00  DEBUG:     File->manifestRecurse(): iDepth = 4, oManifestHashRef = [hash], strPathFileOp = pg_tblspc/1/[TS_PATH-1]/16384, strPathOp = [TEST_PATH]/backup/repo/temp/db.tmp, strPathType = backup:tmp
-P00  DEBUG:     Backup->fileNotInManifest=>: stryFile = (pg_data/PG_VERSION.gz, pg_data/base/1/12000.gz, pg_data/base/1/PG_VERSION.gz, pg_data/base/16384/17000.gz, pg_data/base/16384/PG_VERSION.gz, pg_data/base/32768/33000.gz, pg_data/base/32768/PG_VERSION.gz, pg_data/global/pg_control.gz, pg_data/pg_stat/global.stat.gz, pg_data/postgresql.conf.gz)
+P00  DEBUG:     Backup->fileNotInManifest=>: stryFile = (pg_data/PG_VERSION.gz, pg_data/base/1/12000.gz, pg_data/base/1/PG_VERSION.gz, pg_data/base/16384/17000.gz, pg_data/base/16384/PG_VERSION.gz, pg_data/base/32768/33000.gz, pg_data/base/32768/PG_VERSION.gz, pg_data/global/pg_control.gz, pg_data/pg_stat/global.stat.gz, pg_data/pg_tblspc/1, pg_data/postgresql.conf.gz)
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/postgresql.conf.gz
+P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/pg_tblspc/1
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/pg_stat/global.stat.gz
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/global/pg_control.gz
 P00  DEBUG:     Backup->tmpClean: remove file [TEST_PATH]/backup/repo/temp/db.tmp/pg_data/base/32768/PG_VERSION.gz
@@ -1365,6 +1367,8 @@ P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <f
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2, strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2/[TS_PATH-1], strPathType = backup:tmp
 P00  DEBUG:     File->pathCreate(): bCreateParents = <false>, bIgnoreExists = <false>, strMode = <0750>, strPath = pg_tblspc/2/[TS_PATH-1]/32768, strPathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/1, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/1, strSourcePathType = backup:tmp
+P00  DEBUG:     File->linkCreate(): bHard = false, bPathCreate = <true>, bRelative = true, strDestinationFile = pg_data/pg_tblspc/2, strDestinationPathType = backup:tmp, strSourceFile = pg_tblspc/2, strSourcePathType = backup:tmp
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/global/pg_control to [BACKUP-FULL-2]
 P00  DEBUG:     File->linkCreate(): bHard = true, bPathCreate = true, bRelative = false, strDestinationFile = pg_data/global/pg_control, strDestinationPathType = backup:tmp, strSourceFile = [BACKUP-FULL-2]/pg_data/global/pg_control, strSourcePathType = backup:cluster
 P00  DEBUG:     Backup->processManifest: hardlink pg_data/postgresql.conf to [BACKUP-FULL-2]

test/lib/pgBackRestTest/Backup/Common/HostBackupTest.pm

@@ -145,6 +145,9 @@ sub new
     # Set the backup destination
     $self->{strBackupDestination} = $$oParam{strBackupDestination};
 
+    # Default hardlink to false
+    $self->{bHardLink} = false;
+
     # Create the local file object
     $self->{oFile} = new pgBackRest::File(
         $self->stanza(),
@@ -263,8 +266,73 @@ sub backupEnd
 
     my $strBackup = $self->backupLast();
 
-    # Only do compare for synthetic backups
+    # If a real backup then load the expected manifest from the actual manifest.  An expected manifest can't be generated perfectly
-    if (defined($oExpectedManifest))
+    # because a running database is always in flux.  Even so, it allows us test many things.
+    if (!$self->synthetic())
+    {
+        $oExpectedManifest =
+            iniLoad($self->repoPath() . '/backup/' . $self->stanza() . "/${strBackup}/" . FILE_MANIFEST, $oExpectedManifest);
+    }
+
+    # Make sure tablespace links are correct
+    if ($strType eq BACKUP_TYPE_FULL || $self->hardLink())
+    {
+        my $hTablespaceManifest = $self->{oFile}->manifest(
+            PATH_BACKUP_CLUSTER, "${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC);
+
+        # Iterate file links
+        for my $strFile (sort(keys(%{$hTablespaceManifest})))
+        {
+            next if ($strFile eq '.' || $strFile eq '..');
+
+            # Make sure the link is in the expected manifest
+            my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"};
+
+            if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK ||
+                $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile)
+            {
+                confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id");
+            }
+
+            # Make sure the link really is a link
+            if ($hTablespaceManifest->{$strFile}{type} ne 'l')
+            {
+                confess &log(ERROR, "'${strFile}' in tablespace directory is not a link");
+            }
+
+            # Make sure the link destination is correct
+            my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}";
+
+            if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination)
+            {
+                confess &log(ERROR,
+                    "'${strFile}' link should reference '${strLinkDestination}' but actually references " .
+                    "'$hTablespaceManifest->{$strFile}{link_destination}'");
+            }
+        }
+
+        # Iterate manifest targets
+        for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}})))
+        {
+            my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget};
+            my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID};
+
+            # Make sure the target exists as a link on disk
+            if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) &&
+                !defined($hTablespaceManifest->{$strTablespaceId}))
+            {
+                confess &log(ERROR, "target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'");
+            }
+        }
+    }
+    # Else there should not be a tablespace directory at all
+    elsif ($self->{oFile}->exists(PATH_BACKUP_CLUSTER, "${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))
+    {
+        confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory');
+    }
+
+    # Only do compare for synthetic backups since for real backups the expected manifest *is* the actual manifest.
+    if ($self->synthetic())
     {
         # Set backup type in the expected manifest
         ${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TYPE} = $strType;
@@ -765,6 +833,7 @@ sub configCreate
     {
         if (defined($$oParam{bHardlink}) && $$oParam{bHardlink})
         {
+            $self->{bHardLink} = true;
             $oParamHash{&CONFIG_SECTION_GLOBAL . ':' . &CMD_BACKUP}{&OPTION_HARDLINK} = 'y';
         }
 
@@ -1024,6 +1093,7 @@ sub infoRestore
 sub backrestConfig {return shift->paramGet(HOST_PARAM_BACKREST_CONFIG);}
 sub backupDestination {return shift->{strBackupDestination};}
 sub backrestExe {return shift->paramGet(HOST_PARAM_BACKREST_EXE);}
+sub hardLink {return shift->{bHardLink};}
 sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet();}
 sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER;}
 sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY;}