db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-09 00:45:49 +02:00
Code Issues Releases Activity

Added redacted manifest to testBackupValidate().

Browse Source 
The manifest is excellent for validation but including the entire manifest is too noisy and some values are architecture/algorithm dependent.

Output a redacted version that contains the most important information which can be improved on over time.
This commit is contained in:
David Steele
2020-03-18 10:10:10 -04:00
parent b8cd1b6790
commit 4ec04e5163
2 changed files with 263 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

298
test/src/module/command/backupTest.c
View File

@ -5,6 +5,7 @@ Test Backup Command
#include "command/stanza/create.h"
#include "command/stanza/upgrade.h"
#include "common/crypto/hash.h"
#include "common/io/bufferRead.h"
#include "common/io/bufferWrite.h"
#include "common/io/io.h"
@ -16,13 +17,14 @@ Test Backup Command
#include "common/harnessPq.h"
/***********************************************************************************************************************************
Get a list of all files in the backup
Get a list of all files in the backup and a redacted version of the manifest that can be tested against a static string
***********************************************************************************************************************************/
typedef struct TestBackupValidateCallbackData
{
const Storage *storage; // Storage object when needed (e.g. fileCompressed = true)
const String *path; // Subpath when storage is specified
const Manifest *manifest; // Manifest check for files/links/paths
const Manifest *manifest; // Manifest to check for files/links/paths
const ManifestData *manifestData; // Manifest data
String *content; // String where content should be added
} TestBackupValidateCallbackData;
@ -40,6 +42,9 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
(strEqZ(info->name, BACKUP_MANIFEST_FILE) || strEqZ(info->name, BACKUP_MANIFEST_FILE INFO_COPY_EXT)))
return;
// Get manifest name
const String *manifestName = info->name;
strCatFmt(data->content, "%s {", strPtr(info->name));
switch (info->type)
@ -48,36 +53,62 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
{
strCat(data->content, "file");
uint64_t size = info->size;
const String *manifestName = info->name;
// Calculate checksum/size and decompress if needed
// ---------------------------------------------------------------------------------------------------------------------
StorageRead *read = storageNewReadP(
data->storage,
data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name);
// If the file is compressed then decompress to get the real size
CompressType compressType = compressTypeFromName(info->name);
if (compressType != compressTypeNone)
if (data->manifestData->backupOptionCompressType != compressTypeNone)
{
ASSERT(data->storage != NULL);
StorageRead *read = storageNewReadP(
data->storage,
data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name);
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), decompressFilter(compressType));
size = bufUsed(storageGetP(read));
manifestName = strSubN(info->name, 0, strSize(info->name) - strSize(compressExtStr(compressType)));
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)), decompressFilter(data->manifestData->backupOptionCompressType));
manifestName = strSubN(
info->name, 0, strSize(info->name) - strSize(compressExtStr(data->manifestData->backupOptionCompressType)));
}
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(HASH_TYPE_SHA1_STR));
uint64_t size = bufUsed(storageGetP(read));
const String *checksum = varStr(
ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE_STR));
strCatFmt(data->content, ", s=%" PRIu64, size);
// Check against the manifest
// ---------------------------------------------------------------------------------------------------------------------
const ManifestFile *file = manifestFileFind(data->manifest, manifestName);
// Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even the same
// compression algorithm can give slightly different results based on the version so repo-size is not deterministic for
// compression.
if (size != file->size)
THROW_FMT(AssertError, "'%s' size does match manifest", strPtr(manifestName));
if (info->size != file->sizeRepo)
{
THROW_FMT(AssertError, "'%s' repo size does match manifest", strPtr(manifestName));
}
if (data->manifestData->backupOptionCompressType != compressTypeNone)
((ManifestFile *)file)->sizeRepo = file->size;
// Test the checksum. pg_control and WAL headers have different checksums depending on cpu architecture so remove
// the checksum from the test output.
if (!strEqZ(checksum, file->checksumSha1))
{
THROW_FMT(AssertError, "'%s' checksum does match manifest", strPtr(manifestName));
}
if (strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) ||
strBeginsWith(
manifestName, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strPtr(pgWalPath(data->manifestData->pgVersion)))))
{
((ManifestFile *)file)->checksumSha1[0] = '\0';
}
// Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
// mode and current user/group.
if (info->mode != 0640)
THROW_FMT(AssertError, "'%s' mode is not 0640", strPtr(manifestName));
@ -101,8 +132,11 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
strCat(data->content, "path");
// Check against the manifest
// ---------------------------------------------------------------------------------------------------------------------
manifestPathFind(data->manifest, info->name);
// Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
// mode and current user/group.
if (info->mode != 0750)
THROW_FMT(AssertError, "'%s' mode is not 00750", strPtr(info->name));
@ -137,14 +171,8 @@ testBackupValidate(const Storage *storage, const String *path)
MEM_CONTEXT_TEMP_BEGIN()
{
// Make sure both backup.manifest files exist
if (!storageExistsP(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE, strPtr(path))))
THROW(AssertError, BACKUP_MANIFEST_FILE " is missing");
if (!storageExistsP(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strPtr(path))))
THROW(AssertError, BACKUP_MANIFEST_FILE INFO_COPY_EXT " is missing");
// Build a list of files in the backup path and verify against the manifest
// -------------------------------------------------------------------------------------------------------------------------
Manifest *manifest = manifestLoadFile(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE, strPtr(path)), cipherTypeNone, NULL);
TestBackupValidateCallbackData callbackData =
@ -153,9 +181,57 @@ testBackupValidate(const Storage *storage, const String *path)
.path = path,
.content = result,
.manifest = manifest,
.manifestData = manifestData(manifest),
};
storageInfoListP(storage, path, testBackupValidateCallback, &callbackData, .recurse = true, .sortOrder = sortOrderAsc);
// Make sure both backup.manifest files exist since we skipped them in the callback above
if (!storageExistsP(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE, strPtr(path))))
THROW(AssertError, BACKUP_MANIFEST_FILE " is missing");
if (!storageExistsP(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE INFO_COPY_EXT, strPtr(path))))
THROW(AssertError, BACKUP_MANIFEST_FILE INFO_COPY_EXT " is missing");
// Output the manifest to a string and exclude sections that don't need validation. Note that each of these sections should
// be considered from automatic validation but adding them to the output will make the tests too noisy. One good technique
// would be to remove it from the output only after validation so new values will cause changes in the output.
// -------------------------------------------------------------------------------------------------------------------------
Buffer *manifestSaveBuffer = bufNew(0);
manifestSave(manifest, ioBufferWriteNew(manifestSaveBuffer));
String *manifestEdit = strNew("");
StringList *manifestLine = strLstNewSplitZ(strTrim(strNewBuf(manifestSaveBuffer)), "\n");
bool bSkipSection = false;
for (unsigned int lineIdx = 0; lineIdx < strLstSize(manifestLine); lineIdx++)
{
const String *line = strTrim(strLstGet(manifestLine, lineIdx));
if (strChr(line, '[') == 0)
{
const String *section = strSubN(line, 1, strSize(line) - 2);
if (strEq(section, INFO_SECTION_BACKREST_STR) ||
strEq(section, MANIFEST_SECTION_BACKUP_STR) ||
strEq(section, MANIFEST_SECTION_BACKUP_DB_STR) ||
strEq(section, MANIFEST_SECTION_BACKUP_OPTION_STR) ||
strEq(section, MANIFEST_SECTION_DB_STR) ||
strEq(section, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR) ||
strEq(section, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR) ||
strEq(section, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR))
{
bSkipSection = true;
}
else
bSkipSection = false;
}
if (!bSkipSection)
strCatFmt(manifestEdit, "%s\n", strPtr(line));
}
strCatFmt(result, "--------\n%s\n", strPtr(strTrim(manifestEdit)));
}
MEM_CONTEXT_TEMP_END();
@ -1579,7 +1655,7 @@ testRun(void)
"P00 INFO: backup stop archive = 0000000105D944C000000000, lsn = 5d944c0/800000\n"
"P00 INFO: new backup label = 20191002-070640F");
TEST_RESULT_STR_Z(
TEST_RESULT_STR_Z_KEYRPL(
testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
". {link, d=20191002-070640F}\n"
"pg_data {path}\n"
@ -1587,7 +1663,22 @@ testRun(void)
"pg_data/global {path}\n"
"pg_data/global/pg_control {file, s=8192}\n"
"pg_data/pg_xlog {path}\n"
"pg_data/postgresql.conf {file, s=11}\n",
"pg_data/postgresql.conf {file, s=11}\n"
"--------\n"
"[backup:target]\n"
"pg_data={\"path\":\"{[path]}/pg1\",\"type\":\"path\"}\n"
"\n"
"[target:file]\n"
"pg_data/PG_VERSION={\"checksum\":\"06d06bb31b570b94d7b4325f511f853dbe771c21\",\"size\":3"
",\"timestamp\":1570000000}\n"
"pg_data/global/pg_control={\"size\":8192,\"timestamp\":1570000000}\n"
"pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11"
",\"timestamp\":1570000000}\n"
"\n"
"[target:path]\n"
"pg_data={}\n"
"pg_data/global={}\n"
"pg_data/pg_xlog={}\n",
"compare file list");
}
@ -1736,7 +1827,7 @@ testRun(void)
"P00 INFO: check archive for segment(s) 0000000105D95D3000000000:0000000105D95D3000000000\n"
"P00 INFO: new backup label = 20191003-105320F");
TEST_RESULT_STR_Z(
TEST_RESULT_STR_Z_KEYRPL(
testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
". {link, d=20191003-105320F}\n"
"pg_data {path}\n"
@ -1749,7 +1840,30 @@ testRun(void)
"pg_data/postgresql.conf.gz {file, s=11}\n"
"pg_data/size-mismatch.gz {file, s=4}\n"
"pg_data/time-mismatch.gz {file, s=4}\n"
"pg_data/zero-size.gz {file, s=0}\n",
"pg_data/zero-size.gz {file, s=0}\n"
"--------\n"
"[backup:target]\n"
"pg_data={\"path\":\"{[path]}/pg1\",\"type\":\"path\"}\n"
"\n"
"[target:file]\n"
"pg_data/PG_VERSION={\"checksum\":\"06d06bb31b570b94d7b4325f511f853dbe771c21\",\"size\":3"
",\"timestamp\":1570000000}\n"
"pg_data/global/pg_control={\"size\":8192,\"timestamp\":1570000000}\n"
"pg_data/not-in-resume={\"checksum\":\"984816fd329622876e14907634264e6f332e9fb3\",\"size\":4"
",\"timestamp\":1570100000}\n"
"pg_data/pg_xlog/0000000105D95D3000000000={\"size\":16777216,\"timestamp\":1570100002}\n"
"pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11"
",\"timestamp\":1570000000}\n"
"pg_data/size-mismatch={\"checksum\":\"984816fd329622876e14907634264e6f332e9fb3\",\"size\":4"
",\"timestamp\":1570100000}\n"
"pg_data/time-mismatch={\"checksum\":\"984816fd329622876e14907634264e6f332e9fb3\",\"size\":4"
",\"timestamp\":1570100000}\n"
"pg_data/zero-size={\"size\":0,\"timestamp\":1570100000}\n"
"\n"
"[target:path]\n"
"pg_data={}\n"
"pg_data/global={}\n"
"pg_data/pg_xlog={}\n",
"compare file list");
// Remove test files
@ -1876,7 +1990,7 @@ testRun(void)
"P00 INFO: new backup label = 20191003-105320F_20191004-144000D");
// Check repo directory
TEST_RESULT_STR_Z(
TEST_RESULT_STR_Z_KEYRPL(
testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
". {link, d=20191003-105320F_20191004-144000D}\n"
"pg_data {path}\n"
@ -1886,7 +2000,25 @@ testRun(void)
"pg_data/pg_xlog {path}\n"
"pg_data/postgresql.conf.gz {file, s=11}\n"
"pg_data/resume-ref.gz {file, s=0}\n"
"pg_data/time-mismatch2.gz {file, s=4}\n",
"pg_data/time-mismatch2.gz {file, s=4}\n"
"--------\n"
"[backup:target]\n"
"pg_data={\"path\":\"{[path]}/pg1\",\"type\":\"path\"}\n"
"\n"
"[target:file]\n"
"pg_data/PG_VERSION={\"checksum\":\"06d06bb31b570b94d7b4325f511f853dbe771c21\",\"reference\":\"20191003-105320F\""
",\"size\":3,\"timestamp\":1570000000}\n"
"pg_data/global/pg_control={\"reference\":\"20191003-105320F\",\"size\":8192,\"timestamp\":1570000000}\n"
"pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\""
",\"reference\":\"20191003-105320F\",\"size\":11,\"timestamp\":1570000000}\n"
"pg_data/resume-ref={\"size\":0,\"timestamp\":1570200000}\n"
"pg_data/time-mismatch2={\"checksum\":\"984816fd329622876e14907634264e6f332e9fb3\",\"size\":4"
",\"timestamp\":1570200100}\n"
"\n"
"[target:path]\n"
"pg_data={}\n"
"pg_data/global={}\n"
"pg_data/pg_xlog={}\n",
"compare file list");
// Remove test files
@ -1974,7 +2106,7 @@ testRun(void)
TEST_RESULT_LOG(
"P00 WARN: no prior backup exists, incr backup has been changed to full");
TEST_RESULT_STR_Z(
TEST_RESULT_STR_Z_KEYRPL(
testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
". {link, d=20191016-042640F}\n"
"pg_data {path}\n"
@ -1988,7 +2120,31 @@ testRun(void)
"pg_data/global/pg_control {file, s=8192}\n"
"pg_data/pg_xlog {path}\n"
"pg_data/pg_xlog/0000000105DA69C000000000 {file, s=16777216}\n"
"pg_data/postgresql.conf {file, s=11}\n",
"pg_data/postgresql.conf {file, s=11}\n"
"--------\n"
"[backup:target]\n"
"pg_data={\"path\":\"{[path]}/pg1\",\"type\":\"path\"}\n"
"\n"
"[target:file]\n"
"pg_data/PG_VERSION={\"checksum\":\"f5b7e6d36dc0113f61b36c700817d42b96f7b037\",\"size\":3"
",\"timestamp\":1571200000}\n"
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
",\"timestamp\":1571200002}\n"
"pg_data/base/1/1={\"checksum\":\"580393f5a94fb469585f5dd2a6859a4aab899f37\",\"master\":false,\"size\":4"
",\"timestamp\":1571200000}\n"
"pg_data/base/1/2={\"checksum\":\"580393f5a94fb469585f5dd2a6859a4aab899f37\",\"master\":false,\"size\":4"
",\"timestamp\":1571200000}\n"
"pg_data/global/pg_control={\"size\":8192,\"timestamp\":1571200000}\n"
"pg_data/pg_xlog/0000000105DA69C000000000={\"size\":16777216,\"timestamp\":1571200002}\n"
"pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11"
",\"timestamp\":1570000000}\n"
"\n"
"[target:path]\n"
"pg_data={}\n"
"pg_data/base={}\n"
"pg_data/base/1={}\n"
"pg_data/global={}\n"
"pg_data/pg_xlog={}\n",
"compare file list");
// Remove test files
@ -2129,7 +2285,7 @@ testRun(void)
"P00 INFO: check archive for segment(s) 0000000105DB5DE000000000:0000000105DB5DE000000002\n"
"P00 INFO: new backup label = 20191027-181320F");
TEST_RESULT_STR_Z(
TEST_RESULT_STR_Z_KEYRPL(
testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/20191027-181320F")),
"pg_data {path}\n"
"pg_data/PG_VERSION.gz {file, s=2}\n"
@ -2152,7 +2308,48 @@ testRun(void)
"pg_tblspc/32768 {path}\n"
"pg_tblspc/32768/PG_11_201809051 {path}\n"
"pg_tblspc/32768/PG_11_201809051/1 {path}\n"
"pg_tblspc/32768/PG_11_201809051/1/5.gz {file, s=0}\n",
"pg_tblspc/32768/PG_11_201809051/1/5.gz {file, s=0}\n"
"--------\n"
"[backup:target]\n"
"pg_data={\"path\":\"{[path]}/pg1\",\"type\":\"path\"}\n"
"pg_tblspc/32768={\"path\":\"../../pg1-tblspc/32768\",\"tablespace-id\":\"32768\",\"tablespace-name\":\"tblspc32768\",\"type\":\"link\"}\n"
"\n"
"[target:file]\n"
"pg_data/PG_VERSION={\"checksum\":\"17ba0791499db908433b80f37c5fbc89b870084b\",\"size\":2"
",\"timestamp\":1572200000}\n"
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
",\"timestamp\":1572200002}\n"
"pg_data/base/1/1={\"checksum\":\"0631457264ff7f8d5fb1edc2c0211992a67c73e6\",\"checksum-page\":true"
",\"master\":false,\"size\":8192,\"timestamp\":1572200000}\n"
"pg_data/base/1/2={\"checksum\":\"8beb58e08394fe665fb04a17b4003faa3802760b\",\"checksum-page\":false"
",\"master\":false,\"size\":8193,\"timestamp\":1572200000}\n"
"pg_data/base/1/3={\"checksum\":\"73e537a445ad34eab4b292ac6aa07b8ce14e8421\",\"checksum-page\":false"
",\"checksum-page-error\":[0,[2,3]],\"master\":false,\"size\":32768,\"timestamp\":1572200000}\n"
"pg_data/base/1/4={\"checksum\":\"ba233be7198b3115f0480fa5274448f2a2fc2af1\",\"checksum-page\":false"
",\"checksum-page-error\":[1],\"master\":false,\"size\":24576,\"timestamp\":1572200000}\n"
"pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572200000}\n"
"pg_data/pg_wal/0000000105DB5DE000000000={\"size\":1048576,\"timestamp\":1572200002}\n"
"pg_data/pg_wal/0000000105DB5DE000000001={\"size\":1048576,\"timestamp\":1572200002}\n"
"pg_data/pg_wal/0000000105DB5DE000000002={\"size\":1048576,\"timestamp\":1572200002}\n"
"pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\",\"size\":11"
",\"timestamp\":1570000000}\n"
"pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"master\":false,\"size\":0"
",\"timestamp\":1572200000}\n"
"\n"
"[target:link]\n"
"pg_data/pg_tblspc/32768={\"destination\":\"../../pg1-tblspc/32768\"}\n"
"\n"
"[target:path]\n"
"pg_data={}\n"
"pg_data/base={}\n"
"pg_data/base/1={}\n"
"pg_data/global={}\n"
"pg_data/pg_tblspc={}\n"
"pg_data/pg_wal={}\n"
"pg_tblspc={}\n"
"pg_tblspc/32768={}\n"
"pg_tblspc/32768/PG_11_201809051={}\n"
"pg_tblspc/32768/PG_11_201809051/1={}\n",
"compare file list");
// Remove test files
@ -2240,7 +2437,7 @@ testRun(void)
"P00 INFO: check archive for segment(s) 0000002C05DB8EB000000000:0000002C05DB8EB000000000\n"
"P00 INFO: new backup label = 20191027-181320F_20191030-014640I");
TEST_RESULT_STR_Z(
TEST_RESULT_STR_Z_KEYRPL(
testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
". {link, d=20191027-181320F_20191030-014640I}\n"
"pg_data {path}\n"
@ -2257,7 +2454,36 @@ testRun(void)
"pg_tblspc/32768 {path}\n"
"pg_tblspc/32768/PG_11_201809051 {path}\n"
"pg_tblspc/32768/PG_11_201809051/1 {path}\n"
"pg_tblspc/32768/PG_11_201809051/1/5.gz {file, s=0}\n",
"pg_tblspc/32768/PG_11_201809051/1/5.gz {file, s=0}\n"
"--------\n"
"[backup:target]\n"
"pg_data={\"path\":\"{[path]}/pg1\",\"type\":\"path\"}\n"
"pg_tblspc/32768={\"path\":\"../../pg1-tblspc/32768\",\"tablespace-id\":\"32768\",\"tablespace-name\":\"tblspc32768\",\"type\":\"link\"}\n"
"\n"
"[target:file]\n"
"pg_data/PG_VERSION={\"checksum\":\"17ba0791499db908433b80f37c5fbc89b870084b\",\"reference\":\"20191027-181320F\""
",\"size\":2,\"timestamp\":1572200000}\n"
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
",\"timestamp\":1572400002}\n"
"pg_data/global/pg_control={\"reference\":\"20191027-181320F\",\"size\":8192,\"timestamp\":1572400000}\n"
"pg_data/postgresql.conf={\"checksum\":\"e3db315c260e79211b7b52587123b7aa060f30ab\""
",\"reference\":\"20191027-181320F\",\"size\":11,\"timestamp\":1570000000}\n"
"pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"master\":false,\"reference\":\"20191027-181320F\""
",\"size\":0,\"timestamp\":1572200000}\n"
"\n"
"[target:link]\n"
"pg_data/pg_tblspc/32768={\"destination\":\"../../pg1-tblspc/32768\"}\n"
"\n"
"[target:path]\n"
"pg_data={}\n"
"pg_data/base={}\n"
"pg_data/global={}\n"
"pg_data/pg_tblspc={}\n"
"pg_data/pg_wal={}\n"
"pg_tblspc={}\n"
"pg_tblspc/32768={}\n"
"pg_tblspc/32768/PG_11_201809051={}\n"
"pg_tblspc/32768/PG_11_201809051/1={}\n",
"compare file list");
// Remove test files