diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 54b679460..709e5c612 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -16,7 +16,7 @@ -

Enable S3 storage for archive-get command in C.

+

Enable S3 storage and encryption for archive-get command in C.

 diff --git a/src/Makefile b/src/Makefile index 3fee1689a..2ffa50d76 100644 --- a/src/Makefile +++ b/src/Makefile @@ -165,10 +165,10 @@ install: pgbackrest command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h $(CC) $(CFLAGS) -c command/archive/common.c -o command/archive/common.o -command/archive/get/file.o: command/archive/get/file.c command/archive/common.h command/archive/get/file.h command/control/control.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h compress/gzip.h compress/gzipDecompress.h config/config.auto.h config/config.h config/define.auto.h config/define.h crypto/crypto.h info/infoArchive.h info/infoPg.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h +command/archive/get/file.o: command/archive/get/file.c command/archive/common.h command/archive/get/file.h command/control/control.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h compress/gzip.h compress/gzipDecompress.h config/config.auto.h config/config.h config/define.auto.h config/define.h crypto/cipherBlock.h crypto/crypto.h info/infoArchive.h info/infoPg.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h $(CC) $(CFLAGS) -c command/archive/get/file.c -o command/archive/get/file.o -command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/archive/get/file.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h +command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/archive/get/file.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h crypto/crypto.h perl/exec.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h $(CC) $(CFLAGS) -c command/archive/get/get.c -o command/archive/get/get.o command/archive/push/push.o: command/archive/push/push.c command/archive/common.h command/command.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h diff --git a/src/command/archive/get/file.c b/src/command/archive/get/file.c index b08f3a177..497cade89 100644 --- a/src/command/archive/get/file.c +++ b/src/command/archive/get/file.c @@ -10,7 +10,7 @@ Archive Get File #include "compress/gzip.h" #include "compress/gzipDecompress.h" #include "config/config.h" -#include "crypto/crypto.h" +#include "crypto/cipherBlock.h" #include "info/infoArchive.h" #include "postgres/interface.h" #include "storage/helper.h" @@ -19,16 +19,29 @@ Archive Get File /*********************************************************************************************************************************** Check if a WAL file exists in the repository ***********************************************************************************************************************************/ -static String * -archiveGetCheck(const String *archiveFile) +#define FUNCTION_DEBUG_ARCHIVE_GET_CHECK_RESULT_TYPE \ + ArchiveGetCheckResult +#define FUNCTION_DEBUG_ARCHIVE_GET_CHECK_RESULT_FORMAT(value, buffer, bufferSize) \ + objToLog(&value, "ArchiveGetCheckResult", buffer, bufferSize) + +typedef struct ArchiveGetCheckResult +{ + String *archiveFileActual; + String *cipherPass; +} ArchiveGetCheckResult; + +ArchiveGetCheckResult +archiveGetCheck(const String *archiveFile, CipherType cipherType, const String *cipherPass) { FUNCTION_DEBUG_BEGIN(logLevelDebug); FUNCTION_DEBUG_PARAM(STRING, archiveFile); + FUNCTION_DEBUG_PARAM(ENUM, cipherType); + // cipherPass omitted for security FUNCTION_TEST_ASSERT(archiveFile != NULL); FUNCTION_DEBUG_END(); - String *result = NULL; + ArchiveGetCheckResult result = {0}; MEM_CONTEXT_TEMP_BEGIN() { @@ -37,7 +50,7 @@ archiveGetCheck(const String *archiveFile) // Attempt to load the archive info file InfoArchive *info = infoArchiveNew( - storageRepo(), STRING_CONST(STORAGE_REPO_ARCHIVE "/" INFO_ARCHIVE_FILE), false, cipherTypeNone, NULL); + storageRepo(), STRING_CONST(STORAGE_REPO_ARCHIVE "/" INFO_ARCHIVE_FILE), false, cipherType, cipherPass); // Loop through the pg history in case the WAL we need is not in the most recent archive id String *archiveId = NULL; @@ -85,23 +98,27 @@ archiveGetCheck(const String *archiveFile) if (archiveFileActual != NULL) { memContextSwitch(MEM_CONTEXT_OLD()); - result = strNewFmt("%s/%s", strPtr(archiveId), strPtr(archiveFileActual)); + result.archiveFileActual = strNewFmt("%s/%s", strPtr(archiveId), strPtr(archiveFileActual)); + result.cipherPass = strDup(infoArchiveCipherPass(info)); + memContextSwitch(MEM_CONTEXT_TEMP()); } } MEM_CONTEXT_TEMP_END(); - FUNCTION_DEBUG_RESULT(STRING, result); + FUNCTION_DEBUG_RESULT(ARCHIVE_GET_CHECK_RESULT, result); } /*********************************************************************************************************************************** Copy a file from the archive to the specified destination ***********************************************************************************************************************************/ int -archiveGetFile(const String *archiveFile, const String *walDestination) +archiveGetFile(const String *archiveFile, const String *walDestination, CipherType cipherType, const String *cipherPass) { FUNCTION_DEBUG_BEGIN(logLevelDebug); FUNCTION_DEBUG_PARAM(STRING, archiveFile); FUNCTION_DEBUG_PARAM(STRING, walDestination); + FUNCTION_DEBUG_PARAM(ENUM, cipherType); + // cipherPass omitted for security FUNCTION_TEST_ASSERT(archiveFile != NULL); FUNCTION_TEST_ASSERT(walDestination != NULL); @@ -116,25 +133,39 @@ archiveGetFile(const String *archiveFile, const String *walDestination) MEM_CONTEXT_TEMP_BEGIN() { // Make sure the file exists and other checks pass - String *archiveFileActual = archiveGetCheck(archiveFile); + ArchiveGetCheckResult archiveGetCheckResult = archiveGetCheck(archiveFile, cipherType, cipherPass); - if (archiveFileActual != NULL) + if (archiveGetCheckResult.archiveFileActual != NULL) { StorageFileWrite *destination = storageNewWriteP( storageLocalWrite(), walDestination, .noCreatePath = true, .noSyncFile = true, .noSyncPath = true, .noAtomic = true); - // If file is gzipped then add the decompression filter - if (strEndsWithZ(archiveFileActual, "." GZIP_EXT)) + // Add filters + IoFilterGroup *filterGroup = ioFilterGroupNew(); + + // If there is a cipher then add the decrypt filter + if (cipherType != cipherTypeNone) { - IoFilterGroup *filterGroup = ioFilterGroupNew(); - ioFilterGroupAdd(filterGroup, gzipDecompressFilter(gzipDecompressNew(false))); - ioWriteFilterGroupSet(storageFileWriteIo(destination), filterGroup); + ioFilterGroupAdd( + filterGroup, + cipherBlockFilter( + cipherBlockNew(cipherModeDecrypt, cipherType, bufNewStr(archiveGetCheckResult.cipherPass), NULL))); } + // If file is gzipped then add the decompression filter + if (strEndsWithZ(archiveGetCheckResult.archiveFileActual, "." GZIP_EXT)) + { + ioFilterGroupAdd(filterGroup, gzipDecompressFilter(gzipDecompressNew(false))); + } + + ioWriteFilterGroupSet(storageFileWriteIo(destination), filterGroup); + // Copy the file storageCopyNP( - storageNewReadNP(storageRepo(), strNewFmt("%s/%s", STORAGE_REPO_ARCHIVE, strPtr(archiveFileActual))), destination); + storageNewReadNP( + storageRepo(), strNewFmt("%s/%s", STORAGE_REPO_ARCHIVE, strPtr(archiveGetCheckResult.archiveFileActual))), + destination); // The WAL file was found result = 0; diff --git a/src/command/archive/get/file.h b/src/command/archive/get/file.h index a2819ce7f..c5717ccf9 100644 --- a/src/command/archive/get/file.h +++ b/src/command/archive/get/file.h @@ -5,10 +5,11 @@ Archive Get File #define COMMAND_ARCHIVE_GET_FILE_H #include "common/type/string.h" +#include "crypto/crypto.h" /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ -int archiveGetFile(const String *archiveFile, const String *walDestination); +int archiveGetFile(const String *archiveFile, const String *walDestination, CipherType cipherType, const String *cipherPass); #endif diff --git a/src/command/archive/get/get.c b/src/command/archive/get/get.c index 2cf61ae2a..8927e622f 100644 --- a/src/command/archive/get/get.c +++ b/src/command/archive/get/get.c @@ -271,14 +271,15 @@ cmdArchiveGet(void) // Disable async if it was enabled cfgOptionSet(cfgOptArchiveAsync, cfgOptionSource(cfgOptArchiveAsync), varNewBool(false)); - // If repo server is not remote and not encrypted then this can be done entirely in C - if (!cfgOptionTest(cfgOptRepoHost) && strEqZ(cfgOptionStr(cfgOptRepoCipherType), "none")) + // If repo server is not remote then this can be done entirely in C + if (!cfgOptionTest(cfgOptRepoHost)) // {uncovered - Perl code is covered in unit tests} { - result = archiveGetFile(walSegment, walDestination); + result = archiveGetFile( + walSegment, walDestination, cipherType(cfgOptionStr(cfgOptRepoCipherType)), cfgOptionStr(cfgOptRepoCipherPass)); } // Else do it in Perl else - result = perlExec(); // {uncovered - Perl code is covered in unit tests} + result = perlExec(); // {+uncovered} } // Log whether or not the file was found diff --git a/test/expect/mock-archive-001.log b/test/expect/mock-archive-001.log index 5eaf1b3f7..8164c4a34 100644 --- a/test/expect/mock-archive-001.log +++ b/test/expect/mock-archive-001.log @@ -1,9 +1,9 @@ -run 001 - rmt 0, s3 0, enc 0 +run 001 - rmt 0, s3 0, enc 1 ============================ > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/archive/db/archive.info or [TEST_PATH]/db-master/repo/archive/db/archive.info.copy P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL segments HINT: is archive_command configured in postgresql.conf? @@ -13,7 +13,7 @@ P00 INFO: archive-push command end: aborted with exception [055] > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/archive/db/archive.info or [TEST_PATH]/db-master/repo/archive/db/archive.info.copy HINT: archive.info does not exist but is required to push/get WAL segments. HINT: is archive_command configured in postgresql.conf? @@ -24,7 +24,7 @@ P00 INFO: archive-get command end: aborted with exception [055] stanza-create db - stanza create (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: stanza-create command end: completed successfully + supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info @@ -34,6 +34,9 @@ backrest-checksum="[CHECKSUM]" backrest-format=5 backrest-version="[VERSION-1]" +[cipher] +cipher-pass=[CIPHER-PASS-1] + [db] db-catalog-version=201409291 db-control-version=942 @@ -51,6 +54,9 @@ backrest-checksum="[CHECKSUM]" backrest-format=5 backrest-version="[VERSION-1]" +[cipher] +cipher-pass=[CIPHER-PASS-2] + [db] db-id=1 db-system-id=1000000000000000094 @@ -61,7 +67,7 @@ db-version="9.4" > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress --log-level-console=debug [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: perl/exec::perlExec: (void) P00 DEBUG: Archive::Push::Push->process(): strWalPathFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 @@ -69,7 +75,7 @@ P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false P00 DEBUG: Archive::Push::File::archivePushFile(): bCompress = true, iCompressLevel = 3, strWalFile = 000000010000000100000001, strWalPath = [TEST_PATH]/db-master/db/base/pg_xlog P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = -P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp +P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = , strCipherType = aes-256-cbc, strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp P00 DEBUG: Archive::Common::walInfo(): strWalFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 P00 DEBUG: Archive::Common::walInfo=>: strDbVersion = 9.4, ullDbSysId = 1000000000000000094 P00 DEBUG: Archive::Push::File::archivePushCheck(): strArchiveFile = 000000010000000100000001, strDbVersion = 9.4, strWalFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, ullDbSysId = 1000000000000000094 @@ -77,10 +83,10 @@ P00 DEBUG: Archive::Info->new(): bIgnoreMissing = , bLoad = , P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp P00 DEBUG: Storage::Local->encrypted(): bIgnoreMissing = true, strFileName = [TEST_PATH]/db-master/repo/archive/db/archive.info -P00 DEBUG: Storage::Local->encrypted=>: bEncrypted = false -P00 DEBUG: Storage::Local->encryptionValid(): bEncrypted = false +P00 DEBUG: Storage::Local->encrypted=>: bEncrypted = true +P00 DEBUG: Storage::Local->encryptionValid(): bEncrypted = true P00 DEBUG: Storage::Local->encryptionValid=>: bValid = true -P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], strCipherPass = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info +P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], strCipherPass = , xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info P00 DEBUG: Storage::Base->get(): strCipherPass = [undef], xFile = [object] P00 DEBUG: Archive::Info->check(): bRequired = , strDbVersion = 9.4, ullDbSysId = 1000000000000000094 P00 DEBUG: Archive::Info->archiveId(): strDbVersion = [undef], ullDbSysId = [undef] @@ -90,14 +96,14 @@ P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStor P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPathExp = /9.4-1/0000000100000001, strSortOrder = P00 DEBUG: Storage::Local->list=>: stryFileList = () P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = [undef] -P00 DEBUG: Archive::Push::File::archivePushCheck=>: strArchiveId = 9.4-1, strChecksum = [undef], strCipherPass = [undef], strWarning = [undef] +P00 DEBUG: Archive::Push::File::archivePushCheck=>: strArchiveId = 9.4-1, strChecksum = [undef], strCipherPass = , strWarning = [undef] P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp P00 DEBUG: Storage::Local->hashSize(): bIgnoreMissing = , xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = false, rhyFilter = [undef], strCipherPass = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 P00 DEBUG: Storage::Local->hashSize=>: lSize = 16777216, strHash = ceb021d9bb41f220511e413b095d2b0d89fec113 P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = , rhyFilter = ({rxyParam => ({iLevel => 3}), strClass => pgBackRest::Storage::Filter::Gzip}), strCipherPass = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 -P00 DEBUG: Storage::Local->openWrite(): bAtomic = true, bPathCreate = true, lTimestamp = [undef], rhyFilter = [undef], strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /9.4-1/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz +P00 DEBUG: Storage::Local->openWrite(): bAtomic = true, bPathCreate = true, lTimestamp = [undef], rhyFilter = [undef], strCipherPass = , strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = /9.4-1/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] P00 DEBUG: Archive::Push::File::archivePushFile=>: strWarning = [undef] P00 INFO: pushed WAL segment 000000010000000100000001 @@ -114,17 +120,17 @@ P00 DEBUG: main::main: => 0 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get --log-level-console=debug 700000007000000070000000 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-get command begin [BACKREST-VERSION]: [700000007000000070000000, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-get command begin [BACKREST-VERSION]: [700000007000000070000000, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) -P00 DEBUG: command/archive/get/file::archiveGetFile: (archiveFile: {"700000007000000070000000"}, walDestination: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}) +P00 DEBUG: command/archive/get/file::archiveGetFile: (archiveFile: {"700000007000000070000000"}, walDestination: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, cipherType: 1) P00 DEBUG: command/control/control::lockStopTest: (void) P00 DEBUG: storage/storage::storageExists: (this: {type: posix, path: {"/"}, write: false}, pathExp: {"[TEST_PATH]/db-master/lock/db.stop"}, param.timeout: 0) P00 DEBUG: storage/storage::storageExists: => false P00 DEBUG: storage/storage::storageExists: (this: {type: posix, path: {"/"}, write: false}, pathExp: {"[TEST_PATH]/db-master/lock/all.stop"}, param.timeout: 0) P00 DEBUG: storage/storage::storageExists: => false P00 DEBUG: command/control/control::lockStopTest: => void -P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"700000007000000070000000"}) +P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"700000007000000070000000"}, cipherType: 1) P00 DEBUG: postgres/interface::pgControlFromFile: (pgPath: {"[TEST_PATH]/db-master/db/base"}) P00 DEBUG: storage/storage::storageNewRead: (this: {type: posix, path: {"/"}, write: false}, fileExp: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, param.ignoreMissing: false, param.filterGroup: null) P00 DEBUG: storage/storage::storageNewRead: => {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false} @@ -133,9 +139,9 @@ P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: {version: 90400, systemId: 1000000000000000094, walSegmentSize: 16777216, pageChecksum: true} P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"[TEST_PATH]/db-master/repo"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *)) P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix} -P00 DEBUG: info/infoArchive::infoArchiveNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, ignoreMissing: false, cipherType: 0) -P00 DEBUG: info/infoPg::infoPgNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, type: 0, cipherType: 0) -P00 DEBUG: info/info::infoNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, cipherType: 0) +P00 DEBUG: info/infoArchive::infoArchiveNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, ignoreMissing: false, cipherType: 1) +P00 DEBUG: info/infoPg::infoPgNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, type: 0, cipherType: 1) +P00 DEBUG: info/info::infoNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, cipherType: 1) P00 DEBUG: info/info::infoNew: => {Info} P00 DEBUG: info/infoPg::infoPgNew: => {InfoPg} P00 DEBUG: info/infoArchive::infoArchiveNew: => {InfoArchive} @@ -143,7 +149,7 @@ P00 DEBUG: command/archive/common::walSegmentFind: (storage: {type: posix, P00 DEBUG: storage/storage::storageList: (this: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, pathExp: {"/9.4-1/7000000070000000"}, param.errorOnMissing: false, param.expression: {"^700000007000000070000000-[0-f]{40}(\.gz){0,1}$"}) P00 DEBUG: storage/storage::storageList: => null P00 DEBUG: command/archive/common::walSegmentFind: => null -P00 DEBUG: command/archive/get/file::archiveGetCheck: => null +P00 DEBUG: command/archive/get/file::archiveGetCheck: => {ArchiveGetCheckResult} P00 DEBUG: command/archive/get/file::archiveGetFile: => 1 P00 INFO: unable to find 700000007000000070000000 in the archive P00 DEBUG: command/archive/get/get::cmdArchiveGet: => 1 @@ -156,17 +162,17 @@ P00 DEBUG: main::main: => 1 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get --log-level-console=debug 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) -P00 DEBUG: command/archive/get/file::archiveGetFile: (archiveFile: {"000000010000000100000001"}, walDestination: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}) +P00 DEBUG: command/archive/get/file::archiveGetFile: (archiveFile: {"000000010000000100000001"}, walDestination: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, cipherType: 1) P00 DEBUG: command/control/control::lockStopTest: (void) P00 DEBUG: storage/storage::storageExists: (this: {type: posix, path: {"/"}, write: false}, pathExp: {"[TEST_PATH]/db-master/lock/db.stop"}, param.timeout: 0) P00 DEBUG: storage/storage::storageExists: => false P00 DEBUG: storage/storage::storageExists: (this: {type: posix, path: {"/"}, write: false}, pathExp: {"[TEST_PATH]/db-master/lock/all.stop"}, param.timeout: 0) P00 DEBUG: storage/storage::storageExists: => false P00 DEBUG: command/control/control::lockStopTest: => void -P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000001"}) +P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000001"}, cipherType: 1) P00 DEBUG: postgres/interface::pgControlFromFile: (pgPath: {"[TEST_PATH]/db-master/db/base"}) P00 DEBUG: storage/storage::storageNewRead: (this: {type: posix, path: {"/"}, write: false}, fileExp: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, param.ignoreMissing: false, param.filterGroup: null) P00 DEBUG: storage/storage::storageNewRead: => {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false} @@ -175,9 +181,9 @@ P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: {version: 90400, systemId: 1000000000000000094, walSegmentSize: 16777216, pageChecksum: true} P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"[TEST_PATH]/db-master/repo"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *)) P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix} -P00 DEBUG: info/infoArchive::infoArchiveNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, ignoreMissing: false, cipherType: 0) -P00 DEBUG: info/infoPg::infoPgNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, type: 0, cipherType: 0) -P00 DEBUG: info/info::infoNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, cipherType: 0) +P00 DEBUG: info/infoArchive::infoArchiveNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, ignoreMissing: false, cipherType: 1) +P00 DEBUG: info/infoPg::infoPgNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, type: 0, cipherType: 1) +P00 DEBUG: info/info::infoNew: (storage: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileName: {"/archive.info"}, cipherType: 1) P00 DEBUG: info/info::infoNew: => {Info} P00 DEBUG: info/infoPg::infoPgNew: => {InfoPg} P00 DEBUG: info/infoArchive::infoArchiveNew: => {InfoArchive} @@ -185,13 +191,15 @@ P00 DEBUG: command/archive/common::walSegmentFind: (storage: {type: posix, P00 DEBUG: storage/storage::storageList: (this: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, pathExp: {"/9.4-1/0000000100000001"}, param.errorOnMissing: false, param.expression: {"^000000010000000100000001-[0-f]{40}(\.gz){0,1}$"}) P00 DEBUG: storage/storage::storageList: => {["000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"]} P00 DEBUG: command/archive/common::walSegmentFind: => {"000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"} -P00 DEBUG: command/archive/get/file::archiveGetCheck: => {"9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"} +P00 DEBUG: command/archive/get/file::archiveGetCheck: => {ArchiveGetCheckResult} P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"/"}, modeFile: 0640, modePath: 0750, write: true, pathExpressionFunction: null) P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix} P00 DEBUG: storage/storage::storageNewWrite: (this: {type: posix, path: {"/"}, write: true}, fileExp: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, param.modeFile: 0000, param.modePath: 0000, param.noCreatePath: true, param.noSyncFile: true, param.noSyncPath: true, param.noAtomic: true, param.filterGroup: null) P00 DEBUG: storage/storage::storageNewWrite: => {type: posix, name: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, modeFile: 0640, modePath: 0750, createPath: false, syncFile: false, syncPath: false, atomic: false} P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: (this: {inputSame: false, done: true}, filter: {IoFilter}) P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: => {inputSame: false, done: true} +P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: (this: {inputSame: false, done: true}, filter: {IoFilter}) +P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: => {inputSame: false, done: true} P00 DEBUG: storage/storage::storageNewRead: (this: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, fileExp: {"/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, param.ignoreMissing: false, param.filterGroup: null) P00 DEBUG: storage/storage::storageNewRead: => {type: posix, name: {"[TEST_PATH]/db-master/repo/archive/db/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false} P00 DEBUG: storage/storage::storageCopy: (source: {type: posix, name: {"[TEST_PATH]/db-master/repo/archive/db/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false}, destination: {type: posix, name: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, modeFile: 0640, modePath: 0750, createPath: false, syncFile: false, syncPath: false, atomic: false}) @@ -208,69 +216,69 @@ P00 DEBUG: main::main: => 0 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db P00 INFO: pushed WAL segment 000000010000000100000002 asynchronously P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [044]: WAL segment version 9.4 does not match archive version 8.0 HINT: are you archiving to the correct stanza? P00 INFO: archive-push command end: aborted with exception [044] > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: found 000000010000000100000001 in the archive P00 INFO: archive-get command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [044]: unable to retrieve the archive id for database version '9.4' and system-id '1000000000000000094' P00 INFO: archive-get command end: aborted with exception [044] > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [044]: WAL segment system-id 1000000000000000094 does not match archive system-id 5000900090001855000 HINT: are you archiving to the correct stanza? P00 INFO: archive-push command end: aborted with exception [044] > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [044]: unable to retrieve the archive id for database version '9.4' and system-id '1000000000000000094' P00 INFO: archive-get command end: aborted with exception [044] stop db stanza (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db stop ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: stop command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: stop command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: stop command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [062]: stop file exists for stanza db P00 INFO: archive-push command end: aborted with exception [062] > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [062]: stop file exists for stanza db P00 INFO: archive-get command end: aborted with exception [062] start db stanza (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db start ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: start command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: start command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: start command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 WARN: WAL segment 000000010000000100000002 already exists in the archive with the same checksum HINT: this is valid in some recovery scenarios but may also indicate a problem. P00 INFO: pushed WAL segment 000000010000000100000002 @@ -278,25 +286,25 @@ P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [045]: WAL segment 000000010000000100000002 already exists in the archive P00 INFO: archive-push command end: aborted with exception [045] > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get --archive-async --repo-type=cifs --archive-timeout=5 000000010000000100000002 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --archive-async --archive-timeout=5 --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --repo1-type=cifs --spool-path=[TEST_PATH]/db-master/spool --stanza=db +P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --archive-async --archive-timeout=5 --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --repo1-type=cifs --spool-path=[TEST_PATH]/db-master/spool --stanza=db P00 INFO: found 000000010000000100000002 in the archive P00 INFO: archive-get command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: pushed WAL segment 000000010000000100000002.partial P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 WARN: WAL segment 000000010000000100000002.partial already exists in the archive with the same checksum HINT: this is valid in some recovery scenarios but may also indicate a problem. P00 INFO: pushed WAL segment 000000010000000100000002.partial @@ -304,6 +312,6 @@ P00 INFO: archive-push command end: completed successfully > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 ERROR: [045]: WAL segment 000000010000000100000002.partial already exists in the archive P00 INFO: archive-push command end: aborted with exception [045] diff --git a/test/expect/mock-archive-002.log b/test/expect/mock-archive-002.log index 90340f991..b5077935e 100644 --- a/test/expect/mock-archive-002.log +++ b/test/expect/mock-archive-002.log @@ -1,4 +1,4 @@ -run 002 - rmt 1, s3 0, enc 1 +run 002 - rmt 1, s3 0, enc 0 ============================ > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 @@ -22,7 +22,7 @@ P00 INFO: archive-get command end: aborted with exception [055] stanza-create db - stanza create (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create ------------------------------------------------------------------------------------------------------------------------------------ -P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-2] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/backup/repo --stanza=db +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --force --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-2] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/backup/repo --stanza=db P00 INFO: stanza-create command end: completed successfully + supplemental file: [TEST_PATH]/backup/repo/backup/db/backup.info @@ -32,9 +32,6 @@ backrest-checksum="[CHECKSUM]" backrest-format=5 backrest-version="[VERSION-1]" -[cipher] -cipher-pass=[CIPHER-PASS-1] - [db] db-catalog-version=201409291 db-control-version=942 @@ -52,9 +49,6 @@ backrest-checksum="[CHECKSUM]" backrest-format=5 backrest-version="[VERSION-1]" -[cipher] -cipher-pass=[CIPHER-PASS-2] - [db] db-id=1 db-system-id=1000000000000000094 @@ -84,7 +78,7 @@ P00 DEBUG: Archive::Common::walInfo=>: strDbVersion = 9.4, ullDbSysId = 100 P00 DEBUG: Archive::Push::File::archivePushCheck(): strArchiveFile = 000000010000000100000001, strDbVersion = 9.4, strWalFile = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001, ullDbSysId = 1000000000000000094 P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol -P00 DEBUG: Archive::Push::File::archivePushCheck=>: strArchiveId = 9.4-1, strChecksum = [undef], strCipherPass = , strWarning = [undef] +P00 DEBUG: Archive::Push::File::archivePushCheck=>: strArchiveId = 9.4-1, strChecksum = [undef], strCipherPass = [undef], strWarning = [undef] P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp P00 DEBUG: Storage::Local->hashSize(): bIgnoreMissing = , xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 @@ -133,7 +127,7 @@ P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 1000000000000000094 P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol -P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = [undef], strArchiveId = 9.4-1, strCipherPass = +P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = [undef], strArchiveId = 9.4-1, strCipherPass = [undef] P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 1 P00 DEBUG: perl/exec::perlExec: => 1 P00 INFO: unable to find 700000007000000070000000 in the archive @@ -175,7 +169,7 @@ P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 1000000000000000094 P00 DEBUG: Protocol::Helper::protocolGet(): bCache = , iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = , strRemoteType = backup P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol -P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz, strArchiveId = 9.4-1, strCipherPass = +P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz, strArchiveId = 9.4-1, strCipherPass = [undef] P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = /9.4-1/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = , bPathSync = P00 DEBUG: Storage::Local->new(): bAllowTemp = , hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp diff --git a/test/expect/mock-stanza-001.log b/test/expect/mock-stanza-001.log index 4a16708c5..33d111419 100644 --- a/test/expect/mock-stanza-001.log +++ b/test/expect/mock-stanza-001.log @@ -534,14 +534,14 @@ db-version="9.4" P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) -P00 DEBUG: command/archive/get/file::archiveGetFile: (archiveFile: {"000000010000000100000002"}, walDestination: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}) +P00 DEBUG: command/archive/get/file::archiveGetFile: (archiveFile: {"000000010000000100000002"}, walDestination: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, cipherType: 0) P00 DEBUG: command/control/control::lockStopTest: (void) P00 DEBUG: storage/storage::storageExists: (this: {type: posix, path: {"/"}, write: false}, pathExp: {"[TEST_PATH]/db-master/lock/db.stop"}, param.timeout: 0) P00 DEBUG: storage/storage::storageExists: => false P00 DEBUG: storage/storage::storageExists: (this: {type: posix, path: {"/"}, write: false}, pathExp: {"[TEST_PATH]/db-master/lock/all.stop"}, param.timeout: 0) P00 DEBUG: storage/storage::storageExists: => false P00 DEBUG: command/control/control::lockStopTest: => void -P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000002"}) +P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000002"}, cipherType: 0) P00 DEBUG: postgres/interface::pgControlFromFile: (pgPath: {"[TEST_PATH]/db-master/db/base"}) P00 DEBUG: storage/storage::storageNewRead: (this: {type: posix, path: {"/"}, write: false}, fileExp: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, param.ignoreMissing: false, param.filterGroup: null) P00 DEBUG: storage/storage::storageNewRead: => {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false} @@ -560,7 +560,7 @@ P00 DEBUG: command/archive/common::walSegmentFind: (storage: {type: posix, P00 DEBUG: storage/storage::storageList: (this: {type: posix, path: {"[TEST_PATH]/db-master/repo"}, write: false}, pathExp: {"/9.3-1/0000000100000001"}, param.errorOnMissing: false, param.expression: {"^000000010000000100000002-[0-f]{40}(\.gz){0,1}$"}) P00 DEBUG: storage/storage::storageList: => {["000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz"]} P00 DEBUG: command/archive/common::walSegmentFind: => {"000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz"} -P00 DEBUG: command/archive/get/file::archiveGetCheck: => {"9.3-1/0000000100000001/000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz"} +P00 DEBUG: command/archive/get/file::archiveGetCheck: => {ArchiveGetCheckResult} P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"/"}, modeFile: 0640, modePath: 0750, write: true, pathExpressionFunction: null) P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix} P00 DEBUG: storage/storage::storageNewWrite: (this: {type: posix, path: {"/"}, write: true}, fileExp: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, param.modeFile: 0000, param.modePath: 0000, param.noCreatePath: true, param.noSyncFile: true, param.noSyncPath: true, param.noAtomic: true, param.filterGroup: null) diff --git a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm b/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm index 4769bd080..0bb56ae64 100644 --- a/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm +++ b/test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm @@ -83,7 +83,7 @@ sub run { foreach my $bRemote ($bS3 ? (false) : (false, true)) { - my $bRepoEncrypt = $bRemote && !$bS3 ? true : false; + my $bRepoEncrypt = !$bRemote && !$bS3 ? true : false; # Increment the run, log, and decide whether this unit test should be run if (!$self->begin("rmt ${bRemote}, s3 ${bS3}, enc ${bRepoEncrypt}")) {next} diff --git a/test/src/module/archive/getTest.c b/test/src/module/archive/getTest.c index 3edd56bf5..b81c8cb1f 100644 --- a/test/src/module/archive/getTest.c +++ b/test/src/module/archive/getTest.c @@ -51,7 +51,7 @@ testRun(void) "1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n")); TEST_ERROR( - archiveGetCheck(strNew("876543218765432187654321")), ArchiveMismatchError, + archiveGetCheck(strNew("876543218765432187654321"), cipherTypeNone, NULL), ArchiveMismatchError, "unable to retrieve the archive id for database version '10' and system-id '18072658121562454734'"); // Nothing to find in empty archive dir @@ -70,7 +70,8 @@ testRun(void) "3={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n" "4={\"db-id\":18072658121562454734,\"db-version\":\"10\"}")); - TEST_RESULT_PTR(archiveGetCheck(strNew("876543218765432187654321")), NULL, "no segment found"); + TEST_RESULT_PTR( + archiveGetCheck(strNew("876543218765432187654321"), cipherTypeNone, NULL).archiveFileActual, NULL, "no segment found"); // Write segment into an older archive path // ------------------------------------------------------------------------------------------------------------------------- @@ -82,7 +83,7 @@ testRun(void) NULL); TEST_RESULT_STR( - strPtr(archiveGetCheck(strNew("876543218765432187654321"))), + strPtr(archiveGetCheck(strNew("876543218765432187654321"), cipherTypeNone, NULL).archiveFileActual), "10-2/8765432187654321/876543218765432187654321-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "segment found"); // Write segment into an newer archive path @@ -95,16 +96,21 @@ testRun(void) NULL); TEST_RESULT_STR( - strPtr(archiveGetCheck(strNew("876543218765432187654321"))), + strPtr(archiveGetCheck(strNew("876543218765432187654321"), cipherTypeNone, NULL).archiveFileActual), "10-4/8765432187654321/876543218765432187654321-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "newer segment found"); // Get history file // ------------------------------------------------------------------------------------------------------------------------- - TEST_RESULT_PTR(archiveGetCheck(strNew("00000009.history")), NULL, "history file not found"); + TEST_RESULT_PTR( + archiveGetCheck(strNew("00000009.history"), cipherTypeNone, NULL).archiveFileActual, NULL, "history file not found"); storagePutNP(storageNewWriteNP(storageTest, strNew("repo/archive/test1/10-4/00000009.history")), NULL); - TEST_RESULT_STR(strPtr(archiveGetCheck(strNew("00000009.history"))), "10-4/00000009.history", "history file found"); + TEST_RESULT_STR( + strPtr( + archiveGetCheck( + strNew("00000009.history"), cipherTypeNone, NULL).archiveFileActual), "10-4/00000009.history", + "history file found"); } // ***************************************************************************************************************************** @@ -142,7 +148,7 @@ testRun(void) String *walDestination = strNewFmt("%s/db/pg_wal/RECOVERYXLOG", testPath()); storagePathCreateNP(storageTest, strPath(walDestination)); - TEST_RESULT_INT(archiveGetFile(archiveFile, walDestination), 1, "WAL segment missing"); + TEST_RESULT_INT(archiveGetFile(archiveFile, walDestination, cipherTypeNone, NULL), 1, "WAL segment missing"); // Create a WAL segment to copy // ------------------------------------------------------------------------------------------------------------------------- @@ -157,7 +163,7 @@ testRun(void) "repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")), buffer); - TEST_RESULT_INT(archiveGetFile(archiveFile, walDestination), 0, "WAL segment copied"); + TEST_RESULT_INT(archiveGetFile(archiveFile, walDestination, cipherTypeNone, NULL), 0, "WAL segment copied"); TEST_RESULT_BOOL(storageExistsNP(storageTest, walDestination), true, " check exists"); TEST_RESULT_INT(storageInfoNP(storageTest, walDestination).size, 16 * 1024 * 1024, " check size"); @@ -169,6 +175,28 @@ testRun(void) // Create a compressed WAL segment to copy // ------------------------------------------------------------------------------------------------------------------------- + StorageFileWrite *infoWrite = storageNewWriteNP(storageTest, strNew("repo/archive/test1/archive.info")); + + ioWriteFilterGroupSet( + storageFileWriteIo(infoWrite), + ioFilterGroupAdd( + ioFilterGroupNew(), + cipherBlockFilter(cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, bufNewStr(strNew("12345678")), NULL)))); + + storagePutNP( + infoWrite, + bufNewZ( + "[backrest]\n" + "backrest-checksum=\"60bfcb0a5a2c91d203c11d7f1924e99dcdfa0b80\"\n" + "backrest-format=5\n" + "backrest-version=\"2.06\"\n" + "\n" + "[cipher]\n" + "cipher-pass=\"worstpassphraseever\"\n" + "\n" + "[db:history]\n" + "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}")); + StorageFileWrite *destination = storageNewWriteNP( storageTest, strNew( @@ -176,10 +204,16 @@ testRun(void) IoFilterGroup *filterGroup = ioFilterGroupNew(); ioFilterGroupAdd(filterGroup, gzipCompressFilter(gzipCompressNew(3, false))); + ioFilterGroupAdd( + filterGroup, + cipherBlockFilter( + cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, bufNewStr(strNew("worstpassphraseever")), NULL))); ioWriteFilterGroupSet(storageFileWriteIo(destination), filterGroup); storagePutNP(destination, buffer); - TEST_RESULT_INT(archiveGetFile(archiveFile, walDestination), 0, "WAL segment copied"); + TEST_RESULT_INT( + archiveGetFile( + archiveFile, walDestination, cipherTypeAes256Cbc, strNew("12345678")), 0, "WAL segment copied"); TEST_RESULT_BOOL(storageExistsNP(storageTest, walDestination), true, " check exists"); TEST_RESULT_INT(storageInfoNP(storageTest, walDestination).size, 16 * 1024 * 1024, " check size"); }