db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-03 00:26:59 +02:00
Code Issues Releases Activity

Migrate local, unencrypted, non-S3 archive-get command to C.

Browse Source 
The archive-get command will only be executed in C if the repository is local, unencrypted, and type posix or cifs.  Admittedly a limited use case, but this is just the first step in migrating the archive-get command entirely into C.

This is a direct migration from the Perl code (including messages) to integrate as seamlessly with the remaining Perl code as possible.  It should not be possible to determine if the C version is running unless debug-level logging is enabled.
This commit is contained in:
David Steele
2018-09-11 15:42:31 -04:00
parent 787e7c295f
commit 9b9396c7b7
10 changed files with 502 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@ -48,6 +48,10 @@
</release-improvement-list> </release-improvement-list>
<release-development-list> <release-development-list>
<release-item>
<p>Migrate local, unencrypted, non-S3 <cmd>archive-get</cmd> command to C.</p>
</release-item>
<release-item> <release-item>
<p>Posix file functions now differentiate between open and missing errors.</p> <p>Posix file functions now differentiate between open and missing errors.</p>
</release-item> </release-item>

6
src/Makefile
View File

@ -55,6 +55,7 @@ DESTDIR =
#################################################################################################################################### ####################################################################################################################################
SRCS = \ SRCS = \
command/archive/common.c \ command/archive/common.c \
command/archive/get/file.c \
command/archive/get/get.c \ command/archive/get/get.c \
command/archive/push/push.c \ command/archive/push/push.c \
command/help/help.c \ command/help/help.c \
@ -144,7 +145,10 @@ install: pgbackrest
command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h protocol/storage/helper.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h protocol/storage/helper.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
$(CC) $(CFLAGS) -c command/archive/common.c -o command/archive/common.o $(CC) $(CFLAGS) -c command/archive/common.c -o command/archive/common.o
command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/info.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h command/archive/get/file.o: command/archive/get/file.c command/archive/common.h command/archive/get/file.h command/control/control.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h compress/gzip.h compress/gzipDecompress.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/infoArchive.h info/infoPg.h postgres/info.h protocol/storage/helper.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
$(CC) $(CFLAGS) -c command/archive/get/file.c -o command/archive/get/file.o
command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/archive/get/file.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/info.h protocol/storage/helper.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
$(CC) $(CFLAGS) -c command/archive/get/get.c -o command/archive/get/get.o $(CC) $(CFLAGS) -c command/archive/get/get.c -o command/archive/get/get.o
command/archive/push/push.o: command/archive/push/push.c command/archive/common.h command/command.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h command/archive/push/push.o: command/archive/push/push.c command/archive/common.h command/command.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h

144
src/command/archive/get/file.c Normal file
View File

@ -0,0 +1,144 @@
/***********************************************************************************************************************************
Archive Get File
***********************************************************************************************************************************/
#include "command/archive/get/file.h"
#include "command/archive/common.h"
#include "command/control/control.h"
#include "common/debug.h"
#include "common/io/filter/group.h"
#include "common/log.h"
#include "compress/gzip.h"
#include "compress/gzipDecompress.h"
#include "config/config.h"
#include "info/infoArchive.h"
#include "postgres/info.h"
#include "protocol/storage/helper.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Check if a WAL file exists in the repository
***********************************************************************************************************************************/
static String *
archiveGetCheck(const String *archiveFile)
{
FUNCTION_DEBUG_BEGIN(logLevelDebug);
FUNCTION_DEBUG_PARAM(STRING, archiveFile);
FUNCTION_TEST_ASSERT(archiveFile != NULL);
FUNCTION_DEBUG_END();
String *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
// Get pg control info
PgControlInfo controlInfo = pgControlInfo(cfgOptionStr(cfgOptPgPath));
// Attempt to load the archive info file
InfoArchive *info = infoArchiveNew(storageRepo(), strNew(STORAGE_REPO_ARCHIVE "/" INFO_ARCHIVE_FILE), false);
// Loop through the pg history in case the WAL we need is not in the most recent archive id
String *archiveId = NULL;
const String *archiveFileActual = NULL;
for (unsigned int pgIdx = 0; pgIdx < infoPgDataTotal(infoArchivePg(info)); pgIdx++)
{
InfoPgData pgData = infoPgData(infoArchivePg(info), pgIdx);
// Only use the archive id if it matches the current cluster
if (pgData.systemId == controlInfo.systemId && pgData.version == controlInfo.version)
{
archiveId = infoPgArchiveId(infoArchivePg(info), pgIdx);
// If a WAL segment search among the possible file names
if (walIsSegment(archiveFile))
{
String *walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile);
if (walSegmentFile != NULL)
{
archiveFileActual = strNewFmt("%s/%s", strPtr(strSubN(archiveFile, 0, 16)), strPtr(walSegmentFile));
break;
}
}
// Else if not a WAL segment, see if it exists in the archive dir
else if (
storageExistsNP(
storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(archiveFile))))
{
archiveFileActual = archiveFile;
break;
}
}
}
// Error if no archive id was found -- this indicates a mismatch with the current cluster
if (archiveId == NULL)
{
THROW_FMT(
ArchiveMismatchError, "unable to retrieve the archive id for database version '%s' and system-id '%" PRIu64 "'",
strPtr(pgVersionToStr(controlInfo.version)), controlInfo.systemId);
}
if (archiveFileActual != NULL)
{
memContextSwitch(MEM_CONTEXT_OLD());
result = strNewFmt("%s/%s", strPtr(archiveId), strPtr(archiveFileActual));
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_DEBUG_RESULT(STRING, result);
}
/***********************************************************************************************************************************
Copy a file from the archive to the specified destination
***********************************************************************************************************************************/
int
archiveGetFile(const String *archiveFile, const String *walDestination)
{
FUNCTION_DEBUG_BEGIN(logLevelDebug);
FUNCTION_DEBUG_PARAM(STRING, archiveFile);
FUNCTION_DEBUG_PARAM(STRING, walDestination);
FUNCTION_TEST_ASSERT(archiveFile != NULL);
FUNCTION_TEST_ASSERT(walDestination != NULL);
FUNCTION_DEBUG_END();
// By default result indicates WAL segment not found
int result = 1;
// Test for stop file
lockStopTest();
MEM_CONTEXT_TEMP_BEGIN()
{
// Make sure the file exists and other checks pass
String *archiveFileActual = archiveGetCheck(archiveFile);
if (archiveFileActual != NULL)
{
StorageFileWrite *destination = storageNewWriteP(
storageLocalWrite(), walDestination, .noCreatePath = true, .noSyncFile = true, .noSyncPath = true,
.noAtomic = true);
// If file is gzipped then add the decompression filter
if (strEndsWithZ(archiveFileActual, "." GZIP_EXT))
{
IoFilterGroup *filterGroup = ioFilterGroupNew();
ioFilterGroupAdd(filterGroup, gzipDecompressFilter(gzipDecompressNew(false)));
ioWriteFilterGroupSet(storageFileWriteIo(destination), filterGroup);
}
// Copy the file
storageCopyNP(
storageNewReadNP(storageRepo(), strNewFmt("%s/%s", STORAGE_REPO_ARCHIVE, strPtr(archiveFileActual))), destination);
// The WAL file was found
result = 0;
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_DEBUG_RESULT(INT, result);
}

14
src/command/archive/get/file.h Normal file
View File

@ -0,0 +1,14 @@
/***********************************************************************************************************************************
Archive Get File
***********************************************************************************************************************************/
#ifndef COMMAND_ARCHIVE_GET_FILE_H
#define COMMAND_ARCHIVE_GET_FILE_H
#include "common/type/string.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
int archiveGetFile(const String *archiveFile, const String *walDestination);
#endif

13
src/command/archive/get/get.c
View File

@ -5,6 +5,7 @@ Archive Get Command
#include <unistd.h> #include <unistd.h>
#include "command/archive/common.h" #include "command/archive/common.h"
#include "command/archive/get/file.h"
#include "command/command.h" #include "command/command.h"
#include "common/assert.h" #include "common/assert.h"
#include "common/debug.h" #include "common/debug.h"
@ -17,6 +18,7 @@ Archive Get Command
#include "config/load.h" #include "config/load.h"
#include "perl/exec.h" #include "perl/exec.h"
#include "postgres/info.h" #include "postgres/info.h"
#include "protocol/storage/helper.h"
#include "storage/helper.h" #include "storage/helper.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
@ -271,8 +273,15 @@ cmdArchiveGet(void)
// Disable async if it was enabled // Disable async if it was enabled
cfgOptionSet(cfgOptArchiveAsync, cfgOptionSource(cfgOptArchiveAsync), varNewBool(false)); cfgOptionSet(cfgOptArchiveAsync, cfgOptionSource(cfgOptArchiveAsync), varNewBool(false));
// Call synchronous get // If repo type is not s3 and repo server is not remote and not encrypted then this can be done entirely in C
result = perlExec(); if (!strEqZ(cfgOptionStr(cfgOptRepoType), STORAGE_TYPE_S3) && !cfgOptionTest(cfgOptRepoHost) &&
strEqZ(cfgOptionStr(cfgOptRepoCipherType), "none"))
{
result = archiveGetFile(walSegment, walDestination);
}
// Else do it in Perl
else
result = perlExec(); // {uncovered - Perl code is covered in unit tests}
} }
// Log whether or not the file was found // Log whether or not the file was found

5
src/compress/gzip.h
View File

@ -4,6 +4,11 @@ Gzip Common
#ifndef COMPRESS_GZIP_H #ifndef COMPRESS_GZIP_H
#define COMPRESS_GZIP_H #define COMPRESS_GZIP_H
/***********************************************************************************************************************************
Gzip extension
***********************************************************************************************************************************/
#define GZIP_EXT "gz"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/

3
test/define.yaml
View File

@ -640,10 +640,11 @@ unit:
# ---------------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------------
- name: get - name: get
total: 2 total: 4
perlReq: true perlReq: true
coverage: coverage:
command/archive/get/file: full
command/archive/get/get: full command/archive/get/get: full
# ---------------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------------

143
test/expect/mock-archive-001.log
View File

@ -15,7 +15,7 @@ P00 INFO: archive-push command end: aborted with exception [055]
------------------------------------------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/archive/db/archive.info or [TEST_PATH]/db-master/repo/archive/db/archive.info.copy P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/archive/db/archive.info or [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL segments HINT: archive.info does not exist but is required to push/get WAL segments.
HINT: is archive_command configured in postgresql.conf? HINT: is archive_command configured in postgresql.conf?
HINT: has a stanza-create been performed? HINT: has a stanza-create been performed?
HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme. HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
@ -117,46 +117,41 @@ P00 DEBUG: main::main: => 0
P00 INFO: archive-get command begin [BACKREST-VERSION]: [700000007000000070000000, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: archive-get command begin [BACKREST-VERSION]: [700000007000000070000000, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: config/load::cfgLoad: => void
P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void)
P00 DEBUG: perl/exec::perlExec: (void) P00 DEBUG: command/archive/get/file::archiveGetFile: (archiveFile: {"700000007000000070000000"}, walDestination: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"})
P00 DEBUG: Archive::Get::Get->process(): rstryCommandArg = (700000007000000070000000, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) P00 DEBUG: command/control/control::lockStopTest: (void)
P00 DEBUG: Archive::Get::File::archiveGetFile(): bAtomic = false, strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 700000007000000070000000 P00 DEBUG: storage/storage::storageExists: (this: {path: "/", write: false}, pathExp: {"[TEST_PATH]/db-master/lock/db.stop"}, param.timeout: 0)
P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = <false> P00 DEBUG: storage/storage::storageExists: => false
P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false P00 DEBUG: storage/storage::storageExists: (this: {path: "/", write: false}, pathExp: {"[TEST_PATH]/db-master/lock/all.stop"}, param.timeout: 0)
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true> P00 DEBUG: storage/storage::storageExists: => false
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp P00 DEBUG: command/control/control::lockStopTest: => void
P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 700000007000000070000000, ullDbSysId = [undef] P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"700000007000000070000000"})
P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true P00 DEBUG: postgres/info::pgControlInfo: (pgPath: {"[TEST_PATH]/db-master/db/base"})
P00 DEBUG: Db->new(): iRemoteIdx = 1 P00 DEBUG: storage/storage::storageGet: (file: {StorageFileRead}, param.exactSize: 512)
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512}
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> P00 DEBUG: postgres/info::pgControlInfo: => {PgControlInfo}
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 1000000000000000094 P00 DEBUG: storage/storage::storageNew: (path: {"[TEST_PATH]/db-master/repo"}, param.modeFile: 0000, param.modePath: 0000, param.write: false, param.pathExpressionFunction: (function *))
P00 DEBUG: Archive::Info->new(): bIgnoreMissing = <false>, bLoad = <true>, bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db, strCipherPassSub = [undef] P00 DEBUG: storage/storage::storageNew: => {path: "[TEST_PATH]/db-master/repo", write: false}
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true> P00 DEBUG: info/infoArchive::infoArchiveNew: (fileName: {"<REPO:ARCHIVE>/archive.info"}, ignoreMissing: false)
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp P00 DEBUG: info/infoPg::infoPgNew: (fileName: {"<REPO:ARCHIVE>/archive.info"})
P00 DEBUG: Storage::Local->encrypted(): bIgnoreMissing = true, strFileName = [TEST_PATH]/db-master/repo/archive/db/archive.info P00 DEBUG: info/info::infoNew: (fileName: {"<REPO:ARCHIVE>/archive.info"})
P00 DEBUG: Storage::Local->encrypted=>: bEncrypted = false P00 DEBUG: info/info::infoNew: => {Info}
P00 DEBUG: Storage::Local->encryptionValid(): bEncrypted = false P00 DEBUG: info/info::infoIni: (this: {Info})
P00 DEBUG: Storage::Local->encryptionValid=>: bValid = true P00 DEBUG: info/info::infoIni: => {Ini}
P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], strCipherPass = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info P00 DEBUG: info/infoPg::infoPgAdd: (this: {InfoPg}, infoPgData: {"id: 1, version: 90400, systemId 1000000000000000094, catalog 0, control 0"})
P00 DEBUG: Storage::Base->get(): strCipherPass = [undef], xFile = [object] P00 DEBUG: info/infoPg::infoPgAdd: => 0
P00 DEBUG: Archive::Info->archiveIdList(): strDbVersion = 9.4, ullDbSysId = 1000000000000000094 P00 DEBUG: info/infoPg::infoPgNew: => {InfoPg}
P00 DEBUG: Archive::Info->dbHistoryList=>: hDbHash = [hash] P00 DEBUG: info/infoArchive::infoArchiveNew: => {InfoArchive}
P00 DEBUG: Archive::Info->archiveIdList=>: stryArchiveId = (9.4-1) P00 DEBUG: command/archive/common::walSegmentFind: (storage: {path: "[TEST_PATH]/db-master/repo", write: false}, archiveId: {"9.4-1"}, walSegment: {"700000007000000070000000"})
P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 700000007000000070000000 P00 DEBUG: storage/storage::storageList: (this: {path: "[TEST_PATH]/db-master/repo", write: false}, pathExp: {"<REPO:ARCHIVE>/9.4-1/7000000070000000"}, param.errorOnMissing: false, param.expression: {"^700000007000000070000000-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^700000007000000070000000-[0-f]{40}(\.gz){0,1}$, strPathExp = <REPO:ARCHIVE>/9.4-1/7000000070000000, strSortOrder = <forward> P00 DEBUG: storage/storage::storageList: => null
P00 DEBUG: Storage::Local->list=>: stryFileList = () P00 DEBUG: command/archive/common::walSegmentFind: => null
P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = [undef] P00 DEBUG: command/archive/get/file::archiveGetCheck: => null
P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = [undef], strArchiveId = 9.4-1, strCipherPass = [undef] P00 DEBUG: command/archive/get/file::archiveGetFile: => 1
P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 1
P00 DEBUG: perl/exec::perlExec: => 1
P00 INFO: unable to find 700000007000000070000000 in the archive P00 INFO: unable to find 700000007000000070000000 in the archive
P00 DEBUG: command/archive/get/get::cmdArchiveGet: => 1 P00 DEBUG: command/archive/get/get::cmdArchiveGet: => 1
P00 DEBUG: common/exit::exitSafe: (result: 1, error: false, signalType: 0) P00 DEBUG: common/exit::exitSafe: (result: 1, error: false, signalType: 0)
P00 DEBUG: common/lock::lockRelease: (failOnNoLock: false) P00 DEBUG: common/lock::lockRelease: (failOnNoLock: false)
P00 DEBUG: common/lock::lockRelease: => false P00 DEBUG: common/lock::lockRelease: => false
P00 DEBUG: Main::mainCleanup(): iExitCode = 1
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
P00 INFO: archive-get command end: completed successfully P00 INFO: archive-get command end: completed successfully
P00 DEBUG: common/exit::exitSafe: => 1 P00 DEBUG: common/exit::exitSafe: => 1
P00 DEBUG: main::main: => 1 P00 DEBUG: main::main: => 1
@ -166,51 +161,47 @@ P00 DEBUG: main::main: => 1
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: config/load::cfgLoad: => void
P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void)
P00 DEBUG: perl/exec::perlExec: (void) P00 DEBUG: command/archive/get/file::archiveGetFile: (archiveFile: {"000000010000000100000001"}, walDestination: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"})
P00 DEBUG: Archive::Get::Get->process(): rstryCommandArg = (000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) P00 DEBUG: command/control/control::lockStopTest: (void)
P00 DEBUG: Archive::Get::File::archiveGetFile(): bAtomic = false, strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001 P00 DEBUG: storage/storage::storageExists: (this: {path: "/", write: false}, pathExp: {"[TEST_PATH]/db-master/lock/db.stop"}, param.timeout: 0)
P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = <false> P00 DEBUG: storage/storage::storageExists: => false
P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false P00 DEBUG: storage/storage::storageExists: (this: {path: "/", write: false}, pathExp: {"[TEST_PATH]/db-master/lock/all.stop"}, param.timeout: 0)
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true> P00 DEBUG: storage/storage::storageExists: => false
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp P00 DEBUG: command/control/control::lockStopTest: => void
P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000001, ullDbSysId = [undef] P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000001"})
P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true P00 DEBUG: postgres/info::pgControlInfo: (pgPath: {"[TEST_PATH]/db-master/db/base"})
P00 DEBUG: Db->new(): iRemoteIdx = 1 P00 DEBUG: storage/storage::storageGet: (file: {StorageFileRead}, param.exactSize: 512)
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512}
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> P00 DEBUG: postgres/info::pgControlInfo: => {PgControlInfo}
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 1000000000000000094 P00 DEBUG: storage/storage::storageNew: (path: {"[TEST_PATH]/db-master/repo"}, param.modeFile: 0000, param.modePath: 0000, param.write: false, param.pathExpressionFunction: (function *))
P00 DEBUG: Archive::Info->new(): bIgnoreMissing = <false>, bLoad = <true>, bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db, strCipherPassSub = [undef] P00 DEBUG: storage/storage::storageNew: => {path: "[TEST_PATH]/db-master/repo", write: false}
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true> P00 DEBUG: info/infoArchive::infoArchiveNew: (fileName: {"<REPO:ARCHIVE>/archive.info"}, ignoreMissing: false)
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp P00 DEBUG: info/infoPg::infoPgNew: (fileName: {"<REPO:ARCHIVE>/archive.info"})
P00 DEBUG: Storage::Local->encrypted(): bIgnoreMissing = true, strFileName = [TEST_PATH]/db-master/repo/archive/db/archive.info P00 DEBUG: info/info::infoNew: (fileName: {"<REPO:ARCHIVE>/archive.info"})
P00 DEBUG: Storage::Local->encrypted=>: bEncrypted = false P00 DEBUG: info/info::infoNew: => {Info}
P00 DEBUG: Storage::Local->encryptionValid(): bEncrypted = false P00 DEBUG: info/info::infoIni: (this: {Info})
P00 DEBUG: Storage::Local->encryptionValid=>: bValid = true P00 DEBUG: info/info::infoIni: => {Ini}
P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], strCipherPass = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info P00 DEBUG: info/infoPg::infoPgAdd: (this: {InfoPg}, infoPgData: {"id: 1, version: 90400, systemId 1000000000000000094, catalog 0, control 0"})
P00 DEBUG: Storage::Base->get(): strCipherPass = [undef], xFile = [object] P00 DEBUG: info/infoPg::infoPgAdd: => 0
P00 DEBUG: Archive::Info->archiveIdList(): strDbVersion = 9.4, ullDbSysId = 1000000000000000094 P00 DEBUG: info/infoPg::infoPgNew: => {InfoPg}
P00 DEBUG: Archive::Info->dbHistoryList=>: hDbHash = [hash] P00 DEBUG: info/infoArchive::infoArchiveNew: => {InfoArchive}
P00 DEBUG: Archive::Info->archiveIdList=>: stryArchiveId = (9.4-1) P00 DEBUG: command/archive/common::walSegmentFind: (storage: {path: "[TEST_PATH]/db-master/repo", write: false}, archiveId: {"9.4-1"}, walSegment: {"000000010000000100000001"})
P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.4-1, strWalSegment = 000000010000000100000001 P00 DEBUG: storage/storage::storageList: (this: {path: "[TEST_PATH]/db-master/repo", write: false}, pathExp: {"<REPO:ARCHIVE>/9.4-1/0000000100000001"}, param.errorOnMissing: false, param.expression: {"^000000010000000100000001-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPathExp = <REPO:ARCHIVE>/9.4-1/0000000100000001, strSortOrder = <forward> P00 DEBUG: storage/storage::storageList: => {["000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"]}
P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz) P00 DEBUG: command/archive/common::walSegmentFind: => {"000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}
P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz P00 DEBUG: command/archive/get/file::archiveGetCheck: => {"9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}
P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz, strArchiveId = 9.4-1, strCipherPass = [undef] P00 DEBUG: storage/storage::storageNew: (path: {"/"}, param.modeFile: 0000, param.modePath: 0000, param.write: true, param.pathExpressionFunction: null)
P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = <false>, rhyFilter = [undef], strCipherPass = [undef], xFileExp = <REPO:ARCHIVE>/9.4-1/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz P00 DEBUG: storage/storage::storageNew: => {path: "/", write: true}
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true> P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: (this: {inputSame: false, done: true}, filter: {IoFilter})
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: => void
P00 DEBUG: Storage::Local->openWrite(): bAtomic = false, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG P00 DEBUG: storage/storage::storageCopy: (source: {StorageFileRead}, destination: {StorageFileWrite})
P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] P00 DEBUG: storage/storage::storageCopy: => true
P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0 P00 DEBUG: command/archive/get/file::archiveGetFile: => 0
P00 DEBUG: perl/exec::perlExec: => 0
P00 INFO: found 000000010000000100000001 in the archive P00 INFO: found 000000010000000100000001 in the archive
P00 DEBUG: command/archive/get/get::cmdArchiveGet: => 0 P00 DEBUG: command/archive/get/get::cmdArchiveGet: => 0
P00 DEBUG: common/exit::exitSafe: (result: 0, error: false, signalType: 0) P00 DEBUG: common/exit::exitSafe: (result: 0, error: false, signalType: 0)
P00 DEBUG: common/lock::lockRelease: (failOnNoLock: false) P00 DEBUG: common/lock::lockRelease: (failOnNoLock: false)
P00 DEBUG: common/lock::lockRelease: => false P00 DEBUG: common/lock::lockRelease: => false
P00 DEBUG: Main::mainCleanup(): iExitCode = 0
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
P00 INFO: archive-get command end: completed successfully P00 INFO: archive-get command end: completed successfully
P00 DEBUG: common/exit::exitSafe: => 0 P00 DEBUG: common/exit::exitSafe: => 0
P00 DEBUG: main::main: => 0 P00 DEBUG: main::main: => 0

78
test/expect/mock-stanza-001.log
View File

@ -534,51 +534,49 @@ db-version="9.4"
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 DEBUG: config/load::cfgLoad: => void P00 DEBUG: config/load::cfgLoad: => void
P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void) P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void)
P00 DEBUG: perl/exec::perlExec: (void) P00 DEBUG: command/archive/get/file::archiveGetFile: (archiveFile: {"000000010000000100000002"}, walDestination: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"})
P00 DEBUG: Archive::Get::Get->process(): rstryCommandArg = (000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG) P00 DEBUG: command/control/control::lockStopTest: (void)
P00 DEBUG: Archive::Get::File::archiveGetFile(): bAtomic = false, strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002 P00 DEBUG: storage/storage::storageExists: (this: {path: "/", write: false}, pathExp: {"[TEST_PATH]/db-master/lock/db.stop"}, param.timeout: 0)
P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = <false> P00 DEBUG: storage/storage::storageExists: => false
P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false P00 DEBUG: storage/storage::storageExists: (this: {path: "/", write: false}, pathExp: {"[TEST_PATH]/db-master/lock/all.stop"}, param.timeout: 0)
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true> P00 DEBUG: storage/storage::storageExists: => false
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp P00 DEBUG: command/control/control::lockStopTest: => void
P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef] P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000002"})
P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true P00 DEBUG: postgres/info::pgControlInfo: (pgPath: {"[TEST_PATH]/db-master/db/base"})
P00 DEBUG: Db->new(): iRemoteIdx = 1 P00 DEBUG: storage/storage::storageGet: (file: {StorageFileRead}, param.exactSize: 512)
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef] P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512}
P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base> P00 DEBUG: postgres/info::pgControlInfo: => {PgControlInfo}
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201306121, iDbControlVersion = 937, strDbVersion = 9.3, ullDbSysId = 1000000000000000093 P00 DEBUG: storage/storage::storageNew: (path: {"[TEST_PATH]/db-master/repo"}, param.modeFile: 0000, param.modePath: 0000, param.write: false, param.pathExpressionFunction: (function *))
P00 DEBUG: Archive::Info->new(): bIgnoreMissing = <false>, bLoad = <true>, bRequired = true, strArchiveClusterPath = [TEST_PATH]/db-master/repo/archive/db, strCipherPassSub = [undef] P00 DEBUG: storage/storage::storageNew: => {path: "[TEST_PATH]/db-master/repo", write: false}
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true> P00 DEBUG: info/infoArchive::infoArchiveNew: (fileName: {"<REPO:ARCHIVE>/archive.info"}, ignoreMissing: false)
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp P00 DEBUG: info/infoPg::infoPgNew: (fileName: {"<REPO:ARCHIVE>/archive.info"})
P00 DEBUG: Storage::Local->encrypted(): bIgnoreMissing = true, strFileName = [TEST_PATH]/db-master/repo/archive/db/archive.info P00 DEBUG: info/info::infoNew: (fileName: {"<REPO:ARCHIVE>/archive.info"})
P00 DEBUG: Storage::Local->encrypted=>: bEncrypted = false P00 DEBUG: info/info::infoNew: => {Info}
P00 DEBUG: Storage::Local->encryptionValid(): bEncrypted = false P00 DEBUG: info/info::infoIni: (this: {Info})
P00 DEBUG: Storage::Local->encryptionValid=>: bValid = true P00 DEBUG: info/info::infoIni: => {Ini}
P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = true, rhyFilter = [undef], strCipherPass = [undef], xFileExp = [TEST_PATH]/db-master/repo/archive/db/archive.info P00 DEBUG: info/infoPg::infoPgAdd: (this: {InfoPg}, infoPgData: {"id: 2, version: 90400, systemId 1000000000000000094, catalog 0, control 0"})
P00 DEBUG: Storage::Base->get(): strCipherPass = [undef], xFile = [object] P00 DEBUG: info/infoPg::infoPgAdd: => 0
P00 DEBUG: Archive::Info->archiveIdList(): strDbVersion = 9.3, ullDbSysId = 1000000000000000093 P00 DEBUG: info/infoPg::infoPgAdd: (this: {InfoPg}, infoPgData: {"id: 1, version: 90300, systemId 1000000000000000093, catalog 0, control 0"})
P00 DEBUG: Archive::Info->dbHistoryList=>: hDbHash = [hash] P00 DEBUG: info/infoPg::infoPgAdd: => 1
P00 DEBUG: Archive::Info->archiveIdList=>: stryArchiveId = (9.3-1) P00 DEBUG: info/infoPg::infoPgNew: => {InfoPg}
P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStorageRepo = [object], strArchiveId = 9.3-1, strWalSegment = 000000010000000100000002 P00 DEBUG: info/infoArchive::infoArchiveNew: => {InfoArchive}
P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000002-[0-f]{40}(\.gz){0,1}$, strPathExp = <REPO:ARCHIVE>/9.3-1/0000000100000001, strSortOrder = <forward> P00 DEBUG: command/archive/common::walSegmentFind: (storage: {path: "[TEST_PATH]/db-master/repo", write: false}, archiveId: {"9.3-1"}, walSegment: {"000000010000000100000002"})
P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz) P00 DEBUG: storage/storage::storageList: (this: {path: "[TEST_PATH]/db-master/repo", write: false}, pathExp: {"<REPO:ARCHIVE>/9.3-1/0000000100000001"}, param.errorOnMissing: false, param.expression: {"^000000010000000100000002-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz P00 DEBUG: storage/storage::storageList: => {["000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz"]}
P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = [undef] P00 DEBUG: command/archive/common::walSegmentFind: => {"000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz"}
P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = <false>, rhyFilter = [undef], strCipherPass = [undef], xFileExp = <REPO:ARCHIVE>/9.3-1/000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz P00 DEBUG: command/archive/get/file::archiveGetCheck: => {"9.3-1/0000000100000001/000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz"}
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true> P00 DEBUG: storage/storage::storageNew: (path: {"/"}, param.modeFile: 0000, param.modePath: 0000, param.write: true, param.pathExpressionFunction: null)
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp P00 DEBUG: storage/storage::storageNew: => {path: "/", write: true}
P00 DEBUG: Storage::Local->openWrite(): bAtomic = false, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: (this: {inputSame: false, done: true}, filter: {IoFilter})
P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object] P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: => void
P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0 P00 DEBUG: storage/storage::storageCopy: (source: {StorageFileRead}, destination: {StorageFileWrite})
P00 DEBUG: perl/exec::perlExec: => 0 P00 DEBUG: storage/storage::storageCopy: => true
P00 DEBUG: command/archive/get/file::archiveGetFile: => 0
P00 INFO: found 000000010000000100000002 in the archive P00 INFO: found 000000010000000100000002 in the archive
P00 DEBUG: command/archive/get/get::cmdArchiveGet: => 0 P00 DEBUG: command/archive/get/get::cmdArchiveGet: => 0
P00 DEBUG: common/exit::exitSafe: (result: 0, error: false, signalType: 0) P00 DEBUG: common/exit::exitSafe: (result: 0, error: false, signalType: 0)
P00 DEBUG: common/lock::lockRelease: (failOnNoLock: false) P00 DEBUG: common/lock::lockRelease: (failOnNoLock: false)
P00 DEBUG: common/lock::lockRelease: => false P00 DEBUG: common/lock::lockRelease: => false
P00 DEBUG: Main::mainCleanup(): iExitCode = 0
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
P00 INFO: archive-get command end: completed successfully P00 INFO: archive-get command end: completed successfully
P00 DEBUG: common/exit::exitSafe: => 0 P00 DEBUG: common/exit::exitSafe: => 0
P00 DEBUG: main::main: => 0 P00 DEBUG: main::main: => 0

222
test/src/module/archive/getTest.c
View File

@ -6,6 +6,20 @@ Test Archive Get Command
#include "common/harnessConfig.h" #include "common/harnessConfig.h"
#include "common/harnessFork.h" #include "common/harnessFork.h"
#include "compress/gzipCompress.h"
/***********************************************************************************************************************************
Create test pg_control file
***********************************************************************************************************************************/
static void
testPgControlCreate(const Storage *storage, const String *controlFile, PgControlFile control)
{
Buffer *controlBuffer = bufNew(8192);
memset(bufPtr(controlBuffer), 0, bufSize(controlBuffer));
memcpy(bufPtr(controlBuffer), &control, sizeof(PgControlFile));
bufUsedSet(controlBuffer, bufSize(controlBuffer));
storagePutNP(storageNewWriteNP(storage, controlFile), controlBuffer);
}
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Test Run Test Run
@ -17,6 +31,181 @@ testRun(void)
Storage *storageTest = storageNewP(strNew(testPath()), .write = true); Storage *storageTest = storageNewP(strNew(testPath()), .write = true);
// *****************************************************************************************************************************
if (testBegin("archiveGetCheck()"))
{
// Load Parameters
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath()));
strLstAddZ(argList, "archive-get");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// Create pg_control file
testPgControlCreate(
storageTest, strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL),
(PgControlFile){.systemId = 0xFACEFACEFACEFACE, .controlVersion = 1002, .catalogVersion = 201707211});
// Control and archive info mismatch
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(storageTest, strNew("repo/archive/test1/archive.info")),
bufNewStr(
strNew(
"[backrest]\n"
"backrest-checksum=\"0a415a03fa3faccb4ac171759895478469e9e19e\"\n"
"backrest-format=5\n"
"backrest-version=\"2.06\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n")));
TEST_ERROR(
archiveGetCheck(strNew("876543218765432187654321")), ArchiveMismatchError,
"unable to retrieve the archive id for database version '10' and system-id '18072658121562454734'");
// Nothing to find in empty archive dir
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(storageTest, strNew("repo/archive/test1/archive.info")),
bufNewStr(
strNew(
"[backrest]\n"
"backrest-checksum=\"f7617b5c4c9f212f40b9bc3d8ec7f97edbbf96af\"\n"
"backrest-format=5\n"
"backrest-version=\"2.06\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n"
"2={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n"
"3={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n"
"4={\"db-id\":18072658121562454734,\"db-version\":\"10\"}")));
TEST_RESULT_PTR(archiveGetCheck(strNew("876543218765432187654321")), NULL, "no segment found");
// Write segment into an older archive path
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(
storageTest,
strNew(
"repo/archive/test1/10-2/8765432187654321/876543218765432187654321-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
NULL);
TEST_RESULT_STR(
strPtr(archiveGetCheck(strNew("876543218765432187654321"))),
"10-2/8765432187654321/876543218765432187654321-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "segment found");
// Write segment into an newer archive path
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(
storageTest,
strNew(
"repo/archive/test1/10-4/8765432187654321/876543218765432187654321-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")),
NULL);
TEST_RESULT_STR(
strPtr(archiveGetCheck(strNew("876543218765432187654321"))),
"10-4/8765432187654321/876543218765432187654321-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "newer segment found");
// Get history file
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_PTR(archiveGetCheck(strNew("00000009.history")), NULL, "history file not found");
storagePutNP(storageNewWriteNP(storageTest, strNew("repo/archive/test1/10-4/00000009.history")), NULL);
TEST_RESULT_STR(strPtr(archiveGetCheck(strNew("00000009.history"))), "10-4/00000009.history", "history file found");
// Clear data
storagePathRemoveP(storageTest, strNew("repo"), .recurse = true);
storagePathRemoveP(storageTest, strNew("db"), .recurse = true);
}
// *****************************************************************************************************************************
if (testBegin("archiveGetFile()"))
{
// Load Parameters
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath()));
strLstAddZ(argList, "archive-get");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// Create pg_control file
testPgControlCreate(
storageTest, strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL),
(PgControlFile){.systemId = 0xFACEFACEFACEFACE, .controlVersion = 1002, .catalogVersion = 201707211});
// Create archive.info
storagePutNP(
storageNewWriteNP(storageTest, strNew("repo/archive/test1/archive.info")),
bufNewStr(
strNew(
"[backrest]\n"
"backrest-checksum=\"8a041a4128eaa2c08a23dd1f04934627795946ff\"\n"
"backrest-format=5\n"
"backrest-version=\"2.06\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}")));
// Nothing to copy
// -------------------------------------------------------------------------------------------------------------------------
String *archiveFile = strNew("01ABCDEF01ABCDEF01ABCDEF");
String *walDestination = strNewFmt("%s/db/pg_wal/RECOVERYXLOG", testPath());
storagePathCreateNP(storageTest, strPath(walDestination));
TEST_RESULT_INT(archiveGetFile(archiveFile, walDestination), 1, "WAL segment missing");
// Create a WAL segment to copy
// -------------------------------------------------------------------------------------------------------------------------
Buffer *buffer = bufNew(16 * 1024 * 1024);
memset(bufPtr(buffer), 0, bufSize(buffer));
bufUsedSet(buffer, bufSize(buffer));
storagePutNP(
storageNewWriteNP(
storageTest,
strNew(
"repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
buffer);
TEST_RESULT_INT(archiveGetFile(archiveFile, walDestination), 0, "WAL segment copied");
TEST_RESULT_BOOL(storageExistsNP(storageTest, walDestination), true, " check exists");
TEST_RESULT_INT(storageInfoNP(storageTest, walDestination).size, 16 * 1024 * 1024, " check size");
storageRemoveP(
storageTest,
strNew("repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
.errorOnMissing = true);
storageRemoveP(storageTest, walDestination, .errorOnMissing = true);
// Create a compressed WAL segment to copy
// -------------------------------------------------------------------------------------------------------------------------
StorageFileWrite *destination = storageNewWriteNP(
storageTest,
strNew(
"repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"));
IoFilterGroup *filterGroup = ioFilterGroupNew();
ioFilterGroupAdd(filterGroup, gzipCompressFilter(gzipCompressNew(3, false)));
ioWriteFilterGroupSet(storageFileWriteIo(destination), filterGroup);
storagePutNP(destination, buffer);
TEST_RESULT_INT(archiveGetFile(archiveFile, walDestination), 0, "WAL segment copied");
TEST_RESULT_BOOL(storageExistsNP(storageTest, walDestination), true, " check exists");
TEST_RESULT_INT(storageInfoNP(storageTest, walDestination).size, 16 * 1024 * 1024, " check size");
// Clear data
storagePathRemoveP(storageTest, strNew("repo"), .recurse = true);
storagePathRemoveP(storageTest, strNew("db"), .recurse = true);
}
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("queueNeed()")) if (testBegin("queueNeed()"))
{ {
@ -99,7 +288,8 @@ testRun(void)
strLstAddZ(argList, "--archive-timeout=1"); strLstAddZ(argList, "--archive-timeout=1");
strLstAdd(argList, strNewFmt("--log-path=%s", testPath())); strLstAdd(argList, strNewFmt("--log-path=%s", testPath()));
strLstAdd(argList, strNewFmt("--log-level-file=debug")); strLstAdd(argList, strNewFmt("--log-level-file=debug"));
strLstAddZ(argList, "--stanza=db"); strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAddZ(argList, "--stanza=test1");
strLstAddZ(argList, "archive-get"); strLstAddZ(argList, "archive-get");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
@ -114,13 +304,9 @@ testRun(void)
TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "path to copy WAL segment required"); TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "path to copy WAL segment required");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
String *controlFile = strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL); testPgControlCreate(
PgControlFile control = {.systemId = 0xFACEFACE, .controlVersion = 1002, .catalogVersion = 201707211}; storageTest, strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL),
Buffer *controlBuffer = bufNew(512); (PgControlFile){.systemId = 0xFACEFACEFACEFACE, .controlVersion = 1002, .catalogVersion = 201707211});
memset(bufPtr(controlBuffer), 0, bufSize(controlBuffer));
memcpy(bufPtr(controlBuffer), &control, sizeof(PgControlFile));
bufUsedSet(controlBuffer, bufSize(controlBuffer));
storagePutNP(storageNewWriteNP(storageTest, controlFile), controlBuffer);
storagePathCreateNP(storageTest, strNewFmt("%s/db/pg_wal", testPath())); storagePathCreateNP(storageTest, strNewFmt("%s/db/pg_wal", testPath()));
@ -134,7 +320,15 @@ testRun(void)
{ {
HARNESS_FORK_CHILD() HARNESS_FORK_CHILD()
{ {
TEST_ERROR(cmdArchiveGet(), FileMissingError, "!!!EMBEDDEDPERLERROR!!!"); TEST_ERROR_FMT(
cmdArchiveGet(), FileMissingError,
"unable to open %s/archive/test1/archive.info or %s/archive/test1/archive.info.copy\n"
"HINT: archive.info does not exist but is required to push/get WAL segments.\n"
"HINT: is archive_command configured in postgresql.conf?\n"
"HINT: has a stanza-create been performed?\n"
"HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving"
" scheme.",
strPtr(cfgOptionStr(cfgOptRepoPath)), strPtr(cfgOptionStr(cfgOptRepoPath)));
} }
} }
HARNESS_FORK_END(); HARNESS_FORK_END();
@ -152,7 +346,15 @@ testRun(void)
{ {
HARNESS_FORK_CHILD() HARNESS_FORK_CHILD()
{ {
TEST_ERROR(cmdArchiveGet(), FileMissingError, "!!!EMBEDDEDPERLERROR!!!"); TEST_ERROR_FMT(
cmdArchiveGet(), FileMissingError,
"unable to open %s/archive/test1/archive.info or %s/archive/test1/archive.info.copy\n"
"HINT: archive.info does not exist but is required to push/get WAL segments.\n"
"HINT: is archive_command configured in postgresql.conf?\n"
"HINT: has a stanza-create been performed?\n"
"HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving"
" scheme.",
strPtr(cfgOptionStr(cfgOptRepoPath)), strPtr(cfgOptionStr(cfgOptRepoPath)));
} }
} }
HARNESS_FORK_END(); HARNESS_FORK_END();