1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00

Store base path for remote storage locally.

It wasn't practical for the main process to be ignorant of the remote path, and in any case knowing the path makes debugging easier.

Pull the remote path when connecting and pass the result of local storagePath() to the remote when making calls.
This commit is contained in:
David Steele
2019-11-16 17:12:16 -05:00
parent 6827a13f3a
commit 8d6a8c3bf0
6 changed files with 64 additions and 50 deletions
+2 -2
View File
@@ -588,13 +588,13 @@ storage/posix/write.o: storage/posix/write.c build.auto.h common/assert.h common
storage/read.o: storage/read.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/read.h storage/read.intern.h storage/read.o: storage/read.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/read.h storage/read.intern.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/read.c -o storage/read.o $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/read.c -o storage/read.o
storage/remote/protocol.o: storage/remote/protocol.c build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/sink.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h storage/remote/protocol.o: storage/remote/protocol.c build.auto.h command/backup/pageChecksum.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/sink.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/protocol.c -o storage/remote/protocol.o $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/protocol.c -o storage/remote/protocol.o
storage/remote/read.o: storage/remote/read.c build.auto.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/read.h storage/remote/storage.h storage/remote/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h storage/remote/read.o: storage/remote/read.c build.auto.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/read.h storage/remote/storage.h storage/remote/storage.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/read.c -o storage/remote/read.o $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/read.c -o storage/remote/read.o
storage/remote/storage.o: storage/remote/storage.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/read.h storage/remote/storage.h storage/remote/storage.intern.h storage/remote/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h storage/remote/storage.o: storage/remote/storage.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/read.h storage/remote/storage.h storage/remote/storage.intern.h storage/remote/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/storage.c -o storage/remote/storage.o $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/remote/storage.c -o storage/remote/storage.o
storage/remote/write.o: storage/remote/write.c build.auto.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/storage.h storage/remote/storage.intern.h storage/remote/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h storage/remote/write.o: storage/remote/write.c build.auto.h common/assert.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h storage/info.h storage/read.h storage/read.intern.h storage/remote/protocol.h storage/remote/storage.h storage/remote/storage.intern.h storage/remote/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
+15 -18
View File
@@ -15,6 +15,7 @@ Remote Storage Protocol Handler
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/regExp.h" #include "common/regExp.h"
#include "common/type/json.h"
#include "config/config.h" #include "config/config.h"
#include "storage/remote/protocol.h" #include "storage/remote/protocol.h"
#include "storage/helper.h" #include "storage/helper.h"
@@ -115,28 +116,27 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto
{ {
if (strEq(command, PROTOCOL_COMMAND_STORAGE_EXISTS_STR)) if (strEq(command, PROTOCOL_COMMAND_STORAGE_EXISTS_STR))
{ {
protocolServerResponse(server, VARBOOL( // The unusual line break is to make coverage happy -- not sure why protocolServerResponse(server, VARBOOL(interface.exists(driver, varStr(varLstGet(paramList, 0)))));
interface.exists(driver, storagePathNP(storage, varStr(varLstGet(paramList, 0))))));
} }
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_FEATURE_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_FEATURE_STR))
{ {
protocolServerResponse(server, varNewUInt64(interface.feature)); protocolServerWriteLine(server, jsonFromStr(storagePathNP(storage, NULL)));
protocolServerWriteLine(server, jsonFromUInt64(interface.feature));
protocolServerResponse(server, NULL);
} }
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_LIST_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_LIST_STR))
{ {
protocolServerResponse( protocolServerResponse(
server, server,
varNewVarLst( varNewVarLst(
varLstNewStrLst( varLstNewStrLst(interface.list(driver, varStr(varLstGet(paramList, 0)), varStr(varLstGet(paramList, 1))))));
interface.list(
driver, storagePathNP(storage, varStr(varLstGet(paramList, 0))), varStr(varLstGet(paramList, 1))))));
} }
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR))
{ {
// Create the read object // Create the read object
IoRead *fileRead = storageReadIo( IoRead *fileRead = storageReadIo(
interface.newRead( interface.newRead(driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), false));
driver, storagePathNP(storage, varStr(varLstGet(paramList, 0))), varBool(varLstGet(paramList, 1)), false));
// Set filter group based on passed filters // Set filter group based on passed filters
storageRemoteFilterGroup(ioReadFilterGroup(fileRead), varLstGet(paramList, 2)); storageRemoteFilterGroup(ioReadFilterGroup(fileRead), varLstGet(paramList, 2));
@@ -181,7 +181,7 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto
// Create the write object // Create the write object
IoWrite *fileWrite = storageWriteIo( IoWrite *fileWrite = storageWriteIo(
interface.newWrite( interface.newWrite(
driver, storagePathNP(storage, varStr(varLstGet(paramList, 0))), varUIntForce(varLstGet(paramList, 1)), driver, varStr(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)),
varUIntForce(varLstGet(paramList, 2)), varStr(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)), varUIntForce(varLstGet(paramList, 2)), varStr(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)),
(time_t)varIntForce(varLstGet(paramList, 5)), varBool(varLstGet(paramList, 6)), (time_t)varIntForce(varLstGet(paramList, 5)), varBool(varLstGet(paramList, 6)),
varBool(varLstGet(paramList, 7)), varBool(varLstGet(paramList, 8)), varBool(varLstGet(paramList, 9)), false)); varBool(varLstGet(paramList, 7)), varBool(varLstGet(paramList, 8)), varBool(varLstGet(paramList, 9)), false));
@@ -241,8 +241,8 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_CREATE_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_CREATE_STR))
{ {
interface.pathCreate( interface.pathCreate(
driver, storagePathNP(storage, varStr(varLstGet(paramList, 0))), varBool(varLstGet(paramList, 1)), driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varBool(varLstGet(paramList, 2)),
varBool(varLstGet(paramList, 2)), varUIntForce(varLstGet(paramList, 3))); varUIntForce(varLstGet(paramList, 3)));
protocolServerResponse(server, NULL); protocolServerResponse(server, NULL);
} }
@@ -251,25 +251,22 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto
// Not all drivers implement pathExists() // Not all drivers implement pathExists()
CHECK(interface.pathExists != NULL); CHECK(interface.pathExists != NULL);
protocolServerResponse(server, VARBOOL( // The unusual line break is to make coverage happy -- not sure why protocolServerResponse(server, VARBOOL(interface.pathExists(driver, varStr(varLstGet(paramList, 0)))));
interface.pathExists(driver, storagePathNP(storage, varStr(varLstGet(paramList, 0))))));
} }
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE_STR))
{ {
protocolServerResponse(server, protocolServerResponse(server,
VARBOOL( VARBOOL(interface.pathRemove(driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)))));
interface.pathRemove(
driver, storagePathNP(storage, varStr(varLstGet(paramList, 0))), varBool(varLstGet(paramList, 1)))));
} }
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_SYNC_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_SYNC_STR))
{ {
interface.pathSync(driver, storagePathNP(storage, varStr(varLstGet(paramList, 0)))); interface.pathSync(driver, varStr(varLstGet(paramList, 0)));
protocolServerResponse(server, NULL); protocolServerResponse(server, NULL);
} }
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_REMOVE_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_REMOVE_STR))
{ {
interface.remove(driver, storagePathNP(storage, varStr(varLstGet(paramList, 0))), varBool(varLstGet(paramList, 1))); interface.remove(driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)));
protocolServerResponse(server, NULL); protocolServerResponse(server, NULL);
} }
+20 -3
View File
@@ -7,6 +7,7 @@ Remote Storage
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/object.h" #include "common/object.h"
#include "common/type/json.h"
#include "storage/remote/protocol.h" #include "storage/remote/protocol.h"
#include "storage/remote/read.h" #include "storage/remote/read.h"
#include "storage/remote/storage.intern.h" #include "storage/remote/storage.intern.h"
@@ -94,6 +95,7 @@ storageRemoteList(THIS_VOID, const String *path, const String *expression)
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(path != NULL);
StringList *result = NULL; StringList *result = NULL;
@@ -126,6 +128,7 @@ storageRemoteNewRead(THIS_VOID, const String *file, bool ignoreMissing, bool com
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(file != NULL);
FUNCTION_LOG_RETURN( FUNCTION_LOG_RETURN(
STORAGE_READ, STORAGE_READ,
@@ -216,6 +219,7 @@ storageRemotePathExists(THIS_VOID, const String *path)
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(path != NULL);
bool result = false; bool result = false;
@@ -352,17 +356,30 @@ storageRemoteNew(
driver->compressLevel = compressLevel; driver->compressLevel = compressLevel;
uint64_t feature = 0; uint64_t feature = 0;
const String *path = NULL;
// Get storage features from the remote // Get storage features from the remote
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
feature = varUInt64( // Send command
protocolClientExecute(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE_STR), true)); protocolClientWriteCommand(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE_STR));
// Read values
path = jsonToStr(protocolClientReadLine(driver->client));
feature = jsonToUInt64(protocolClientReadLine(driver->client));
// Acknowledge command completed
protocolClientReadOutput(driver->client, false);
// Dup path into parent context
memContextSwitch(MEM_CONTEXT_OLD());
path = strDup(path);
memContextSwitch(MEM_CONTEXT_TEMP());
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
this = storageNewP( this = storageNewP(
STORAGE_REMOTE_TYPE_STR, NULL, modeFile, modePath, write, pathExpressionFunction, driver, .feature = feature, STORAGE_REMOTE_TYPE_STR, path, modeFile, modePath, write, pathExpressionFunction, driver, .feature = feature,
.exists = storageRemoteExists, .info = storageRemoteInfo, .list = storageRemoteList, .newRead = storageRemoteNewRead, .exists = storageRemoteExists, .info = storageRemoteInfo, .list = storageRemoteList, .newRead = storageRemoteNewRead,
.newWrite = storageRemoteNewWrite, .pathCreate = storageRemotePathCreate, .pathExists = storageRemotePathExists, .newWrite = storageRemoteNewWrite, .pathCreate = storageRemotePathCreate, .pathExists = storageRemotePathExists,
.pathRemove = storageRemotePathRemove, .pathSync = storageRemotePathSync, .remove = storageRemoteRemove); .pathRemove = storageRemotePathRemove, .pathSync = storageRemotePathSync, .remove = storageRemoteRemove);
+3 -5
View File
@@ -56,7 +56,7 @@ storageNew(
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(type != NULL); ASSERT(type != NULL);
ASSERT(path == NULL || (strSize(path) >= 1 && strPtr(path)[0] == '/')); ASSERT(strSize(path) >= 1 && strPtr(path)[0] == '/');
ASSERT(driver != NULL); ASSERT(driver != NULL);
ASSERT(interface.exists != NULL); ASSERT(interface.exists != NULL);
ASSERT(interface.list != NULL); ASSERT(interface.list != NULL);
@@ -666,7 +666,7 @@ storagePath(const Storage *this, const String *pathExp)
if ((strPtr(pathExp))[0] == '/') if ((strPtr(pathExp))[0] == '/')
{ {
// Make sure the base storage path is contained within the path expression // Make sure the base storage path is contained within the path expression
if (this->path != NULL && !strEqZ(this->path, "/")) if (!strEqZ(this->path, "/"))
{ {
if (this->pathEnforce && (!strBeginsWith(pathExp, this->path) || if (this->pathEnforce && (!strBeginsWith(pathExp, this->path) ||
!(strSize(pathExp) == strSize(this->path) || *(strPtr(pathExp) + strSize(this->path)) == '/'))) !(strSize(pathExp) == strSize(this->path) || *(strPtr(pathExp) + strSize(this->path)) == '/')))
@@ -730,9 +730,7 @@ storagePath(const Storage *this, const String *pathExp)
strFree(path); strFree(path);
} }
if (this->path == NULL) if (strEqZ(this->path, "/"))
result = strDup(pathExp);
else if (strEqZ(this->path, "/"))
result = strNewFmt("/%s", strPtr(pathExp)); result = strNewFmt("/%s", strPtr(pathExp));
else else
result = strNewFmt("%s/%s", strPtr(this->path), strPtr(pathExp)); result = strNewFmt("%s/%s", strPtr(this->path), strPtr(pathExp));
+2 -2
View File
@@ -4,7 +4,7 @@ run 002 - rmt 1, s3 1, enc 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 ERROR: [055]: unable to load info file 'archive/db/archive.info' or 'archive/db/archive.info.copy': P00 ERROR: [055]: unable to load info file '/archive/db/archive.info' or '/archive/db/archive.info.copy':
FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info': No such file or directory FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info': No such file or directory
FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info.copy': No such file or directory FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info.copy': No such file or directory
HINT: archive.info cannot be opened but is required to push/get WAL segments. HINT: archive.info cannot be opened but is required to push/get WAL segments.
@@ -16,7 +16,7 @@ P00 INFO: archive-push command end: aborted with exception [055]
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 ERROR: [055]: unable to load info file 'archive/db/archive.info' or 'archive/db/archive.info.copy': P00 ERROR: [055]: unable to load info file '/archive/db/archive.info' or '/archive/db/archive.info.copy':
FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info': No such file or directory FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info': No such file or directory
FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info.copy': No such file or directory FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info.copy': No such file or directory
HINT: archive.info cannot be opened but is required to push/get WAL segments. HINT: archive.info cannot be opened but is required to push/get WAL segments.
+22 -20
View File
@@ -62,13 +62,15 @@ testRun(void)
TEST_RESULT_UINT(storageInterface(storageRemote).feature, storageInterface(storageTest).feature, " check features"); TEST_RESULT_UINT(storageInterface(storageRemote).feature, storageInterface(storageTest).feature, " check features");
TEST_RESULT_BOOL(storageFeature(storageRemote, storageFeaturePath), true, " check path feature"); TEST_RESULT_BOOL(storageFeature(storageRemote, storageFeaturePath), true, " check path feature");
TEST_RESULT_BOOL(storageFeature(storageRemote, storageFeatureCompress), true, " check compress feature"); TEST_RESULT_BOOL(storageFeature(storageRemote, storageFeatureCompress), true, " check compress feature");
TEST_RESULT_STR_STR(storagePath(storageRemote, NULL), strNewFmt("%s/repo", testPath()), " check path");
// Check protocol function directly // Check protocol function directly
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_FEATURE_STR, varLstNew(), server), true, "protocol feature"); storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_FEATURE_STR, varLstNew(), server), true, "protocol feature");
TEST_RESULT_STR( TEST_RESULT_STR_STR(
strPtr(strNewBuf(serverWrite)), strPtr(strNewFmt("{\"out\":%" PRIu64 "}\n", storageInterface(storageTest).feature)), strNewBuf(serverWrite),
strNewFmt(".\"%s/repo\"\n.%" PRIu64 "\n{}\n", testPath(), storageInterface(storageTest).feature),
"check result"); "check result");
bufUsedSet(serverWrite, 0); bufUsedSet(serverWrite, 0);
@@ -97,7 +99,7 @@ testRun(void)
cfgOptionValidSet(cfgOptType, true); cfgOptionValidSet(cfgOptType, true);
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNew("test.txt"))); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test.txt", testPath())));
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_EXISTS_STR, paramList, server), true, "protocol exists"); storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_EXISTS_STR, paramList, server), true, "protocol exists");
@@ -131,7 +133,7 @@ testRun(void)
// Check protocol function directly // Check protocol function directly
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, NULL); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo", testPath())));
varLstAdd(paramList, varNewStr(strNew("^testy$"))); varLstAdd(paramList, varNewStr(strNew("^testy$")));
TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_LIST_STR, paramList, server), true, "protocol list"); TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_LIST_STR, paramList, server), true, "protocol list");
@@ -170,7 +172,7 @@ testRun(void)
TEST_ASSIGN(fileRead, storageNewReadNP(storageRemote, strNew("test.txt")), "new file"); TEST_ASSIGN(fileRead, storageNewReadNP(storageRemote, strNew("test.txt")), "new file");
TEST_RESULT_BOOL(bufEq(storageGetNP(fileRead), contentBuf), true, "get file"); TEST_RESULT_BOOL(bufEq(storageGetNP(fileRead), contentBuf), true, "get file");
TEST_RESULT_BOOL(storageReadIgnoreMissing(fileRead), false, "check ignore missing"); TEST_RESULT_BOOL(storageReadIgnoreMissing(fileRead), false, "check ignore missing");
TEST_RESULT_STR(strPtr(storageReadName(fileRead)), "test.txt", "check name"); TEST_RESULT_STR_Z(storageReadName(fileRead), hrnReplaceKey("{[path]}/repo/test.txt"), "check name");
TEST_RESULT_SIZE( TEST_RESULT_SIZE(
storageReadRemote(storageRead(fileRead), bufNew(32), false), 0, storageReadRemote(storageRead(fileRead), bufNew(32), false), 0,
"nothing more to read"); "nothing more to read");
@@ -213,7 +215,7 @@ testRun(void)
ioBufferSizeSet(4); ioBufferSizeSet(4);
paramList = varLstNew(); paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNew("test.txt"))); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test.txt", testPath())));
varLstAdd(paramList, varNewBool(false)); varLstAdd(paramList, varNewBool(false));
// Create filters to test filter logic // Create filters to test filter logic
@@ -246,7 +248,7 @@ testRun(void)
// Check protocol function directly (file exists but all data goes to sink) // Check protocol function directly (file exists but all data goes to sink)
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
paramList = varLstNew(); paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNew("test.txt"))); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test.txt", testPath())));
varLstAdd(paramList, varNewBool(false)); varLstAdd(paramList, varNewBool(false));
// Create filters to test filter logic // Create filters to test filter logic
@@ -270,7 +272,7 @@ testRun(void)
// Check for error on a bogus filter // Check for error on a bogus filter
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
paramList = varLstNew(); paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNew("test.txt"))); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test.txt", testPath())));
varLstAdd(paramList, varNewBool(false)); varLstAdd(paramList, varNewBool(false));
varLstAdd(paramList, varNewVarLst(varLstAdd(varLstNew(), varNewKv(kvAdd(kvNew(), varNewStrZ("bogus"), NULL))))); varLstAdd(paramList, varNewVarLst(varLstAdd(varLstNew(), varNewKv(kvAdd(kvNew(), varNewStrZ("bogus"), NULL)))));
@@ -309,7 +311,7 @@ testRun(void)
TEST_RESULT_BOOL(storageWriteCreatePath(write), true, "path will be created"); TEST_RESULT_BOOL(storageWriteCreatePath(write), true, "path will be created");
TEST_RESULT_UINT(storageWriteModeFile(write), STORAGE_MODE_FILE_DEFAULT, "file mode is default"); TEST_RESULT_UINT(storageWriteModeFile(write), STORAGE_MODE_FILE_DEFAULT, "file mode is default");
TEST_RESULT_UINT(storageWriteModePath(write), STORAGE_MODE_PATH_DEFAULT, "path mode is default"); TEST_RESULT_UINT(storageWriteModePath(write), STORAGE_MODE_PATH_DEFAULT, "path mode is default");
TEST_RESULT_STR(strPtr(storageWriteName(write)), "test.txt", "check file name"); TEST_RESULT_STR(strPtr(storageWriteName(write)), hrnReplaceKey("{[path]}/repo/test.txt"), "check file name");
TEST_RESULT_BOOL(storageWriteSyncFile(write), true, "file is synced"); TEST_RESULT_BOOL(storageWriteSyncFile(write), true, "file is synced");
TEST_RESULT_BOOL(storageWriteSyncPath(write), true, "path is synced"); TEST_RESULT_BOOL(storageWriteSyncPath(write), true, "path is synced");
@@ -350,7 +352,7 @@ testRun(void)
ioBufferSizeSet(10); ioBufferSizeSet(10);
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNew("test3.txt"))); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test3.txt", testPath())));
varLstAdd(paramList, varNewUInt64(0640)); varLstAdd(paramList, varNewUInt64(0640));
varLstAdd(paramList, varNewUInt64(0750)); varLstAdd(paramList, varNewUInt64(0750));
varLstAdd(paramList, NULL); varLstAdd(paramList, NULL);
@@ -391,7 +393,7 @@ testRun(void)
ioBufferSizeSet(10); ioBufferSizeSet(10);
paramList = varLstNew(); paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNew("test4.txt"))); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test4.txt", testPath())));
varLstAdd(paramList, varNewUInt64(0640)); varLstAdd(paramList, varNewUInt64(0640));
varLstAdd(paramList, varNewUInt64(0750)); varLstAdd(paramList, varNewUInt64(0750));
varLstAdd(paramList, NULL); varLstAdd(paramList, NULL);
@@ -432,7 +434,7 @@ testRun(void)
// Check protocol function directly // Check protocol function directly
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNew("test"))); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test.txt", testPath())));
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR, paramList, server), true, "protocol path exists"); storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR, paramList, server), true, "protocol path exists");
@@ -460,7 +462,7 @@ testRun(void)
// Check protocol function directly // Check protocol function directly
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(path)); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/%s", testPath(), strPtr(path))));
varLstAdd(paramList, varNewBool(true)); // errorOnExists varLstAdd(paramList, varNewBool(true)); // errorOnExists
varLstAdd(paramList, varNewBool(true)); // noParentCreate (true=error if it does not have a parent, false=create parent) varLstAdd(paramList, varNewBool(true)); // noParentCreate (true=error if it does not have a parent, false=create parent)
varLstAdd(paramList, varNewUInt64(0)); // path mode varLstAdd(paramList, varNewUInt64(0)); // path mode
@@ -473,7 +475,7 @@ testRun(void)
// Error if parent path not exist // Error if parent path not exist
path = strNew("parent/testpath"); path = strNew("parent/testpath");
paramList = varLstNew(); paramList = varLstNew();
varLstAdd(paramList, varNewStr(path)); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/%s", testPath(), strPtr(path))));
varLstAdd(paramList, varNewBool(false)); // errorOnExists varLstAdd(paramList, varNewBool(false)); // errorOnExists
varLstAdd(paramList, varNewBool(true)); // noParentCreate (true=error if it does not have a parent, false=create parent) varLstAdd(paramList, varNewBool(true)); // noParentCreate (true=error if it does not have a parent, false=create parent)
varLstAdd(paramList, varNewUInt64(0)); // path mode varLstAdd(paramList, varNewUInt64(0)); // path mode
@@ -485,7 +487,7 @@ testRun(void)
// Create parent and path with default mode // Create parent and path with default mode
paramList = varLstNew(); paramList = varLstNew();
varLstAdd(paramList, varNewStr(path)); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/%s", testPath(), strPtr(path))));
varLstAdd(paramList, varNewBool(true)); // errorOnExists varLstAdd(paramList, varNewBool(true)); // errorOnExists
varLstAdd(paramList, varNewBool(false)); // noParentCreate (true=error if it does not have a parent, false=create parent) varLstAdd(paramList, varNewBool(false)); // noParentCreate (true=error if it does not have a parent, false=create parent)
varLstAdd(paramList, varNewUInt64(0777)); // path mode varLstAdd(paramList, varNewUInt64(0777)); // path mode
@@ -517,7 +519,7 @@ testRun(void)
// Check protocol function directly // Check protocol function directly
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(path)); // path varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/%s", testPath(), strPtr(path))));
varLstAdd(paramList, varNewBool(true)); // recurse varLstAdd(paramList, varNewBool(true)); // recurse
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
@@ -560,7 +562,7 @@ testRun(void)
// Check protocol function directly // Check protocol function directly
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(file)); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/%s", testPath(), strPtr(file))));
varLstAdd(paramList, varNewBool(true)); varLstAdd(paramList, varNewBool(true));
TEST_ERROR_FMT( TEST_ERROR_FMT(
@@ -569,7 +571,7 @@ testRun(void)
"[2] No such file or directory", testPath()); "[2] No such file or directory", testPath());
paramList = varLstNew(); paramList = varLstNew();
varLstAdd(paramList, varNewStr(file)); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/%s", testPath(), strPtr(file))));
varLstAdd(paramList, varNewBool(false)); varLstAdd(paramList, varNewBool(false));
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
@@ -603,7 +605,7 @@ testRun(void)
// Check protocol function directly // Check protocol function directly
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(path)); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/%s", testPath(), strPtr(path))));
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_PATH_SYNC_STR, paramList, server), true, storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_PATH_SYNC_STR, paramList, server), true,
@@ -612,7 +614,7 @@ testRun(void)
bufUsedSet(serverWrite, 0); bufUsedSet(serverWrite, 0);
paramList = varLstNew(); paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNew("anewpath"))); varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/anewpath", testPath())));
TEST_ERROR_FMT( TEST_ERROR_FMT(
storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_PATH_SYNC_STR, paramList, server), PathMissingError, storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_PATH_SYNC_STR, paramList, server), PathMissingError,
"raised from remote-0 protocol on 'localhost': " STORAGE_ERROR_PATH_SYNC_MISSING, "raised from remote-0 protocol on 'localhost': " STORAGE_ERROR_PATH_SYNC_MISSING,