diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 477a0160e..997d6cfc9 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -15,6 +15,14 @@ + + + + + +

Migrate backup manifest load/save to C.

+ + diff --git a/src/Makefile.in b/src/Makefile.in index 0a77f26d9..bf43aff39 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -136,6 +136,7 @@ SRCS = \ info/info.c \ info/infoArchive.c \ info/infoBackup.c \ + info/manifest.c \ info/infoPg.c \ perl/config.c \ perl/exec.c \ @@ -261,7 +262,7 @@ command/control/start.o: command/control/start.c build.auto.h command/control/co command/control/stop.o: command/control/stop.c build.auto.h command/control/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h storage/helper.h storage/info.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/control/stop.c -o command/control/stop.o -command/expire/expire.o: command/expire/expire.c build.auto.h command/archive/common.h command/backup/common.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/expire/expire.o: command/expire/expire.c build.auto.h command/archive/common.h command/backup/common.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/expire/expire.c -o command/expire/expire.o command/help/help.o: command/help/help.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h @@ -288,7 +289,7 @@ command/stanza/common.o: command/stanza/common.c build.auto.h command/check/comm command/stanza/create.o: command/stanza/create.c build.auto.h command/control/common.h command/stanza/common.h command/stanza/create.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/stanza/create.c -o command/stanza/create.o -command/stanza/delete.o: command/stanza/delete.c build.auto.h command/backup/common.h command/control/common.h command/stanza/delete.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +command/stanza/delete.o: command/stanza/delete.c build.auto.h command/backup/common.h command/control/common.h command/stanza/delete.h common/assert.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/stanza/delete.c -o command/stanza/delete.o command/stanza/upgrade.o: command/stanza/upgrade.c build.auto.h command/control/common.h command/stanza/common.h command/stanza/upgrade.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h @@ -483,12 +484,15 @@ info/info.o: info/info.c build.auto.h common/assert.h common/crypto/hash.h commo info/infoArchive.o: info/infoArchive.c build.auto.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h info/info.h info/infoArchive.h info/infoPg.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/infoArchive.c -o info/infoArchive.o -info/infoBackup.o: info/infoBackup.c build.auto.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h info/info.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h +info/infoBackup.o: info/infoBackup.c build.auto.h command/backup/common.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h info/info.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/infoBackup.c -o info/infoBackup.o info/infoPg.o: info/infoPg.c build.auto.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h info/info.h info/infoPg.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/infoPg.c -o info/infoPg.o +info/manifest.o: info/manifest.c build.auto.h command/backup/common.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/mcv.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h info/info.h info/manifest.h postgres/interface.h postgres/version.h storage/info.h storage/read.h storage/storage.h storage/write.h + $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/manifest.c -o info/manifest.o + main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/check/check.h command/command.h command/control/start.h command/control/stop.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/stanza/create.h command/stanza/delete.h command/stanza/upgrade.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c main.c -o main.o diff --git a/src/info/manifest.c b/src/info/manifest.c new file mode 100644 index 000000000..d53ddb4d0 --- /dev/null +++ b/src/info/manifest.c @@ -0,0 +1,2004 @@ +/*********************************************************************************************************************************** +Backup Manifest Handler +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include + +#include "common/crypto/cipherBlock.h" +#include "common/debug.h" +#include "common/log.h" +#include "common/type/json.h" +#include "common/type/list.h" +#include "common/type/mcv.h" +#include "info/info.h" +#include "info/manifest.h" +#include "postgres/interface.h" +#include "postgres/version.h" +#include "storage/storage.h" + +/*********************************************************************************************************************************** +Constants +***********************************************************************************************************************************/ +STRING_EXTERN(BACKUP_MANIFEST_FILE_STR, BACKUP_MANIFEST_FILE); + +STRING_EXTERN(MANIFEST_TARGET_PGDATA_STR, MANIFEST_TARGET_PGDATA); +STRING_EXTERN(MANIFEST_TARGET_PGTBLSPC_STR, MANIFEST_TARGET_PGTBLSPC); + +STRING_STATIC(MANIFEST_TARGET_TYPE_LINK_STR, "link"); +STRING_STATIC(MANIFEST_TARGET_TYPE_PATH_STR, "path"); + +STRING_STATIC(MANIFEST_SECTION_BACKUP_STR, "backup"); +STRING_STATIC(MANIFEST_SECTION_BACKUP_DB_STR, "backup:db"); +STRING_STATIC(MANIFEST_SECTION_BACKUP_OPTION_STR, "backup:option"); +STRING_STATIC(MANIFEST_SECTION_BACKUP_TARGET_STR, "backup:target"); + +STRING_STATIC(MANIFEST_SECTION_DB_STR, "db"); + +STRING_STATIC(MANIFEST_SECTION_TARGET_FILE_STR, "target:file"); +STRING_STATIC(MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, "target:file:default"); + +STRING_STATIC(MANIFEST_SECTION_TARGET_LINK_STR, "target:link"); +STRING_STATIC(MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR, "target:link:default"); + +STRING_STATIC(MANIFEST_SECTION_TARGET_PATH_STR, "target:path"); +STRING_STATIC(MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, "target:path:default"); + +#define MANIFEST_KEY_BACKUP_ARCHIVE_START "backup-archive-start" + STRING_STATIC(MANIFEST_KEY_BACKUP_ARCHIVE_START_STR, MANIFEST_KEY_BACKUP_ARCHIVE_START); +#define MANIFEST_KEY_BACKUP_ARCHIVE_STOP "backup-archive-stop" + STRING_STATIC(MANIFEST_KEY_BACKUP_ARCHIVE_STOP_STR, MANIFEST_KEY_BACKUP_ARCHIVE_STOP); +#define MANIFEST_KEY_BACKUP_LABEL "backup-label" + STRING_STATIC(MANIFEST_KEY_BACKUP_LABEL_STR, MANIFEST_KEY_BACKUP_LABEL); +#define MANIFEST_KEY_BACKUP_LSN_START "backup-lsn-start" + STRING_STATIC(MANIFEST_KEY_BACKUP_LSN_START_STR, MANIFEST_KEY_BACKUP_LSN_START); +#define MANIFEST_KEY_BACKUP_LSN_STOP "backup-lsn-stop" + STRING_STATIC(MANIFEST_KEY_BACKUP_LSN_STOP_STR, MANIFEST_KEY_BACKUP_LSN_STOP); +#define MANIFEST_KEY_BACKUP_PRIOR "backup-prior" + STRING_STATIC(MANIFEST_KEY_BACKUP_PRIOR_STR, MANIFEST_KEY_BACKUP_PRIOR); +#define MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START "backup-timestamp-copy-start" + STRING_STATIC(MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START); +#define MANIFEST_KEY_BACKUP_TIMESTAMP_START "backup-timestamp-start" + STRING_STATIC(MANIFEST_KEY_BACKUP_TIMESTAMP_START_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_START); +#define MANIFEST_KEY_BACKUP_TIMESTAMP_STOP "backup-timestamp-stop" + STRING_STATIC(MANIFEST_KEY_BACKUP_TIMESTAMP_STOP_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_STOP); +#define MANIFEST_KEY_BACKUP_TYPE "backup-type" + STRING_STATIC(MANIFEST_KEY_BACKUP_TYPE_STR, MANIFEST_KEY_BACKUP_TYPE); +#define MANIFEST_KEY_CHECKSUM "checksum" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_CHECKSUM_VAR, MANIFEST_KEY_CHECKSUM); +#define MANIFEST_KEY_CHECKSUM_PAGE "checksum-page" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_CHECKSUM_PAGE_VAR, MANIFEST_KEY_CHECKSUM_PAGE); +#define MANIFEST_KEY_CHECKSUM_PAGE_ERROR "checksum-page-error" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, MANIFEST_KEY_CHECKSUM_PAGE_ERROR); +#define MANIFEST_KEY_DB_ID "db-id" + STRING_STATIC(MANIFEST_KEY_DB_ID_STR, MANIFEST_KEY_DB_ID); + VARIANT_STRDEF_STATIC(MANIFEST_KEY_DB_ID_VAR, MANIFEST_KEY_DB_ID); +#define MANIFEST_KEY_DB_LAST_SYSTEM_ID "db-last-system-id" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_DB_LAST_SYSTEM_ID_VAR, MANIFEST_KEY_DB_LAST_SYSTEM_ID); +#define MANIFEST_KEY_DB_SYSTEM_ID "db-system-id" + STRING_STATIC(MANIFEST_KEY_DB_SYSTEM_ID_STR, MANIFEST_KEY_DB_SYSTEM_ID); +#define MANIFEST_KEY_DB_VERSION "db-version" + STRING_STATIC(MANIFEST_KEY_DB_VERSION_STR, MANIFEST_KEY_DB_VERSION); +#define MANIFEST_KEY_DESTINATION "destination" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_DESTINATION_VAR, MANIFEST_KEY_DESTINATION); +#define MANIFEST_KEY_FILE "file" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_FILE_VAR, MANIFEST_KEY_FILE); +#define MANIFEST_KEY_GROUP "group" + STRING_STATIC(MANIFEST_KEY_GROUP_STR, MANIFEST_KEY_GROUP); + VARIANT_STRDEF_STATIC(MANIFEST_KEY_GROUP_VAR, MANIFEST_KEY_GROUP); +#define MANIFEST_KEY_PRIMARY "ma" "st" "er" + STRING_STATIC(MANIFEST_KEY_PRIMARY_STR, MANIFEST_KEY_PRIMARY); + VARIANT_STRDEF_STATIC(MANIFEST_KEY_PRIMARY_VAR, MANIFEST_KEY_PRIMARY); +#define MANIFEST_KEY_MODE "mode" + STRING_STATIC(MANIFEST_KEY_MODE_STR, MANIFEST_KEY_MODE); + VARIANT_STRDEF_STATIC(MANIFEST_KEY_MODE_VAR, MANIFEST_KEY_MODE); +#define MANIFEST_KEY_PATH "path" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_PATH_VAR, MANIFEST_KEY_PATH); +#define MANIFEST_KEY_REFERENCE "reference" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_REFERENCE_VAR, MANIFEST_KEY_REFERENCE); +#define MANIFEST_KEY_SIZE "size" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_SIZE_VAR, MANIFEST_KEY_SIZE); +#define MANIFEST_KEY_SIZE_REPO "repo-size" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_SIZE_REPO_VAR, MANIFEST_KEY_SIZE_REPO); +#define MANIFEST_KEY_TABLESPACE_ID "tablespace-id" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_TABLESPACE_ID_VAR, MANIFEST_KEY_TABLESPACE_ID); +#define MANIFEST_KEY_TABLESPACE_NAME "tablespace-name" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_TABLESPACE_NAME_VAR, MANIFEST_KEY_TABLESPACE_NAME); +#define MANIFEST_KEY_TIMESTAMP "timestamp" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_TIMESTAMP_VAR, MANIFEST_KEY_TIMESTAMP); +#define MANIFEST_KEY_TYPE "type" + VARIANT_STRDEF_STATIC(MANIFEST_KEY_TYPE_VAR, MANIFEST_KEY_TYPE); +#define MANIFEST_KEY_USER "user" + STRING_STATIC(MANIFEST_KEY_USER_STR, MANIFEST_KEY_USER); + VARIANT_STRDEF_STATIC(MANIFEST_KEY_USER_VAR, MANIFEST_KEY_USER); + +#define MANIFEST_KEY_OPTION_ARCHIVE_CHECK "option-archive-check" + STRING_STATIC(MANIFEST_KEY_OPTION_ARCHIVE_CHECK_STR, MANIFEST_KEY_OPTION_ARCHIVE_CHECK); +#define MANIFEST_KEY_OPTION_ARCHIVE_COPY "option-archive-copy" + STRING_STATIC(MANIFEST_KEY_OPTION_ARCHIVE_COPY_STR, MANIFEST_KEY_OPTION_ARCHIVE_COPY); +#define MANIFEST_KEY_OPTION_BACKUP_STANDBY "option-backup-standby" + STRING_STATIC(MANIFEST_KEY_OPTION_BACKUP_STANDBY_STR, MANIFEST_KEY_OPTION_BACKUP_STANDBY); +#define MANIFEST_KEY_OPTION_BUFFER_SIZE "option-buffer-size" + STRING_STATIC(MANIFEST_KEY_OPTION_BUFFER_SIZE_STR, MANIFEST_KEY_OPTION_BUFFER_SIZE); +#define MANIFEST_KEY_OPTION_CHECKSUM_PAGE "option-checksum-page" + STRING_STATIC(MANIFEST_KEY_OPTION_CHECKSUM_PAGE_STR, MANIFEST_KEY_OPTION_CHECKSUM_PAGE); +#define MANIFEST_KEY_OPTION_COMPRESS "option-compress" + STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_STR, MANIFEST_KEY_OPTION_COMPRESS); +#define MANIFEST_KEY_OPTION_COMPRESS_LEVEL "option-compress-level" + STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_LEVEL_STR, MANIFEST_KEY_OPTION_COMPRESS_LEVEL); +#define MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK "option-compress-level-network" + STRING_STATIC(MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK_STR, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK); +#define MANIFEST_KEY_OPTION_DELTA "option-delta" + STRING_STATIC(MANIFEST_KEY_OPTION_DELTA_STR, MANIFEST_KEY_OPTION_DELTA); +#define MANIFEST_KEY_OPTION_HARDLINK "option-hardlink" + STRING_STATIC(MANIFEST_KEY_OPTION_HARDLINK_STR, MANIFEST_KEY_OPTION_HARDLINK); +#define MANIFEST_KEY_OPTION_ONLINE "option-online" + STRING_STATIC(MANIFEST_KEY_OPTION_ONLINE_STR, MANIFEST_KEY_OPTION_ONLINE); +#define MANIFEST_KEY_OPTION_PROCESS_MAX "option-process-max" + STRING_STATIC(MANIFEST_KEY_OPTION_PROCESS_MAX_STR, MANIFEST_KEY_OPTION_PROCESS_MAX); + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +struct Manifest +{ + MemContext *memContext; // Context that contains the Manifest + + Info *info; // Base info object + StringList *ownerList; // List of users/groups + StringList *referenceList; // List of file references + + ManifestData data; // Manifest data and options + List *targetList; // List of targets + List *pathList; // List of paths + List *fileList; // List of files + List *linkList; // List of links + List *dbList; // List of databases +}; + +/*********************************************************************************************************************************** +Internal functions to add types to their lists +***********************************************************************************************************************************/ +// Helper to add owner to the owner list if it is not there already and return the pointer. This saves a lot of space. +static const String * +manifestOwnerCache(Manifest *this, const String *owner) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, owner); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (owner != NULL) + FUNCTION_TEST_RETURN(strLstAddIfMissing(this->ownerList, owner)); + + FUNCTION_TEST_RETURN(NULL); +} + +static void +manifestDbAdd(Manifest *this, const ManifestDb *db) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(MANIFEST_DB, db); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(db != NULL); + ASSERT(db->name != NULL); + + MEM_CONTEXT_BEGIN(lstMemContext(this->dbList)) + { + ManifestDb dbAdd = + { + .id = db->id, + .lastSystemId = db->lastSystemId, + .name = strDup(db->name), + }; + + lstAdd(this->dbList, &dbAdd); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +static void +manifestFileAdd(Manifest *this, const ManifestFile *file) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(MANIFEST_FILE, file); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(file != NULL); + ASSERT(file->name != NULL); + + MEM_CONTEXT_BEGIN(lstMemContext(this->fileList)) + { + ManifestFile fileAdd = + { + .checksumPage = file->checksumPage, + .checksumPageError = file->checksumPageError, + .checksumPageErrorList = varLstDup(file->checksumPageErrorList), + .group = manifestOwnerCache(this, file->group), + .mode = file->mode, + .name = strDup(file->name), + .primary = file->primary, + .size = file->size, + .sizeRepo = file->sizeRepo, + .timestamp = file->timestamp, + .user = manifestOwnerCache(this, file->user), + }; + + memcpy(fileAdd.checksumSha1, file->checksumSha1, HASH_TYPE_SHA1_SIZE_HEX + 1); + + if (file->reference != NULL) + { + // Search for the reference in the list + for (unsigned int referenceIdx = 0; referenceIdx < strLstSize(this->referenceList); referenceIdx++) + { + const String *found = strLstGet(this->referenceList, referenceIdx); + + if (strEq(file->reference, found)) + { + fileAdd.reference = found; + break; + } + } + + // If not found then add it + if (fileAdd.reference == NULL) + { + strLstAdd(this->referenceList, file->reference); + fileAdd.reference = strLstGet(this->referenceList, strLstSize(this->referenceList) - 1); + } + } + + lstAdd(this->fileList, &fileAdd); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +static void +manifestLinkAdd(Manifest *this, const ManifestLink *link) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(MANIFEST_LINK, link); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(link != NULL); + ASSERT(link->name != NULL); + ASSERT(link->destination != NULL); + + MEM_CONTEXT_BEGIN(lstMemContext(this->linkList)) + { + ManifestLink linkAdd = + { + .destination = strDup(link->destination), + .name = strDup(link->name), + .group = manifestOwnerCache(this, link->group), + .user = manifestOwnerCache(this, link->user), + }; + + lstAdd(this->linkList, &linkAdd); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +static void +manifestPathAdd(Manifest *this, const ManifestPath *path) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(MANIFEST_PATH, path); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(path != NULL); + ASSERT(path->name != NULL); + + MEM_CONTEXT_BEGIN(lstMemContext(this->pathList)) + { + ManifestPath pathAdd = + { + .mode = path->mode, + .name = strDup(path->name), + .group = manifestOwnerCache(this, path->group), + .user = manifestOwnerCache(this, path->user), + }; + + lstAdd(this->pathList, &pathAdd); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +static void +manifestTargetAdd(Manifest *this, const ManifestTarget *target) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(MANIFEST_TARGET, target); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(target != NULL); + ASSERT(target->path != NULL); + + MEM_CONTEXT_BEGIN(lstMemContext(this->targetList)) + { + ManifestTarget targetAdd = + { + .file = strDup(target->file), + .name = strDup(target->name), + .path = strDup(target->path), + .tablespaceId = target->tablespaceId, + .tablespaceName = strDup(target->tablespaceName), + .type = target->type, + }; + + lstAdd(this->targetList, &targetAdd); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Create new object +***********************************************************************************************************************************/ +static Manifest * +manifestNewInternal(void) +{ + FUNCTION_TEST_VOID(); + + // Create object + Manifest *this = memNew(sizeof(Manifest)); + this->memContext = memContextCurrent(); + + // Create lists + this->dbList = lstNewP(sizeof(ManifestDb), .comparator = lstComparatorStr); + this->fileList = lstNewP(sizeof(ManifestFile), .comparator = lstComparatorStr); + this->linkList = lstNewP(sizeof(ManifestLink), .comparator = lstComparatorStr); + this->pathList = lstNewP(sizeof(ManifestPath), .comparator = lstComparatorStr); + this->ownerList = strLstNew(); + this->referenceList = strLstNew(); + this->targetList = lstNewP(sizeof(ManifestTarget), .comparator = lstComparatorStr); + + FUNCTION_TEST_RETURN(this); +} + +/*********************************************************************************************************************************** +Load manifest +***********************************************************************************************************************************/ +// Keep track of which values were found during load and which need to be loaded from defaults. There is no point in having +// multiple structs since most of the fields are the same and the size shouldn't be more than 4/8 bytes. +typedef struct ManifestLoadFound +{ + bool group:1; + bool mode:1; + bool primary:1; + bool user:1; +} ManifestLoadFound; + +typedef struct ManifestLoadData +{ + MemContext *memContext; // Mem context for data needed only during load + Manifest *manifest; // Manifest info + + List *fileFoundList; // Values found in files + const Variant *fileGroupDefault; // File default group + mode_t fileModeDefault; // File default mode + bool filePrimaryDefault; // File default primary + const Variant *fileUserDefault; // File default user + + List *linkFoundList; // Values found in links + const Variant *linkGroupDefault; // Link default group + const Variant *linkUserDefault; // Link default user + + List *pathFoundList; // Values found in paths + const Variant *pathGroupDefault; // Path default group + mode_t pathModeDefault; // Path default mode + const Variant *pathUserDefault; // Path default user +} ManifestLoadData; + +// Helper to transform a variant that could be boolean or string into a string. If the boolean is false return NULL else return +// the string. The boolean cannot be true. +static const String * +manifestOwnerGet(const Variant *owner) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(VARIANT, owner); + FUNCTION_TEST_END(); + + ASSERT(owner != NULL); + + // If bool then it should be false. This indicates that the owner could not be mapped to a name during the backup. + if (varType(owner) == varTypeBool) + { + CHECK(!varBool(owner)); + FUNCTION_TEST_RETURN(NULL); + } + + FUNCTION_TEST_RETURN(varStr(owner)); +} + +// Helper to convert default owner to a variant. Input could be boolean false (meaning there is no owner) or a string (there is an +// owner). +static const Variant * +manifestOwnerDefaultGet(const String *ownerDefault) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, ownerDefault); + FUNCTION_TEST_END(); + + ASSERT(ownerDefault != NULL); + + FUNCTION_TEST_RETURN(strEq(ownerDefault, FALSE_STR) ? BOOL_FALSE_VAR : varNewStr(jsonToStr(ownerDefault))); +} + +static void +manifestLoadCallback(void *callbackData, const String *section, const String *key, const String *value) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM_P(VOID, callbackData); + FUNCTION_TEST_PARAM(STRING, section); + FUNCTION_TEST_PARAM(STRING, key); + FUNCTION_TEST_PARAM(STRING, value); + FUNCTION_TEST_END(); + + ASSERT(callbackData != NULL); + ASSERT(section != NULL); + ASSERT(key != NULL); + ASSERT(value != NULL); + + ManifestLoadData *loadData = (ManifestLoadData *)callbackData; + Manifest *manifest = loadData->manifest; + + // ----------------------------------------------------------------------------------------------------------------------------- + if (strEq(section, MANIFEST_SECTION_TARGET_FILE_STR)) + { + KeyValue *fileKv = varKv(jsonToVar(value)); + + MEM_CONTEXT_BEGIN(lstMemContext(manifest->fileList)) + { + ManifestLoadFound valueFound = {0}; + + ManifestFile file = + { + .name = key, + .reference = varStr(kvGetDefault(fileKv, MANIFEST_KEY_REFERENCE_VAR, NULL)), + .size = varUInt64(kvGet(fileKv, MANIFEST_KEY_SIZE_VAR)), + .timestamp = (time_t)varUInt64(kvGet(fileKv, MANIFEST_KEY_TIMESTAMP_VAR)), + }; + + file.sizeRepo = varUInt64(kvGetDefault(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, VARUINT64(file.size))); + + if (file.size == 0) + memcpy(file.checksumSha1, HASH_TYPE_SHA1_ZERO, HASH_TYPE_SHA1_SIZE_HEX + 1); + else + { + memcpy( + file.checksumSha1, strPtr(varStr(kvGet(fileKv, MANIFEST_KEY_CHECKSUM_VAR))), HASH_TYPE_SHA1_SIZE_HEX + 1); + } + + const Variant *checksumPage = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, NULL); + + if (checksumPage != NULL) + { + file.checksumPage = true; + file.checksumPageError = varBool(checksumPage); + + const Variant *checksumPageErrorList = kvGetDefault(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, NULL); + + if (checksumPageErrorList != NULL) + file.checksumPageErrorList = varVarLst(checksumPageErrorList); + } + + if (kvKeyExists(fileKv, MANIFEST_KEY_GROUP_VAR)) + { + valueFound.group = true; + file.group = manifestOwnerGet(kvGet(fileKv, MANIFEST_KEY_GROUP_VAR)); + } + + if (kvKeyExists(fileKv, MANIFEST_KEY_MODE_VAR)) + { + valueFound.mode = true; + file.mode = cvtZToMode(strPtr(varStr(kvGet(fileKv, MANIFEST_KEY_MODE_VAR)))); + } + + if (kvKeyExists(fileKv, MANIFEST_KEY_PRIMARY_VAR)) + { + valueFound.primary = true; + file.primary = varBool(kvGet(fileKv, MANIFEST_KEY_PRIMARY_VAR)); + } + + if (kvKeyExists(fileKv, MANIFEST_KEY_USER_VAR)) + { + valueFound.user = true; + file.user = manifestOwnerGet(kvGet(fileKv, MANIFEST_KEY_USER_VAR)); + } + + lstAdd(loadData->fileFoundList, &valueFound); + manifestFileAdd(manifest, &file); + } + MEM_CONTEXT_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_TARGET_PATH_STR)) + { + KeyValue *pathKv = varKv(jsonToVar(value)); + + MEM_CONTEXT_BEGIN(lstMemContext(manifest->pathList)) + { + ManifestLoadFound valueFound = {0}; + + ManifestPath path = + { + .name = key, + }; + + if (kvKeyExists(pathKv, MANIFEST_KEY_GROUP_VAR)) + { + valueFound.group = true; + path.group = manifestOwnerGet(kvGet(pathKv, MANIFEST_KEY_GROUP_VAR)); + } + + if (kvKeyExists(pathKv, MANIFEST_KEY_MODE_VAR)) + { + valueFound.mode = true; + path.mode = cvtZToMode(strPtr(varStr(kvGet(pathKv, MANIFEST_KEY_MODE_VAR)))); + } + + if (kvKeyExists(pathKv, MANIFEST_KEY_USER_VAR)) + { + valueFound.user = true; + path.user = manifestOwnerGet(kvGet(pathKv, MANIFEST_KEY_USER_VAR)); + } + + lstAdd(loadData->pathFoundList, &valueFound); + manifestPathAdd(manifest, &path); + } + MEM_CONTEXT_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_TARGET_LINK_STR)) + { + KeyValue *linkKv = varKv(jsonToVar(value)); + + MEM_CONTEXT_BEGIN(lstMemContext(manifest->linkList)) + { + ManifestLoadFound valueFound = {0}; + + ManifestLink link = + { + .name = key, + .destination = varStr(kvGet(linkKv, MANIFEST_KEY_DESTINATION_VAR)), + }; + + if (kvKeyExists(linkKv, MANIFEST_KEY_GROUP_VAR)) + { + valueFound.group = true; + link.group = manifestOwnerGet(kvGet(linkKv, MANIFEST_KEY_GROUP_VAR)); + } + + if (kvKeyExists(linkKv, MANIFEST_KEY_USER_VAR)) + { + valueFound.user = true; + link.user = manifestOwnerGet(kvGet(linkKv, MANIFEST_KEY_USER_VAR)); + } + + lstAdd(loadData->linkFoundList, &valueFound); + manifestLinkAdd(manifest, &link); + } + MEM_CONTEXT_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR)) + { + MEM_CONTEXT_BEGIN(loadData->memContext) + { + if (strEq(key, MANIFEST_KEY_GROUP_STR)) + loadData->fileGroupDefault = manifestOwnerDefaultGet(value); + else if (strEq(key, MANIFEST_KEY_MODE_STR)) + loadData->fileModeDefault = cvtZToMode(strPtr(jsonToStr(value))); + else if (strEq(key, MANIFEST_KEY_PRIMARY_STR)) + loadData->filePrimaryDefault = jsonToBool(value); + else if (strEq(key, MANIFEST_KEY_USER_STR)) + loadData->fileUserDefault = manifestOwnerDefaultGet(value); + } + MEM_CONTEXT_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR)) + { + MEM_CONTEXT_BEGIN(loadData->memContext) + { + if (strEq(key, MANIFEST_KEY_GROUP_STR)) + loadData->pathGroupDefault = manifestOwnerDefaultGet(value); + else if (strEq(key, MANIFEST_KEY_MODE_STR)) + loadData->pathModeDefault = cvtZToMode(strPtr(jsonToStr(value))); + else if (strEq(key, MANIFEST_KEY_USER_STR)) + loadData->pathUserDefault = manifestOwnerDefaultGet(value); + } + MEM_CONTEXT_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR)) + { + MEM_CONTEXT_BEGIN(loadData->memContext) + { + if (strEq(key, MANIFEST_KEY_GROUP_STR)) + loadData->linkGroupDefault = manifestOwnerDefaultGet(value); + else if (strEq(key, MANIFEST_KEY_USER_STR)) + loadData->linkUserDefault = manifestOwnerDefaultGet(value); + } + MEM_CONTEXT_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_BACKUP_TARGET_STR)) + { + KeyValue *targetKv = varKv(jsonToVar(value)); + const String *targetType = varStr(kvGet(targetKv, MANIFEST_KEY_TYPE_VAR)); + + ASSERT(strEq(targetType, MANIFEST_TARGET_TYPE_LINK_STR) || strEq(targetType, MANIFEST_TARGET_TYPE_PATH_STR)); + + ManifestTarget target = + { + .name = key, + .file = varStr(kvGetDefault(targetKv, MANIFEST_KEY_FILE_VAR, NULL)), + .path = varStr(kvGet(targetKv, MANIFEST_KEY_PATH_VAR)), + .tablespaceId = + cvtZToUInt(strPtr(varStr(kvGetDefault(targetKv, MANIFEST_KEY_TABLESPACE_ID_VAR, VARSTRDEF("0"))))), + .tablespaceName = varStr(kvGetDefault(targetKv, MANIFEST_KEY_TABLESPACE_NAME_VAR, NULL)), + .type = strEq(targetType, MANIFEST_TARGET_TYPE_PATH_STR) ? manifestTargetTypePath : manifestTargetTypeLink, + }; + + manifestTargetAdd(manifest, &target); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_DB_STR)) + { + KeyValue *dbKv = varKv(jsonToVar(value)); + + MEM_CONTEXT_BEGIN(lstMemContext(manifest->dbList)) + { + ManifestDb db = + { + .name = strDup(key), + .id = varUIntForce(kvGet(dbKv, MANIFEST_KEY_DB_ID_VAR)), + .lastSystemId = varUIntForce(kvGet(dbKv, MANIFEST_KEY_DB_LAST_SYSTEM_ID_VAR)), + }; + + manifestDbAdd(manifest, &db); + } + MEM_CONTEXT_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_BACKUP_STR)) + { + MEM_CONTEXT_BEGIN(manifest->memContext) + { + if (strEq(key, MANIFEST_KEY_BACKUP_ARCHIVE_START_STR)) + manifest->data.archiveStart = jsonToStr(value); + else if (strEq(key, MANIFEST_KEY_BACKUP_ARCHIVE_STOP_STR)) + manifest->data.archiveStop = jsonToStr(value); + else if (strEq(key, MANIFEST_KEY_BACKUP_LABEL_STR)) + manifest->data.backupLabel = jsonToStr(value); + else if (strEq(key, MANIFEST_KEY_BACKUP_LSN_START_STR)) + manifest->data.lsnStart = jsonToStr(value); + else if (strEq(key, MANIFEST_KEY_BACKUP_LSN_STOP_STR)) + manifest->data.lsnStop = jsonToStr(value); + else if (strEq(key, MANIFEST_KEY_BACKUP_PRIOR_STR)) + manifest->data.backupLabelPrior = jsonToStr(value); + else if (strEq(key, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START_STR)) + manifest->data.backupTimestampCopyStart = (time_t)jsonToUInt64(value); + else if (strEq(key, MANIFEST_KEY_BACKUP_TIMESTAMP_START_STR)) + manifest->data.backupTimestampStart = (time_t)jsonToUInt64(value); + else if (strEq(key, MANIFEST_KEY_BACKUP_TIMESTAMP_STOP_STR)) + manifest->data.backupTimestampStop = (time_t)jsonToUInt64(value); + else if (strEq(key, MANIFEST_KEY_BACKUP_TYPE_STR)) + manifest->data.backupType = backupType(jsonToStr(value)); + } + MEM_CONTEXT_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_BACKUP_DB_STR)) + { + if (strEq(key, MANIFEST_KEY_DB_ID_STR)) + manifest->data.pgId = jsonToUInt(value); + else if (strEq(key, MANIFEST_KEY_DB_SYSTEM_ID_STR)) + manifest->data.pgSystemId = jsonToUInt64(value); + else if (strEq(key, MANIFEST_KEY_DB_VERSION_STR)) + manifest->data.pgVersion = pgVersionFromStr(jsonToStr(value)); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + else if (strEq(section, MANIFEST_SECTION_BACKUP_OPTION_STR)) + { + MEM_CONTEXT_BEGIN(manifest->memContext) + { + // Required options + if (strEq(key, MANIFEST_KEY_OPTION_ARCHIVE_CHECK_STR)) + manifest->data.backupOptionArchiveCheck = jsonToBool(value); + else if (strEq(key, MANIFEST_KEY_OPTION_ARCHIVE_COPY_STR)) + manifest->data.backupOptionArchiveCopy = jsonToBool(value); + else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_STR)) + manifest->data.backupOptionCompress = jsonToBool(value); + else if (strEq(key, MANIFEST_KEY_OPTION_HARDLINK_STR)) + manifest->data.backupOptionHardLink = jsonToBool(value); + else if (strEq(key, MANIFEST_KEY_OPTION_ONLINE_STR)) + manifest->data.backupOptionOnline = jsonToBool(value); + + // Options that were added after v1.00 and may not be present in every manifest + else if (strEq(key, MANIFEST_KEY_OPTION_BACKUP_STANDBY_STR)) + manifest->data.backupOptionStandby = varNewBool(jsonToBool(value)); + else if (strEq(key, MANIFEST_KEY_OPTION_BUFFER_SIZE_STR)) + manifest->data.backupOptionBufferSize = varNewUInt(jsonToUInt(value)); + else if (strEq(key, MANIFEST_KEY_OPTION_CHECKSUM_PAGE_STR)) + manifest->data.backupOptionChecksumPage = varNewBool(jsonToBool(value)); + else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_STR)) + manifest->data.backupOptionCompressLevel = varNewUInt(jsonToUInt(value)); + else if (strEq(key, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK_STR)) + manifest->data.backupOptionCompressLevelNetwork = varNewUInt(jsonToUInt(value)); + else if (strEq(key, MANIFEST_KEY_OPTION_DELTA_STR)) + manifest->data.backupOptionDelta = varNewBool(jsonToBool(value)); + else if (strEq(key, MANIFEST_KEY_OPTION_PROCESS_MAX_STR)) + manifest->data.backupOptionProcessMax = varNewUInt(jsonToUInt(value)); + } + MEM_CONTEXT_END(); + } + + FUNCTION_TEST_RETURN_VOID(); +} + +Manifest * +manifestNewLoad(IoRead *read) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(IO_READ, read); + FUNCTION_LOG_END(); + + ASSERT(read != NULL); + + Manifest *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("Manifest") + { + this = manifestNewInternal(); + + // Load the manifest + ManifestLoadData loadData = + { + .memContext = memContextNew("load"), + .manifest = this, + }; + + MEM_CONTEXT_BEGIN(loadData.memContext) + { + loadData.fileFoundList = lstNew(sizeof(ManifestLoadFound)); + loadData.linkFoundList = lstNew(sizeof(ManifestLoadFound)); + loadData.pathFoundList = lstNew(sizeof(ManifestLoadFound)); + } + MEM_CONTEXT_END(); + + this->info = infoNewLoad(read, manifestLoadCallback, &loadData); + + // Process file defaults + for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) + { + ManifestFile *file = lstGet(this->fileList, fileIdx); + ManifestLoadFound *found = lstGet(loadData.fileFoundList, fileIdx); + + if (!found->group) + file->group = manifestOwnerCache(this, manifestOwnerGet(loadData.fileGroupDefault)); + + if (!found->mode) + file->mode = loadData.fileModeDefault; + + if (!found->primary) + file->primary = loadData.filePrimaryDefault; + + if (!found->user) + file->user = manifestOwnerCache(this, manifestOwnerGet(loadData.fileUserDefault)); + } + + // Process link defaults + for (unsigned int linkIdx = 0; linkIdx < manifestLinkTotal(this); linkIdx++) + { + ManifestLink *link = lstGet(this->linkList, linkIdx); + ManifestLoadFound *found = lstGet(loadData.linkFoundList, linkIdx); + + if (!found->group) + link->group = manifestOwnerCache(this, manifestOwnerGet(loadData.linkGroupDefault)); + + if (!found->user) + link->user = manifestOwnerCache(this, manifestOwnerGet(loadData.linkUserDefault)); + } + + // Process path defaults + for (unsigned int pathIdx = 0; pathIdx < manifestPathTotal(this); pathIdx++) + { + ManifestPath *path = lstGet(this->pathList, pathIdx); + ManifestLoadFound *found = lstGet(loadData.pathFoundList, pathIdx); + + if (!found->group) + path->group = manifestOwnerCache(this, manifestOwnerGet(loadData.pathGroupDefault)); + + if (!found->mode) + path->mode = loadData.pathModeDefault; + + if (!found->user) + path->user = manifestOwnerCache(this, manifestOwnerGet(loadData.pathUserDefault)); + } + + // Sort the lists. They should already be sorted in the file but it is possible that this system has a different collation + // that renders that sort useless. + // + // This must happen *after* the default processing because found lists are in natural file order and it is not worth writing + // comparator routines for them. + lstSort(this->dbList, sortOrderAsc); + lstSort(this->fileList, sortOrderAsc); + lstSort(this->linkList, sortOrderAsc); + lstSort(this->pathList, sortOrderAsc); + lstSort(this->targetList, sortOrderAsc); + + // Make sure the base path exists + manifestTargetBase(this); + + // Free the context holding temporary load data + memContextFree(loadData.memContext); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_LOG_RETURN(MANIFEST, this); +} + +/*********************************************************************************************************************************** +Save manifest +***********************************************************************************************************************************/ +typedef struct ManifestSaveData +{ + Manifest *manifest; // Manifest object to be saved + + const Variant *fileGroupDefault; // File default group + mode_t fileModeDefault; // File default mode + bool filePrimaryDefault; // File default primary + const Variant *fileUserDefault; // File default user + + const Variant *linkGroupDefault; // Link default group + const Variant *linkUserDefault; // Link default user + + const Variant *pathGroupDefault; // Path default group + mode_t pathModeDefault; // Path default mode + const Variant *pathUserDefault; // Path default user +} ManifestSaveData; + +// Helper to convert the owner MCV to a default. If the input is NULL boolean false should be returned, else the owner string. +static const Variant * +manifestOwnerVar(const String *ownerDefault) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, ownerDefault); + FUNCTION_TEST_END(); + + FUNCTION_TEST_RETURN(ownerDefault == NULL ? BOOL_FALSE_VAR : varNewStr(ownerDefault)); +} + +static void +manifestSaveCallback(void *callbackData, const String *sectionNext, InfoSave *infoSaveData) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM_P(VOID, callbackData); + FUNCTION_TEST_PARAM(STRING, sectionNext); + FUNCTION_TEST_PARAM(INFO_SAVE, infoSaveData); + FUNCTION_TEST_END(); + + ASSERT(callbackData != NULL); + ASSERT(infoSaveData != NULL); + + ManifestSaveData *saveData = (ManifestSaveData *)callbackData; + Manifest *manifest = saveData->manifest; + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_BACKUP_STR, sectionNext)) + { + if (manifest->data.archiveStart != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_ARCHIVE_START_STR, + jsonFromStr(manifest->data.archiveStart)); + } + + if (manifest->data.archiveStop != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_ARCHIVE_STOP_STR, + jsonFromStr(manifest->data.archiveStop)); + } + + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_LABEL_STR, + jsonFromStr(manifest->data.backupLabel)); + + if (manifest->data.lsnStart != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_LSN_START_STR, + jsonFromStr(manifest->data.lsnStart)); + } + + if (manifest->data.lsnStop != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_LSN_STOP_STR, + jsonFromStr(manifest->data.lsnStop)); + } + + if (manifest->data.backupLabelPrior != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_PRIOR_STR, + jsonFromStr(manifest->data.backupLabelPrior)); + } + + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_COPY_START_STR, + jsonFromInt64(manifest->data.backupTimestampCopyStart)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_START_STR, + jsonFromInt64(manifest->data.backupTimestampStart)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_TIMESTAMP_STOP_STR, + jsonFromInt64(manifest->data.backupTimestampStop)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_STR, MANIFEST_KEY_BACKUP_TYPE_STR, + jsonFromStr(backupTypeStr(manifest->data.backupType))); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, sectionNext)) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, STRDEF("db-catalog-version"), + jsonFromUInt(pgCatalogVersion(manifest->data.pgVersion))); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, STRDEF("db-control-version"), + jsonFromUInt(pgControlVersion(manifest->data.pgVersion))); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, MANIFEST_KEY_DB_ID_STR, jsonFromUInt(manifest->data.pgId)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, MANIFEST_KEY_DB_SYSTEM_ID_STR, + jsonFromUInt64(manifest->data.pgSystemId)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_DB_STR, MANIFEST_KEY_DB_VERSION_STR, + jsonFromStr(pgVersionToStr(manifest->data.pgVersion))); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, sectionNext)) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_ARCHIVE_CHECK_STR, + jsonFromBool(manifest->data.backupOptionArchiveCheck)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_ARCHIVE_COPY_STR, + jsonFromBool(manifest->data.backupOptionArchiveCopy)); + + if (manifest->data.backupOptionStandby != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_BACKUP_STANDBY_STR, + jsonFromVar(manifest->data.backupOptionStandby, 0)); + } + + if (manifest->data.backupOptionBufferSize != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_BUFFER_SIZE_STR, + jsonFromVar(manifest->data.backupOptionBufferSize, 0)); + } + + if (manifest->data.backupOptionChecksumPage != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_CHECKSUM_PAGE_STR, + jsonFromVar(manifest->data.backupOptionChecksumPage, 0)); + } + + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_COMPRESS_STR, + jsonFromBool(manifest->data.backupOptionCompress)); + + if (manifest->data.backupOptionCompressLevel != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_STR, + jsonFromVar(manifest->data.backupOptionCompressLevel, 0)); + } + + if (manifest->data.backupOptionCompressLevelNetwork != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_COMPRESS_LEVEL_NETWORK_STR, + jsonFromVar(manifest->data.backupOptionCompressLevelNetwork, 0)); + } + + if (manifest->data.backupOptionDelta != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_DELTA_STR, + jsonFromVar(manifest->data.backupOptionDelta, 0)); + } + + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_HARDLINK_STR, + jsonFromBool(manifest->data.backupOptionHardLink)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_ONLINE_STR, + jsonFromBool(manifest->data.backupOptionOnline)); + + if (manifest->data.backupOptionProcessMax != NULL) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_BACKUP_OPTION_STR, MANIFEST_KEY_OPTION_PROCESS_MAX_STR, + jsonFromVar(manifest->data.backupOptionProcessMax, 0)); + } + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_BACKUP_TARGET_STR, sectionNext)) + { + MEM_CONTEXT_TEMP_RESET_BEGIN() + { + for (unsigned int targetIdx = 0; targetIdx < manifestTargetTotal(manifest); targetIdx++) + { + const ManifestTarget *target = manifestTarget(manifest, targetIdx); + KeyValue *targetKv = kvNew(); + + if (target->file != NULL) + kvPut(targetKv, MANIFEST_KEY_FILE_VAR, VARSTR(target->file)); + + kvPut(targetKv, MANIFEST_KEY_PATH_VAR, VARSTR(target->path)); + + if (target->tablespaceId != 0) + kvPut(targetKv, MANIFEST_KEY_TABLESPACE_ID_VAR, VARSTR(strNewFmt("%u", target->tablespaceId))); + + if (target->tablespaceName != NULL) + kvPut(targetKv, MANIFEST_KEY_TABLESPACE_NAME_VAR, VARSTR(target->tablespaceName)); + + kvPut( + targetKv, MANIFEST_KEY_TYPE_VAR, + VARSTR( + target->type == manifestTargetTypePath ? + MANIFEST_TARGET_TYPE_PATH_STR : MANIFEST_TARGET_TYPE_LINK_STR)); + + infoSaveValue(infoSaveData, MANIFEST_SECTION_BACKUP_TARGET_STR, target->name, jsonFromKv(targetKv, 0)); + + MEM_CONTEXT_TEMP_RESET(1000); + } + } + MEM_CONTEXT_TEMP_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_DB_STR, sectionNext)) + { + MEM_CONTEXT_TEMP_RESET_BEGIN() + { + for (unsigned int dbIdx = 0; dbIdx < manifestDbTotal(manifest); dbIdx++) + { + const ManifestDb *db = manifestDb(manifest, dbIdx); + KeyValue *dbKv = kvNew(); + + kvPut(dbKv, MANIFEST_KEY_DB_ID_VAR, VARUINT(db->id)); + kvPut(dbKv, MANIFEST_KEY_DB_LAST_SYSTEM_ID_VAR, VARUINT(db->lastSystemId)); + + infoSaveValue(infoSaveData, MANIFEST_SECTION_DB_STR, db->name, jsonFromKv(dbKv, 0)); + + MEM_CONTEXT_TEMP_RESET(1000); + } + } + MEM_CONTEXT_TEMP_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, sectionNext)) + { + MEM_CONTEXT_TEMP_RESET_BEGIN() + { + for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) + { + const ManifestFile *file = manifestFile(manifest, fileIdx); + KeyValue *fileKv = kvNew(); + + if (file->size != 0) + kvPut(fileKv, MANIFEST_KEY_CHECKSUM_VAR, VARSTRZ(file->checksumSha1)); + + if (file->checksumPage) + { + kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_VAR, VARBOOL(file->checksumPageError)); + + if (file->checksumPageErrorList != NULL) + kvPut(fileKv, MANIFEST_KEY_CHECKSUM_PAGE_ERROR_VAR, varNewVarLst(file->checksumPageErrorList)); + } + + if (!varEq(manifestOwnerVar(file->group), saveData->fileGroupDefault)) + kvPut(fileKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(file->group)); + + if (file->primary != saveData->filePrimaryDefault) + kvPut(fileKv, MANIFEST_KEY_PRIMARY_VAR, VARBOOL(file->primary)); + + if (file->mode != saveData->fileModeDefault) + kvPut(fileKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", file->mode))); + + if (file->reference != NULL) + kvPut(fileKv, MANIFEST_KEY_REFERENCE_VAR, VARSTR(file->reference)); + + if (file->sizeRepo != file->size) + kvPut(fileKv, MANIFEST_KEY_SIZE_REPO_VAR, varNewUInt64(file->sizeRepo)); + + kvPut(fileKv, MANIFEST_KEY_SIZE_VAR, varNewUInt64(file->size)); + + kvPut(fileKv, MANIFEST_KEY_TIMESTAMP_VAR, varNewUInt64((uint64_t)file->timestamp)); + + if (!varEq(manifestOwnerVar(file->user), saveData->fileUserDefault)) + kvPut(fileKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(file->user)); + + infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_FILE_STR, file->name, jsonFromKv(fileKv, 0)); + + MEM_CONTEXT_TEMP_RESET(1000); + } + } + MEM_CONTEXT_TEMP_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, sectionNext)) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, MANIFEST_KEY_GROUP_STR, + jsonFromVar(saveData->fileGroupDefault, 0)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, MANIFEST_KEY_PRIMARY_STR, + jsonFromBool(saveData->filePrimaryDefault)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, MANIFEST_KEY_MODE_STR, + jsonFromStr(strNewFmt("%04o", saveData->fileModeDefault))); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR, MANIFEST_KEY_USER_STR, + jsonFromVar(saveData->fileUserDefault, 0)); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_LINK_STR, sectionNext)) + { + MEM_CONTEXT_TEMP_RESET_BEGIN() + { + for (unsigned int linkIdx = 0; linkIdx < manifestLinkTotal(manifest); linkIdx++) + { + const ManifestLink *link = manifestLink(manifest, linkIdx); + KeyValue *linkKv = kvNew(); + + if (!varEq(manifestOwnerVar(link->user), saveData->linkUserDefault)) + kvPut(linkKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(link->user)); + + if (!varEq(manifestOwnerVar(link->group), saveData->linkGroupDefault)) + kvPut(linkKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(link->group)); + + kvPut(linkKv, MANIFEST_KEY_DESTINATION_VAR, VARSTR(link->destination)); + + infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_LINK_STR, link->name, jsonFromKv(linkKv, 0)); + + MEM_CONTEXT_TEMP_RESET(1000); + } + } + MEM_CONTEXT_TEMP_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR, sectionNext)) + { + if (manifestLinkTotal(manifest) > 0) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR, MANIFEST_KEY_GROUP_STR, + jsonFromVar(saveData->linkGroupDefault, 0)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR, MANIFEST_KEY_USER_STR, + jsonFromVar(saveData->linkUserDefault, 0)); + } + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_PATH_STR, sectionNext)) + { + MEM_CONTEXT_TEMP_RESET_BEGIN() + { + for (unsigned int pathIdx = 0; pathIdx < manifestPathTotal(manifest); pathIdx++) + { + const ManifestPath *path = manifestPath(manifest, pathIdx); + KeyValue *pathKv = kvNew(); + + if (!varEq(manifestOwnerVar(path->group), saveData->pathGroupDefault)) + kvPut(pathKv, MANIFEST_KEY_GROUP_VAR, manifestOwnerVar(path->group)); + + if (path->mode != saveData->pathModeDefault) + kvPut(pathKv, MANIFEST_KEY_MODE_VAR, VARSTR(strNewFmt("%04o", path->mode))); + + if (!varEq(manifestOwnerVar(path->user), saveData->pathUserDefault)) + kvPut(pathKv, MANIFEST_KEY_USER_VAR, manifestOwnerVar(path->user)); + + infoSaveValue(infoSaveData, MANIFEST_SECTION_TARGET_PATH_STR, path->name, jsonFromKv(pathKv, 0)); + + MEM_CONTEXT_TEMP_RESET(1000); + } + } + MEM_CONTEXT_TEMP_END(); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (infoSaveSection(infoSaveData, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, sectionNext)) + { + infoSaveValue( + infoSaveData, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, MANIFEST_KEY_GROUP_STR, + jsonFromVar(saveData->pathGroupDefault, 0)); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, MANIFEST_KEY_MODE_STR, + jsonFromStr(strNewFmt("%04o", saveData->pathModeDefault))); + infoSaveValue( + infoSaveData, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR, MANIFEST_KEY_USER_STR, + jsonFromVar(saveData->pathUserDefault, 0)); + } + + FUNCTION_TEST_RETURN_VOID(); +} + +void +manifestSave(Manifest *this, IoWrite *write) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(MANIFEST, this); + FUNCTION_LOG_PARAM(IO_WRITE, write); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(write != NULL); + + MEM_CONTEXT_TEMP_BEGIN() + { + ManifestSaveData saveData = + { + .manifest = this, + }; + + // Get default file values + MostCommonValue *fileGroupMcv = mcvNew(); + MostCommonValue *fileModeMcv = mcvNew(); + MostCommonValue *filePrimaryMcv = mcvNew(); + MostCommonValue *fileUserMcv = mcvNew(); + + ASSERT(manifestFileTotal(this) > 0); + + for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(this); fileIdx++) + { + const ManifestFile *file = manifestFile(this, fileIdx); + + mcvUpdate(fileGroupMcv, VARSTR(file->group)); + mcvUpdate(fileModeMcv, VARUINT(file->mode)); + mcvUpdate(filePrimaryMcv, VARBOOL(file->primary)); + mcvUpdate(fileUserMcv, VARSTR(file->user)); + } + + saveData.fileGroupDefault = manifestOwnerVar(varStr(mcvResult(fileGroupMcv))); + saveData.fileModeDefault = varUInt(mcvResult(fileModeMcv)); + saveData.filePrimaryDefault = varBool(mcvResult(filePrimaryMcv)); + saveData.fileUserDefault = manifestOwnerVar(varStr(mcvResult(fileUserMcv))); + + // Get default link values + if (manifestLinkTotal(this) > 0) + { + MostCommonValue *linkGroupMcv = mcvNew(); + MostCommonValue *linkUserMcv = mcvNew(); + + for (unsigned int linkIdx = 0; linkIdx < manifestLinkTotal(this); linkIdx++) + { + const ManifestLink *link = manifestLink(this, linkIdx); + + mcvUpdate(linkGroupMcv, VARSTR(link->group)); + mcvUpdate(linkUserMcv, VARSTR(link->user)); + } + + saveData.linkGroupDefault = manifestOwnerVar(varStr(mcvResult(linkGroupMcv))); + saveData.linkUserDefault = manifestOwnerVar(varStr(mcvResult(linkUserMcv))); + } + + // Get default path values + MostCommonValue *pathGroupMcv = mcvNew(); + MostCommonValue *pathModeMcv = mcvNew(); + MostCommonValue *pathUserMcv = mcvNew(); + + ASSERT(manifestPathTotal(this) > 0); + + for (unsigned int pathIdx = 0; pathIdx < manifestPathTotal(this); pathIdx++) + { + const ManifestPath *path = manifestPath(this, pathIdx); + + mcvUpdate(pathGroupMcv, VARSTR(path->group)); + mcvUpdate(pathModeMcv, VARUINT(path->mode)); + mcvUpdate(pathUserMcv, VARSTR(path->user)); + } + + saveData.pathGroupDefault = manifestOwnerVar(varStr(mcvResult(pathGroupMcv))); + saveData.pathModeDefault = varUInt(mcvResult(pathModeMcv)); + saveData.pathUserDefault = manifestOwnerVar(varStr(mcvResult(pathUserMcv))); + + // Save manifest + infoSave(this->info, write, manifestSaveCallback, &saveData); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Ensure that symlinks do not point to the same directory or a subdirectory of another link +***********************************************************************************************************************************/ +void +manifestLinkCheck(const Manifest *this) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(MANIFEST, this); + FUNCTION_LOG_END(); + + for (unsigned int linkIdx1 = 0; linkIdx1 < manifestTargetTotal(this); linkIdx1++) + { + const ManifestTarget *link1 = manifestTarget(this, linkIdx1); + + if (link1->type == manifestTargetTypeLink) + { + for (unsigned int linkIdx2 = 0; linkIdx2 < manifestTargetTotal(this); linkIdx2++) + { + const ManifestTarget *link2 = manifestTarget(this, linkIdx2); + + if (link2->type == manifestTargetTypeLink && link1 != link2) + { + if (!(link1->file != NULL && link2->file != NULL) && + strBeginsWith( + strNewFmt("%s/", strPtr(manifestTargetPath(this, link1))), + strNewFmt("%s/", strPtr(manifestTargetPath(this, link2))))) + { + THROW_FMT( + LinkDestinationError, + "link '%s' (%s) destination is a subdirectory of or the same directory as link '%s' (%s)", + strPtr(manifestPgPath(link1->name)), strPtr(manifestTargetPath(this, link1)), + strPtr(manifestPgPath(link2->name)), strPtr(manifestTargetPath(this, link2))); + } + } + } + } + } + + FUNCTION_LOG_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Return the base target, i.e. the target that is the data directory +***********************************************************************************************************************************/ +const ManifestTarget * +manifestTargetBase(const Manifest *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(manifestTargetFind(this, MANIFEST_TARGET_PGDATA_STR)); +} + +/*********************************************************************************************************************************** +Return an absolute path to the target +***********************************************************************************************************************************/ +String * +manifestTargetPath(const Manifest *this, const ManifestTarget *target) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(MANIFEST_TARGET, target); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(target != NULL); + + // If the target path is already absolute then just return it + if (strBeginsWith(target->path, FSLASH_STR)) + FUNCTION_TEST_RETURN(strDup(target->path)); + + // Construct it from the base pg path and a relative path + String *result = NULL; + + MEM_CONTEXT_TEMP_BEGIN() + { + String *pgPath = strPath(manifestPgPath(target->name)); + + if (strSize(pgPath) != 0) + strCat(pgPath, "/"); + + strCat(pgPath, strPtr(target->path)); + + memContextSwitch(MEM_CONTEXT_OLD()); + result = strPathAbsolute(pgPath, manifestTargetBase(this)->path); + memContextSwitch(MEM_CONTEXT_TEMP()); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_TEST_RETURN(result); +} + +/*********************************************************************************************************************************** +Return the data directory relative path for any manifest file/link/path/target name +***********************************************************************************************************************************/ +String * +manifestPgPath(const String *manifestPath) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STRING, manifestPath); + FUNCTION_TEST_END(); + + ASSERT(manifestPath != NULL); + + // If something in pg_data/ + if (strBeginsWith(manifestPath, STRDEF(MANIFEST_TARGET_PGDATA "/"))) + { + FUNCTION_TEST_RETURN(strNew(strPtr(manifestPath) + sizeof(MANIFEST_TARGET_PGDATA))); + } + // Else not pg_data (this is faster since the length of everything else will be different than pg_data) + else if (!strEq(manifestPath, MANIFEST_TARGET_PGDATA_STR)) + { + // A tablespace target is the only valid option if not pg_data or pg_data/ + ASSERT( + strEq(manifestPath, MANIFEST_TARGET_PGTBLSPC_STR) || strBeginsWith(manifestPath, STRDEF(MANIFEST_TARGET_PGTBLSPC "/"))); + + FUNCTION_TEST_RETURN(strDup(manifestPath)); + } + + FUNCTION_TEST_RETURN(NULL); +} + +/*********************************************************************************************************************************** +Get the cipher sub-passphrase +***********************************************************************************************************************************/ +const String * +manifestCipherSubPass(const Manifest *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(infoCipherPass(this->info)); +} + +/*********************************************************************************************************************************** +Return manifest configuration and options +***********************************************************************************************************************************/ +const ManifestData * +manifestData(const Manifest *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(&this->data); +} + +/*********************************************************************************************************************************** +Db functions and getters/setters +***********************************************************************************************************************************/ +const ManifestDb * +manifestDb(const Manifest *this, unsigned int dbIdx) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(UINT, dbIdx); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstGet(this->dbList, dbIdx)); +} + +const ManifestDb * +manifestDbFind(const Manifest *this, const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + const ManifestDb *result = lstFind(this->dbList, &name); + + if (result == NULL) + THROW_FMT(AssertError, "unable to find '%s' in manifest db list", strPtr(name)); + + FUNCTION_TEST_RETURN(result); +} + +// If the database requested is not found in the list, return the default passed rather than throw an error +const ManifestDb * +manifestDbFindDefault(const Manifest *this, const String *name, const ManifestDb *dbDefault) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_PARAM(MANIFEST_DB, dbDefault); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + FUNCTION_TEST_RETURN(lstFindDefault(this->dbList, &name, (void *)dbDefault)); +} + +unsigned int +manifestDbTotal(const Manifest *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstSize(this->dbList)); +} + +/*********************************************************************************************************************************** +File functions and getters/setters +***********************************************************************************************************************************/ +const ManifestFile * +manifestFile(const Manifest *this, unsigned int fileIdx) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(UINT, fileIdx); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstGet(this->fileList, fileIdx)); +} + +const ManifestFile * +manifestFileFind(const Manifest *this, const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + const ManifestFile *result = lstFind(this->fileList, &name); + + if (result == NULL) + THROW_FMT(AssertError, "unable to find '%s' in manifest file list", strPtr(name)); + + FUNCTION_TEST_RETURN(result); +} + +// If the file requested is not found in the list, return the default passed rather than throw an error +const ManifestFile * +manifestFileFindDefault(const Manifest *this, const String *name, const ManifestFile *fileDefault) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_PARAM(MANIFEST_TARGET, fileDefault); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + FUNCTION_TEST_RETURN(lstFindDefault(this->fileList, &name, (void *)fileDefault)); +} + +void +manifestFileRemove(const Manifest *this, const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + if (!lstRemove(this->fileList, &name)) + THROW_FMT(AssertError, "unable to remove '%s' from manifest file list", strPtr(name)); + + FUNCTION_TEST_RETURN_VOID(); +} + +unsigned int +manifestFileTotal(const Manifest *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstSize(this->fileList)); +} + +/*********************************************************************************************************************************** +Link functions and getters/setters +***********************************************************************************************************************************/ +const ManifestLink * +manifestLink(const Manifest *this, unsigned int linkIdx) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(UINT, linkIdx); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstGet(this->linkList, linkIdx)); +} + +const ManifestLink * +manifestLinkFind(const Manifest *this, const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + const ManifestLink *result = lstFind(this->linkList, &name); + + if (result == NULL) + THROW_FMT(AssertError, "unable to find '%s' in manifest link list", strPtr(name)); + + FUNCTION_TEST_RETURN(result); +} + +// If the link requested is not found in the list, return the default passed rather than throw an error +const ManifestLink * +manifestLinkFindDefault(const Manifest *this, const String *name, const ManifestLink *linkDefault) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_PARAM(MANIFEST_TARGET, linkDefault); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + FUNCTION_TEST_RETURN(lstFindDefault(this->linkList, &name, (void *)linkDefault)); +} + +void +manifestLinkRemove(const Manifest *this, const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + if (!lstRemove(this->linkList, &name)) + THROW_FMT(AssertError, "unable to remove '%s' from manifest link list", strPtr(name)); + + FUNCTION_TEST_RETURN_VOID(); +} + +unsigned int +manifestLinkTotal(const Manifest *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstSize(this->linkList)); +} + +void +manifestLinkUpdate(const Manifest *this, const String *name, const String *destination) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_PARAM(STRING, destination); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + ASSERT(destination != NULL); + + ManifestLink *link = (ManifestLink *)manifestLinkFind(this, name); + + MEM_CONTEXT_BEGIN(lstMemContext(this->linkList)) + { + if (!strEq(link->destination, destination)) + link->destination = strDup(destination); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Path functions and getters/setters +***********************************************************************************************************************************/ +const ManifestPath * +manifestPath(const Manifest *this, unsigned int pathIdx) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(UINT, pathIdx); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstGet(this->pathList, pathIdx)); +} + +const ManifestPath * +manifestPathFind(const Manifest *this, const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + const ManifestPath *result = lstFind(this->pathList, &name); + + if (result == NULL) + THROW_FMT(AssertError, "unable to find '%s' in manifest path list", strPtr(name)); + + FUNCTION_TEST_RETURN(result); +} + +// If the path requested is not found in the list, return the default passed rather than throw an error +const ManifestPath * +manifestPathFindDefault(const Manifest *this, const String *name, const ManifestPath *pathDefault) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_PARAM(MANIFEST_TARGET, pathDefault); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + FUNCTION_TEST_RETURN(lstFindDefault(this->pathList, &name, (void *)pathDefault)); +} + +unsigned int +manifestPathTotal(const Manifest *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstSize(this->pathList)); +} + +/*********************************************************************************************************************************** +Target functions and getters/setters +***********************************************************************************************************************************/ +const ManifestTarget * +manifestTarget(const Manifest *this, unsigned int targetIdx) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(UINT, targetIdx); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstGet(this->targetList, targetIdx)); +} + +const ManifestTarget * +manifestTargetFind(const Manifest *this, const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + const ManifestTarget *result = lstFind(this->targetList, &name); + + if (result == NULL) + THROW_FMT(AssertError, "unable to find '%s' in manifest target list", strPtr(name)); + + FUNCTION_TEST_RETURN(result); +} + +void +manifestTargetRemove(const Manifest *this, const String *name) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + + if (!lstRemove(this->targetList, &name)) + THROW_FMT(AssertError, "unable to remove '%s' from manifest target list", strPtr(name)); + + FUNCTION_TEST_RETURN_VOID(); +} + +unsigned int +manifestTargetTotal(const Manifest *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(lstSize(this->targetList)); +} + +void +manifestTargetUpdate(const Manifest *this, const String *name, const String *path, const String *file) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(MANIFEST, this); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_PARAM(STRING, path); + FUNCTION_TEST_PARAM(STRING, file); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(name != NULL); + ASSERT(path != NULL); + + ManifestTarget *target = (ManifestTarget *)manifestTargetFind(this, name); + + ASSERT((target->file == NULL && file == NULL) || (target->file != NULL && file != NULL)); + + MEM_CONTEXT_BEGIN(lstMemContext(this->targetList)) + { + if (!strEq(target->path, path)) + target->path = strDup(path); + + if (!strEq(target->file, file)) + target->file = strDup(file); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Helper function to load backup manifest files +***********************************************************************************************************************************/ +typedef struct ManifestLoadFileData +{ + MemContext *memContext; // Mem context + const Storage *storage; // Storage to load from + const String *fileName; // Base filename + CipherType cipherType; // Cipher type + const String *cipherPass; // Cipher passphrase + Manifest *manifest; // Loaded manifest object +} ManifestLoadFileData; + +static bool +manifestLoadFileCallback(void *data, unsigned int try) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM_P(VOID, data); + FUNCTION_LOG_PARAM(UINT, try); + FUNCTION_LOG_END(); + + ASSERT(data != NULL); + + ManifestLoadFileData *loadData = (ManifestLoadFileData *)data; + bool result = false; + + if (try < 2) + { + // Construct filename based on try + const String *fileName = try == 0 ? loadData->fileName : strNewFmt("%s" INFO_COPY_EXT, strPtr(loadData->fileName)); + + // Attempt to load the file + IoRead *read = storageReadIo(storageNewReadNP(loadData->storage, fileName)); + cipherBlockFilterGroupAdd(ioReadFilterGroup(read), loadData->cipherType, cipherModeDecrypt, loadData->cipherPass); + + MEM_CONTEXT_BEGIN(loadData->memContext) + { + loadData->manifest = manifestNewLoad(read); + result = true; + } + MEM_CONTEXT_END(); + } + + FUNCTION_LOG_RETURN(BOOL, result); +} + +Manifest * +manifestLoadFile(const Storage *storage, const String *fileName, CipherType cipherType, const String *cipherPass) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(STORAGE, storage); + FUNCTION_LOG_PARAM(STRING, fileName); + FUNCTION_LOG_PARAM(ENUM, cipherType); + FUNCTION_TEST_PARAM(STRING, cipherPass); + FUNCTION_LOG_END(); + + ASSERT(storage != NULL); + ASSERT(fileName != NULL); + ASSERT((cipherType == cipherTypeNone && cipherPass == NULL) || (cipherType != cipherTypeNone && cipherPass != NULL)); + + ManifestLoadFileData data = + { + .memContext = memContextCurrent(), + .storage = storage, + .fileName = fileName, + .cipherType = cipherType, + .cipherPass = cipherPass, + }; + + MEM_CONTEXT_TEMP_BEGIN() + { + const char *fileNamePath = strPtr(storagePathNP(storage, fileName)); + + infoLoad( + strNewFmt("unable to load backup manifest file '%s' or '%s" INFO_COPY_EXT "'", fileNamePath, fileNamePath), + manifestLoadFileCallback, &data); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(MANIFEST, data.manifest); +} diff --git a/src/info/manifest.h b/src/info/manifest.h index aaa09bc3d..afe4bbf3d 100644 --- a/src/info/manifest.h +++ b/src/info/manifest.h @@ -1,12 +1,246 @@ /*********************************************************************************************************************************** -Manifest Handler +Backup Manifest Handler + +The backup manifest stores a complete list of all files, links, and paths in a backup along with metadata such as checksums, sizes, +timestamps, etc. A list of databases is also included for selective restore. + +The purpose of the manifest is to allow the restore command to confidently reconstruct the PostgreSQL data directory and ensure that +nothing is missing or corrupt. It is also useful for reporting, e.g. size of backup, backup time, etc. ***********************************************************************************************************************************/ #ifndef INFO_MANIFEST_H #define INFO_MANIFEST_H +#include "command/backup/common.h" +#include "common/crypto/common.h" +#include "common/type/variantList.h" + /*********************************************************************************************************************************** Constants ***********************************************************************************************************************************/ #define BACKUP_MANIFEST_FILE "backup.manifest" + STRING_DECLARE(BACKUP_MANIFEST_FILE_STR); + +#define MANIFEST_TARGET_PGDATA "pg_data" + STRING_DECLARE(MANIFEST_TARGET_PGDATA_STR); +#define MANIFEST_TARGET_PGTBLSPC "pg_tblspc" + STRING_DECLARE(MANIFEST_TARGET_PGTBLSPC_STR); + +/*********************************************************************************************************************************** +Object type +***********************************************************************************************************************************/ +typedef struct Manifest Manifest; + +#include "common/crypto/hash.h" +#include "storage/storage.h" + +/*********************************************************************************************************************************** +Manifest data +***********************************************************************************************************************************/ +typedef struct ManifestData +{ + const String *backupLabel; // Backup label (unique identifier for the backup) + const String *backupLabelPrior; // Backup label for backup this diff/incr is based on + time_t backupTimestampCopyStart; // When did the file copy start? + time_t backupTimestampStart; // When did the backup start? + time_t backupTimestampStop; // When did the backup stop? + BackupType backupType; // Type of backup: full, diff, incr + + // ??? Note that these fields are redundant and verbose since storing the start/stop lsn as a uint64 would be sufficient. + // However, we currently lack the functions to transform these values back and forth so this will do for now. + const String *archiveStart; // First WAL file in the backup + const String *archiveStop; // Last WAL file in the backup + const String *lsnStart; // Start LSN for the backup + const String *lsnStop; // Stop LSN for the backup + + unsigned int pgId; // PostgreSQL id in backup.info + unsigned int pgVersion; // PostgreSQL version + uint64_t pgSystemId; // PostgreSQL system identifier + + bool backupOptionArchiveCheck; // Will WAL segments be checked at the end of the backup? + bool backupOptionArchiveCopy; // Will WAL segments be copied to the backup? + const Variant *backupOptionStandby; // Will the backup be performed from a standby? + const Variant *backupOptionBufferSize; // Buffer size used for file/protocol operations + const Variant *backupOptionChecksumPage; // Will page checksums be verified? + bool backupOptionCompress; // Will compression be used for backup? + const Variant *backupOptionCompressLevel; // Level to use for compression + const Variant *backupOptionCompressLevelNetwork; // Level to use for network compression + const Variant *backupOptionDelta; // Will a checksum delta be performed? + bool backupOptionHardLink; // Will hardlinks be created in the backup? + bool backupOptionOnline; // Will an online backup be performed? + const Variant *backupOptionProcessMax; // How many processes will be used for backup? +} ManifestData; + +/*********************************************************************************************************************************** +Db type +***********************************************************************************************************************************/ +typedef struct ManifestDb +{ + const String *name; // Db name (must be first member in struct) + unsigned int id; // Db oid + unsigned int lastSystemId; // Highest oid used by system objects in this database +} ManifestDb; + +/*********************************************************************************************************************************** +File type +***********************************************************************************************************************************/ +typedef struct ManifestFile +{ + const String *name; // File name (must be first member in struct) + bool primary:1; // Should this file be copied from the primary? + bool checksumPage:1; // Does this file have page checksums? + bool checksumPageError:1; // Is there an error in the page checksum? + mode_t mode; // File mode + char checksumSha1[HASH_TYPE_SHA1_SIZE_HEX + 1]; // SHA1 checksum + const VariantList *checksumPageErrorList; // List of page checksum errors if there are any + const String *user; // User name + const String *group; // Group name + const String *reference; // Reference to a prior backup + uint64_t size; // Original size + uint64_t sizeRepo; // Size in repo + time_t timestamp; // Original timestamp +} ManifestFile; + +/*********************************************************************************************************************************** +Link type +***********************************************************************************************************************************/ +typedef struct ManifestLink +{ + const String *name; // Link name (must be first member in struct) + const String *destination; // Link destination + const String *user; // User name + const String *group; // Group name +} ManifestLink; + +/*********************************************************************************************************************************** +Path type +***********************************************************************************************************************************/ +typedef struct ManifestPath +{ + const String *name; // Path name (must be first member in struct) + mode_t mode; // Directory mode + const String *user; // User name + const String *group; // Group name +} ManifestPath; + +/*********************************************************************************************************************************** +Target type +***********************************************************************************************************************************/ +typedef enum +{ + manifestTargetTypePath, + manifestTargetTypeLink, +} ManifestTargetType; + +typedef struct ManifestTarget +{ + const String *name; // Target name (must be first member in struct) + ManifestTargetType type; // Target type + const String *path; // Target path (if path or link) + const String *file; // Target file (if file link) + unsigned int tablespaceId; // Oid if this link is a tablespace + const String *tablespaceName; // Name of the tablespace +} ManifestTarget; + +/*********************************************************************************************************************************** +Constructor +***********************************************************************************************************************************/ +Manifest *manifestNewLoad(IoRead *read); + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +void manifestLinkCheck(const Manifest *this); +void manifestSave(Manifest *this, IoWrite *write); + +/*********************************************************************************************************************************** +Db functions and getters/setters +***********************************************************************************************************************************/ +const ManifestDb *manifestDb(const Manifest *this, unsigned int dbIdx); +const ManifestDb *manifestDbFind(const Manifest *this, const String *name); +const ManifestDb *manifestDbFindDefault(const Manifest *this, const String *name, const ManifestDb *dbDefault); +unsigned int manifestDbTotal(const Manifest *this); + +/*********************************************************************************************************************************** +File functions and getters/setters +***********************************************************************************************************************************/ +const ManifestFile *manifestFile(const Manifest *this, unsigned int fileIdx); +const ManifestFile *manifestFileFind(const Manifest *this, const String *name); +const ManifestFile *manifestFileFindDefault(const Manifest *this, const String *name, const ManifestFile *fileDefault); +void manifestFileRemove(const Manifest *this, const String *name); +unsigned int manifestFileTotal(const Manifest *this); + +/*********************************************************************************************************************************** +Link functions and getters/setters +***********************************************************************************************************************************/ +const ManifestLink *manifestLink(const Manifest *this, unsigned int linkIdx); +const ManifestLink *manifestLinkFind(const Manifest *this, const String *name); +const ManifestLink *manifestLinkFindDefault(const Manifest *this, const String *name, const ManifestLink *linkDefault); +void manifestLinkRemove(const Manifest *this, const String *name); +unsigned int manifestLinkTotal(const Manifest *this); +void manifestLinkUpdate(const Manifest *this, const String *name, const String *path); + +/*********************************************************************************************************************************** +Path functions and getters/setters +***********************************************************************************************************************************/ +const ManifestPath *manifestPath(const Manifest *this, unsigned int pathIdx); +const ManifestPath *manifestPathFind(const Manifest *this, const String *name); +const ManifestPath *manifestPathFindDefault(const Manifest *this, const String *name, const ManifestPath *pathDefault); +unsigned int manifestPathTotal(const Manifest *this); + +/*********************************************************************************************************************************** +Target functions and getters/setters +***********************************************************************************************************************************/ +const ManifestTarget *manifestTarget(const Manifest *this, unsigned int targetIdx); +const ManifestTarget *manifestTargetFind(const Manifest *this, const String *name); +void manifestTargetRemove(const Manifest *this, const String *name); +unsigned int manifestTargetTotal(const Manifest *this); +void manifestTargetUpdate(const Manifest *this, const String *name, const String *path, const String *file); + +/*********************************************************************************************************************************** +Getters +***********************************************************************************************************************************/ +const String *manifestCipherSubPass(const Manifest *this); +const ManifestData *manifestData(const Manifest *this); +String *manifestPgPath(const String *manifestPath); +const ManifestTarget *manifestTargetBase(const Manifest *this); +String *manifestTargetPath(const Manifest *this, const ManifestTarget *target); + +/*********************************************************************************************************************************** +Helper functions +***********************************************************************************************************************************/ +Manifest *manifestLoadFile(const Storage *storage, const String *fileName, CipherType cipherType, const String *cipherPass); + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +#define FUNCTION_LOG_MANIFEST_TYPE \ + Manifest * +#define FUNCTION_LOG_MANIFEST_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "Manifest", buffer, bufferSize) + +#define FUNCTION_LOG_MANIFEST_DB_TYPE \ + ManifestDb * +#define FUNCTION_LOG_MANIFEST_DB_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "ManifestDb", buffer, bufferSize) + +#define FUNCTION_LOG_MANIFEST_FILE_TYPE \ + ManifestFile * +#define FUNCTION_LOG_MANIFEST_FILE_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "ManifestFile", buffer, bufferSize) + +#define FUNCTION_LOG_MANIFEST_LINK_TYPE \ + ManifestLink * +#define FUNCTION_LOG_MANIFEST_LINK_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "ManifestLink", buffer, bufferSize) + +#define FUNCTION_LOG_MANIFEST_PATH_TYPE \ + ManifestPath * +#define FUNCTION_LOG_MANIFEST_PATH_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "ManifestPath", buffer, bufferSize) + +#define FUNCTION_LOG_MANIFEST_TARGET_TYPE \ + ManifestTarget * +#define FUNCTION_LOG_MANIFEST_TARGET_FORMAT(value, buffer, bufferSize) \ + objToLog(value, "ManifestTarget", buffer, bufferSize) #endif diff --git a/test/define.yaml b/test/define.yaml index 639c50b71..77d1190b7 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -549,6 +549,13 @@ unit: coverage: info/infoBackup: full + # ---------------------------------------------------------------------------------------------------------------------------- + - name: manifest + total: 4 + + coverage: + info/manifest: full + # ---------------------------------------------------------------------------------------------------------------------------- - name: info-backup-perl total: 3 diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c new file mode 100644 index 000000000..a1ca0503e --- /dev/null +++ b/test/src/module/info/manifestTest.c @@ -0,0 +1,559 @@ +/*********************************************************************************************************************************** +Test Backup Manifest Handler +***********************************************************************************************************************************/ +#include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" +#include "info/infoBackup.h" +#include "storage/posix/storage.h" + +#include "common/harnessInfo.h" + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void +testRun(void) +{ + Storage *storageTest = storagePosixNew( + strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL); + + // ***************************************************************************************************************************** + if (testBegin("struct sizes")) + { + // Make sure the size of structs doesn't change without us knowing about it + // ------------------------------------------------------------------------------------------------------------------------- + TEST_RESULT_UINT(sizeof(ManifestLoadFound), TEST_64BIT() ? 1 : 1, "check size of ManifestLoadFound"); + TEST_RESULT_UINT(sizeof(ManifestPath), TEST_64BIT() ? 32 : 16, "check size of ManifestPath"); + TEST_RESULT_UINT(sizeof(ManifestFile), TEST_64BIT() ? 120 : 92, "check size of ManifestFile"); + } + + // ***************************************************************************************************************************** + if (testBegin("manifestNewLoad() and manifestSave()")) + { + Manifest *manifest = NULL; + + // Manifest with minimal features + // ------------------------------------------------------------------------------------------------------------------------- + const Buffer *contentLoad = harnessInfoChecksumZ + ( + "[backup]\n" + "backup-label=\"20190808-163540F\"\n" + "backup-timestamp-copy-start=1565282141\n" + "backup-timestamp-start=1565282140\n" + "backup-timestamp-stop=1565282142\n" + "backup-type=\"full\"\n" + "\n" + "[backup:db]\n" + "db-catalog-version=201409291\n" + "db-control-version=942\n" + "db-id=1\n" + "db-system-id=1000000000000000094\n" + "db-version=\"9.4\"\n" + "\n" + "[backup:option]\n" + "option-archive-check=true\n" + "option-archive-copy=true\n" + "option-compress=false\n" + "option-hardlink=false\n" + "option-online=false\n" + "\n" + "[backup:target]\n" + "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" + "\n" + "[cipher]\n" + "cipher-pass=\"somepass\"\n" + "\n" + "[target:file]\n" + "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"size\":4,\"timestamp\":1565282114}\n" + "\n" + "[target:file:default]\n" + "group=\"group1\"\n" + "master=true\n" + "mode=\"0600\"\n" + "user=\"user1\"\n" + "\n" + "[target:path]\n" + "pg_data={}\n" + "\n" + "[target:path:default]\n" + "group=\"group1\"\n" + "mode=\"0700\"\n" + "user=\"user1\"\n" + ); + + TEST_ASSIGN(manifest, manifestNewLoad(ioBufferReadNew(contentLoad)), "load manifest"); + + TEST_ERROR( + manifestTargetFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest target list"); + TEST_RESULT_STR_Z(manifestData(manifest)->backupLabel, "20190808-163540F", " check manifest data"); + + TEST_RESULT_STR_Z(manifestCipherSubPass(manifest), "somepass", " check cipher subpass"); + + TEST_RESULT_VOID( + manifestTargetUpdate(manifest, MANIFEST_TARGET_PGDATA_STR, STRDEF("/pg/base"), NULL), " update target no change"); + TEST_RESULT_VOID( + manifestTargetUpdate(manifest, MANIFEST_TARGET_PGDATA_STR, STRDEF("/path2"), NULL), " update target"); + TEST_RESULT_STR_Z( + manifestTargetFind(manifest, MANIFEST_TARGET_PGDATA_STR)->path, "/path2", " check target path"); + TEST_RESULT_VOID( + manifestTargetUpdate(manifest, MANIFEST_TARGET_PGDATA_STR, STRDEF("/pg/base"), NULL), " fix target path"); + + Buffer *contentSave = bufNew(0); + + TEST_RESULT_VOID(manifestSave(manifest, ioBufferWriteNew(contentSave)), "save manifest"); + TEST_RESULT_STR_STR(strNewBuf(contentSave), strNewBuf(contentLoad), " check save"); + + // Manifest with all features + // ------------------------------------------------------------------------------------------------------------------------- + #define TEST_MANIFEST_HEADER \ + "[backup]\n" \ + "backup-archive-start=\"000000030000028500000089\"\n" \ + "backup-archive-stop=\"000000030000028500000089\"\n" \ + "backup-label=\"20190818-084502F_20190820-084502D\"\n" \ + "backup-lsn-start=\"285/89000028\"\n" \ + "backup-lsn-stop=\"285/89001F88\"\n" \ + "backup-prior=\"20190818-084502F\"\n" \ + "backup-timestamp-copy-start=1565282141\n" \ + "backup-timestamp-start=1565282140\n" \ + "backup-timestamp-stop=1565282142\n" \ + "backup-type=\"full\"\n" \ + "\n" \ + "[backup:db]\n" \ + "db-catalog-version=201409291\n" \ + "db-control-version=942\n" \ + "db-id=1\n" \ + "db-system-id=1000000000000000094\n" \ + "db-version=\"9.4\"\n" \ + "\n" \ + "[backup:option]\n" \ + "option-archive-check=true\n" \ + "option-archive-copy=true\n" \ + "option-backup-standby=false\n" \ + "option-buffer-size=16384\n" \ + "option-checksum-page=true\n" \ + "option-compress=false\n" \ + "option-compress-level=3\n" \ + "option-compress-level-network=3\n" \ + "option-delta=false\n" \ + "option-hardlink=false\n" \ + "option-online=false\n" \ + "option-process-max=32\n" + + #define TEST_MANIFEST_TARGET \ + "\n" \ + "[backup:target]\n" \ + "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" \ + "pg_data/base/1={\"path\":\"../../base-1\",\"type\":\"link\"}\n" \ + "pg_data/pg_hba.conf={\"file\":\"pg_hba.conf\",\"path\":\"../pg_config\",\"type\":\"link\"}\n" \ + "pg_data/pg_stat={\"path\":\"../pg_stat\",\"type\":\"link\"}\n" \ + "pg_data/postgresql.conf={\"file\":\"postgresql.conf\",\"path\":\"../pg_config\",\"type\":\"link\"}\n" \ + "pg_tblspc/1={\"path\":\"/tblspc/ts1\",\"tablespace-id\":\"1\",\"tablespace-name\":\"ts1\",\"type\":\"link\"}\n" + + #define TEST_MANIFEST_DB \ + "\n" \ + "[db]\n" \ + "mail={\"db-id\":16456,\"db-last-system-id\":12168}\n" \ + "postgres={\"db-id\":12173,\"db-last-system-id\":12168}\n" \ + "template0={\"db-id\":12168,\"db-last-system-id\":12168}\n" \ + "template1={\"db-id\":1,\"db-last-system-id\":12168}\n" \ + + #define TEST_MANIFEST_FILE \ + "\n" \ + "[target:file]\n" \ + "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"master\":true" \ + ",\"reference\":\"20190818-084502F_20190819-084506D\",\"size\":4,\"timestamp\":1565282114}\n" \ + "pg_data/base/16384/17000={\"checksum\":\"e0101dd8ffb910c9c202ca35b5f828bcb9697bed\",\"checksum-page\":false" \ + ",\"checksum-page-error\":[1],\"repo-size\":4096,\"size\":8192,\"timestamp\":1565282114}\n" \ + "pg_data/base/16384/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"group\":false,\"size\":4" \ + ",\"timestamp\":1565282115}\n" \ + "pg_data/base/32768/33000={\"checksum\":\"7a16d165e4775f7c92e8cdf60c0af57313f0bf90\",\"checksum-page\":true" \ + ",\"reference\":\"20190818-084502F\",\"size\":1073741824,\"timestamp\":1565282116}\n" \ + "pg_data/base/32768/33000.32767={\"checksum\":\"6e99b589e550e68e934fd235ccba59fe5b592a9e\",\"checksum-page\":true" \ + ",\"reference\":\"20190818-084502F\",\"size\":32768,\"timestamp\":1565282114}\n" \ + "pg_data/postgresql.conf={\"checksum\":\"6721d92c9fcdf4248acff1f9a1377127d9064807\",\"master\":true,\"size\":4457" \ + ",\"timestamp\":1565282114}\n" \ + "pg_data/special={\"master\":true,\"mode\":\"0640\",\"size\":0,\"timestamp\":1565282120,\"user\":false}\n" + + #define TEST_MANIFEST_FILE_DEFAULT \ + "\n" \ + "[target:file:default]\n" \ + "group=\"group1\"\n" \ + "master=false\n" \ + "mode=\"0600\"\n" \ + "user=\"user1\"\n" + + #define TEST_MANIFEST_LINK \ + "\n" \ + "[target:link]\n" \ + "pg_data/pg_stat={\"destination\":\"../pg_stat\"}\n" \ + "pg_data/postgresql.conf={\"destination\":\"../pg_config/postgresql.conf\",\"group\":false,\"user\":\"user1\"}\n" + + #define TEST_MANIFEST_LINK_DEFAULT \ + "\n" \ + "[target:link:default]\n" \ + "group=\"group1\"\n" \ + "user=false\n" + + #define TEST_MANIFEST_PATH \ + "\n" \ + "[target:path]\n" \ + "pg_data={\"user\":\"user2\"}\n" \ + "pg_data/base={\"group\":\"group2\"}\n" \ + "pg_data/base/16384={\"mode\":\"0750\"}\n" \ + "pg_data/base/32768={}\n" \ + "pg_data/base/65536={\"user\":false}\n" + + #define TEST_MANIFEST_PATH_DEFAULT \ + "\n" \ + "[target:path:default]\n" \ + "group=false\n" \ + "mode=\"0700\"\n" \ + "user=\"user1\"\n" + + contentLoad = harnessInfoChecksumZ + ( + TEST_MANIFEST_HEADER + TEST_MANIFEST_TARGET + TEST_MANIFEST_DB + TEST_MANIFEST_FILE + TEST_MANIFEST_FILE_DEFAULT + TEST_MANIFEST_LINK + TEST_MANIFEST_LINK_DEFAULT + TEST_MANIFEST_PATH + TEST_MANIFEST_PATH_DEFAULT + ); + + TEST_ASSIGN(manifest, manifestNewLoad(ioBufferReadNew(contentLoad)), "load manifest"); + + TEST_RESULT_STR_Z(manifestPgPath(STRDEF("pg_data")), NULL, "check pg_data path"); + TEST_RESULT_STR_Z(manifestPgPath(STRDEF("pg_data/PG_VERSION")), "PG_VERSION", "check pg_data path/file"); + TEST_RESULT_STR_Z(manifestPgPath(STRDEF("pg_tblspc/1")), "pg_tblspc/1", "check pg_tblspc path/file"); + + TEST_RESULT_PTR(manifestCipherSubPass(manifest), NULL, " check cipher subpass"); + + // Absolute target paths + TEST_RESULT_STR_Z(manifestTargetPath(manifest, manifestTargetBase(manifest)), "/pg/base", "base target path"); + TEST_RESULT_STR_Z( + manifestTargetPath(manifest, manifestTargetFind(manifest, STRDEF("pg_data/pg_hba.conf"))), "/pg/pg_config", + "relative file link target path"); + TEST_RESULT_STR_Z( + manifestTargetPath(manifest, manifestTargetFind(manifest, STRDEF("pg_data/pg_stat"))), "/pg/pg_stat", + "relative path link target path"); + TEST_RESULT_STR_Z( + manifestTargetPath(manifest, manifestTargetFind(manifest, STRDEF("pg_data/base/1"))), "/pg/base-1", + "relative path link target path"); + + // Link check + TEST_RESULT_VOID(manifestLinkCheck(manifest), "successful link check"); + manifestTargetAdd( + manifest, &(ManifestTarget){ + .name = STRDEF("pg_data/base/2"), .type = manifestTargetTypeLink, .path = STRDEF("../../base-1/base-2")}); + TEST_ERROR( + manifestLinkCheck(manifest), LinkDestinationError, + "link 'base/2' (/pg/base-1/base-2) destination is a subdirectory of or the same directory as" + " link 'base/1' (/pg/base-1)"); + manifestTargetRemove(manifest, STRDEF("pg_data/base/2")); + + // ManifestFile getters + const ManifestFile *file = NULL; + TEST_ERROR( + manifestFileFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest file list"); + TEST_ASSIGN(file, manifestFileFind(manifest, STRDEF("pg_data/PG_VERSION")), "manifestFileFind()"); + TEST_RESULT_STR_Z(file->name, "pg_data/PG_VERSION", " find file"); + TEST_RESULT_STR_Z( + manifestFileFindDefault(manifest, STRDEF("bogus"), file)->name, "pg_data/PG_VERSION", + "manifestFileFindDefault() - return default"); + TEST_RESULT_STR_Z( + manifestFileFindDefault(manifest, STRDEF("pg_data/special"), file)->name, "pg_data/special", + "manifestFileFindDefault() - return found"); + TEST_ASSIGN(file, manifestFileFindDefault(manifest, STRDEF("bogus"), NULL), "manifestFileFindDefault()"); + TEST_RESULT_PTR(file, NULL, " return default NULL"); + + // ManifestDb getters + const ManifestDb *db = NULL; + TEST_ERROR( + manifestDbFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest db list"); + TEST_ASSIGN(db, manifestDbFind(manifest, STRDEF("postgres")), "manifestDbFind()"); + TEST_RESULT_STR_Z(db->name, "postgres", " check name"); + TEST_RESULT_STR_Z( + manifestDbFindDefault(manifest, STRDEF("bogus"), db)->name, "postgres", "manifestDbFindDefault() - return default"); + TEST_RESULT_UINT( + manifestDbFindDefault(manifest, STRDEF("template0"), db)->id, 12168, "manifestDbFindDefault() - return found"); + TEST_ASSIGN(db, manifestDbFindDefault(manifest, STRDEF("bogus"), NULL), "manifestDbFindDefault()"); + TEST_RESULT_PTR(db, NULL, " return default NULL"); + + // ManifestLink getters + const ManifestLink *link = NULL; + TEST_ERROR( + manifestLinkFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest link list"); + TEST_ASSIGN(link, manifestLinkFind(manifest, STRDEF("pg_data/pg_stat")), "find link"); + TEST_RESULT_VOID(manifestLinkUpdate(manifest, STRDEF("pg_data/pg_stat"), STRDEF("../pg_stat")), " no update"); + TEST_RESULT_STR_Z(link->destination, "../pg_stat", " check link"); + TEST_RESULT_VOID(manifestLinkUpdate(manifest, STRDEF("pg_data/pg_stat"), STRDEF("../pg_stat2")), " update"); + TEST_RESULT_STR_Z(link->destination, "../pg_stat2", " check link"); + TEST_RESULT_VOID(manifestLinkUpdate(manifest, STRDEF("pg_data/pg_stat"), STRDEF("../pg_stat")), " fix link destination"); + TEST_RESULT_STR_Z( + manifestLinkFindDefault(manifest, STRDEF("bogus"), link)->name, "pg_data/pg_stat", + "manifestLinkFindDefault() - return default"); + TEST_RESULT_STR_Z( + manifestLinkFindDefault(manifest, STRDEF("pg_data/postgresql.conf"), link)->destination, "../pg_config/postgresql.conf", + "manifestLinkFindDefault() - return found"); + TEST_ASSIGN(link, manifestLinkFindDefault(manifest, STRDEF("bogus"), NULL), "manifestLinkFindDefault()"); + TEST_RESULT_PTR(link, NULL, " return default NULL"); + + // ManifestPath getters + const ManifestPath *path = NULL; + TEST_ERROR( + manifestPathFind(manifest, STRDEF("bogus")), AssertError, "unable to find 'bogus' in manifest path list"); + TEST_ASSIGN(path, manifestPathFind(manifest, STRDEF("pg_data")), "manifestPathFind()"); + TEST_RESULT_STR_Z(path->name, "pg_data", " check path"); + TEST_RESULT_STR_Z( + manifestPathFindDefault(manifest, STRDEF("bogus"), path)->name, "pg_data", + "manifestPathFindDefault() - return default"); + TEST_RESULT_STR_Z( + manifestPathFindDefault(manifest, STRDEF("pg_data/base"), path)->group, "group2", + "manifestPathFindDefault() - return found"); + TEST_ASSIGN(path, manifestPathFindDefault(manifest, STRDEF("bogus"), NULL), "manifestPathFindDefault()"); + TEST_RESULT_PTR(path, NULL, " return default NULL"); + + const ManifestTarget *target = NULL; + TEST_ASSIGN(target, manifestTargetFind(manifest, STRDEF("pg_data/pg_hba.conf")), "find target"); + TEST_RESULT_VOID( + manifestTargetUpdate(manifest, target->name, target->path, STRDEF("pg_hba2.conf")), " update target file"); + TEST_RESULT_STR_Z(target->file, "pg_hba2.conf", " check target file"); + TEST_RESULT_VOID(manifestTargetUpdate(manifest, target->name, target->path, STRDEF("pg_hba.conf")), " fix target file"); + + contentSave = bufNew(0); + + TEST_RESULT_VOID(manifestSave(manifest, ioBufferWriteNew(contentSave)), "save manifest"); + TEST_RESULT_STR_STR(strNewBuf(contentSave), strNewBuf(contentLoad), " check save"); + + TEST_RESULT_VOID(manifestFileRemove(manifest, STRDEF("pg_data/PG_VERSION")), "remove file"); + TEST_ERROR( + manifestFileRemove(manifest, STRDEF("pg_data/PG_VERSION")), AssertError, + "unable to remove 'pg_data/PG_VERSION' from manifest file list"); + + TEST_RESULT_VOID(manifestLinkRemove(manifest, STRDEF("pg_data/pg_stat")), "remove link"); + TEST_ERROR( + manifestLinkRemove(manifest, STRDEF("pg_data/pg_stat")), AssertError, + "unable to remove 'pg_data/pg_stat' from manifest link list"); + + TEST_RESULT_VOID(manifestTargetRemove(manifest, STRDEF("pg_data/pg_hba.conf")), "remove target"); + TEST_ERROR( + manifestTargetRemove(manifest, STRDEF("pg_data/pg_hba.conf")), AssertError, + "unable to remove 'pg_data/pg_hba.conf' from manifest target list"); + } + + // ***************************************************************************************************************************** + if (testBegin("manifestLoadFile()")) + { + Manifest *manifest = NULL; + + TEST_ERROR_FMT( + manifestLoadFile(storageTest, BACKUP_MANIFEST_FILE_STR, cipherTypeNone, NULL), FileMissingError, + "unable to load backup manifest file '%s/backup.manifest' or '%s/backup.manifest.copy':\n" + "FileMissingError: unable to open missing file '%s/backup.manifest' for read\n" + "FileMissingError: unable to open missing file '%s/backup.manifest.copy' for read", + testPath(), testPath(), testPath(), testPath()); + + // Also use this test to check that extra sections/keys are ignored using coverage. + // ------------------------------------------------------------------------------------------------------------------------- + const Buffer *content = harnessInfoChecksumZ + ( + "[backup]\n" + "backup-label=\"20190808-163540F\"\n" + "backup-timestamp-copy-start=1565282141\n" + "backup-timestamp-start=1565282140\n" + "backup-timestamp-stop=1565282142\n" + "backup-type=\"full\"\n" + "ignore-key=ignore-value\n" + "\n" + "[backup:db]\n" + "db-catalog-version=201409291\n" + "db-control-version=942\n" + "db-id=1\n" + "db-system-id=1000000000000000094\n" + "db-version=\"9.4\"\n" + "ignore-key=ignore-value\n" + "\n" + "[backup:option]\n" + "ignore-key=ignore-value\n" + "option-archive-check=true\n" + "option-archive-copy=true\n" + "option-compress=false\n" + "option-hardlink=false\n" + "option-online=false\n" + "\n" + "[backup:target]\n" + "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n" + "\n" + "[ignore-section]\n" + "ignore-key=ignore-value\n" + "\n" + "[target:file]\n" + "pg_data/PG_VERSION={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"size\":4,\"timestamp\":1565282114}\n" + "\n" + "[target:file:default]\n" + "group=\"group1\"\n" + "ignore-key=ignore-value\n" + "master=true\n" + "mode=\"0600\"\n" + "user=\"user1\"\n" + "\n" + "[target:link:default]\n" + "ignore-key=ignore-value\n" + "\n" + "[target:path]\n" + "pg_data={}\n" + "\n" + "[target:path:default]\n" + "group=\"group1\"\n" + "ignore-key=ignore-value\n" + "mode=\"0700\"\n" + "user=\"user1\"\n" + ); + + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageTest, strNew(BACKUP_MANIFEST_FILE INFO_COPY_EXT)), content), "write copy"); + TEST_ASSIGN(manifest, manifestLoadFile(storageTest, STRDEF(BACKUP_MANIFEST_FILE), cipherTypeNone, NULL), "load copy"); + TEST_RESULT_UINT(manifestData(manifest)->pgSystemId, 1000000000000000094, " check file loaded"); + + storageRemoveP(storageTest, strNew(BACKUP_MANIFEST_FILE INFO_COPY_EXT), .errorOnMissing = true); + + TEST_RESULT_VOID( + storagePutNP(storageNewWriteNP(storageTest, BACKUP_MANIFEST_FILE_STR), content), "write main"); + TEST_ASSIGN(manifest, manifestLoadFile(storageTest, STRDEF(BACKUP_MANIFEST_FILE), cipherTypeNone, NULL), "load main"); + TEST_RESULT_UINT(manifestData(manifest)->pgSystemId, 1000000000000000094, " check file loaded"); + } + + // Load/save a larger manifest to test performance and memory usage. The default sizing is for a "typical" cluster but this can + // be scaled to test larger cluster sizes. + // ***************************************************************************************************************************** + if (testBegin("manifestNewLoad()/manifestSave() performance")) + { + Manifest *manifest = NULL; + + // Manifest with all features + // ------------------------------------------------------------------------------------------------------------------------- + String *manifestStr = strNew + ( + "[backup]\n" + "backup-label=\"20190818-084502F_20190820-084502D\"\n" + "backup-prior=\"20190818-084502F\"\n" + "backup-timestamp-copy-start=1566290707\n" + "backup-timestamp-start=1566290702\n" + "backup-timestamp-stop=1566290710\n" + "backup-type=\"diff\"\n" + "\n" + "[backup:db]\n" + "db-catalog-version=201809051\n" + "db-control-version=1100\n" + "db-id=2\n" + "db-system-id=6689162560678426440\n" + "db-version=\"11\"\n" + "\n" + "[backup:option]\n" + "option-archive-check=true\n" + "option-archive-copy=false\n" + "option-backup-standby=false\n" + "option-buffer-size=4194304\n" + "option-checksum-page=true\n" + "option-compress=true\n" + "option-compress-level=9\n" + "option-compress-level-network=3\n" + "option-delta=false\n" + "option-hardlink=false\n" + "option-online=false\n" + "option-process-max=2\n" + "\n" + "[backup:target]\n" + "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n"); + + for (unsigned int linkIdx = 0; linkIdx < 1; linkIdx++) + strCatFmt(manifestStr, "pg_data/pg_stat%u={\"path\":\"../pg_stat\",\"type\":\"link\"}\n", linkIdx); + + strCat( + manifestStr, + "\n" + "[target:file]\n"); + + for (unsigned int fileIdx = 0; fileIdx < 10; fileIdx++) + strCatFmt( + manifestStr, + "pg_data/base/16384/%u={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"size\":16384" + ",\"timestamp\":1565282114}\n", + 16384 + fileIdx); + + strCat( + manifestStr, + "\n" + "[target:file:default]\n" + "group=\"postgres\"\n" + "master=false\n" + "mode=\"0600\"\n" + "user=\"postgres\"\n" + "\n" + "[target:link]\n" + "pg_data/pg_stat={\"destination\":\"../pg_stat\"}\n" + "\n" + "[target:link:default]\n" + "group=\"postgres\"\n" + "user=\"postgres\"\n" + "\n" + "[target:path]\n" + "pg_data={}\n" + "pg_data/base={}\n" + "pg_data/base/1={}\n" + "pg_data/base/13124={}\n" + "pg_data/base/13125={}\n" + "pg_data/base/16391={}\n" + "pg_data/global={}\n" + "pg_data/pg_commit_ts={}\n" + "pg_data/pg_dynshmem={}\n" + "pg_data/pg_logical={}\n" + "pg_data/pg_logical/mappings={}\n" + "pg_data/pg_logical/snapshots={}\n" + "pg_data/pg_multixact={}\n" + "pg_data/pg_multixact/members={}\n" + "pg_data/pg_multixact/offsets={}\n" + "pg_data/pg_notify={}\n" + "pg_data/pg_replslot={}\n" + "pg_data/pg_serial={}\n" + "pg_data/pg_snapshots={}\n" + "pg_data/pg_stat={}\n" + "pg_data/pg_stat_tmp={}\n" + "pg_data/pg_subtrans={}\n" + "pg_data/pg_tblspc={}\n" + "pg_data/pg_twophase={}\n" + "pg_data/pg_wal={}\n" + "pg_data/pg_wal/archive_status={}\n" + "pg_data/pg_xact={}\n" + "\n" + "[target:path:default]\n" + "group=\"postgres\"\n" + "mode=\"0700\"\n" + "user=\"postgres\"\n" + ); + + const Buffer *contentLoad = harnessInfoChecksum(manifestStr); + + TEST_ASSIGN(manifest, manifestNewLoad(ioBufferReadNew(contentLoad)), "load manifest"); + (void)manifest; + + Buffer *contentSave = bufNew(0); + + TEST_RESULT_VOID(manifestSave(manifest, ioBufferWriteNew(contentSave)), "save manifest"); + + if (!bufEq(contentSave, contentLoad)) + { + TEST_RESULT_VOID( // {uncovered - only for debugging} + storagePutNP(storageNewWriteNP(storageTest, strNew(BACKUP_MANIFEST_FILE ".expected")), contentLoad), + "write expected manifest"); + TEST_RESULT_VOID( // {uncovered - only for debugging} + storagePutNP(storageNewWriteNP(storageTest, strNew(BACKUP_MANIFEST_FILE ".actual")), contentSave), + "write actual manifest"); + } + + TEST_RESULT_BOOL(bufEq(contentSave, contentLoad), true, " check save"); + } +}