db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Add reference filter and output to manifest command.

Browse Source 
This allows the file list to be filtered by reference. The reference is output when it is not the default reference for the backup.
This commit is contained in:
David Steele 2023-03-21 12:29:45 +08:00
parent 1807bfcff5
commit c8ec114c8c
7 changed files with 300 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
doc/xml/release.xml
View File

@ -15,6 +15,19 @@
<release-list>
<release date="XXXX-XX-XX" version="2.46dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-development-list>
<release-item>
<commit subject="Add reference filter and output to manifest command."/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
</release-item-contributor-list>
<p>Improve internal <cmd>manifest</cmd> command.</p>
</release-item>
</release-development-list>
</release-core-list>
</release>
<release date="2023-03-20" version="2.45" title="Block Incremental Backup (BETA)">

8
src/build/config/config.yaml
View File

@ -332,6 +332,14 @@ option:
command-role:
main: {}
reference:
type: string
required: false
command:
manifest: {}
command-role:
main: {}
set:
type: string
required: false

10
src/build/help/help.xml
View File

@ -2066,6 +2066,16 @@
<example>1</example>
</option>
<option id="reference" name="Filter on Reference">
<summary>Filter on reference.</summary>
<text>
<p>Show only files with the specified reference. Use <id>latest</id> for files that were updated in the backup specified by <br-option>--set</br-option>.</p>
</text>
<example>20150131-153358F_20150131-153401I</example>
</option>
<option id="set" name="Set">
<summary>Manifest to query.</summary>

26
src/command/manifest/manifest.c
View File

@ -363,6 +363,10 @@ cmdManifestFileRender(const Manifest *const manifest, const ManifestFile *const
if (json)
{
strCatFmt(result, "{\"name\":%s", strZ(jsonFromVar(VARSTR(file->name))));
if (file->reference != NULL)
strCatFmt(result, ",\"reference\":%s", strZ(jsonFromVar(VARSTR(file->reference))));
strCatFmt(result, ",\"size\":%" PRIu64, file->size);
strCatFmt(
result, ",\"checksum\":\"%s\"", strZ(strNewEncode(encodingHex, BUF(file->checksumSha1, HASH_TYPE_SHA1_SIZE))));
@ -387,6 +391,10 @@ cmdManifestFileRender(const Manifest *const manifest, const ManifestFile *const
else
{
strCatFmt(result, " - %s\n", strZ(file->name));
if (file->reference != NULL)
strCatFmt(result, " reference: %s\n", strZ(file->reference));
strCatFmt(
result, " size: %s, repo %s\n", strZ(strSizeFormat(file->size)), strZ(strSizeFormat(file->sizeRepo)));
strCatFmt(result, " checksum: %s\n", strZ(strNewEncode(encodingHex, BUF(file->checksumSha1, HASH_TYPE_SHA1_SIZE))));
@ -500,17 +508,33 @@ cmdManifestRender(void)
// Multiple files
else
{
bool first = true;
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
const ManifestFile file = manifestFile(manifest, fileIdx);
if (fileIdx != 0)
// Filter on reference
if (cfgOptionTest(cfgOptReference))
{
if (strEqZ(cfgOptionStr(cfgOptReference), "latest"))
{
if (file.reference != NULL)
continue;
}
else if (!strEq(cfgOptionStr(cfgOptReference), file.reference))
continue;
}
if (!first)
{
if (json)
strCatChr(result, ',');
else
strCatChr(result, '\n');
}
else
first = false;
strCat(result, cmdManifestFileRender(manifest, &file, false, json));
}

4
src/config/config.auto.h
View File

@ -104,6 +104,7 @@ Option constants
#define CFGOPT_RAW "raw"
#define CFGOPT_RECOVERY_OPTION "recovery-option"
#define CFGOPT_RECURSE "recurse"
#define CFGOPT_REFERENCE "reference"
#define CFGOPT_REMOTE_TYPE "remote-type"
#define CFGOPT_REPO "repo"
#define CFGOPT_RESUME "resume"
@ -133,7 +134,7 @@ Option constants
#define CFGOPT_TYPE "type"
#define CFGOPT_VERBOSE "verbose"
#define CFG_OPTION_TOTAL 165
#define CFG_OPTION_TOTAL 166
/***********************************************************************************************************************************
Option value constants
@ -441,6 +442,7 @@ typedef enum
cfgOptRaw,
cfgOptRecoveryOption,
cfgOptRecurse,
cfgOptReference,
cfgOptRemoteType,
cfgOptRepo,
cfgOptRepoAzureAccount,

14
src/config/parse.auto.c.inc
View File

@ -4439,6 +4439,19 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
), // opt/recurse
), // opt/recurse
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/reference
( // opt/reference
PARSE_RULE_OPTION_NAME("reference"), // opt/reference
PARSE_RULE_OPTION_TYPE(cfgOptTypeString), // opt/reference
PARSE_RULE_OPTION_REQUIRED(false), // opt/reference
PARSE_RULE_OPTION_SECTION(cfgSectionCommandLine), // opt/reference
// opt/reference
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/reference
( // opt/reference
PARSE_RULE_OPTION_COMMAND(cfgCmdManifest) // opt/reference
), // opt/reference
), // opt/reference
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/remote-type
( // opt/remote-type
PARSE_RULE_OPTION_NAME("remote-type"), // opt/remote-type
@ -9874,6 +9887,7 @@ static const uint8_t optionResolveOrder[] =
cfgOptProtocolTimeout, // opt-resolve-order
cfgOptRaw, // opt-resolve-order
cfgOptRecurse, // opt-resolve-order
cfgOptReference, // opt-resolve-order
cfgOptRemoteType, // opt-resolve-order
cfgOptRepo, // opt-resolve-order
cfgOptRepoBundle, // opt-resolve-order

230
test/src/module/command/manifestTest.c
View File

@ -79,6 +79,10 @@ testRun(void)
{
const time_t backupTimeStart = BACKUP_EPOCH;
// Version file that will not be updated after the full backup
HRN_STORAGE_PUT_Z(
storagePgWrite(), PG_PATH_BASE "/" PG_FILE_PGVERSION, PG_VERSION_95_STR, .timeModified = backupTimeStart);
// Zeroed file large enough to use block incr
Buffer *relation = bufNew(8 * 8192);
memset(bufPtr(relation), 0, bufSize(relation));
@ -125,6 +129,22 @@ testRun(void)
hrnBackupPqScriptP(PG_VERSION_95, backupTimeStart, .noArchiveCheck = true, .noWal = true);
TEST_RESULT_VOID(hrnCmdBackup(), "backup repo1");
// Munge the pg_control checksum since it will vary by architecture
Manifest *manifest = manifestLoadFile(
storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/20191002-070640F_20191003-105320D/" BACKUP_MANIFEST_FILE),
cipherTypeNone, NULL);
ManifestFile file = manifestFileFind(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL));
file.checksumSha1 = bufPtr(bufNewDecode(encodingHex, STRDEF("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")));
manifestFileUpdate(manifest, &file);
manifestSave(
manifest,
storageWriteIo(
storageNewWriteP(
storageRepoWrite(),
STRDEF(STORAGE_REPO_BACKUP "/20191002-070640F_20191003-105320D/" BACKUP_MANIFEST_FILE))));
// Backup to repo2
hrnCfgArgRawZ(argList, cfgOptRepo, "2");
HRN_CFG_LOAD(cfgCmdBackup, argList);
@ -138,6 +158,8 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("manifest block map");
{
TEST_TITLE("single file block map from full");
StringList *argList = strLstDup(argListCommon);
hrnCfgArgRawZ(argList, cfgOptFilter, PG_PATH_BASE "/1/2");
hrnCfgArgRawZ(argList, cfgOptSet, "20191002-070640F");
@ -197,7 +219,7 @@ testRun(void)
"},"
"\"bundle\":{"
"\"id\":1,"
"\"offset\":8192"
"\"offset\":8195"
"},"
"\"block\":{"
"\"size\":8192,"
@ -232,6 +254,207 @@ testRun(void)
TEST_RESULT_VOID(jsonToVar(cmdManifestRender()), "check json");
// ---------------------------------------------------------------------------------------------------------------------
TEST_TITLE("multiple files from diff");
argList = strLstDup(argListCommon);
HRN_CFG_LOAD(cfgCmdManifest, argList);
TEST_RESULT_STR_Z(
cmdManifestRender(),
"label: 20191002-070640F_20191003-105320D\n"
"reference: 20191002-070640F, 20191002-070640F_20191003-105320D\n"
"type: diff\n"
"time: start: 2019-10-03 10:53:20, stop: 2019-10-04 01:27:07, duration: 14:33:47\n"
"bundle: true\n"
"block: true\n"
"\n"
"file list:\n"
" - pg_data/base/1/2\n"
" size: 96KB, repo 64.1KB\n"
" checksum: d4976e362696a43fb09e7d4e780d7d9352a2ec2e\n"
" bundle: 1\n"
" block: size 8KB, map size 99B, checksum size 6B\n"
"\n"
" - pg_data/base/PG_VERSION\n"
" reference: 20191002-070640F\n"
" size: 3B, repo 3B\n"
" checksum: 06d06bb31b570b94d7b4325f511f853dbe771c21\n"
" bundle: 1\n"
"\n"
" - pg_data/global/pg_control\n"
" size: 8KB, repo 8KB\n"
" checksum: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
" bundle: 1\n",
"repo 1 text");
// ---------------------------------------------------------------------------------------------------------------------
TEST_TITLE("reference from diff");
argList = strLstDup(argListCommon);
hrnCfgArgRawZ(argList, cfgOptReference, "20191002-070640F");
HRN_CFG_LOAD(cfgCmdManifest, argList);
TEST_RESULT_STR_Z(
cmdManifestRender(),
"label: 20191002-070640F_20191003-105320D\n"
"reference: 20191002-070640F, 20191002-070640F_20191003-105320D\n"
"type: diff\n"
"time: start: 2019-10-03 10:53:20, stop: 2019-10-04 01:27:07, duration: 14:33:47\n"
"bundle: true\n"
"block: true\n"
"\n"
"file list:\n"
" - pg_data/base/PG_VERSION\n"
" reference: 20191002-070640F\n"
" size: 3B, repo 3B\n"
" checksum: 06d06bb31b570b94d7b4325f511f853dbe771c21\n"
" bundle: 1\n",
"repo 1 text");
hrnCfgArgRawZ(argList, cfgOptOutput, "json");
HRN_CFG_LOAD(cfgCmdManifest, argList);
TEST_RESULT_STR_Z(
cmdManifestRender(),
// {uncrustify_off - indentation}
"{"
"\"label\":\"20191002-070640F_20191003-105320D\","
"\"reference\":["
"\"20191002-070640F\","
"\"20191002-070640F_20191003-105320D\""
"],"
"\"type\":\"diff\","
"\"time\":{"
"\"start\":1570100000,"
"\"copy\":1570100001,"
"\"stop\":1570152427"
"},"
"\"bundle\":{"
"\"bundle\":true,"
"\"raw\":true"
"},"
"\"block\":{"
"\"block\":true"
"},"
"\"fileList\":["
"{"
"\"name\":\"pg_data/base/PG_VERSION\","
"\"reference\":\"20191002-070640F\","
"\"size\":3,"
"\"checksum\":\"06d06bb31b570b94d7b4325f511f853dbe771c21\","
"\"repo\":{"
"\"size\":3"
"},"
"\"bundle\":{"
"\"id\":1,"
"\"offset\":8192"
"}"
"}"
"]"
"}",
// {uncrustify_on}
"repo 2 test");
TEST_RESULT_VOID(jsonToVar(cmdManifestRender()), "check json");
// ---------------------------------------------------------------------------------------------------------------------
TEST_TITLE("latest reference from diff");
argList = strLstDup(argListCommon);
hrnCfgArgRawZ(argList, cfgOptReference, "latest");
HRN_CFG_LOAD(cfgCmdManifest, argList);
TEST_RESULT_STR_Z(
cmdManifestRender(),
"label: 20191002-070640F_20191003-105320D\n"
"reference: 20191002-070640F, 20191002-070640F_20191003-105320D\n"
"type: diff\n"
"time: start: 2019-10-03 10:53:20, stop: 2019-10-04 01:27:07, duration: 14:33:47\n"
"bundle: true\n"
"block: true\n"
"\n"
"file list:\n"
" - pg_data/base/1/2\n"
" size: 96KB, repo 64.1KB\n"
" checksum: d4976e362696a43fb09e7d4e780d7d9352a2ec2e\n"
" bundle: 1\n"
" block: size 8KB, map size 99B, checksum size 6B\n"
"\n"
" - pg_data/global/pg_control\n"
" size: 8KB, repo 8KB\n"
" checksum: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
" bundle: 1\n",
"repo 1 text");
hrnCfgArgRawZ(argList, cfgOptOutput, "json");
HRN_CFG_LOAD(cfgCmdManifest, argList);
TEST_RESULT_STR_Z(
cmdManifestRender(),
// {uncrustify_off - indentation}
"{"
"\"label\":\"20191002-070640F_20191003-105320D\","
"\"reference\":["
"\"20191002-070640F\","
"\"20191002-070640F_20191003-105320D\""
"],"
"\"type\":\"diff\","
"\"time\":{"
"\"start\":1570100000,"
"\"copy\":1570100001,"
"\"stop\":1570152427"
"},"
"\"bundle\":{"
"\"bundle\":true,"
"\"raw\":true"
"},"
"\"block\":{"
"\"block\":true"
"},"
"\"fileList\":["
"{"
"\"name\":\"pg_data/base/1/2\","
"\"size\":98304,"
"\"checksum\":\"d4976e362696a43fb09e7d4e780d7d9352a2ec2e\","
"\"repo\":{"
"\"size\":65647"
"},"
"\"bundle\":{"
"\"id\":1,"
"\"offset\":8192"
"},"
"\"block\":{"
"\"size\":8192,"
"\"map\":{"
"\"size\":99"
"},"
"\"checksum\":{"
"\"size\":6"
"}"
"}"
"},"
"{"
"\"name\":\"pg_data/global/pg_control\","
"\"size\":8192,"
"\"checksum\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\","
"\"repo\":{"
"\"size\":8192"
"},"
"\"bundle\":{"
"\"id\":1,"
"\"offset\":0"
"}"
"}"
"]"
"}",
// {uncrustify_on}
"repo 2 test");
TEST_RESULT_VOID(jsonToVar(cmdManifestRender()), "check json");
// ---------------------------------------------------------------------------------------------------------------------
TEST_TITLE("single file block map from diff");
argList = strLstDup(argListCommon);
hrnCfgArgRawZ(argList, cfgOptFilter, PG_PATH_BASE "/1/2");
HRN_CFG_LOAD(cfgCmdManifest, argList);
@ -257,10 +480,11 @@ testRun(void)
" total read: 2/128KB, superBlock: 3/128KB, block: 12/96KB\n",
"repo 1 text");
// Block map from repo 2
// ---------------------------------------------------------------------------------------------------------------------
TEST_TITLE("single file block map delta from diff");
argList = strLstDup(argListCommon);
hrnCfgArgRawZ(argList, cfgOptFilter, PG_PATH_BASE "/1/2");
// hrnCfgArgRawZ(argList, cfgOptSet, "20191002-070640F");
hrnCfgArgRawZ(argList, cfgOptRepo, "2");
hrnCfgArgRawZ(argList, cfgOptPg, "1");
HRN_CFG_LOAD(cfgCmdManifest, argList);