db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-01-18 04:58:51 +02:00
Code Issues Releases Activity

Add test to show behavior of bundled files truncated during backup.

Browse Source 
This behavior violates an assertion but is completely possible with the current implementation. This behavior will be fixed in a future commit, but for now at least test how it works correctly and remove the assertion so the test runs without error.

Also add a new harness that allows changes during the backup to be scripted.
This commit is contained in:
David Steele 2023-11-24 12:25:40 -03:00
parent ac78b96583
commit 337da35ab2
5 changed files with 166 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/info/manifest.c
View File

@ -3050,7 +3050,6 @@ manifestFileUpdate(Manifest *const this, const ManifestFile *const file)
(!file->checksumPage && !file->checksumPageError && file->checksumPageErrorList == NULL) ||
(file->checksumPage && !file->checksumPageError && file->checksumPageErrorList == NULL) ||
(file->checksumPage && file->checksumPageError));
ASSERT(file->size != 0 || (file->bundleId == 0 && file->bundleOffset == 0));
ManifestFilePack **const filePack = manifestFilePackFindInternal(this, file->name);
manifestFilePackUpdate(this, filePack, file);

7
test/define.yaml
View File

@ -879,7 +879,12 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: backup
total: 13
harness: backup
harness:
name: backup
shim:
command/backup/backup:
function:
- backupProcess
coverage:
- command/backup/backup

122
test/src/common/harnessBackup.c
View File

@ -12,15 +12,135 @@ Harness for Creating Test Backups
#include "common/lock.h"
#include "config/config.h"
#include "info/infoArchive.h"
#include "info/manifest.h"
#include "postgres/interface.h"
#include "storage/helper.h"
#include "storage/posix/storage.h"
#include "common/harnessBackup.h"
#include "common/harnessDebug.h"
#include "common/harnessPostgres.h"
#include "common/harnessPq.h"
#include "common/harnessStorage.h"
#include "common/harnessTest.h"
#include "common/harnessTest.intern.h"
/***********************************************************************************************************************************
Include shimmed C modules
***********************************************************************************************************************************/
{[SHIM_MODULE]}
/***********************************************************************************************************************************
Local variables
***********************************************************************************************************************************/
static struct HrnBackupLocal
{
MemContext *memContext; // Script mem context
// Script that defines how shim functions operate
HrnBackupScript script[1024];
unsigned int scriptSize;
unsigned int scriptIdx;
} hrnBackupLocal;
/**********************************************************************************************************************************/
static void
hrnBackupScriptAdd(const HrnBackupScript *const script, const unsigned int scriptSize)
{
if (scriptSize == 0)
THROW(AssertError, "backup script must have entries");
if (hrnBackupLocal.scriptSize == 0)
{
MEM_CONTEXT_BEGIN(memContextTop())
{
MEM_CONTEXT_NEW_BEGIN(hrnBackupLocal, .childQty = MEM_CONTEXT_QTY_MAX)
{
hrnBackupLocal.memContext = MEM_CONTEXT_NEW();
}
MEM_CONTEXT_NEW_END();
}
MEM_CONTEXT_END();
hrnBackupLocal.scriptIdx = 0;
}
// Copy records into local storage
MEM_CONTEXT_BEGIN(hrnBackupLocal.memContext)
{
for (unsigned int scriptIdx = 0; scriptIdx < scriptSize; scriptIdx++)
{
ASSERT(script[scriptIdx].op != 0);
ASSERT(script[scriptIdx].file != NULL);
hrnBackupLocal.script[hrnBackupLocal.scriptSize] = script[scriptIdx];
hrnBackupLocal.script[hrnBackupLocal.scriptSize].file = strDup(script[scriptIdx].file);
if (script[scriptIdx].content != NULL)
hrnBackupLocal.script[hrnBackupLocal.scriptSize].content = bufDup(script[scriptIdx].content);
hrnBackupLocal.scriptSize++;
}
}
MEM_CONTEXT_END();
}
void
hrnBackupScriptSet(const HrnBackupScript *const script, const unsigned int scriptSize)
{
if (hrnBackupLocal.scriptSize != 0)
THROW(AssertError, "previous pq script has not yet completed");
hrnBackupScriptAdd(script, scriptSize);
}
/**********************************************************************************************************************************/
static void
backupProcess(const BackupData *const backupData, Manifest *const manifest, const String *const cipherPassBackup)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(BACKUP_DATA, backupData);
FUNCTION_HARNESS_PARAM(MANIFEST, manifest);
FUNCTION_HARNESS_PARAM(STRING, cipherPassBackup);
FUNCTION_HARNESS_END();
// If any file changes are scripted then make them
if (hrnBackupLocal.scriptSize != 0)
{
MEM_CONTEXT_TEMP_BEGIN()
{
Storage *const storageTest = storagePosixNewP(strNewZ(testPath()), .write = true);
for (unsigned int scriptIdx = 0; scriptIdx < hrnBackupLocal.scriptSize; scriptIdx++)
{
switch (hrnBackupLocal.script[scriptIdx].op)
{
// Update file
case hrnBackupScriptOpUpdate:
storagePutP(
storageNewWriteP(
storageTest, hrnBackupLocal.script[scriptIdx].file,
.timeModified = hrnBackupLocal.script[scriptIdx].time),
hrnBackupLocal.script[scriptIdx].content == NULL ?
BUFSTRDEF("") : hrnBackupLocal.script[scriptIdx].content);
break;
default:
THROW_FMT(
AssertError, "unknown backup script op '%s'", strZ(strIdToStr(hrnBackupLocal.script[scriptIdx].op)));
}
}
}
MEM_CONTEXT_TEMP_END();
// Free script
memContextFree(hrnBackupLocal.memContext);
hrnBackupLocal.scriptSize = 0;
}
backupProcess_SHIMMED(backupData, manifest, cipherPassBackup);
FUNCTION_HARNESS_RETURN_VOID();
}
/**********************************************************************************************************************************/
void

28
test/src/common/harnessBackup.h
View File

@ -10,6 +10,22 @@ Harness for Creating Test Backups
// start time. Backups added to the beginning of the test will need to subtract from the epoch.
#define BACKUP_EPOCH 1570000000
/***********************************************************************************************************************************
Structure for scripting backup storage changes
***********************************************************************************************************************************/
typedef enum
{
hrnBackupScriptOpUpdate = STRID5("update", 0xb4092150),
} HrnBackupScriptOp;
typedef struct HrnBackupScript
{
HrnBackupScriptOp op; // Operation to perform
const String *file; // File to operate on
const Buffer *content; // New content (valid for update op)
time_t time; // New modified time (valid for update op)
} HrnBackupScript;
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
@ -41,4 +57,16 @@ typedef struct HrnBackupPqScriptParam
void hrnBackupPqScript(unsigned int pgVersion, time_t backupTimeStart, HrnBackupPqScriptParam param);
// Generate storage scripts for modifying files during a backup. The modifications will happen after the manifest is built but
// before backupProcess() is called so they can be used to simulate changes made by PostgreSQL while the backup is in progress.
#define HRN_BACKUP_SCRIPT_SET(...) \
do \
{ \
const HrnBackupScript script[] = {__VA_ARGS__}; \
hrnBackupScriptSet(script, LENGTH_OF(script)); \
} \
while (0)
void hrnBackupScriptSet(const HrnBackupScript *script, unsigned int scriptSize);
#endif

12
test/src/module/command/backupTest.c
View File

@ -3960,7 +3960,13 @@ testRun(void)
HRN_STORAGE_PUT(storagePgWrite(), "grow-to-block-incr", file, .timeModified = backupTimeStart);
// File that gets truncated to zero during the backup
HRN_STORAGE_PUT(storagePgWrite(), "truncate-to-zero", BUFSTRDEF("DATA"), .timeModified = backupTimeStart);
// Run backup
HRN_BACKUP_SCRIPT_SET(
{.op = hrnBackupScriptOpUpdate, .file = storagePathP(storagePg(), STRDEF("truncate-to-zero")),
.time = backupTimeStart + 1});
hrnBackupPqScriptP(
PG_VERSION_11, backupTimeStart, .walCompressType = compressTypeGz, .walTotal = 2, .walSwitch = true,
.tablespace = true);
@ -3973,6 +3979,7 @@ testRun(void)
"P00 INFO: check archive for segment 0000000105DC213000000000\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-larger (1.4MB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (128KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/truncate-to-zero (bundle 1/0, 4B->0B, [PCT])\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16411, 8KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24603, 16.0KB, [PCT]) checksum [SHA1]\n"
@ -3983,7 +3990,7 @@ testRun(void)
"P00 DETAIL: wrote 'tablespace_map' file returned from backup stop function\n"
"P00 INFO: check archive for segment(s) 0000000105DC213000000000:0000000105DC213000000001\n"
"P00 INFO: new backup label = 20191103-165320F_20191106-002640D\n"
"P00 INFO: diff backup size = [SIZE], file total = 8");
"P00 INFO: diff backup size = [SIZE], file total = 9");
TEST_RESULT_STR_Z(
testBackupValidateP(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
@ -3992,6 +3999,7 @@ testRun(void)
"bundle/1/pg_data/block-incr-shrink {file, s=16383}\n"
"bundle/1/pg_data/global/pg_control {file, s=8192}\n"
"bundle/1/pg_data/grow-to-block-incr {file, m=1:{0,1,2}, s=16385}\n"
"bundle/1/pg_data/truncate-to-zero {file, s=0}\n"
"pg_data {path}\n"
"pg_data/backup_label {file, s=17}\n"
"pg_data/block-incr-grow.pgbi {file, m=0:{0},1:{0},0:{2},1:{1,2,3,4,5,6,7,8,9,10,11,12,13}, s=131072}\n"
@ -4018,6 +4026,7 @@ testRun(void)
"\"size\":16385,\"timestamp\":1573000000}\n"
"pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19"
",\"timestamp\":1573000002}\n"
"pg_data/truncate-to-zero={\"size\":0,\"szo\":4,\"timestamp\":1573000000}\n"
"\n"
"[target:path]\n"
"pg_data={}\n"
@ -4028,6 +4037,7 @@ testRun(void)
HRN_STORAGE_REMOVE(storagePgWrite(), "block-incr-larger");
HRN_STORAGE_REMOVE(storagePgWrite(), "block-incr-shrink");
HRN_STORAGE_REMOVE(storagePgWrite(), "grow-to-block-incr");
HRN_STORAGE_REMOVE(storagePgWrite(), "truncate-to-zero");
}
// It is better to put as few tests here as possible because cmp/enc makes tests more expensive (especially with valgrind)