db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-03 00:26:59 +02:00
Code Issues Releases Activity
Files
3df075bf4022e84f0238591b18bc5e866d6a2aa8
pgbackrest/test/src/module/command/backupCommonTest.c

259 lines
14 KiB
C
Raw Normal View History

Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
/***********************************************************************************************************************************
Test Common Functions and Definitions for Backup and Expire Commands
***********************************************************************************************************************************/
#include "common/harnessConfig.h"
Migrate page checksum filter to C. This filter exactly mimics the behavior of the Perl filter so is a drop-in replacement. The filter is not integrated yet since it requires the Perl-to-C storage layer interface coming in a future commit.
2019-06-17 07:52:03 -04:00
#include "common/io/bufferWrite.h"
#include "common/regExp.h"
#include "common/type/json.h"
#include "postgres/interface.h"
Simplify storage object names. Remove "File" and "Driver" from object names so they are shorter and easier to keep consistent. Also remove the "driver" directory so storage implementations are visible directly under "storage".
2019-05-03 15:46:15 -04:00
#include "storage/posix/storage.h"
Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
Migrate page checksum filter to C. This filter exactly mimics the behavior of the Perl filter so is a drop-in replacement. The filter is not integrated yet since it requires the Perl-to-C storage layer interface coming in a future commit.
2019-06-17 07:52:03 -04:00
/***********************************************************************************************************************************
Need these structures to mock up test data
***********************************************************************************************************************************/
typedef struct
{
uint32_t walid; // high bits
uint32_t xrecoff; // low bits
} PageWalRecPtr;
typedef struct PageHeaderData
{
// LSN is member of *any* block, not only page-organized ones
PageWalRecPtr pd_lsn; // Lsn for last change to this page
uint16_t pd_checksum; // checksum
uint16_t pd_flags; // flag bits, see below
uint16_t pd_lower; // offset to start of free space
uint16_t pd_upper; // offset to end of free space
} PageHeaderData;
Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("backupRegExp()"))
{
String *full = strNew("20181119-152138F");
String *incr = strNew("20181119-152138F_20181119-152152I");
String *diff = strNew("20181119-152138F_20181119-152152D");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(
backupRegExpP(0),
AssertError, "assertion 'param.full || param.differential || param.incremental' failed");
// -------------------------------------------------------------------------------------------------------------------------
String *filter = backupRegExpP(.full = true);
TEST_RESULT_STR(strPtr(filter), "^[0-9]{8}\\-[0-9]{6}F$", "full backup regex with anchors");
TEST_RESULT_BOOL(regExpMatchOne(filter, incr), false, " does not exactly match incr");
TEST_RESULT_BOOL(regExpMatchOne(filter, diff), false, " does not exactly match diff");
TEST_RESULT_BOOL(regExpMatchOne(filter, full), true, " exactly matches full");
// -------------------------------------------------------------------------------------------------------------------------
filter = backupRegExpP(.full = true, .incremental = true);
Add linefeeds for clarity.
2019-05-16 09:56:48 -04:00
Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
TEST_RESULT_STR(
strPtr(filter),
"^[0-9]{8}\\-[0-9]{6}F(\\_[0-9]{8}\\-[0-9]{6}I){0,1}$", "full and optional incr backup regex with anchors");
TEST_RESULT_BOOL(regExpMatchOne(filter, incr), true, " match incr");
TEST_RESULT_BOOL(regExpMatchOne(filter, diff), false, " does not match diff");
TEST_RESULT_BOOL(regExpMatchOne(filter, full), true, " match full");
TEST_RESULT_BOOL(
regExpMatchOne(
filter, strNew("12341234-123123F_12341234-123123IG")), false, " does not match with trailing character");
TEST_RESULT_BOOL(
regExpMatchOne(
filter, strNew("A12341234-123123F_12341234-123123I")), false, " does not match with leading character");
// -------------------------------------------------------------------------------------------------------------------------
filter = backupRegExpP(.full = true, .differential = true);
Add linefeeds for clarity.
2019-05-16 09:56:48 -04:00
Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
TEST_RESULT_STR(
strPtr(filter),
"^[0-9]{8}\\-[0-9]{6}F(\\_[0-9]{8}\\-[0-9]{6}D){0,1}$", "full and optional diff backup regex with anchors");
TEST_RESULT_BOOL(regExpMatchOne(filter, incr), false, " does not match incr");
TEST_RESULT_BOOL(regExpMatchOne(filter, diff), true, " match diff");
TEST_RESULT_BOOL(regExpMatchOne(filter, full), true, " match full");
// -------------------------------------------------------------------------------------------------------------------------
filter = backupRegExpP(.full = true, .incremental = true, .differential = true);
Add linefeeds for clarity.
2019-05-16 09:56:48 -04:00
Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
TEST_RESULT_STR(
strPtr(filter),
"^[0-9]{8}\\-[0-9]{6}F(\\_[0-9]{8}\\-[0-9]{6}(D|I)){0,1}$", "full, optional diff and incr backup regex with anchors");
TEST_RESULT_BOOL(regExpMatchOne(filter, incr), true, " match incr");
TEST_RESULT_BOOL(regExpMatchOne(filter, diff), true, " match diff");
TEST_RESULT_BOOL(regExpMatchOne(filter, full), true, " match full");
// -------------------------------------------------------------------------------------------------------------------------
filter = backupRegExpP(.incremental = true, .differential = true);
Add linefeeds for clarity.
2019-05-16 09:56:48 -04:00
Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
TEST_RESULT_STR(
strPtr(filter),
"^[0-9]{8}\\-[0-9]{6}F\\_[0-9]{8}\\-[0-9]{6}(D|I)$", "diff and incr backup regex with anchors");
TEST_RESULT_BOOL(regExpMatchOne(filter, incr), true, " match incr");
TEST_RESULT_BOOL(regExpMatchOne(filter, diff), true, " match diff");
TEST_RESULT_BOOL(regExpMatchOne(filter, full), false, " does not match full");
TEST_RESULT_BOOL(
regExpMatchOne(
filter, strNew("12341234-123123F_12341234-123123DG")), false, " does not match with trailing character");
TEST_RESULT_BOOL(
regExpMatchOne(
filter, strNew("A12341234-123123F_12341234-123123I")), false, " does not match with leading character");
// -------------------------------------------------------------------------------------------------------------------------
filter = backupRegExpP(.incremental = true);
Add linefeeds for clarity.
2019-05-16 09:56:48 -04:00
Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
TEST_RESULT_STR(
strPtr(filter),
"^[0-9]{8}\\-[0-9]{6}F\\_[0-9]{8}\\-[0-9]{6}I$", "incr backup regex with anchors");
TEST_RESULT_BOOL(regExpMatchOne(filter, incr), true, " match incr");
TEST_RESULT_BOOL(regExpMatchOne(filter, diff), false, " does not match diff");
TEST_RESULT_BOOL(regExpMatchOne(filter, full), false, " does not match full");
// -------------------------------------------------------------------------------------------------------------------------
filter = backupRegExpP(.differential = true);
Add linefeeds for clarity.
2019-05-16 09:56:48 -04:00
Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
TEST_RESULT_STR(
strPtr(filter),
"^[0-9]{8}\\-[0-9]{6}F\\_[0-9]{8}\\-[0-9]{6}D$", "diff backup regex with anchors");
TEST_RESULT_BOOL(regExpMatchOne(filter, incr), false, " does not match incr");
TEST_RESULT_BOOL(regExpMatchOne(filter, diff), true, " match diff");
TEST_RESULT_BOOL(regExpMatchOne(filter, full), false, " does not match full");
}
Migrate page checksum filter to C. This filter exactly mimics the behavior of the Perl filter so is a drop-in replacement. The filter is not integrated yet since it requires the Perl-to-C storage layer interface coming in a future commit.
2019-06-17 07:52:03 -04:00
// *****************************************************************************************************************************
if (testBegin("PageChecksum"))
{
TEST_RESULT_UINT(PG_SEGMENT_PAGE_DEFAULT, 131072, "check pages per segment");
// Test pages with all zeros (these are considered valid)
// -------------------------------------------------------------------------------------------------------------------------
Buffer *buffer = bufNew(PG_PAGE_SIZE_DEFAULT * 3);
Buffer *bufferOut = bufNew(0);
bufUsedSet(buffer, bufSize(buffer));
memset(bufPtr(buffer), 0, bufSize(buffer));
Allow protocol compression when read/writing remote files. If the file is compressible (i.e. not encrypted or already compressed) it can be marked as such in storageNewRead()/storageNewWrite(). If the file is being read from/written to a remote it will be compressed in transit using gzip. Simplify filter group handling by having the IoRead/IoWrite objects create the filter group automatically. This removes the need for a lot of NULL checking and has a negligible effect on performance since a filter group needs to be created eventually unless the source file is missing. Allow filters to be created using a VariantList so filter parameters can be passed to the remote.
2019-06-24 10:20:47 -04:00
IoWrite *write = ioBufferWriteNew(bufferOut);
ioFilterGroupAdd(ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0));
Migrate page checksum filter to C. This filter exactly mimics the behavior of the Perl filter so is a drop-in replacement. The filter is not integrated yet since it requires the Perl-to-C storage layer interface coming in a future commit.
2019-06-17 07:52:03 -04:00
ioWriteOpen(write);
ioWrite(write, buffer);
ioWriteClose(write);
TEST_RESULT_STR(
strPtr(jsonFromVar(ioFilterGroupResult(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE_STR), 0)),
"{\"align\":true,\"valid\":true}", "all zero pages");
// Single checksum error
// -------------------------------------------------------------------------------------------------------------------------
buffer = bufNew(PG_PAGE_SIZE_DEFAULT * 1);
bufUsedSet(buffer, bufSize(buffer));
memset(bufPtr(buffer), 0, bufSize(buffer));
// Page 0 has bogus checksum
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)))->pd_upper = 0x01;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)))->pd_lsn.walid = 0xF0F0F0F0;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)))->pd_lsn.xrecoff = 0xF0F0F0F0;
Allow protocol compression when read/writing remote files. If the file is compressible (i.e. not encrypted or already compressed) it can be marked as such in storageNewRead()/storageNewWrite(). If the file is being read from/written to a remote it will be compressed in transit using gzip. Simplify filter group handling by having the IoRead/IoWrite objects create the filter group automatically. This removes the need for a lot of NULL checking and has a negligible effect on performance since a filter group needs to be created eventually unless the source file is missing. Allow filters to be created using a VariantList so filter parameters can be passed to the remote.
2019-06-24 10:20:47 -04:00
write = ioBufferWriteNew(bufferOut);
Allow multiple filters to be pushed to the remote and return results. Previously only a single filter could be pushed to the remote since order was not being maintained. Now the filters are strictly ordered. Results are returned from the remote and set in the local IoFilterGroup so they can be retrieved. Expand remote filter support to include all filters.
2019-07-15 16:49:46 -04:00
Allow protocol compression when read/writing remote files. If the file is compressible (i.e. not encrypted or already compressed) it can be marked as such in storageNewRead()/storageNewWrite(). If the file is being read from/written to a remote it will be compressed in transit using gzip. Simplify filter group handling by having the IoRead/IoWrite objects create the filter group automatically. This removes the need for a lot of NULL checking and has a negligible effect on performance since a filter group needs to be created eventually unless the source file is missing. Allow filters to be created using a VariantList so filter parameters can be passed to the remote.
2019-06-24 10:20:47 -04:00
ioFilterGroupAdd(
Allow multiple filters to be pushed to the remote and return results. Previously only a single filter could be pushed to the remote since order was not being maintained. Now the filters are strictly ordered. Results are returned from the remote and set in the local IoFilterGroup so they can be retrieved. Expand remote filter support to include all filters.
2019-07-15 16:49:46 -04:00
ioWriteFilterGroup(write),
pageChecksumNewVar(
varVarLst(
jsonToVar(
strNewFmt("[0,%u,%u,%" PRIu64 "]", PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0xFACEFACE00000000)))));
Migrate page checksum filter to C. This filter exactly mimics the behavior of the Perl filter so is a drop-in replacement. The filter is not integrated yet since it requires the Perl-to-C storage layer interface coming in a future commit.
2019-06-17 07:52:03 -04:00
ioWriteOpen(write);
ioWrite(write, buffer);
ioWriteClose(write);
TEST_RESULT_STR(
strPtr(jsonFromVar(ioFilterGroupResult(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE_STR), 0)),
"{\"align\":true,\"error\":[0],\"valid\":false}", "single checksum error");
// Various checksum errors some of which will be skipped because of the LSN
// -------------------------------------------------------------------------------------------------------------------------
buffer = bufNew(PG_PAGE_SIZE_DEFAULT * 8 - (PG_PAGE_SIZE_DEFAULT - 512));
bufUsedSet(buffer, bufSize(buffer));
memset(bufPtr(buffer), 0, bufSize(buffer));
// Page 0 has bogus checksum
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)))->pd_upper = 0x01;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)))->pd_lsn.walid = 0xF0F0F0F0;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)))->pd_lsn.xrecoff = 0xF0F0F0F0;
// Page 1 has bogus checksum but lsn above the limit
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x01)))->pd_upper = 0x01;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x01)))->pd_lsn.walid = 0xFACEFACE;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x01)))->pd_lsn.xrecoff = 0x00000000;
// Page 2 has bogus checksum
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x02)))->pd_upper = 0x01;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x02)))->pd_lsn.xrecoff = 0x2;
// Page 3 has bogus checksum
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x03)))->pd_upper = 0x01;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x03)))->pd_lsn.xrecoff = 0x3;
// Page 4 has bogus checksum
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x04)))->pd_upper = 0x01;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x04)))->pd_lsn.xrecoff = 0x4;
// Page 6 has bogus checksum
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x06)))->pd_upper = 0x01;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x06)))->pd_lsn.xrecoff = 0x6;
// Page 7 has bogus checksum (and is misaligned but large enough to test)
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x07)))->pd_upper = 0x01;
((PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x07)))->pd_lsn.xrecoff = 0x7;
Allow protocol compression when read/writing remote files. If the file is compressible (i.e. not encrypted or already compressed) it can be marked as such in storageNewRead()/storageNewWrite(). If the file is being read from/written to a remote it will be compressed in transit using gzip. Simplify filter group handling by having the IoRead/IoWrite objects create the filter group automatically. This removes the need for a lot of NULL checking and has a negligible effect on performance since a filter group needs to be created eventually unless the source file is missing. Allow filters to be created using a VariantList so filter parameters can be passed to the remote.
2019-06-24 10:20:47 -04:00
write = ioBufferWriteNew(bufferOut);
ioFilterGroupAdd(
ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0xFACEFACE00000000));
Migrate page checksum filter to C. This filter exactly mimics the behavior of the Perl filter so is a drop-in replacement. The filter is not integrated yet since it requires the Perl-to-C storage layer interface coming in a future commit.
2019-06-17 07:52:03 -04:00
ioWriteOpen(write);
ioWrite(write, buffer);
ioWriteClose(write);
TEST_RESULT_STR(
strPtr(jsonFromVar(ioFilterGroupResult(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE_STR), 0)),
"{\"align\":false,\"error\":[0,[2,4],[6,7]],\"valid\":false}", "various checksum errors");
// Impossibly misaligned page
// -------------------------------------------------------------------------------------------------------------------------
buffer = bufNew(256);
bufUsedSet(buffer, bufSize(buffer));
memset(bufPtr(buffer), 0, bufSize(buffer));
Allow protocol compression when read/writing remote files. If the file is compressible (i.e. not encrypted or already compressed) it can be marked as such in storageNewRead()/storageNewWrite(). If the file is being read from/written to a remote it will be compressed in transit using gzip. Simplify filter group handling by having the IoRead/IoWrite objects create the filter group automatically. This removes the need for a lot of NULL checking and has a negligible effect on performance since a filter group needs to be created eventually unless the source file is missing. Allow filters to be created using a VariantList so filter parameters can be passed to the remote.
2019-06-24 10:20:47 -04:00
write = ioBufferWriteNew(bufferOut);
ioFilterGroupAdd(
ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0xFACEFACE00000000));
Migrate page checksum filter to C. This filter exactly mimics the behavior of the Perl filter so is a drop-in replacement. The filter is not integrated yet since it requires the Perl-to-C storage layer interface coming in a future commit.
2019-06-17 07:52:03 -04:00
ioWriteOpen(write);
ioWrite(write, buffer);
ioWriteClose(write);
TEST_RESULT_STR(
strPtr(jsonFromVar(ioFilterGroupResult(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE_STR), 0)),
"{\"align\":false,\"valid\":false}", "misalignment");
// Two misaligned buffers in a row
// -------------------------------------------------------------------------------------------------------------------------
buffer = bufNew(513);
bufUsedSet(buffer, bufSize(buffer));
memset(bufPtr(buffer), 0, bufSize(buffer));
Allow protocol compression when read/writing remote files. If the file is compressible (i.e. not encrypted or already compressed) it can be marked as such in storageNewRead()/storageNewWrite(). If the file is being read from/written to a remote it will be compressed in transit using gzip. Simplify filter group handling by having the IoRead/IoWrite objects create the filter group automatically. This removes the need for a lot of NULL checking and has a negligible effect on performance since a filter group needs to be created eventually unless the source file is missing. Allow filters to be created using a VariantList so filter parameters can be passed to the remote.
2019-06-24 10:20:47 -04:00
write = ioBufferWriteNew(bufferOut);
ioFilterGroupAdd(
ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0xFACEFACE00000000));
Migrate page checksum filter to C. This filter exactly mimics the behavior of the Perl filter so is a drop-in replacement. The filter is not integrated yet since it requires the Perl-to-C storage layer interface coming in a future commit.
2019-06-17 07:52:03 -04:00
ioWriteOpen(write);
ioWrite(write, buffer);
TEST_ERROR(ioWrite(write, buffer), AssertError, "should not be possible to see two misaligned pages in a row");
}
Migrate backupRegExp() to C. Removed the "anchor" parameter because it was never used in any calls in the Perl code so it was just a dead parameter that always defaulted to true. Contributed by Cynthia Shang.
2019-04-15 08:29:25 -04:00
FUNCTION_HARNESS_RESULT_VOID();
}
Reference in New Issue Copy Permalink