db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-17 01:12:23 +02:00
Code Issues Releases Activity

Use PG_PAGE_SIZE_DEFAULT constant instead of pageSize variable.

Browse Source 
Page size is passed around a lot but in fact it can only have one value, PG_PAGE_SIZE_DEFAULT, which is checked when pg_control is loaded. There may be an argument for supporting multiple page sizes in the future but for now just use the constant to simplify the code.

There is also a significant performance benefit.  Because pageSize was being used in pageChecksumBlock() the main loop was neither unrolled nor vectorized (-funroll-loops -ftree-vectorize) as it is now with a constant loop boundary.
This commit is contained in:
David Steele
2020-03-05 09:14:27 -05:00
parent 91f321fb86
commit 4ab8943ca8
13 changed files with 48 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
libc/xs/postgres/pageChecksum.xs
View File

@ -5,16 +5,15 @@
MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC MODULE = pgBackRest::LibC PACKAGE = pgBackRest::LibC
U16 U16
pgPageChecksum(page, blockNo, pageSize) pgPageChecksum(page, blockNo)
const char *page const char *page
U32 blockNo U32 blockNo
U32 pageSize
CODE: CODE:
RETVAL = 0; RETVAL = 0;
ERROR_XS_BEGIN() ERROR_XS_BEGIN()
{ {
RETVAL = pgPageChecksum((const unsigned char *)page, blockNo, pageSize); RETVAL = pgPageChecksum((const unsigned char *)page, blockNo);
} }
ERROR_XS_END(); ERROR_XS_END();
OUTPUT: OUTPUT:

2
src/Makefile.in
View File

@ -242,7 +242,7 @@ command/backup/common.o: command/backup/common.c build.auto.h command/backup/com
command/backup/file.o: command/backup/file.c build.auto.h command/backup/file.h command/backup/pageChecksum.h common/assert.h common/compress/gz/common.h common/compress/gz/compress.h common/compress/gz/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h command/backup/file.o: command/backup/file.c build.auto.h command/backup/file.h command/backup/pageChecksum.h common/assert.h common/compress/gz/common.h common/compress/gz/compress.h common/compress/gz/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/io.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/file.c -o command/backup/file.o $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/file.c -o command/backup/file.o
command/backup/pageChecksum.o: command/backup/pageChecksum.c build.auto.h command/backup/pageChecksum.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h postgres/pageChecksum.h command/backup/pageChecksum.o: command/backup/pageChecksum.c build.auto.h command/backup/pageChecksum.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h postgres/interface.h postgres/pageChecksum.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/pageChecksum.c -o command/backup/pageChecksum.o $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/pageChecksum.c -o command/backup/pageChecksum.o
command/backup/protocol.o: command/backup/protocol.c build.auto.h command/backup/file.h command/backup/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h command/backup/protocol.o: command/backup/protocol.c build.auto.h command/backup/file.h command/backup/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h

9
src/command/backup/backup.c
View File

@ -193,7 +193,6 @@ typedef struct BackupData
const String *hostStandby; // Host name of the standby const String *hostStandby; // Host name of the standby
unsigned int version; // PostgreSQL version unsigned int version; // PostgreSQL version
unsigned int pageSize; // PostgreSQL page size
unsigned int walSegmentSize; // PostgreSQL wal segment size unsigned int walSegmentSize; // PostgreSQL wal segment size
} BackupData; } BackupData;
@ -258,7 +257,6 @@ backupInit(const InfoBackup *infoBackup)
PgControl pgControl = pgControlFromFile(result->storagePrimary); PgControl pgControl = pgControlFromFile(result->storagePrimary);
result->version = pgControl.version; result->version = pgControl.version;
result->pageSize = pgControl.pageSize;
result->walSegmentSize = pgControl.walSegmentSize; result->walSegmentSize = pgControl.walSegmentSize;
// Validate pg_control info against the stanza // Validate pg_control info against the stanza
@ -1040,7 +1038,7 @@ Log the results of a job and throw errors
static uint64_t static uint64_t
backupJobResult( backupJobResult(
Manifest *manifest, const String *host, const String *const fileName, StringList *fileRemove, ProtocolParallelJob *const job, Manifest *manifest, const String *host, const String *const fileName, StringList *fileRemove, ProtocolParallelJob *const job,
const uint64_t sizeTotal, uint64_t sizeCopied, unsigned int pageSize) const uint64_t sizeTotal, uint64_t sizeCopied)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(MANIFEST, manifest); FUNCTION_LOG_PARAM(MANIFEST, manifest);
@ -1050,7 +1048,6 @@ backupJobResult(
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, job); FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, job);
FUNCTION_LOG_PARAM(UINT64, sizeTotal); FUNCTION_LOG_PARAM(UINT64, sizeTotal);
FUNCTION_LOG_PARAM(UINT64, sizeCopied); FUNCTION_LOG_PARAM(UINT64, sizeCopied);
FUNCTION_LOG_PARAM(UINT, pageSize);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(manifest != NULL); ASSERT(manifest != NULL);
@ -1144,7 +1141,7 @@ backupJobResult(
// ??? Update formatting after migration // ??? Update formatting after migration
LOG_WARN_FMT( LOG_WARN_FMT(
"page misalignment in file %s: file size %" PRIu64 " is not divisible by page size %u", "page misalignment in file %s: file size %" PRIu64 " is not divisible by page size %u",
strPtr(fileLog), copySize, pageSize); strPtr(fileLog), copySize, PG_PAGE_SIZE_DEFAULT);
} }
else else
{ {
@ -1617,7 +1614,7 @@ backupProcess(BackupData *backupData, Manifest *manifest, const String *lsnStart
storagePathP( storagePathP(
protocolParallelJobProcessId(job) > 1 ? storagePgId(pgId) : backupData->storagePrimary, protocolParallelJobProcessId(job) > 1 ? storagePgId(pgId) : backupData->storagePrimary,
manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job)))->name)), manifestPathPg(manifestFileFind(manifest, varStr(protocolParallelJobKey(job)))->name)),
fileRemove, job, sizeTotal, sizeCopied, backupData->pageSize); fileRemove, job, sizeTotal, sizeCopied);
} }
// A keep-alive is required here for the remote holding open the backup connection // A keep-alive is required here for the remote holding open the backup connection

2
src/command/backup/file.c
View File

@ -203,7 +203,7 @@ backupFile(
{ {
ioFilterGroupAdd( ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)), pageChecksumNew(segmentNumber(pgFile), PG_SEGMENT_PAGE_DEFAULT, ioReadFilterGroup(storageReadIo(read)), pageChecksumNew(segmentNumber(pgFile), PG_SEGMENT_PAGE_DEFAULT,
PG_PAGE_SIZE_DEFAULT, pgFileChecksumPageLsnLimit)); pgFileChecksumPageLsnLimit));
} }
// Add compression // Add compression

18
src/command/backup/pageChecksum.c
View File

@ -10,6 +10,7 @@ Page Checksum Filter
#include "common/memContext.h" #include "common/memContext.h"
#include "common/object.h" #include "common/object.h"
#include "postgres/pageChecksum.h" #include "postgres/pageChecksum.h"
#include "postgres/interface.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Filter type constant Filter type constant
@ -24,7 +25,6 @@ typedef struct PageChecksum
MemContext *memContext; // Mem context of filter MemContext *memContext; // Mem context of filter
unsigned int pageNoOffset; // Page number offset for subsequent segments unsigned int pageNoOffset; // Page number offset for subsequent segments
size_t pageSize; // Page size
uint64_t lsnLimit; // Lower limit of pages that could be torn uint64_t lsnLimit; // Lower limit of pages that could be torn
bool valid; // Is the relation structure valid? bool valid; // Is the relation structure valid?
@ -66,10 +66,10 @@ pageChecksumProcess(THIS_VOID, const Buffer *input)
ASSERT(input != NULL); ASSERT(input != NULL);
// Calculate total pages in the buffer // Calculate total pages in the buffer
unsigned int pageTotal = (unsigned int)(bufUsed(input) / this->pageSize); unsigned int pageTotal = (unsigned int)(bufUsed(input) / PG_PAGE_SIZE_DEFAULT);
// If there is a partial page make sure there is enough of it to validate the checksum // If there is a partial page make sure there is enough of it to validate the checksum
unsigned int pageRemainder = (unsigned int)(bufUsed(input) % this->pageSize); unsigned int pageRemainder = (unsigned int)(bufUsed(input) % PG_PAGE_SIZE_DEFAULT);
if (pageRemainder != 0) if (pageRemainder != 0)
{ {
@ -95,9 +95,9 @@ pageChecksumProcess(THIS_VOID, const Buffer *input)
{ {
for (unsigned int pageIdx = 0; pageIdx < pageTotal; pageIdx++) for (unsigned int pageIdx = 0; pageIdx < pageTotal; pageIdx++)
{ {
unsigned char *pagePtr = bufPtr(input) + (pageIdx * this->pageSize); unsigned char *pagePtr = bufPtr(input) + (pageIdx * PG_PAGE_SIZE_DEFAULT);
unsigned int pageNo = this->pageNoOffset + pageIdx; unsigned int pageNo = this->pageNoOffset + pageIdx;
size_t pageSize = this->align || pageIdx < pageTotal - 1 ? this->pageSize : pageRemainder; size_t pageSize = this->align || pageIdx < pageTotal - 1 ? PG_PAGE_SIZE_DEFAULT : pageRemainder;
if (!pgPageChecksumTest( if (!pgPageChecksumTest(
pagePtr, pageNo, (unsigned int)pageSize, (unsigned int)(this->lsnLimit >> 32), pagePtr, pageNo, (unsigned int)pageSize, (unsigned int)(this->lsnLimit >> 32),
@ -206,12 +206,11 @@ pageChecksumResult(THIS_VOID)
New object New object
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
IoFilter * IoFilter *
pageChecksumNew(unsigned int segmentNo, unsigned int segmentPageTotal, size_t pageSize, uint64_t lsnLimit) pageChecksumNew(unsigned int segmentNo, unsigned int segmentPageTotal, uint64_t lsnLimit)
{ {
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(UINT, segmentNo); FUNCTION_LOG_PARAM(UINT, segmentNo);
FUNCTION_LOG_PARAM(UINT, segmentPageTotal); FUNCTION_LOG_PARAM(UINT, segmentPageTotal);
FUNCTION_LOG_PARAM(SIZE, pageSize);
FUNCTION_LOG_PARAM(UINT64, lsnLimit); FUNCTION_LOG_PARAM(UINT64, lsnLimit);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
@ -225,7 +224,6 @@ pageChecksumNew(unsigned int segmentNo, unsigned int segmentPageTotal, size_t pa
{ {
.memContext = memContextCurrent(), .memContext = memContextCurrent(),
.pageNoOffset = segmentNo * segmentPageTotal, .pageNoOffset = segmentNo * segmentPageTotal,
.pageSize = pageSize,
.lsnLimit = lsnLimit, .lsnLimit = lsnLimit,
.valid = true, .valid = true,
.align = true, .align = true,
@ -235,7 +233,6 @@ pageChecksumNew(unsigned int segmentNo, unsigned int segmentPageTotal, size_t pa
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewUInt(segmentNo)); varLstAdd(paramList, varNewUInt(segmentNo));
varLstAdd(paramList, varNewUInt(segmentPageTotal)); varLstAdd(paramList, varNewUInt(segmentPageTotal));
varLstAdd(paramList, varNewUInt64(pageSize));
varLstAdd(paramList, varNewUInt64(lsnLimit)); varLstAdd(paramList, varNewUInt64(lsnLimit));
this = ioFilterNewP( this = ioFilterNewP(
@ -250,6 +247,5 @@ IoFilter *
pageChecksumNewVar(const VariantList *paramList) pageChecksumNewVar(const VariantList *paramList)
{ {
return pageChecksumNew( return pageChecksumNew(
varUIntForce(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)), varUIntForce(varLstGet(paramList, 2)), varUIntForce(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)));
varUInt64(varLstGet(paramList, 3)));
} }

2
src/command/backup/pageChecksum.h
View File

@ -17,7 +17,7 @@ Filter type constant
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Constructor Constructor
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
IoFilter *pageChecksumNew(unsigned int segmentNo, unsigned int segmentPageTotal, size_t pageSize, uint64_t lsnLimit); IoFilter *pageChecksumNew(unsigned int segmentNo, unsigned int segmentPageTotal, uint64_t lsnLimit);
IoFilter *pageChecksumNewVar(const VariantList *paramList); IoFilter *pageChecksumNewVar(const VariantList *paramList);
#endif #endif

12
src/postgres/pageChecksum.c
View File

@ -101,11 +101,10 @@ do { \
} while (0) } while (0)
static uint32_t static uint32_t
pageChecksumBlock(const unsigned char *page, unsigned int pageSize) pageChecksumBlock(const unsigned char *page)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(UCHARDATA, page); FUNCTION_TEST_PARAM_P(UCHARDATA, page);
FUNCTION_TEST_PARAM(UINT, pageSize);
FUNCTION_TEST_END(); FUNCTION_TEST_END();
ASSERT(page != NULL); ASSERT(page != NULL);
@ -119,7 +118,7 @@ pageChecksumBlock(const unsigned char *page, unsigned int pageSize)
memcpy(sums, checksumBaseOffsets, sizeof(checksumBaseOffsets)); memcpy(sums, checksumBaseOffsets, sizeof(checksumBaseOffsets));
/* main checksum calculation */ /* main checksum calculation */
for (i = 0; i < pageSize / sizeof(uint32_t) / N_SUMS; i++) for (i = 0; i < PG_PAGE_SIZE_DEFAULT / sizeof(uint32_t) / N_SUMS; i++)
for (j = 0; j < N_SUMS; j++) for (j = 0; j < N_SUMS; j++)
CHECKSUM_COMP(sums[j], dataArray[i][j]); CHECKSUM_COMP(sums[j], dataArray[i][j]);
@ -142,12 +141,11 @@ The checksum includes the block number (to detect the case where a page is someh
(excluding the checksum itself), and the page data. (excluding the checksum itself), and the page data.
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
uint16_t uint16_t
pgPageChecksum(const unsigned char *page, unsigned int blockNo, unsigned int pageSize) pgPageChecksum(const unsigned char *page, unsigned int blockNo)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(UCHARDATA, page); FUNCTION_TEST_PARAM_P(UCHARDATA, page);
FUNCTION_TEST_PARAM(UINT, blockNo); FUNCTION_TEST_PARAM(UINT, blockNo);
FUNCTION_TEST_PARAM(UINT, pageSize);
FUNCTION_TEST_END(); FUNCTION_TEST_END();
ASSERT(page != NULL); ASSERT(page != NULL);
@ -158,7 +156,7 @@ pgPageChecksum(const unsigned char *page, unsigned int blockNo, unsigned int pag
uint16_t originalChecksum = pageHeader->pd_checksum; uint16_t originalChecksum = pageHeader->pd_checksum;
pageHeader->pd_checksum = 0; pageHeader->pd_checksum = 0;
uint32_t checksum = pageChecksumBlock(page, pageSize); uint32_t checksum = pageChecksumBlock(page);
pageHeader->pd_checksum = originalChecksum; pageHeader->pd_checksum = originalChecksum;
// Mix in the block number to detect transposed pages // Mix in the block number to detect transposed pages
@ -204,5 +202,5 @@ pgPageChecksumTest(
// LSN is after the backup started so checksum is not tested because pages may be torn // LSN is after the backup started so checksum is not tested because pages may be torn
(((PageHeader)page)->pd_lsn.xlogid >= ignoreWalId && ((PageHeader)page)->pd_lsn.xrecoff >= ignoreWalOffset) || (((PageHeader)page)->pd_lsn.xlogid >= ignoreWalId && ((PageHeader)page)->pd_lsn.xrecoff >= ignoreWalOffset) ||
// Checksum is valid if a full page // Checksum is valid if a full page
(pageSize == PG_PAGE_SIZE_DEFAULT && ((PageHeader)page)->pd_checksum == pgPageChecksum(page, blockNo, pageSize))); (pageSize == PG_PAGE_SIZE_DEFAULT && ((PageHeader)page)->pd_checksum == pgPageChecksum(page, blockNo)));
} }

2
src/postgres/pageChecksum.h
View File

@ -9,7 +9,7 @@ Checksum Implementation for Data Pages
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
uint16_t pgPageChecksum(const unsigned char *page, unsigned int blockNo, unsigned int pageSize); uint16_t pgPageChecksum(const unsigned char *page, unsigned int blockNo);
uint64_t pgPageLsn(const unsigned char *page); uint64_t pgPageLsn(const unsigned char *page);
bool pgPageChecksumTest( bool pgPageChecksumTest(
const unsigned char *page, unsigned int blockNo, unsigned int pageSize, uint32_t ignoreWalId, uint32_t ignoreWalOffset); const unsigned char *page, unsigned int blockNo, unsigned int pageSize, uint32_t ignoreWalId, uint32_t ignoreWalOffset);

2
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
View File

@ -49,7 +49,7 @@ sub pageBuild
my $iWalOffset = shift; my $iWalOffset = shift;
my $tPage = defined($iWalId) ? pack('I', $iWalId) . pack('I', $iWalOffset) . substr($tPageSource, 8) : $tPageSource; my $tPage = defined($iWalId) ? pack('I', $iWalId) . pack('I', $iWalOffset) . substr($tPageSource, 8) : $tPageSource;
my $iChecksum = pgPageChecksum($tPage, $iBlockNo, length($tPage)); my $iChecksum = pgPageChecksum($tPage, $iBlockNo);
return substr($tPage, 0, 8) . pack('S', $iChecksum) . substr($tPage, 10); return substr($tPage, 0, 8) . pack('S', $iChecksum) . substr($tPage, 10);
} }

16
test/src/module/command/backupCommonTest.c
View File

@ -113,7 +113,7 @@ testRun(void)
*(PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x02)) = (PageHeaderData){.pd_upper = 0}; *(PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x02)) = (PageHeaderData){.pd_upper = 0};
IoWrite *write = ioBufferWriteNew(bufferOut); IoWrite *write = ioBufferWriteNew(bufferOut);
ioFilterGroupAdd(ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0)); ioFilterGroupAdd(ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, 0));
ioWriteOpen(write); ioWriteOpen(write);
ioWrite(write, buffer); ioWrite(write, buffer);
ioWriteClose(write); ioWriteClose(write);
@ -143,10 +143,7 @@ testRun(void)
ioFilterGroupAdd( ioFilterGroupAdd(
ioWriteFilterGroup(write), ioWriteFilterGroup(write),
pageChecksumNewVar( pageChecksumNewVar(varVarLst(jsonToVar(strNewFmt("[0,%u,%" PRIu64 "]", PG_SEGMENT_PAGE_DEFAULT, 0xFACEFACE00000000)))));
varVarLst(
jsonToVar(
strNewFmt("[0,%u,%u,%" PRIu64 "]", PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0xFACEFACE00000000)))));
ioWriteOpen(write); ioWriteOpen(write);
ioWrite(write, buffer); ioWrite(write, buffer);
ioWriteClose(write); ioWriteClose(write);
@ -237,8 +234,7 @@ testRun(void)
}; };
write = ioBufferWriteNew(bufferOut); write = ioBufferWriteNew(bufferOut);
ioFilterGroupAdd( ioFilterGroupAdd(ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, 0xFACEFACE00000000));
ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0xFACEFACE00000000));
ioWriteOpen(write); ioWriteOpen(write);
ioWrite(write, buffer); ioWrite(write, buffer);
ioWriteClose(write); ioWriteClose(write);
@ -256,8 +252,7 @@ testRun(void)
*(PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)) = (PageHeaderData){.pd_upper = 0}; *(PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)) = (PageHeaderData){.pd_upper = 0};
write = ioBufferWriteNew(bufferOut); write = ioBufferWriteNew(bufferOut);
ioFilterGroupAdd( ioFilterGroupAdd(ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, 0xFACEFACE00000000));
ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0xFACEFACE00000000));
ioWriteOpen(write); ioWriteOpen(write);
ioWrite(write, buffer); ioWrite(write, buffer);
ioWriteClose(write); ioWriteClose(write);
@ -275,8 +270,7 @@ testRun(void)
*(PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)) = (PageHeaderData){.pd_upper = 0}; *(PageHeaderData *)(bufPtr(buffer) + (PG_PAGE_SIZE_DEFAULT * 0x00)) = (PageHeaderData){.pd_upper = 0};
write = ioBufferWriteNew(bufferOut); write = ioBufferWriteNew(bufferOut);
ioFilterGroupAdd( ioFilterGroupAdd(ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, 0xFACEFACE00000000));
ioWriteFilterGroup(write), pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0xFACEFACE00000000));
ioWriteOpen(write); ioWriteOpen(write);
ioWrite(write, buffer); ioWrite(write, buffer);
TEST_ERROR(ioWrite(write, buffer), AssertError, "should not be possible to see two misaligned pages in a row"); TEST_ERROR(ioWrite(write, buffer), AssertError, "should not be possible to see two misaligned pages in a row");

4
test/src/module/command/backupTest.c
View File

@ -1284,7 +1284,7 @@ testRun(void)
ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("key"), protocolCommandNew(STRDEF("command"))); ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("key"), protocolCommandNew(STRDEF("command")));
protocolParallelJobErrorSet(job, errorTypeCode(&AssertError), STRDEF("error message")); protocolParallelJobErrorSet(job, errorTypeCode(&AssertError), STRDEF("error message"));
TEST_ERROR(backupJobResult((Manifest *)1, NULL, STRDEF("log"), strLstNew(), job, 0, 0, 0), AssertError, "error message"); TEST_ERROR(backupJobResult((Manifest *)1, NULL, STRDEF("log"), strLstNew(), job, 0, 0), AssertError, "error message");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("report host/100% progress on noop result"); TEST_TITLE("report host/100% progress on noop result");
@ -1306,7 +1306,7 @@ testRun(void)
manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF("pg_data/test")}); manifestFileAdd(manifest, &(ManifestFile){.name = STRDEF("pg_data/test")});
TEST_RESULT_UINT( TEST_RESULT_UINT(
backupJobResult(manifest, STRDEF("host"), STRDEF("log-test"), strLstNew(), job, 0, 0, 0), 0, "log noop result"); backupJobResult(manifest, STRDEF("host"), STRDEF("log-test"), strLstNew(), job, 0, 0), 0, "log noop result");
TEST_RESULT_LOG("P00 DETAIL: match file from prior backup host:log-test (0B, 100%)"); TEST_RESULT_LOG("P00 DETAIL: match file from prior backup host:log-test (0B, 100%)");
} }

37
test/src/module/postgres/pageChecksumTest.c
View File

@ -5,7 +5,6 @@ Test Page Checksums
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Page data for testing -- use 8192 for page size since this is the most common value Page data for testing -- use 8192 for page size since this is the most common value
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
#define TEST_PAGE_SIZE 8192
#define TEST_PAGE_TOTAL 16 #define TEST_PAGE_TOTAL 16
// GCC doesn't like this elements of this array being used as both char * and struct * so wrap it in a function to disable the // GCC doesn't like this elements of this array being used as both char * and struct * so wrap it in a function to disable the
@ -13,7 +12,7 @@ Page data for testing -- use 8192 for page size since this is the most common va
unsigned char * unsigned char *
testPage(unsigned int pageIdx) testPage(unsigned int pageIdx)
{ {
static unsigned char testPageBuffer[TEST_PAGE_TOTAL][TEST_PAGE_SIZE]; static unsigned char testPageBuffer[TEST_PAGE_TOTAL][PG_PAGE_SIZE_DEFAULT];
return testPageBuffer[pageIdx]; return testPageBuffer[pageIdx];
} }
@ -29,42 +28,42 @@ testRun(void)
if (testBegin("pgPageChecksum()")) if (testBegin("pgPageChecksum()"))
{ {
// Checksum for 0x00 fill, page 0x00 // Checksum for 0x00 fill, page 0x00
memset(testPage(0), 0, TEST_PAGE_SIZE); memset(testPage(0), 0, PG_PAGE_SIZE_DEFAULT);
TEST_RESULT_U16_HEX(pgPageChecksum(testPage(0), 0, TEST_PAGE_SIZE), 0xC6AA, "check for 0x00 filled page, block 0"); TEST_RESULT_U16_HEX(pgPageChecksum(testPage(0), 0), 0xC6AA, "check for 0x00 filled page, block 0");
// Checksum for 0xFF fill, page 0x00 // Checksum for 0xFF fill, page 0x00
memset(testPage(0), 0xFF, TEST_PAGE_SIZE); memset(testPage(0), 0xFF, PG_PAGE_SIZE_DEFAULT);
TEST_RESULT_U16_HEX(pgPageChecksum(testPage(0), 0, TEST_PAGE_SIZE), 0x0E1C, "check for 0xFF filled page, block 0"); TEST_RESULT_U16_HEX(pgPageChecksum(testPage(0), 0), 0x0E1C, "check for 0xFF filled page, block 0");
// Checksum for 0xFF fill, page 0xFF // Checksum for 0xFF fill, page 0xFF
memset(testPage(0), 0xFF, TEST_PAGE_SIZE); memset(testPage(0), 0xFF, PG_PAGE_SIZE_DEFAULT);
TEST_RESULT_U16_HEX(pgPageChecksum(testPage(0), 999, TEST_PAGE_SIZE), 0x0EC3, "check for 0xFF filled page, block 999"); TEST_RESULT_U16_HEX(pgPageChecksum(testPage(0), 999), 0x0EC3, "check for 0xFF filled page, block 999");
} }
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("pgPageChecksumTest() and pgPageLsn()")) if (testBegin("pgPageChecksumTest() and pgPageLsn()"))
{ {
// Zero the pages // Zero the pages
memset(testPage(0), 0, TEST_PAGE_TOTAL * TEST_PAGE_SIZE); memset(testPage(0), 0, TEST_PAGE_TOTAL * PG_PAGE_SIZE_DEFAULT);
for (unsigned int pageIdx = 0; pageIdx < TEST_PAGE_TOTAL; pageIdx++) for (unsigned int pageIdx = 0; pageIdx < TEST_PAGE_TOTAL; pageIdx++)
*(PageHeader)testPage(pageIdx) = (PageHeaderData){.pd_upper = 0x0}; *(PageHeader)testPage(pageIdx) = (PageHeaderData){.pd_upper = 0x0};
// Pages with pd_upper = 0 should always return true no matter the block no // Pages with pd_upper = 0 should always return true no matter the block no
TEST_RESULT_BOOL(pgPageChecksumTest(testPage(0), 0, TEST_PAGE_SIZE, 0, 0), true, "pd_upper is 0, block 0"); TEST_RESULT_BOOL(pgPageChecksumTest(testPage(0), 0, PG_PAGE_SIZE_DEFAULT, 0, 0), true, "pd_upper is 0, block 0");
TEST_RESULT_BOOL(pgPageChecksumTest(testPage(1), 999, TEST_PAGE_SIZE, 0, 0), true, "pd_upper is 0, block 999"); TEST_RESULT_BOOL(pgPageChecksumTest(testPage(1), 999, PG_PAGE_SIZE_DEFAULT, 0, 0), true, "pd_upper is 0, block 999");
// Partial pages are always invalid // Partial pages are always invalid
*(PageHeader)testPage(0) = (PageHeaderData){.pd_upper = 0x00FF}; *(PageHeader)testPage(0) = (PageHeaderData){.pd_upper = 0x00FF};
((PageHeader)testPage(0))->pd_checksum = pgPageChecksum(testPage(0), 0, TEST_PAGE_SIZE); ((PageHeader)testPage(0))->pd_checksum = pgPageChecksum(testPage(0), 0);
TEST_RESULT_BOOL(pgPageChecksumTest(testPage(0), 0, TEST_PAGE_SIZE, 1, 1), true, "valid page"); TEST_RESULT_BOOL(pgPageChecksumTest(testPage(0), 0, PG_PAGE_SIZE_DEFAULT, 1, 1), true, "valid page");
TEST_RESULT_BOOL(pgPageChecksumTest(testPage(0), 0, TEST_PAGE_SIZE / 2, 1, 1), false, "invalid partial page"); TEST_RESULT_BOOL(pgPageChecksumTest(testPage(0), 0, PG_PAGE_SIZE_DEFAULT / 2, 1, 1), false, "invalid partial page");
// Update pd_upper and check for failure no matter the block no // Update pd_upper and check for failure no matter the block no
*(PageHeader)testPage(0) = (PageHeaderData){.pd_upper = 0x00FF, .pd_checksum = 0}; *(PageHeader)testPage(0) = (PageHeaderData){.pd_upper = 0x00FF, .pd_checksum = 0};
TEST_RESULT_BOOL(pgPageChecksumTest(testPage(0), 0, TEST_PAGE_SIZE, 0xFFFF, 0xFFFF), false, "badchecksum, page 0"); TEST_RESULT_BOOL(pgPageChecksumTest(testPage(0), 0, PG_PAGE_SIZE_DEFAULT, 0xFFFF, 0xFFFF), false, "badchecksum, page 0");
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
pgPageChecksumTest(testPage(0), 9999, TEST_PAGE_SIZE, 0xFFFF, 0xFFFF), false, "badchecksum, page 9999"); pgPageChecksumTest(testPage(0), 9999, PG_PAGE_SIZE_DEFAULT, 0xFFFF, 0xFFFF), false, "badchecksum, page 9999");
// Update LSNs and check that page checksums past the ignore limits are successful // Update LSNs and check that page checksums past the ignore limits are successful
((PageHeader)testPage(0))->pd_lsn.xlogid = 0x8888; ((PageHeader)testPage(0))->pd_lsn.xlogid = 0x8888;
@ -73,11 +72,11 @@ testRun(void)
TEST_RESULT_UINT(pgPageLsn(testPage(0)), 0x0000888800008888, "check page lsn"); TEST_RESULT_UINT(pgPageLsn(testPage(0)), 0x0000888800008888, "check page lsn");
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
pgPageChecksumTest(testPage(0), 0, TEST_PAGE_SIZE, 0x8888, 0x8888), true, "bad checksum past ignore limit"); pgPageChecksumTest(testPage(0), 0, PG_PAGE_SIZE_DEFAULT, 0x8888, 0x8888), true, "bad checksum past ignore limit");
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
pgPageChecksumTest(testPage(0), 0, TEST_PAGE_SIZE, 0x8888, 0x8889), false, "bad checksum before ignore limit"); pgPageChecksumTest(testPage(0), 0, PG_PAGE_SIZE_DEFAULT, 0x8888, 0x8889), false, "bad checksum before ignore limit");
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
pgPageChecksumTest(testPage(0), 0, TEST_PAGE_SIZE, 0x8889, 0x8889), false, "bad checksum before ignore limit"); pgPageChecksumTest(testPage(0), 0, PG_PAGE_SIZE_DEFAULT, 0x8889, 0x8889), false, "bad checksum before ignore limit");
} }
FUNCTION_HARNESS_RESULT_VOID(); FUNCTION_HARNESS_RESULT_VOID();

2
test/src/module/storage/remoteTest.c
View File

@ -444,7 +444,7 @@ testRun(void)
IoFilterGroup *filterGroup = ioFilterGroupNew(); IoFilterGroup *filterGroup = ioFilterGroupNew();
ioFilterGroupAdd(filterGroup, ioSizeNew()); ioFilterGroupAdd(filterGroup, ioSizeNew());
ioFilterGroupAdd(filterGroup, cryptoHashNew(HASH_TYPE_SHA1_STR)); ioFilterGroupAdd(filterGroup, cryptoHashNew(HASH_TYPE_SHA1_STR));
ioFilterGroupAdd(filterGroup, pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0)); ioFilterGroupAdd(filterGroup, pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, 0));
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL)); ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL));
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL)); ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL));
ioFilterGroupAdd(filterGroup, gzCompressNew(3)); ioFilterGroupAdd(filterGroup, gzCompressNew(3));