mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2024-12-12 10:04:14 +02:00
Retry reads of pg_control until checksum is valid.
On certain file systems (e.g. ext4) pg_control may appear torn if there is a concurrent write while reading the file. To prevent an invalid read, retry until the checksum matches the control data. Special handling is required for the pg-version-force feature since the offset of the checksum is not known. In this case, scan from the default position to the end of the data looking for a checksum match. This is a bit imprecise, but better than nothing, and the chance of a random collision in the control data seems very remote considering the ratio of data size (< 512 bytes) to checksum size (4 bytes). This was discovered and a possible solution proposed for PostgreSQL in [1]. The proposed solution may work for backup, but pgBackRest needs to be able to read pg_control reliably outside of backup. So no matter what fix is adopted for PostgreSQL, pgBackRest need retries. Further adjustment may be required as the PostgreSQL fix evolves. [1] https://www.postgresql.org/message-id/20221123014224.xisi44byq3cf5psi%40awork3.anarazel.de
This commit is contained in:
David Steele
GitHub
parent
c1805134b3
commit
f42d927d2d
@ -26,6 +26,18 @@
|
||||
<p>Multi-stanza check command.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<github-pull-request id="2131"/>
|
||||
|
||||
<release-item-contributor-list>
|
||||
<release-item-contributor id="david.steele"/>
|
||||
<release-item-reviewer id="stefan.fercot"/>
|
||||
<release-item-reviewer id="stephen.frost"/>
|
||||
</release-item-contributor-list>
|
||||
|
||||
<p>Retry reads of <file>pg_control</file> until checksum is valid.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<github-pull-request id="2162"/>
|
||||
|
||||
|
||||
@ -162,6 +162,7 @@ SRCS = \
|
||||
info/infoPg.c \
|
||||
postgres/client.c \
|
||||
postgres/interface.c \
|
||||
postgres/interface/crc32.c \
|
||||
postgres/interface/page.c \
|
||||
protocol/client.c \
|
||||
protocol/command.c \
|
||||
|
||||
@ -1990,6 +1990,7 @@ backupJobCallback(void *const data, const unsigned int clientIdx)
|
||||
pckWriteI32P(param, jobData->compressLevel);
|
||||
pckWriteU64P(param, jobData->cipherSubPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc);
|
||||
pckWriteStrP(param, jobData->cipherSubPass);
|
||||
pckWriteStrP(param, cfgOptionStrNull(cfgOptPgVersionForce));
|
||||
}
|
||||
|
||||
pckWriteStrP(param, manifestPathPg(file.name));
|
||||
|
||||
@ -11,6 +11,7 @@ Backup File
|
||||
#include "common/crypto/cipherBlock.h"
|
||||
#include "common/crypto/hash.h"
|
||||
#include "common/debug.h"
|
||||
#include "common/io/bufferRead.h"
|
||||
#include "common/io/filter/group.h"
|
||||
#include "common/io/filter/size.h"
|
||||
#include "common/io/io.h"
|
||||
@ -41,7 +42,7 @@ FN_EXTERN List *
|
||||
backupFile(
|
||||
const String *const repoFile, const uint64_t bundleId, const bool bundleRaw, const unsigned int blockIncrReference,
|
||||
const CompressType repoFileCompressType, const int repoFileCompressLevel, const CipherType cipherType,
|
||||
const String *const cipherPass, const List *const fileList)
|
||||
const String *const cipherPass, const String *const pgVersionForce, const List *const fileList)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||
FUNCTION_LOG_PARAM(STRING, repoFile); // Repo file
|
||||
@ -52,6 +53,7 @@ backupFile(
|
||||
FUNCTION_LOG_PARAM(INT, repoFileCompressLevel); // Compression level for repo file
|
||||
FUNCTION_LOG_PARAM(STRING_ID, cipherType); // Encryption type
|
||||
FUNCTION_TEST_PARAM(STRING, cipherPass); // Password to access the repo file if encrypted
|
||||
FUNCTION_LOG_PARAM(STRING, pgVersionForce); // Force pg version
|
||||
FUNCTION_LOG_PARAM(LIST, fileList); // List of files to backup
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
@ -193,13 +195,20 @@ backupFile(
|
||||
{
|
||||
// Setup pg file for read. Only read as many bytes as passed in pgFileSize. If the file is growing it does no
|
||||
// good to copy data past the end of the size recorded in the manifest since those blocks will need to be
|
||||
// replayed from WAL during recovery.
|
||||
// replayed from WAL during recovery. pg_control requires special handling since it needs to be retried on crc
|
||||
// validation failure.
|
||||
bool repoChecksum = false;
|
||||
IoRead *readIo;
|
||||
|
||||
StorageRead *const read = storageNewReadP(
|
||||
if (strEqZ(file->pgFile, PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))
|
||||
readIo = ioBufferReadNew(pgControlBufferFromFile(storagePg(), pgVersionForce));
|
||||
else
|
||||
{
|
||||
readIo = storageReadIo(
|
||||
storageNewReadP(
|
||||
storagePg(), file->pgFile, .ignoreMissing = file->pgFileIgnoreMissing, .compressible = compressible,
|
||||
.limit = file->pgFileCopyExactSize ? VARUINT64(file->pgFileSize) : NULL);
|
||||
IoRead *const readIo = storageReadIo(read);
|
||||
.limit = file->pgFileCopyExactSize ? VARUINT64(file->pgFileSize) : NULL));
|
||||
}
|
||||
|
||||
ioFilterGroupAdd(ioReadFilterGroup(readIo), cryptoHashNew(hashTypeSha1));
|
||||
ioFilterGroupAdd(ioReadFilterGroup(readIo), ioSizeNew());
|
||||
|
||||
@ -62,6 +62,6 @@ typedef struct BackupFileResult
|
||||
|
||||
FN_EXTERN List *backupFile(
|
||||
const String *repoFile, uint64_t bundleId, bool bundleRaw, unsigned int blockIncrReference, CompressType repoFileCompressType,
|
||||
int repoFileCompressLevel, CipherType cipherType, const String *cipherPass, const List *fileList);
|
||||
int repoFileCompressLevel, CipherType cipherType, const String *cipherPass, const String *pgVersionForce, const List *fileList);
|
||||
|
||||
#endif
|
||||
|
||||
@ -36,6 +36,7 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
|
||||
const int repoFileCompressLevel = pckReadI32P(param);
|
||||
const CipherType cipherType = (CipherType)pckReadU64P(param);
|
||||
const String *const cipherPass = pckReadStrP(param);
|
||||
const String *const pgVersionForce = pckReadStrP(param);
|
||||
|
||||
// Build the file list
|
||||
List *const fileList = lstNewP(sizeof(BackupFile));
|
||||
@ -77,7 +78,7 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
|
||||
// Backup file
|
||||
const List *const result = backupFile(
|
||||
repoFile, bundleId, bundleRaw, blockIncrReference, repoFileCompressType, repoFileCompressLevel, cipherType, cipherPass,
|
||||
fileList);
|
||||
pgVersionForce, fileList);
|
||||
|
||||
// Return result
|
||||
PackWrite *const resultPack = protocolPackNew();
|
||||
|
||||
@ -227,6 +227,7 @@ src_pgbackrest = [
|
||||
'info/infoPg.c',
|
||||
'postgres/client.c',
|
||||
'postgres/interface.c',
|
||||
'postgres/interface/crc32.c',
|
||||
'postgres/interface/page.c',
|
||||
'protocol/client.c',
|
||||
'protocol/command.c',
|
||||
|
||||
@ -9,7 +9,9 @@ PostgreSQL Interface
|
||||
#include "common/log.h"
|
||||
#include "common/memContext.h"
|
||||
#include "common/regExp.h"
|
||||
#include "common/time.h"
|
||||
#include "postgres/interface.h"
|
||||
#include "postgres/interface/crc32.h"
|
||||
#include "postgres/interface/static.vendor.h"
|
||||
#include "postgres/version.h"
|
||||
#include "storage/helper.h"
|
||||
@ -44,8 +46,10 @@ STRING_STATIC(PG_NAME_LOCATION_STR, "location");
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
The control file is 8192 bytes but only the first 512 bytes are used to prevent torn pages even on really old storage with 512-byte
|
||||
sectors. This is true across all versions of PostgreSQL.
|
||||
sectors. This is true across all versions of PostgreSQL. Unfortunately, this is not sufficient to prevent torn pages during
|
||||
concurrent reads and writes so retries are required.
|
||||
***********************************************************************************************************************************/
|
||||
#define PG_CONTROL_SIZE 8192
|
||||
#define PG_CONTROL_DATA_SIZE 512
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -64,6 +68,9 @@ typedef struct PgInterface
|
||||
// Convert pg_control to a common data structure
|
||||
PgControl (*control)(const unsigned char *);
|
||||
|
||||
// Get control crc offset
|
||||
size_t (*controlCrcOffset)(void);
|
||||
|
||||
// Get the control version for this version of PostgreSQL
|
||||
uint32_t (*controlVersion)(void);
|
||||
|
||||
@ -222,6 +229,47 @@ pgControlFromBuffer(const Buffer *controlFile, const String *const pgVersionForc
|
||||
PgControl result = interface->control(bufPtrConst(controlFile));
|
||||
result.version = interface->version;
|
||||
|
||||
// Check CRC
|
||||
size_t crcOffset = interface->controlCrcOffset();
|
||||
|
||||
do
|
||||
{
|
||||
// Calculate CRC and retrieve expected CRC
|
||||
const uint32_t crcCalculated =
|
||||
result.version > PG_VERSION_94 ?
|
||||
crc32cOne(bufPtrConst(controlFile), crcOffset) : crc32One(bufPtrConst(controlFile), crcOffset);
|
||||
const uint32_t crcExpected = *((uint32_t *)(bufPtrConst(controlFile) + crcOffset));
|
||||
|
||||
// If CRC does not match
|
||||
if (crcCalculated != crcExpected)
|
||||
{
|
||||
// If version is forced then the CRC might be later in the file (assuming the fork added extra fields to pg_control).
|
||||
// Increment the offset by CRC data size and continue to try again.
|
||||
if (pgVersionForce != NULL)
|
||||
{
|
||||
crcOffset += sizeof(uint32_t);
|
||||
|
||||
if (crcOffset <= bufUsed(controlFile) - sizeof(uint32_t))
|
||||
continue;
|
||||
}
|
||||
|
||||
// If no retry then error
|
||||
THROW_FMT(
|
||||
ChecksumError,
|
||||
"calculated " PG_FILE_PGCONTROL " checksum does not match expected value\n"
|
||||
"HINT: calculated 0x%x but expected value is 0x%x\n"
|
||||
"%s"
|
||||
"HINT: is " PG_FILE_PGCONTROL " corrupt?\n"
|
||||
"HINT: does " PG_FILE_PGCONTROL " have a different layout than expected?",
|
||||
crcCalculated, crcExpected,
|
||||
pgVersionForce == NULL ? "" : "HINT: checksum values may be misleading due to forced version scan\n");
|
||||
}
|
||||
|
||||
// Do not retry if the CRC is valid
|
||||
break;
|
||||
}
|
||||
while (true);
|
||||
|
||||
// Check the segment size
|
||||
pgWalSegmentSizeCheck(result.version, result.walSegmentSize);
|
||||
|
||||
@ -232,8 +280,20 @@ pgControlFromBuffer(const Buffer *controlFile, const String *const pgVersionForc
|
||||
FUNCTION_LOG_RETURN(PG_CONTROL, result);
|
||||
}
|
||||
|
||||
FN_EXTERN PgControl
|
||||
pgControlFromFile(const Storage *storage, const String *const pgVersionForce)
|
||||
// Helper to compare control data to last read
|
||||
static bool
|
||||
pgControlBufferEq(const Buffer *const last, const Buffer *const current)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, last);
|
||||
FUNCTION_TEST_PARAM(BUFFER, current);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RETURN(BOOL, last != NULL && bufEq(last, current));
|
||||
}
|
||||
|
||||
FN_EXTERN Buffer *
|
||||
pgControlBufferFromFile(const Storage *const storage, const String *const pgVersionForce)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||
FUNCTION_LOG_PARAM(STORAGE, storage);
|
||||
@ -242,18 +302,75 @@ pgControlFromFile(const Storage *storage, const String *const pgVersionForce)
|
||||
|
||||
ASSERT(storage != NULL);
|
||||
|
||||
PgControl result = {0};
|
||||
Buffer *result = NULL;
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
// On filesystems that do not implement atomicity of concurrent reads and writes, we might get garbage if the server is
|
||||
// writing the pg_control file at the same time as we try to read it. Keep trying until success or we read the same data
|
||||
// twice in a row. Do not use a timeout here because there are plenty of other errors that can happen and we don't want to
|
||||
// wait for them.
|
||||
Buffer *controlFileLast = NULL;
|
||||
bool done = false;
|
||||
|
||||
do
|
||||
{
|
||||
// Read control file
|
||||
Buffer *controlFile = storageGetP(
|
||||
Buffer *const controlFile = storageGetP(
|
||||
storageNewReadP(storage, STRDEF(PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)), .exactSize = PG_CONTROL_DATA_SIZE);
|
||||
|
||||
result = pgControlFromBuffer(controlFile, pgVersionForce);
|
||||
TRY_BEGIN()
|
||||
{
|
||||
// Check that control data is valid
|
||||
pgControlFromBuffer(controlFile, pgVersionForce);
|
||||
|
||||
// Create a buffer of the correct size to hold pg_control and zero out the remaining portion
|
||||
result = bufNew(PG_CONTROL_SIZE);
|
||||
|
||||
bufCat(result, controlFile);
|
||||
memset(bufPtr(result) + bufUsed(controlFile), 0, bufSize(result) - bufUsed(controlFile));
|
||||
bufUsedSet(result, bufSize(result));
|
||||
bufMove(result, memContextPrior());
|
||||
|
||||
done = true;
|
||||
}
|
||||
CATCH_ANY()
|
||||
{
|
||||
// If we get the same bad data twice in a row then error
|
||||
if (pgControlBufferEq(controlFileLast, controlFile))
|
||||
RETHROW();
|
||||
|
||||
// Copy data to last
|
||||
bufFree(controlFileLast);
|
||||
controlFileLast = controlFile;
|
||||
|
||||
// Sleep to let data stabilize
|
||||
sleepMSec(50);
|
||||
}
|
||||
TRY_END();
|
||||
}
|
||||
while (!done);
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(BUFFER, result);
|
||||
}
|
||||
|
||||
FN_EXTERN PgControl
|
||||
pgControlFromFile(const Storage *const storage, const String *const pgVersionForce)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||
FUNCTION_LOG_PARAM(STORAGE, storage);
|
||||
FUNCTION_LOG_PARAM(STRING, pgVersionForce);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(storage != NULL);
|
||||
|
||||
Buffer *const buffer = pgControlBufferFromFile(storage, pgVersionForce);
|
||||
PgControl result = pgControlFromBuffer(buffer, pgVersionForce);
|
||||
|
||||
bufFree(buffer);
|
||||
|
||||
FUNCTION_LOG_RETURN(PG_CONTROL, result);
|
||||
}
|
||||
|
||||
|
||||
@ -130,6 +130,7 @@ FN_EXTERN bool pgDbIsSystem(const String *name);
|
||||
FN_EXTERN bool pgDbIsSystemId(unsigned int id);
|
||||
|
||||
// Get info from pg_control
|
||||
FN_EXTERN Buffer *pgControlBufferFromFile(const Storage *storage, const String *pgVersionForce);
|
||||
FN_EXTERN PgControl pgControlFromFile(const Storage *storage, const String *pgVersionForce);
|
||||
|
||||
// Get the control version for a PostgreSQL version
|
||||
|
||||
102
src/postgres/interface/crc32.c
Normal file
102
src/postgres/interface/crc32.c
Normal file
@ -0,0 +1,102 @@
|
||||
/***********************************************************************************************************************************
|
||||
CRC-32 Calculation
|
||||
***********************************************************************************************************************************/
|
||||
#include "build.auto.h"
|
||||
|
||||
#include "postgres/interface/crc32.h"
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
static const uint32_t crc32_lookup[256] =
|
||||
{
|
||||
0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
|
||||
0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,
|
||||
0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
|
||||
0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,
|
||||
0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
|
||||
0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
|
||||
0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,
|
||||
0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,
|
||||
0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
|
||||
0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
|
||||
0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,
|
||||
0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
|
||||
0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,
|
||||
0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,
|
||||
0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
|
||||
0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,
|
||||
0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,
|
||||
0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
|
||||
0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,
|
||||
0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
|
||||
0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
|
||||
0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,
|
||||
0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,
|
||||
0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
|
||||
0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
|
||||
0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,
|
||||
0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
|
||||
0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,
|
||||
0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,
|
||||
0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
|
||||
0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
|
||||
0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
|
||||
};
|
||||
|
||||
FN_EXTERN uint32_t
|
||||
crc32One(const unsigned char *data, size_t size)
|
||||
{
|
||||
uint32_t result = 0xffffffff;
|
||||
|
||||
while (size--)
|
||||
result = crc32_lookup[((result >> 24) ^ *data++) & 0xFF] ^ (result << 8);
|
||||
|
||||
return result ^ 0xffffffff;
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
static const uint32_t crc32c_lookup[256] =
|
||||
{
|
||||
0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
|
||||
0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24,
|
||||
0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
|
||||
0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B,
|
||||
0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35,
|
||||
0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
|
||||
0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A,
|
||||
0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595,
|
||||
0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
|
||||
0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198,
|
||||
0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38,
|
||||
0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
|
||||
0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789,
|
||||
0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46,
|
||||
0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
|
||||
0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829,
|
||||
0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93,
|
||||
0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
|
||||
0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC,
|
||||
0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033,
|
||||
0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
|
||||
0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982,
|
||||
0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622,
|
||||
0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
|
||||
0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F,
|
||||
0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0,
|
||||
0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
|
||||
0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F,
|
||||
0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1,
|
||||
0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
|
||||
0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
|
||||
0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351,
|
||||
};
|
||||
|
||||
FN_EXTERN uint32_t
|
||||
crc32cOne(const unsigned char *data, size_t size)
|
||||
{
|
||||
uint32_t result = 0xffffffff;
|
||||
|
||||
while (size--)
|
||||
result = crc32c_lookup[(result ^ *data++) & 0xFF] ^ (result >> 8);
|
||||
|
||||
return result ^ 0xffffffff;
|
||||
}
|
||||
21
src/postgres/interface/crc32.h
Normal file
21
src/postgres/interface/crc32.h
Normal file
@ -0,0 +1,21 @@
|
||||
/***********************************************************************************************************************************
|
||||
CRC-32 Calculation
|
||||
|
||||
CRC-32 and CRC-32C calculations required to validate the integrity of pg_control.
|
||||
***********************************************************************************************************************************/
|
||||
#ifndef POSTGRES_INTERFACE_CRC32_H
|
||||
#define POSTGRES_INTERFACE_CRC32_H
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <stddef.h>
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Functions
|
||||
***********************************************************************************************************************************/
|
||||
// Generate CRC-32 checksum (required by <= 9.4)
|
||||
FN_EXTERN uint32_t crc32One(const unsigned char *data, size_t size);
|
||||
|
||||
// Generate CRC-32C checksum (required by >= 9.5)
|
||||
FN_EXTERN uint32_t crc32cOne(const unsigned char *data, size_t size);
|
||||
|
||||
#endif
|
||||
@ -76,6 +76,22 @@ Read the version specific pg_control into a general data structure
|
||||
|
||||
#endif
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get control crc offset
|
||||
***********************************************************************************************************************************/
|
||||
#if PG_VERSION > PG_VERSION_MAX
|
||||
|
||||
#elif PG_VERSION >= PG_VERSION_93
|
||||
|
||||
#define PG_INTERFACE_CONTROL_CRC_OFFSET(version) \
|
||||
static size_t \
|
||||
pgInterfaceControlCrcOffset##version(void) \
|
||||
{ \
|
||||
return offsetof(ControlFileData, crc); \
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get the control version
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -444,6 +444,7 @@ unit:
|
||||
- config/protocol
|
||||
- command/command
|
||||
- postgres/interface
|
||||
- postgres/interface/crc32
|
||||
- postgres/interface/page
|
||||
- protocol/client
|
||||
- protocol/command
|
||||
@ -645,6 +646,7 @@ unit:
|
||||
|
||||
coverage:
|
||||
- postgres/interface
|
||||
- postgres/interface/crc32
|
||||
- postgres/interface/page
|
||||
|
||||
# ********************************************************************************************************************************
|
||||
|
||||
@ -271,48 +271,6 @@ sub dbCatalogVersion
|
||||
return $hCatalogVersion->{$strPgVersion};
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Get database control version for the db version
|
||||
####################################################################################################################################
|
||||
sub dbControlVersion
|
||||
{
|
||||
my $self = shift;
|
||||
|
||||
# Assign function parameters, defaults, and log debug info
|
||||
my
|
||||
(
|
||||
$strOperation,
|
||||
$strPgVersion,
|
||||
) =
|
||||
logDebugParam
|
||||
(
|
||||
__PACKAGE__ . '->dbControlVersion', \@_,
|
||||
{name => 'strPgVersion', trace => true},
|
||||
);
|
||||
|
||||
my $hControlVersion =
|
||||
{
|
||||
&PG_VERSION_93 => 937,
|
||||
&PG_VERSION_94 => 942,
|
||||
&PG_VERSION_95 => 942,
|
||||
&PG_VERSION_96 => 960,
|
||||
&PG_VERSION_10 => 1002,
|
||||
&PG_VERSION_11 => 1100,
|
||||
&PG_VERSION_12 => 1201,
|
||||
&PG_VERSION_13 => 1300,
|
||||
&PG_VERSION_14 => 1300,
|
||||
&PG_VERSION_15 => 1300,
|
||||
&PG_VERSION_16 => 1300,
|
||||
};
|
||||
|
||||
if (!defined($hControlVersion->{$strPgVersion}))
|
||||
{
|
||||
confess &log(ASSERT, "no control version defined for pg version ${strPgVersion}");
|
||||
}
|
||||
|
||||
return $hControlVersion->{$strPgVersion};
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# Generate control file content
|
||||
####################################################################################################################################
|
||||
@ -332,48 +290,97 @@ sub controlGenerateContent
|
||||
{name => 'strPgVersion', trace => true},
|
||||
);
|
||||
|
||||
my $tControlContent = pack('Q', $self->dbSysId($strPgVersion));
|
||||
$tControlContent .= pack('L', $self->dbControlVersion($strPgVersion));
|
||||
$tControlContent .= pack('L', $self->dbCatalogVersion($strPgVersion));
|
||||
|
||||
# Offset to page size by architecture bits and version
|
||||
my $rhOffsetToPageSize =
|
||||
my $hControlContent =
|
||||
{
|
||||
32 =>
|
||||
{
|
||||
'9.3' => 180 - length($tControlContent),
|
||||
'9.4' => 188 - length($tControlContent),
|
||||
'9.5' => 200 - length($tControlContent),
|
||||
'9.6' => 200 - length($tControlContent),
|
||||
'10' => 200 - length($tControlContent),
|
||||
'11' => 192 - length($tControlContent),
|
||||
'12' => 196 - length($tControlContent),
|
||||
'13' => 196 - length($tControlContent),
|
||||
&PG_VERSION_93 =>
|
||||
"5d0064a7b3b6e00da903000009b0ff0b00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"00000000000000000000000000200000000000000000000000000001000000000000000000000000000000000000000043d77e50",
|
||||
&PG_VERSION_94 =>
|
||||
"5e0064a7b3b6e00dae0300000b43010c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000002000000000000000000000000000010000000000000000000000000000000000000000" .
|
||||
"000000006b0756c8",
|
||||
&PG_VERSION_95 =>
|
||||
"5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
|
||||
"000000000000000000000000000000003bfe413a",
|
||||
&PG_VERSION_96 =>
|
||||
"600064a7b3b6e00dc0030000c34b040c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
|
||||
"000000000000000000000000000000005d135da6",
|
||||
&PG_VERSION_10 =>
|
||||
"640064a7b3b6e00dea030000cbce050c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
|
||||
"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8556c34",
|
||||
},
|
||||
|
||||
64 =>
|
||||
{
|
||||
'9.3' => 192 - length($tControlContent),
|
||||
'9.4' => 200 - length($tControlContent),
|
||||
'9.5' => 216 - length($tControlContent),
|
||||
'9.6' => 216 - length($tControlContent),
|
||||
'10' => 216 - length($tControlContent),
|
||||
'11' => 208 - length($tControlContent),
|
||||
'12' => 212 - length($tControlContent),
|
||||
'13' => 212 - length($tControlContent),
|
||||
&PG_VERSION_93 =>
|
||||
"5d0064a7b3b6e00da903000009b0ff0b00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
|
||||
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000020000000000000000000000000000100000000000000000000000000000000" .
|
||||
"0000000003fff8cc",
|
||||
&PG_VERSION_94 =>
|
||||
"5e0064a7b3b6e00dae0300000b43010c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
|
||||
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
|
||||
"00000000000000000000000000000000ee6cf996",
|
||||
&PG_VERSION_95 =>
|
||||
"5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
|
||||
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" .
|
||||
"0000000000000001000000000000000000000000000000000000000000000000381ec2de",
|
||||
&PG_VERSION_96 =>
|
||||
"600064a7b3b6e00dc0030000c34b040c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
|
||||
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" .
|
||||
"00000000000000010000000000000000000000000000000000000000000000002d96a4c0",
|
||||
&PG_VERSION_10 =>
|
||||
"640064a7b3b6e00dea030000cbce050c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
|
||||
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" .
|
||||
"0000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
|
||||
"00000000000000008d543cdf",
|
||||
},
|
||||
};
|
||||
|
||||
# Fill up to page size and set page size
|
||||
$tControlContent .= ('C' x $rhOffsetToPageSize->{$self->archBits()}{$strPgVersion});
|
||||
$tControlContent .= pack('L', 8192);
|
||||
my $strControlContent = $hControlContent->{$self->archBits()}{$strPgVersion};
|
||||
|
||||
if (!defined($strControlContent))
|
||||
{
|
||||
confess &log(ASSERT, "no control content defined for pg version ${strPgVersion}");
|
||||
}
|
||||
|
||||
my $tControlContent = '';
|
||||
|
||||
for (my $iIdx = 0; $iIdx < length($strControlContent) / 2; $iIdx++)
|
||||
{
|
||||
my $iChar = hex(substr($strControlContent, $iIdx * 2, 2));
|
||||
$tControlContent .= pack('C', $iChar);
|
||||
}
|
||||
|
||||
# Fill up to wal segment size and set wal segment size
|
||||
$tControlContent .= ('C' x 8);
|
||||
$tControlContent .= pack('L', 16 * 1024 * 1024);
|
||||
|
||||
# Pad bytes
|
||||
$tControlContent .= ('C' x (8192 - length($tControlContent)));
|
||||
for (my $iIdx = length($tControlContent); $iIdx < 8192; $iIdx++)
|
||||
{
|
||||
$tControlContent .= pack('C', 0);
|
||||
}
|
||||
|
||||
return \$tControlContent;
|
||||
}
|
||||
|
||||
@ -372,7 +372,7 @@ sub run
|
||||
|
||||
$oHostDbPrimary->manifestFileCreate(
|
||||
\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_PGCONTROL, '[replaceme]',
|
||||
$self->archBits() == 32 ? '8107e546c59c72a8c1818fc3610d7cc1e5623660' : '4c77c900f7af0d9ab13fa9982051a42e0b637f6c',
|
||||
$self->archBits() == 32 ? '59331420592064afff045f352de3a1ce53b0dd21' : 'e8bd9ef18a0b58f89aed7e98a1c32119f03727ee',
|
||||
$lTime - 100, undef, true);
|
||||
|
||||
# Copy pg_control
|
||||
|
||||
@ -78,7 +78,7 @@ hrnBackupPqScript(const unsigned int pgVersion, const time_t backupTimeStart, Hr
|
||||
pgControl.timeline = param.timeline;
|
||||
|
||||
HRN_STORAGE_PUT(
|
||||
storagePgIdxWrite(0), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, hrnPgControlToBuffer(0, pgControl),
|
||||
storagePgIdxWrite(0), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, hrnPgControlToBuffer(0, 0, pgControl),
|
||||
.timeModified = backupTimeStart);
|
||||
|
||||
// Update pg_control on primary with the backup time
|
||||
@ -205,7 +205,7 @@ hrnBackupPqScript(const unsigned int pgVersion, const time_t backupTimeStart, Hr
|
||||
ASSERT(!param.noArchiveCheck);
|
||||
|
||||
// Save pg_control with updated info
|
||||
HRN_STORAGE_PUT(storagePgIdxWrite(1), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, hrnPgControlToBuffer(0, pgControl));
|
||||
HRN_STORAGE_PUT(storagePgIdxWrite(1), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, hrnPgControlToBuffer(0, 0, pgControl));
|
||||
|
||||
if (param.noPriorWal)
|
||||
{
|
||||
|
||||
@ -12,47 +12,47 @@ Harness for PostgreSQL Interface
|
||||
Interface definition
|
||||
***********************************************************************************************************************************/
|
||||
uint32_t hrnPgInterfaceCatalogVersion093(void);
|
||||
void hrnPgInterfaceControl093(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl093(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal093(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion094(void);
|
||||
void hrnPgInterfaceControl094(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl094(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal094(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion095(void);
|
||||
void hrnPgInterfaceControl095(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl095(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal095(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion096(void);
|
||||
void hrnPgInterfaceControl096(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl096(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal096(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion100(void);
|
||||
void hrnPgInterfaceControl100(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl100(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal100(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion110(void);
|
||||
void hrnPgInterfaceControl110(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl110(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal110(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion120(void);
|
||||
void hrnPgInterfaceControl120(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl120(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal120(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion130(void);
|
||||
void hrnPgInterfaceControl130(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl130(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal130(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion140(void);
|
||||
void hrnPgInterfaceControl140(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl140(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal140(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion150(void);
|
||||
void hrnPgInterfaceControl150(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl150(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal150(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
uint32_t hrnPgInterfaceCatalogVersion160(void);
|
||||
void hrnPgInterfaceControl160(unsigned int controlVersion, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceControl160(unsigned int controlVersion, unsigned int crc, PgControl pgControl, unsigned char *buffer);
|
||||
void hrnPgInterfaceWal160(unsigned int magic, PgWal pgWal, unsigned char *buffer);
|
||||
|
||||
typedef struct HrnPgInterface
|
||||
@ -64,7 +64,7 @@ typedef struct HrnPgInterface
|
||||
unsigned int (*catalogVersion)(void);
|
||||
|
||||
// Create pg_control
|
||||
void (*control)(unsigned int, PgControl, unsigned char *);
|
||||
void (*control)(unsigned int, unsigned int, PgControl, unsigned char *);
|
||||
|
||||
// Create WAL header
|
||||
void (*wal)(unsigned int, PgWal, unsigned char *);
|
||||
@ -192,10 +192,11 @@ hrnPgCatalogVersion(unsigned int pgVersion)
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
Buffer *
|
||||
hrnPgControlToBuffer(const unsigned int controlVersion, PgControl pgControl)
|
||||
hrnPgControlToBuffer(const unsigned int controlVersion, const unsigned int crc, PgControl pgControl)
|
||||
{
|
||||
FUNCTION_HARNESS_BEGIN();
|
||||
FUNCTION_HARNESS_PARAM(UINT, controlVersion);
|
||||
FUNCTION_HARNESS_PARAM(UINT, crc);
|
||||
FUNCTION_HARNESS_PARAM(PG_CONTROL, pgControl);
|
||||
FUNCTION_HARNESS_END();
|
||||
|
||||
@ -216,7 +217,7 @@ hrnPgControlToBuffer(const unsigned int controlVersion, PgControl pgControl)
|
||||
bufUsedSet(result, bufSize(result));
|
||||
|
||||
// Generate pg_control
|
||||
hrnPgInterfaceVersion(pgControl.version)->control(controlVersion, pgControl, bufPtr(result));
|
||||
hrnPgInterfaceVersion(pgControl.version)->control(controlVersion, crc, pgControl, bufPtr(result));
|
||||
|
||||
FUNCTION_HARNESS_RETURN(BUFFER, result);
|
||||
}
|
||||
|
||||
@ -48,12 +48,17 @@ Put a control file to storage
|
||||
#define HRN_PG_CONTROL_PUT(storageParam, versionParam, ...) \
|
||||
HRN_STORAGE_PUT( \
|
||||
storageParam, PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, \
|
||||
hrnPgControlToBuffer(0, (PgControl){.version = versionParam, __VA_ARGS__}))
|
||||
hrnPgControlToBuffer(0, 0, (PgControl){.version = versionParam, __VA_ARGS__}))
|
||||
|
||||
#define HRN_PG_CONTROL_OVERRIDE_VERSION_PUT(storageParam, versionParam, controlVersionParam, ...) \
|
||||
HRN_STORAGE_PUT( \
|
||||
storageParam, PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, \
|
||||
hrnPgControlToBuffer(controlVersionParam, (PgControl){.version = versionParam, __VA_ARGS__}))
|
||||
hrnPgControlToBuffer(controlVersionParam, 0, (PgControl){.version = versionParam, __VA_ARGS__}))
|
||||
|
||||
#define HRN_PG_CONTROL_OVERRIDE_CRC_PUT(storageParam, versionParam, crcParam, ...) \
|
||||
HRN_STORAGE_PUT( \
|
||||
storageParam, PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, \
|
||||
hrnPgControlToBuffer(0, crcParam, (PgControl){.version = versionParam, __VA_ARGS__}))
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Copy WAL info to buffer
|
||||
@ -77,7 +82,7 @@ Functions
|
||||
unsigned int hrnPgCatalogVersion(unsigned int pgVersion);
|
||||
|
||||
// Create pg_control
|
||||
Buffer *hrnPgControlToBuffer(unsigned int controlVersion, PgControl pgControl);
|
||||
Buffer *hrnPgControlToBuffer(unsigned int controlVersion, unsigned int crc, PgControl pgControl);
|
||||
|
||||
// Get system id by version
|
||||
FN_INLINE_ALWAYS uint64_t
|
||||
|
||||
@ -6,6 +6,7 @@ Macros to create harness functions per PostgreSQL version.
|
||||
#ifndef TEST_COMMON_HARNESS_POSTGRES_VERSIONINTERN_H
|
||||
#define TEST_COMMON_HARNESS_POSTGRES_VERSIONINTERN_H
|
||||
|
||||
#include "postgres/interface/crc32.h"
|
||||
#include "postgres/interface/version.vendor.h"
|
||||
|
||||
#include "common/harnessPostgres.h"
|
||||
@ -35,7 +36,8 @@ Create a pg_control file
|
||||
|
||||
#define HRN_PG_INTERFACE_CONTROL_TEST(version) \
|
||||
void \
|
||||
hrnPgInterfaceControl##version(const unsigned int controlVersion, const PgControl pgControl, unsigned char *const buffer) \
|
||||
hrnPgInterfaceControl##version( \
|
||||
const unsigned int controlVersion, const unsigned int crc, const PgControl pgControl, unsigned char *const buffer) \
|
||||
{ \
|
||||
ASSERT(buffer != NULL); \
|
||||
\
|
||||
@ -53,6 +55,12 @@ Create a pg_control file
|
||||
.xlog_seg_size = pgControl.walSegmentSize, \
|
||||
.data_checksum_version = pgControl.pageChecksum, \
|
||||
}; \
|
||||
\
|
||||
((ControlFileData *)buffer)->crc = \
|
||||
crc == 0 ? \
|
||||
(PG_VERSION > PG_VERSION_94 ? \
|
||||
crc32cOne(buffer, offsetof(ControlFileData, crc)) : crc32One(buffer, offsetof(ControlFileData, crc))) : \
|
||||
crc; \
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -1337,7 +1337,7 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, NULL, fileList), 0),
|
||||
"pg file missing, ignoreMissing=true, no delta");
|
||||
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy/repo size 0");
|
||||
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file");
|
||||
@ -1362,7 +1362,7 @@ testRun(void)
|
||||
lstAdd(fileList, &file);
|
||||
|
||||
TEST_ERROR(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, fileList), FileMissingError,
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, NULL, fileList), FileMissingError,
|
||||
"unable to open missing file '" TEST_PATH "/pg/missing' for read");
|
||||
|
||||
// Create a pg file to backup
|
||||
@ -1399,7 +1399,7 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, NULL, fileList), 0),
|
||||
"file checksummed with pageChecksum enabled");
|
||||
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
|
||||
TEST_RESULT_UINT(result.repoSize, 9, "repo=pgFile size");
|
||||
@ -1431,7 +1431,7 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, NULL, fileList), 0),
|
||||
"backup file");
|
||||
TEST_RESULT_UINT(result.copySize, 12, "copy size");
|
||||
TEST_RESULT_UINT(result.repoSize, 12, "repo size");
|
||||
@ -1464,7 +1464,7 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, NULL, fileList), 0),
|
||||
"file in db and repo, checksum equal, no ignoreMissing, no pageChecksum, delta, hasReference");
|
||||
TEST_RESULT_UINT(result.copySize, 9, "copy size set");
|
||||
TEST_RESULT_UINT(result.repoSize, 0, "repo size not set since already exists in repo");
|
||||
@ -1498,7 +1498,7 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, NULL, fileList), 0),
|
||||
"file in db and repo, pg checksum not equal, no ignoreMissing, no pageChecksum, delta, hasReference");
|
||||
TEST_RESULT_UINT(result.copySize, 9, "copy 9 bytes");
|
||||
TEST_RESULT_UINT(result.repoSize, 9, "repo=copy size");
|
||||
@ -1533,7 +1533,7 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, NULL, fileList), 0),
|
||||
"db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference");
|
||||
TEST_RESULT_UINT(result.copySize, 12, "copy=pgFile size");
|
||||
TEST_RESULT_UINT(result.repoSize, 12, "repo=pgFile size");
|
||||
@ -1573,7 +1573,7 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, NULL, fileList), 0),
|
||||
"file in repo only, checksum in repo equal, ignoreMissing=true, no pageChecksum, delta, no hasReference");
|
||||
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=0 size");
|
||||
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultSkip, "skip file");
|
||||
@ -1606,7 +1606,7 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeGz, 3, cipherTypeNone, NULL, fileList), 0),
|
||||
backupFile(repoFile, 0, false, 0, compressTypeGz, 3, cipherTypeNone, NULL, NULL, fileList), 0),
|
||||
"pg file exists, no checksum, no ignoreMissing, compression, no pageChecksum, no delta, no hasReference");
|
||||
TEST_RESULT_UINT(result.copySize, 9, "copy=pgFile size");
|
||||
TEST_RESULT_UINT(result.repoSize, 29, "repo compress size");
|
||||
@ -1646,7 +1646,7 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, fileList), 0),
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeNone, NULL, NULL, fileList), 0),
|
||||
"zero-sized pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
|
||||
TEST_RESULT_UINT(result.copySize + result.repoSize, 0, "copy=repo=pgFile size 0");
|
||||
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "copy file");
|
||||
@ -1697,7 +1697,9 @@ testRun(void)
|
||||
TEST_ASSIGN(
|
||||
result,
|
||||
*(BackupFileResult *)lstGet(
|
||||
backupFile(repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), fileList), 0),
|
||||
backupFile(
|
||||
repoFile, 0, false, 0, compressTypeNone, 1, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS), NULL, fileList),
|
||||
0),
|
||||
"pg file exists, no repo file, no ignoreMissing, no pageChecksum, no delta, no hasReference");
|
||||
TEST_RESULT_UINT(result.copySize, 9, "copy size set");
|
||||
TEST_RESULT_UINT(result.repoSize, 32, "repo size set");
|
||||
@ -2307,7 +2309,7 @@ testRun(void)
|
||||
HRN_CFG_LOAD(cfgCmdStanzaCreate, argList);
|
||||
|
||||
// Create pg_control
|
||||
HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_93);
|
||||
HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_94);
|
||||
|
||||
cmdStanzaCreate();
|
||||
TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'test1' on repo1");
|
||||
@ -2359,8 +2361,8 @@ testRun(void)
|
||||
"P00 INFO: new backup label = [FULL-1]\n"
|
||||
"P00 INFO: full backup size = 8KB, file total = 2",
|
||||
TEST_64BIT() ?
|
||||
(TEST_BIG_ENDIAN() ? "542c539fabf827717812b872913d593766848434" : "cc603d502a8ecede30fa96b0b0dc35014950aba2") :
|
||||
"aca8af76589171a70c901e13080e359efedd7a08");
|
||||
(TEST_BIG_ENDIAN() ? "8e756232fbbf97f0d1026d62ba68b3d6602acbd7" : "20df8640327df5ebc0658817cbc4c93624fd451b") :
|
||||
"56d0a8a2d6b0b6dd2880c4b0221fec24e958c1a6");
|
||||
|
||||
// Make pg no longer appear to be running
|
||||
HRN_STORAGE_REMOVE(storagePgWrite(), PG_FILE_POSTMTRPID, .errorOnMissing = true);
|
||||
@ -4212,7 +4214,7 @@ testRun(void)
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-age-to-zero (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-age-multiplier (bundle 1/56, 32KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/184, 8KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (bundle 1/272, 48KB, [PCT]) checksum [SHA1]\n"
|
||||
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (bundle 1/288, 48KB, [PCT]) checksum [SHA1]\n"
|
||||
"P00 DETAIL: reference pg_data/PG_VERSION to 20191108-080000F\n"
|
||||
"P00 DETAIL: reference pg_data/block-incr-wayback to 20191108-080000F\n"
|
||||
"P00 INFO: execute non-exclusive backup stop and wait for all WAL segments to archive\n"
|
||||
|
||||
@ -74,6 +74,33 @@ testRun(void)
|
||||
"unexpected control version = 1501 and catalog version = 202211111\n"
|
||||
"HINT: is this version of PostgreSQL supported?");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("invalid CRC");
|
||||
|
||||
HRN_PG_CONTROL_OVERRIDE_CRC_PUT(storageTest, PG_VERSION_11, 0xFADEFADE);
|
||||
|
||||
TEST_ERROR_FMT(
|
||||
pgControlFromFile(storageTest, NULL), ChecksumError,
|
||||
"calculated pg_control checksum does not match expected value\n"
|
||||
"HINT: calculated 0x%x but expected value is 0xfadefade\n"
|
||||
"HINT: is pg_control corrupt?\n"
|
||||
"HINT: does pg_control have a different layout than expected?",
|
||||
(uint32_t)(TEST_BIG_ENDIAN() ? 0x4e206eeb : (TEST_64BIT() ? 0x4ad387b2 : 0x3ca3a1ec)));
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("invalid CRC on force control version");
|
||||
|
||||
HRN_PG_CONTROL_OVERRIDE_CRC_PUT(storageTest, PG_VERSION_13, 0xFADEFADE);
|
||||
|
||||
TEST_ERROR_FMT(
|
||||
pgControlFromFile(storageTest, STRDEF(PG_VERSION_14_Z)), ChecksumError,
|
||||
"calculated pg_control checksum does not match expected value\n"
|
||||
"HINT: calculated 0x%x but expected value is 0x0\n"
|
||||
"HINT: checksum values may be misleading due to forced version scan\n"
|
||||
"HINT: is pg_control corrupt?\n"
|
||||
"HINT: does pg_control have a different layout than expected?",
|
||||
(uint32_t)(TEST_BIG_ENDIAN() ? 0x27dc2b85 : (TEST_64BIT() ? 0xa9264d94 : 0xb371302a)));
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
HRN_PG_CONTROL_PUT(
|
||||
storageTest, PG_VERSION_11, .systemId = 0xFACEFACE, .checkpoint = 0xEEFFEEFFAABBAABB, .timeline = 47,
|
||||
@ -133,6 +160,21 @@ testRun(void)
|
||||
TEST_RESULT_UINT(info.catalogVersion, 202211111, "check catalog version");
|
||||
TEST_RESULT_UINT(info.checkpoint, 0xAABBAABBEEFFEEFF, "check checkpoint");
|
||||
TEST_RESULT_UINT(info.timeline, 88, "check timeline");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("CRC at greater offset on force control version");
|
||||
|
||||
// Start with an invalid crc but write a valid one at a greater offset to test the forced scan
|
||||
Buffer *control = hrnPgControlToBuffer(0, 0xFADEFADE, (PgControl){.version = PG_VERSION_13, .systemId = 0xAAAA0AAAA});
|
||||
size_t crcOffset = pgInterfaceVersion(PG_VERSION_13)->controlCrcOffset() + sizeof(uint32_t) * 4;
|
||||
*((uint32_t *)(bufPtr(control) + crcOffset)) = crc32cOne(bufPtrConst(control), crcOffset);
|
||||
|
||||
HRN_STORAGE_PUT(storageTest, PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, control);
|
||||
|
||||
TEST_ASSIGN(info, pgControlFromFile(storageTest, STRDEF(PG_VERSION_14_Z)), "get control info force v14");
|
||||
TEST_RESULT_UINT(info.systemId, 0xAAAA0AAAA, "check system id");
|
||||
TEST_RESULT_UINT(info.version, PG_VERSION_14, "check version");
|
||||
TEST_RESULT_UINT(info.catalogVersion, 202007201, "check catalog version");
|
||||
}
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
|
||||
Loading…
Reference in New Issue
Block a user