db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-13 01:00:23 +02:00
Code Issues Releases Activity

Add hex encode/decoding to decode module.

Browse Source 
This replaces the bufHex() function and also allows hex to be decoded.
This commit is contained in:
David Steele
2022-12-11 19:46:48 +07:00
parent 9a9ee8e640
commit 010efffb0c
26 changed files with 282 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/command/archive/push/file.c
View File

@ -160,8 +160,8 @@ archivePushFile(
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
ioReadDrain(read);
const String *const walSegmentChecksum = bufHex(
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)));
const String *const walSegmentChecksum = strNewEncode(
encodingHex, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)));
// Check each repo for the WAL segment
for (unsigned int repoListIdx = 0; repoListIdx < lstSize(repoList); repoListIdx++)

3
src/command/backup/backup.c
View File

@ -984,7 +984,8 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti
};
memcpy(
file.checksumSha1, strZ(bufHex(pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))),
file.checksumSha1,
strZ(strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))),
HASH_TYPE_SHA1_SIZE_HEX + 1);
manifestFileAdd(manifest, &file);

11
src/command/backup/file.c
View File

@ -94,8 +94,8 @@ backupFile(
// If the pg file exists check the checksum/size
if (ioReadDrain(read))
{
const String *const pgTestChecksum = bufHex(
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)));
const String *const pgTestChecksum = strNewEncode(
encodingHex, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)));
uint64_t pgTestSize = pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE));
// Does the pg file match?
@ -160,8 +160,8 @@ backupFile(
ioReadDrain(read);
// Test checksum/size
const String *const pgTestChecksum = bufHex(
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)));
const String *const pgTestChecksum = strNewEncode(
encodingHex, pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)));
uint64_t pgTestSize = pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE));
// No need to recopy if checksum/size match
@ -275,7 +275,8 @@ backupFile(
fileResult->copySize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE, .idx = 0));
fileResult->bundleOffset = bundleOffset;
fileResult->copyChecksum = bufHex(
fileResult->copyChecksum = strNewEncode(
encodingHex,
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE)));
fileResult->repoSize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE, .idx = 1));

10
src/command/restore/file.c
View File

@ -121,7 +121,8 @@ List *restoreFile(
if (file->size == 0 ||
strEq(
file->checksum,
bufHex(
strNewEncode(
encodingHex,
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE)))))
{
// If the hash/size are now the same but the time is not, then set the time back to the backup
@ -268,12 +269,15 @@ List *restoreFile(
storageReadFree(repoFileRead);
// Validate checksum
if (!strEq(file->checksum, bufHex(pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))))
if (!strEq(
file->checksum,
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))))
{
THROW_FMT(
ChecksumError,
"error restoring '%s': actual checksum '%s' does not match expected checksum '%s'", strZ(file->name),
strZ(bufHex(pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))),
strZ(
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))),
strZ(file->checksum));
}
}

4
src/command/verify/file.c
View File

@ -65,7 +65,9 @@ verifyFile(
if (ioReadDrain(read))
{
// Validate checksum
if (!strEq(fileChecksum, bufHex(pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))))
if (!strEq(
fileChecksum,
strNewEncode(encodingHex, pckReadBinP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))))
{
result = verifyChecksumMismatch;
}

2
src/command/verify/verify.c
View File

@ -229,7 +229,7 @@ verifyInfoFile(const String *pathFileName, bool keepFile, const String *cipherPa
MEM_CONTEXT_PRIOR_BEGIN()
{
result.checksum = bufHex(filterResult);
result.checksum = strNewEncode(encodingHex, filterResult);
}
MEM_CONTEXT_PRIOR_END();
}

145
src/common/encode.c
View File

@ -14,7 +14,7 @@ Binary to String Encode/Decode
Assert that encoding type is valid. This needs to be kept up to date with the last item in the enum.
***********************************************************************************************************************************/
#define ASSERT_ENCODE_TYPE_VALID(type) \
ASSERT(type <= encodingBase64Url);
ASSERT(type <= encodingHex);
/***********************************************************************************************************************************
Base64 encoding/decoding
@ -225,7 +225,7 @@ decodeToBinSizeBase64(const char *source)
}
/***********************************************************************************************************************************
Base64 encoding
Base64Url encoding
***********************************************************************************************************************************/
static const char encodeBase64LookupUrl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
@ -314,6 +314,131 @@ encodeToStrSizeBase64Url(size_t sourceSize)
FUNCTION_TEST_RETURN(SIZE, encodeTotal);
}
/***********************************************************************************************************************************
Hex encoding/decoding
***********************************************************************************************************************************/
static const char encodeHexLookup[] = "0123456789abcdef";
static void
encodeToStrHex(const unsigned char *const source, const size_t sourceSize, char *const destination)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(UCHARDATA, source);
FUNCTION_TEST_PARAM(SIZE, sourceSize);
FUNCTION_TEST_PARAM_P(CHARDATA, destination);
FUNCTION_TEST_END();
ASSERT(source != NULL);
ASSERT(destination != NULL);
unsigned int destinationIdx = 0;
// Encode the string from one bytes to two characters
for (unsigned int sourceIdx = 0; sourceIdx < sourceSize; sourceIdx += 1)
{
destination[destinationIdx++] = encodeHexLookup[source[sourceIdx] >> 4];
destination[destinationIdx++] = encodeHexLookup[source[sourceIdx] & 0xF];
}
// Zero-terminate the string
destination[destinationIdx] = 0;
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
static size_t
encodeToStrSizeHex(const size_t sourceSize)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(SIZE, sourceSize);
FUNCTION_TEST_END();
FUNCTION_TEST_RETURN(SIZE, sourceSize * 2);
}
/**********************************************************************************************************************************/
static const int decodeHexLookup[256] =
{
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
};
static void
decodeToBinValidateHex(const char *const source)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRINGZ, source);
FUNCTION_TEST_END();
// Check for the correct length
const size_t sourceSize = strlen(source);
if (sourceSize % 2 != 0)
THROW_FMT(FormatError, "hex size %zu is not evenly divisible by 2", sourceSize);
// Check all characters
for (unsigned int sourceIdx = 0; sourceIdx < sourceSize; sourceIdx++)
{
if (decodeHexLookup[(int)source[sourceIdx]] == -1)
THROW_FMT(FormatError, "hex invalid character found at position %u", sourceIdx);
}
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
static void
decodeToBinHex(const char *const source, unsigned char *const destination)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRINGZ, source);
FUNCTION_TEST_PARAM_P(UCHARDATA, destination);
FUNCTION_TEST_END();
// Validate encoded string
decodeToBinValidateHex(source);
int destinationIdx = 0;
// Decode the binary data from two characters to one byte
for (unsigned int sourceIdx = 0; sourceIdx < strlen(source); sourceIdx += 2)
{
destination[destinationIdx++] =
(unsigned char)(decodeHexLookup[(int)source[sourceIdx]] << 4 | decodeHexLookup[(int)source[sourceIdx + 1]]);
}
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
static size_t
decodeToBinSizeHex(const char *const source)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRINGZ, source);
FUNCTION_TEST_END();
// Validate encoded string
decodeToBinValidateHex(source);
FUNCTION_TEST_RETURN(SIZE, strlen(source) / 2);
}
/***********************************************************************************************************************************
Generic encoding/decoding
***********************************************************************************************************************************/
@ -338,6 +463,10 @@ encodeToStr(const EncodingType type, const unsigned char *const source, const si
case encodingBase64Url:
encodeToStrBase64Url(source, sourceSize, destination);
break;
case encodingHex:
encodeToStrHex(source, sourceSize, destination);
break;
}
FUNCTION_TEST_RETURN_VOID();
@ -365,6 +494,10 @@ encodeToStrSize(const EncodingType type, const size_t sourceSize)
case encodingBase64Url:
destinationSize = encodeToStrSizeBase64Url(sourceSize);
break;
case encodingHex:
destinationSize = encodeToStrSizeHex(sourceSize);
break;
}
FUNCTION_TEST_RETURN(SIZE, destinationSize);
@ -388,6 +521,10 @@ decodeToBin(const EncodingType type, const char *const source, unsigned char *co
decodeToBinBase64(source, destination);
break;
case encodingHex:
decodeToBinHex(source, destination);
break;
default:
ASSERT_MSG("unsupported");
}
@ -414,6 +551,10 @@ decodeToBinSize(const EncodingType type, const char *const source)
destinationSize = decodeToBinSizeBase64(source);
break;
case encodingHex:
destinationSize = decodeToBinSizeHex(source);
break;
default:
ASSERT_MSG("unsupported");
}

1
src/common/encode.h
View File

@ -13,6 +13,7 @@ typedef enum
{
encodingBase64,
encodingBase64Url,
encodingHex,
} EncodingType;
/***********************************************************************************************************************************

18
src/common/type/buffer.c
View File

@ -207,24 +207,6 @@ bufEq(const Buffer *this, const Buffer *compare)
FUNCTION_TEST_RETURN(BOOL, false);
}
/**********************************************************************************************************************************/
String *
bufHex(const Buffer *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BUFFER, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
String *result = strNew();
for (unsigned int bufferIdx = 0; bufferIdx < bufUsed(this); bufferIdx++)
strCatFmt(result, "%02x", bufPtrConst(this)[bufferIdx]);
FUNCTION_TEST_RETURN(STRING, result);
}
/**********************************************************************************************************************************/
Buffer *
bufResize(Buffer *this, size_t size)

3
src/common/type/buffer.h
View File

@ -125,9 +125,6 @@ Buffer *bufCatSub(Buffer *this, const Buffer *cat, size_t catOffset, size_t catS
// Are two buffers equal?
bool bufEq(const Buffer *this, const Buffer *compare);
// Convert the buffer to a hex string
String *bufHex(const Buffer *this);
// Move to a new parent mem context
FN_INLINE_ALWAYS Buffer *
bufMove(Buffer *const this, MemContext *const parentNew)

6
src/common/type/string.c
View File

@ -220,7 +220,13 @@ strNewEncode(const EncodingType type, const Buffer *const buffer)
String *this = strNewFixed(encodeToStrSize(type, bufUsed(buffer)));
// Encode buffer
if (bufUsed(buffer) > 0)
{
encodeToStr(type, bufPtrConst(buffer), bufUsed(buffer), this->pub.buffer);
}
// Else zero-terminate
else
this->pub.buffer[0] = '\0';
FUNCTION_TEST_RETURN(STRING, this);
}

7
src/info/info.c
View File

@ -287,7 +287,8 @@ infoNewLoad(IoRead *read, InfoLoadNewCallback *callbackFunction, void *callbackD
INFO_CHECKSUM_END(data.checksumActual);
// Verify the checksum
const String *const checksumActual = bufHex(pckReadBinP(pckReadNew(ioFilterResult(data.checksumActual))));
const String *const checksumActual = strNewEncode(
encodingHex, pckReadBinP(pckReadNew(ioFilterResult(data.checksumActual))));
if (data.checksumExpected == NULL)
THROW_FMT(ChecksumError, "invalid checksum, actual '%s' but no checksum found", strZ(checksumActual));
@ -426,7 +427,9 @@ infoSave(Info *this, IoWrite *write, InfoSaveCallback *callbackFunction, void *c
INFO_CHECKSUM_END(data.checksum);
ioWrite(data.write, BUFSTRDEF("\n[" INFO_SECTION_BACKREST "]\n" INFO_KEY_CHECKSUM "="));
ioWriteLine(data.write, BUFSTR(jsonFromVar(VARSTR(bufHex(pckReadBinP(pckReadNew(ioFilterResult(data.checksum))))))));
ioWriteLine(
data.write,
BUFSTR(jsonFromVar(VARSTR(strNewEncode(encodingHex, pckReadBinP(pckReadNew(ioFilterResult(data.checksum))))))));
// Close the file
ioWriteClose(data.write);

4
src/storage/gcs/write.c
View File

@ -95,8 +95,8 @@ storageWriteGcsVerify(StorageWriteGcs *const this, HttpResponse *const response)
if (!bufEq(md5actual, md5expected))
{
THROW_FMT(
FormatError, "expected md5 '%s' for '%s' but actual is '%s'", strZ(bufHex(md5expected)), strZ(this->interface.name),
strZ(bufHex(md5actual)));
FormatError, "expected md5 '%s' for '%s' but actual is '%s'", strZ(strNewEncode(encodingHex, md5expected)),
strZ(this->interface.name), strZ(strNewEncode(encodingHex, md5actual)));
}
// Check the size when available

6
src/storage/s3/storage.c
View File

@ -204,7 +204,7 @@ storageS3Auth(
// Generate string to sign
const String *stringToSign = strNewFmt(
AWS4_HMAC_SHA256 "\n%s\n%s/%s/" S3 "/" AWS4_REQUEST "\n%s", strZ(dateTime), strZ(date), strZ(this->region),
strZ(bufHex(cryptoHashOne(hashTypeSha256, BUFSTR(canonicalRequest)))));
strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha256, BUFSTR(canonicalRequest)))));
// Generate signing key. This key only needs to be regenerated every seven days but we'll do it once a day to keep the
// logic simple. It's a relatively expensive operation so we'd rather not do it for every request.
@ -229,7 +229,7 @@ storageS3Auth(
const String *authorization = strNewFmt(
AWS4_HMAC_SHA256 " Credential=%s/%s/%s/" S3 "/" AWS4_REQUEST ",SignedHeaders=%s,Signature=%s",
strZ(this->accessKey), strZ(date), strZ(this->region), strZ(signedHeaders),
strZ(bufHex(cryptoHmacOne(hashTypeSha256, this->signingKey, BUFSTR(stringToSign)))));
strZ(strNewEncode(encodingHex, cryptoHmacOne(hashTypeSha256, this->signingKey, BUFSTR(stringToSign)))));
httpHeaderPut(httpHeader, HTTP_HEADER_AUTHORIZATION_STR, authorization);
}
@ -533,7 +533,7 @@ storageS3RequestAsync(StorageS3 *this, const String *verb, const String *path, S
storageS3Auth(
this, verb, path, param.query, storageS3DateTime(time(NULL)), requestHeader,
param.content == NULL || bufEmpty(param.content) ?
HASH_TYPE_SHA256_ZERO_STR : bufHex(cryptoHashOne(hashTypeSha256, param.content)));
HASH_TYPE_SHA256_ZERO_STR : strNewEncode(encodingHex, cryptoHashOne(hashTypeSha256, param.content)));
// Send request
MEM_CONTEXT_PRIOR_BEGIN()

4
test/define.yaml
View File

@ -122,7 +122,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: encode
total: 2
total: 3
coverage:
- common/encode
@ -168,7 +168,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-buffer
total: 6
total: 5
coverage:
- common/type/buffer

3
test/src/common/harnessInfo.c
View File

@ -96,7 +96,8 @@ harnessInfoChecksum(const String *info)
// Append checksum to buffer
bufCat(result, BUFSTRDEF("\n[backrest]\nbackrest-checksum="));
bufCat(result, BUFSTR(jsonFromVar(VARSTR(bufHex(pckReadBinP(pckReadNew(ioFilterResult(data.checksum))))))));
bufCat(
result, BUFSTR(jsonFromVar(VARSTR(strNewEncode(encodingHex, pckReadBinP(pckReadNew(ioFilterResult(data.checksum))))))));
bufCat(result, BUFSTRDEF("\n"));
bufMove(result, memContextPrior());

2
test/src/common/harnessPack.c
View File

@ -54,7 +54,7 @@ String *hrnPackReadToStr(PackRead *read)
break;
case pckTypeBin:
strCatFmt(result, "%s", strZ(bufHex(pckReadBinP(read, .id = id))));
strCatFmt(result, "%s", strZ(strNewEncode(encodingHex, pckReadBinP(read, .id = id))));
break;
case pckTypeI32:

12
test/src/module/command/archivePushTest.c
View File

@ -314,7 +314,7 @@ testRun(void)
memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 1}, walBuffer1);
const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer1)));
const char *walBuffer1Sha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer1)));
HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1);
@ -375,7 +375,7 @@ testRun(void)
bufUsedSet(walBuffer2, bufSize(walBuffer2));
memset(bufPtr(walBuffer2), 0xFF, bufSize(walBuffer2));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11}, walBuffer2);
const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer2)));
const char *walBuffer2Sha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer2)));
HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer2);
@ -464,7 +464,7 @@ testRun(void)
bufUsedSet(walBuffer2, bufSize(walBuffer2));
memset(bufPtr(walBuffer2), 0xFF, bufSize(walBuffer2));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11}, walBuffer2);
walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer2)));
walBuffer2Sha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer2)));
HRN_STORAGE_PUT(storageTest, "pg/pg_wal/000000010000000100000002", walBuffer2, .comment = "write WAL");
argListTemp = strLstNew();
@ -758,7 +758,7 @@ testRun(void)
bufUsedSet(walBuffer1, bufSize(walBuffer1));
memset(bufPtr(walBuffer1), 0xFF, bufSize(walBuffer1));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94}, walBuffer1);
const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer1)));
const char *walBuffer1Sha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer1)));
HRN_STORAGE_PUT(storagePgWrite(),"pg_xlog/000000010000000100000001", walBuffer1);
@ -863,7 +863,7 @@ testRun(void)
bufUsedSet(walBuffer2, bufSize(walBuffer2));
memset(bufPtr(walBuffer2), 0x0C, bufSize(walBuffer2));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94}, walBuffer2);
const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer2)));
const char *walBuffer2Sha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer2)));
HRN_STORAGE_PUT(storagePgWrite(), "pg_xlog/000000010000000100000002", walBuffer2);
@ -912,7 +912,7 @@ testRun(void)
bufUsedSet(walBuffer3, bufSize(walBuffer3));
memset(bufPtr(walBuffer3), 0x44, bufSize(walBuffer3));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94}, walBuffer3);
const char *walBuffer3Sha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer3)));
const char *walBuffer3Sha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer3)));
HRN_STORAGE_PUT(storagePgWrite(), "pg_xlog/000000010000000100000003", walBuffer3);

11
test/src/module/command/backupTest.c
View File

@ -121,7 +121,8 @@ testBackupValidateList(
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(hashTypeSha1));
uint64_t size = bufUsed(storageGetP(read));
const String *checksum = bufHex(
const String *checksum = strNewEncode(
encodingHex,
pckReadBinP(ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE)));
strCatFmt(result, ", s=%" PRIu64, size);
@ -371,7 +372,7 @@ testBackupPqScript(unsigned int pgVersion, time_t backupTimeStart, TestBackupPqS
bufUsedSet(walBuffer, bufSize(walBuffer));
memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
hrnPgWalToBuffer((PgWal){.version = pgControl.version, .systemId = pgControl.systemId}, walBuffer);
const String *walChecksum = bufHex(cryptoHashOne(hashTypeSha1, walBuffer));
const String *walChecksum = strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer));
for (unsigned int walSegmentIdx = 0; walSegmentIdx < strLstSize(walSegmentList); walSegmentIdx++)
{
@ -2864,7 +2865,7 @@ testRun(void)
*(PageHeaderData *)(bufPtr(relation) + (PG_PAGE_SIZE_DEFAULT * 0x01)) = (PageHeaderData){.pd_upper = 0xFF};
HRN_STORAGE_PUT(storagePgWrite(), PG_PATH_BASE "/1/2", relation, .timeModified = backupTimeStart);
const char *rel1_2Sha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, relation)));
const char *rel1_2Sha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, relation)));
// File with bad page checksums
relation = bufNew(PG_PAGE_SIZE_DEFAULT * 5);
@ -2878,7 +2879,7 @@ testRun(void)
bufUsedSet(relation, bufSize(relation));
HRN_STORAGE_PUT(storagePgWrite(), PG_PATH_BASE "/1/3", relation, .timeModified = backupTimeStart);
const char *rel1_3Sha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, relation)));
const char *rel1_3Sha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, relation)));
// File with bad page checksum
relation = bufNew(PG_PAGE_SIZE_DEFAULT * 3);
@ -2891,7 +2892,7 @@ testRun(void)
bufUsedSet(relation, bufSize(relation));
HRN_STORAGE_PUT(storagePgWrite(), PG_PATH_BASE "/1/4", relation, .timeModified = backupTimeStart);
const char *rel1_4Sha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, relation)));
const char *rel1_4Sha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, relation)));
// Add a tablespace
HRN_STORAGE_PATH_CREATE(storagePgWrite(), PG_PATH_PGTBLSPC);

12
test/src/module/command/verifyTest.c
View File

@ -1004,7 +1004,7 @@ testRun(void)
bufUsedSet(walBuffer, bufSize(walBuffer));
memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .size = 1024 * 1024}, walBuffer);
const char *walBufferSha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer)));
const char *walBufferSha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer)));
HRN_STORAGE_PUT(
storageRepoIdxWrite(0),
@ -1060,7 +1060,7 @@ testRun(void)
storageRepoIdxWrite(0),
zNewFmt(
STORAGE_REPO_ARCHIVE "/11-2/0000000200000007/000000020000000700000FFF-%s",
strZ(bufHex(cryptoHashOne(hashTypeSha1, BUFSTRDEF("invalidsize"))))),
strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, BUFSTRDEF("invalidsize"))))),
BUFSTRDEF("invalidsize"), .comment = "WAL - invalid size");
HRN_STORAGE_PUT(
storageRepoIdxWrite(0),
@ -1638,7 +1638,7 @@ testRun(void)
bufUsedSet(walBuffer, bufSize(walBuffer));
memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .size = 1024 * 1024}, walBuffer);
const char *walBufferSha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer)));
const char *walBufferSha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer)));
HRN_STORAGE_PUT(
storageRepoWrite(),
@ -1850,7 +1850,7 @@ testRun(void)
bufUsedSet(walBuffer, bufSize(walBuffer));
memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .size = 1024 * 1024}, walBuffer);
const char *walBufferSha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer)));
const char *walBufferSha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer)));
HRN_STORAGE_PUT(
storageRepoIdxWrite(0),
@ -1921,7 +1921,7 @@ testRun(void)
bufUsedSet(walBuffer, bufSize(walBuffer));
memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .size = 1024 * 1024}, walBuffer);
const char *walBufferSha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer)));
const char *walBufferSha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer)));
HRN_STORAGE_PUT(
storageRepoIdxWrite(0),
@ -1976,7 +1976,7 @@ testRun(void)
bufUsedSet(walBuffer, bufSize(walBuffer));
memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .size = 1024 * 1024}, walBuffer);
const char *walBufferSha1 = strZ(bufHex(cryptoHashOne(hashTypeSha1, walBuffer)));
const char *walBufferSha1 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer)));
HRN_STORAGE_PUT(
storageRepoIdxWrite(0),

29
test/src/module/common/cryptoTest.c
View File

@ -321,9 +321,11 @@ testRun(void)
pckWriteEndP(packWrite);
TEST_ASSIGN(hash, cryptoHashNewPack(pckWriteResult(packWrite)), "create sha1 hash");
TEST_RESULT_STR_Z(bufHex(cryptoHash((CryptoHash *)ioFilterDriver(hash))), HASH_TYPE_SHA1_ZERO, " check empty hash");
TEST_RESULT_STR_Z(
bufHex(cryptoHash((CryptoHash *)ioFilterDriver(hash))), HASH_TYPE_SHA1_ZERO, " check empty hash again");
strNewEncode(encodingHex, cryptoHash((CryptoHash *)ioFilterDriver(hash))), HASH_TYPE_SHA1_ZERO, " check empty hash");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, cryptoHash((CryptoHash *)ioFilterDriver(hash))), HASH_TYPE_SHA1_ZERO,
" check empty hash again");
TEST_RESULT_VOID(ioFilterFree(hash), " free hash");
// -------------------------------------------------------------------------------------------------------------------------
@ -335,7 +337,7 @@ testRun(void)
TEST_RESULT_VOID(ioFilterProcessIn(hash, BUFSTRDEF("5")), " add 5");
TEST_RESULT_STR_Z(
bufHex(pckReadBinP(pckReadNew(ioFilterResult(hash)))), "8cb2237d0679ca88db6464eac60da96345513964",
strNewEncode(encodingHex, pckReadBinP(pckReadNew(ioFilterResult(hash)))), "8cb2237d0679ca88db6464eac60da96345513964",
" check small hash");
TEST_RESULT_VOID(ioFilterFree(hash), " free hash");
@ -343,7 +345,8 @@ testRun(void)
TEST_TITLE("md5 hash - zero bytes");
TEST_ASSIGN(hash, cryptoHashNew(hashTypeMd5), "create md5 hash");
TEST_RESULT_STR_Z(bufHex(pckReadBinP(pckReadNew(ioFilterResult(hash)))), HASH_TYPE_MD5_ZERO, "check empty hash");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(pckReadNew(ioFilterResult(hash)))), HASH_TYPE_MD5_ZERO, "check empty hash");
// Exercise most of the conditions in the local MD5 code
// -------------------------------------------------------------------------------------------------------------------------
@ -362,7 +365,9 @@ testRun(void)
TEST_RESULT_VOID(
ioFilterProcessIn(hash, BUFSTRZ("12345678901234567890123456789001234567890012345678901234")), "add 58 bytes");
TEST_RESULT_STR_Z(bufHex(pckReadBinP(pckReadNew(ioFilterResult(hash)))), "3318600bc9c1d379e91e4bae90721243", "check hash");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(pckReadNew(ioFilterResult(hash)))), "3318600bc9c1d379e91e4bae90721243",
"check hash");
// Full coverage of local MD5 requires processing > 511MB of data but that makes the test run too long. Instead we'll cheat
// a bit and initialize the context at 511MB to start. This does not produce a valid MD5 hash but does provide coverage of
@ -374,22 +379,26 @@ testRun(void)
((CryptoHash *)ioFilterDriver(hash))->md5Context->lo = 0x1fffffff;
TEST_RESULT_VOID(ioFilterProcessIn(hash, BUFSTRZ("1")), "add 1");
TEST_RESULT_STR_Z(bufHex(pckReadBinP(pckReadNew(ioFilterResult(hash)))), "5c99876f9cafa7f485eac9c7a8a2764c", "check hash");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(pckReadNew(ioFilterResult(hash)))), "5c99876f9cafa7f485eac9c7a8a2764c",
"check hash");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(hash, cryptoHashNew(hashTypeSha256), "create sha256 hash");
TEST_RESULT_STR_Z(bufHex(pckReadBinP(pckReadNew(ioFilterResult(hash)))), HASH_TYPE_SHA256_ZERO, " check empty hash");
TEST_RESULT_STR_Z(
strNewEncode(encodingHex, pckReadBinP(pckReadNew(ioFilterResult(hash)))), HASH_TYPE_SHA256_ZERO, " check empty hash");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_STR_Z(
bufHex(cryptoHashOne(hashTypeSha1, BUFSTRDEF("12345"))), "8cb2237d0679ca88db6464eac60da96345513964",
strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, BUFSTRDEF("12345"))), "8cb2237d0679ca88db6464eac60da96345513964",
" check small hash");
TEST_RESULT_STR_Z(
bufHex(cryptoHashOne(hashTypeSha1, BUFSTRDEF(""))), HASH_TYPE_SHA1_ZERO, " check empty hash");
strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, BUFSTRDEF(""))), HASH_TYPE_SHA1_ZERO, " check empty hash");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_STR_Z(
bufHex(
strNewEncode(
encodingHex,
cryptoHmacOne(
hashTypeSha256,
BUFSTRDEF("AWS4wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"),

52
test/src/module/common/encodeTest.c
View File

@ -139,5 +139,57 @@ testRun(void)
TEST_ERROR(decodeToBin(encodingBase64Url, "c3", destinationDecode), AssertError, "unsupported");
}
// *****************************************************************************************************************************
if (testBegin("hex"))
{
TEST_TITLE("encode");
const unsigned char *encode = (const unsigned char *)"string_to_encode\r\n";
char destinationEncode[256];
encodeToStr(encodingHex, encode, 1, destinationEncode);
TEST_RESULT_Z(destinationEncode, "73", "1 character encode");
TEST_RESULT_UINT(encodeToStrSize(encodingHex, 1), strlen(destinationEncode), "check size");
encodeToStr(encodingHex, encode, strlen((char *)encode), destinationEncode);
TEST_RESULT_Z(destinationEncode, "737472696e675f746f5f656e636f64650d0a", "encode full string with \\r\\n");
TEST_RESULT_UINT(encodeToStrSize(encodingHex, strlen((char *)encode)), strlen(destinationEncode), "check size");
encodeToStr(encodingHex, encode, strlen((char *)encode) + 1, destinationEncode);
TEST_RESULT_Z(destinationEncode, "737472696e675f746f5f656e636f64650d0a00", "encode full string with \\r\\n and null");
TEST_RESULT_UINT(encodeToStrSize(encodingHex, strlen((char *)encode) + 1), strlen(destinationEncode), "check size");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("decode");
unsigned char destinationDecode[256];
memset(destinationDecode, 0xFF, sizeof(destinationDecode));
const char *decode = "737472696e675f746f5f656e636f64650d0a00";
decodeToBin(encodingHex, decode, destinationDecode);
TEST_RESULT_Z((char *)destinationDecode, (char *)encode, "full string with \\r\\n and null decode");
TEST_RESULT_INT(destinationDecode[strlen((char *)encode) + 1], 0xFF, "check for overrun");
TEST_RESULT_UINT(decodeToBinSize(encodingHex, decode), strlen((char *)encode) + 1, "check size");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("decode/encode with mixed case");
const char *decodeMixed = "0123456789AaBbCcDdEeFf";
TEST_RESULT_VOID(decodeToBin(encodingHex, decodeMixed, destinationDecode), "decode");
TEST_RESULT_VOID(
encodeToStr(encodingHex, destinationDecode, decodeToBinSize(encodingHex, decodeMixed), destinationEncode), "encode");
TEST_RESULT_Z(destinationEncode, "0123456789aabbccddeeff", "check encoded hex");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("decode errors");
TEST_ERROR(decodeToBin(encodingHex, "c", destinationDecode), FormatError, "hex size 1 is not evenly divisible by 2");
TEST_ERROR(
decodeToBin(encodingHex, "hh", destinationDecode), FormatError,
"hex invalid character found at position 0");
}
FUNCTION_HARNESS_RETURN_VOID();
}

6
test/src/module/common/typeBufferTest.c
View File

@ -133,12 +133,6 @@ testRun(void)
TEST_RESULT_BOOL(bufEq(bufDup(BUFSTRZ("123")), BUF("123", 3)), true, "buffers equal");
}
// *****************************************************************************************************************************
if (testBegin("bufHex()"))
{
TEST_RESULT_STR_Z(bufHex(BUFSTRDEF("ABC-CBA")), "4142432d434241", "buffer to hex");
}
// *****************************************************************************************************************************
if (testBegin("bufCat*()"))
{

6
test/src/module/common/typePackTest.c
View File

@ -173,7 +173,7 @@ testRun(void)
"check pack string");
TEST_RESULT_STR_Z(
bufHex(pack),
strNewEncode(encodingHex, pack),
"98e803" // 1, u64, 750
"98fd9fad8f07" // 2, u64, 1911246845
"9c01ffffffffffffffffff01" // 7, u64, 0xFFFFFFFFFFFFFFFF
@ -304,7 +304,7 @@ testRun(void)
TEST_RESULT_INT(pckReadI32P(packRead, .id = 7, .defaultValue = 1), 1, "read default 1");
TEST_RESULT_VOID(pckReadArrayEndP(packRead), "read array end");
TEST_RESULT_STR_Z(bufHex(pckReadBinP(packRead)), "050403020100", "read bin");
TEST_RESULT_STR_Z(strNewEncode(encodingHex, pckReadBinP(packRead)), "050403020100", "read bin");
TEST_RESULT_PTR(pckReadBinP(packRead), NULL, "read bin null");
TEST_RESULT_UINT(bufSize(pckReadBinP(packRead)), 0, "read bin zero length");
@ -372,7 +372,7 @@ testRun(void)
TEST_RESULT_BOOL(pckReadNext(packRead), true, "next pack");
TEST_RESULT_UINT(pckReadSize(packRead), 4, "pack size");
TEST_RESULT_STR_Z(bufHex(BUF(pckReadBufPtr(packRead), pckReadSize(packRead))), "98890600", "pack hex");
TEST_RESULT_STR_Z(strNewEncode(encodingHex, BUF(pckReadBufPtr(packRead), pckReadSize(packRead))), "98890600", "pack hex");
TEST_RESULT_UINT(pckReadU64P(pckReadNewC(pckReadBufPtr(packRead), pckReadSize(packRead))), 777, "u64 value");
TEST_RESULT_VOID(pckReadConsume(packRead), "consume pack");

1
test/src/module/common/typeStringTest.c
View File

@ -90,6 +90,7 @@ testRun(void)
TEST_TITLE("strNewEncode()");
TEST_RESULT_STR_Z(strNewEncode(encodingBase64, BUFSTRDEF("zz")), "eno=", "encode base64");
TEST_RESULT_STR_Z(strNewEncode(encodingHex, bufNew(0)), "", "encode empty hex");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("fixed string large enough to need separate allocation");

3
test/src/module/storage/s3Test.c
View File

@ -117,7 +117,8 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
request,
"x-amz-content-sha256:%s\r\n"
"x-amz-date:????????T??????Z" "\r\n",
param.content == NULL ? HASH_TYPE_SHA256_ZERO : strZ(bufHex(cryptoHashOne(hashTypeSha256, BUFSTRZ(param.content)))));
param.content == NULL ?
HASH_TYPE_SHA256_ZERO : strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha256, BUFSTRZ(param.content)))));
// Add security token
if (securityToken != NULL)