db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Refactor common/ini module to remove callbacks and duplicated code.

Browse Source 
The callbacks in iniLoad() made the downstream code more complicated than it needed to be so use an iterator model instead.

Combine the two functions that were used to load the ini data to remove code duplication. In theory it would be nice to use iniValueNext() in the config/parse module rather than loading a KeyValue store but this would mean a big change to the parser, which does not seem worthwhile at this time.
This commit is contained in:
David Steele 2023-01-12 21:24:28 +07:00 committed by GitHub
parent 9ca492cecf
commit 34e4835ff3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 450 additions and 588 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/xml/release.xml
View File

@ -98,6 +98,17 @@
<p>Add meson unity build and tests.</p>
</release-item>
<release-item>
<github-pull-request id="1968"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="stefan.fercot"/>
</release-item-contributor-list>
<p>Refactor <id>common/ini</id> module to remove callbacks and duplicated code.</p>
</release-item>
</release-development-list>
</release-core-list>

414
src/common/ini.c
View File

@ -19,13 +19,21 @@ Object type
struct Ini
{
KeyValue *store; // Key value store that contains the ini data
IoRead *read; // Read object for ini data
IniValue value; // Current value
unsigned int lineIdx; // Current line used for error reporting
bool strict; // Expect all values to be JSON and do not trim
};
/**********************************************************************************************************************************/
FN_EXTERN Ini *
iniNew(void)
iniNew(IoRead *const read, const IniNewParam param)
{
FUNCTION_TEST_VOID();
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(IO_READ, read);
FUNCTION_LOG_PARAM(BOOL, param.strict);
FUNCTION_LOG_PARAM(BOOL, param.store);
FUNCTION_LOG_END();
Ini *this = NULL;
@ -35,12 +43,183 @@ iniNew(void)
*this = (Ini)
{
.store = kvNew(),
.read = read,
.value =
{
.section = strNew(),
.key = strNew(),
.value = strNew(),
},
.strict = param.strict,
};
ioReadOpen(this->read);
// Build KeyValue store
if (param.store)
{
this->store = kvNew();
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
const IniValue *value = iniValueNext(this);
// Add values while not done
while (value != NULL)
{
const Variant *const sectionKey = VARSTR(value->section);
KeyValue *sectionKv = varKv(kvGet(this->store, sectionKey));
if (sectionKv == NULL)
sectionKv = kvPutKv(this->store, sectionKey);
kvAdd(sectionKv, VARSTR(value->key), VARSTR(value->value));
MEM_CONTEXT_TEMP_RESET(1000);
value = iniValueNext(this);
}
}
MEM_CONTEXT_TEMP_END();
}
}
OBJ_NEW_END();
FUNCTION_TEST_RETURN(INI, this);
FUNCTION_LOG_RETURN(INI, this);
}
/**********************************************************************************************************************************/
FN_EXTERN const IniValue *
iniValueNext(Ini *const this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INI, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
const IniValue *result = NULL;
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
// Read until a value is found or eof
while (result == NULL && !ioReadEof(this->read))
{
// Read the line and trim if needed
String *const line = ioReadLineParam(this->read, true);
if (!this->strict)
strTrim(line);
const char *const linePtr = strZ(line);
this->lineIdx++;
// Only interested in lines that are not blank or comments
if (strSize(line) > 0 && (this->strict || linePtr[0] != '#'))
{
// The line is a section. Since the value must be valid JSON this means that the value must never be an array.
if (linePtr[0] == '[' && (!this->strict || linePtr[strSize(line) - 1] == ']'))
{
// Make sure the section ends with ]
if (!this->strict && linePtr[strSize(line) - 1] != ']')
THROW_FMT(FormatError, "ini section should end with ] at line %u: %s", this->lineIdx, linePtr);
// Store the section
strCatZN(strTrunc(this->value.section), linePtr + 1, strSize(line) - 2);
if (strEmpty(this->value.section))
THROW_FMT(FormatError, "invalid empty section at line %u: %s", this->lineIdx, linePtr);
}
// Else it is a key/value
else
{
if (strEmpty(this->value.section))
THROW_FMT(FormatError, "key/value found outside of section at line %u: %s", this->lineIdx, linePtr);
// Find the =
const char *lineEqual = strstr(linePtr, "=");
if (lineEqual == NULL)
THROW_FMT(FormatError, "missing '=' in key/value at line %u: %s", this->lineIdx, linePtr);
// Extract the key/value. For strict this may require some retries if the key includes an = character since this
// is also the separator. We know the value must be valid JSON so if it isn't then add the characters up to the
// next = to the key and try to parse the value as JSON again. If the value never becomes valid JSON then an
// error is thrown.
bool retry;
do
{
retry = false;
// Get key/value
strCatZN(strTrunc(this->value.key), linePtr, (size_t)(lineEqual - linePtr));
strCatZ(strTrunc(this->value.value), lineEqual + 1);
// Value is expected to be valid JSON
if (this->strict)
{
TRY_BEGIN()
{
jsonValidate(this->value.value);
}
CATCH(JsonFormatError)
{
// If value is not valid JSON look for another =. If not found then nothing to retry.
lineEqual = strstr(lineEqual + 1, "=");
if (lineEqual == NULL)
{
THROW_FMT(
FormatError, "invalid JSON value at line %u '%s': %s", this->lineIdx, linePtr,
errorMessage());
}
// Try again with = in new position
retry = true;
}
TRY_END();
}
// Else just trim
else
{
strTrim(this->value.key);
strTrim(this->value.value);
}
}
while (retry);
// Key may not be zero-length
if (strSize(this->value.key) == 0)
THROW_FMT(FormatError, "key is zero-length at line %u: %s", this->lineIdx, linePtr);
// A value was found so return it
result = &this->value;
}
}
MEM_CONTEXT_TEMP_RESET(1000);
}
// If no more values found then close
if (result == NULL)
ioReadClose(this->read);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_CONST(INI_VALUE, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
iniValid(Ini *const this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INI, this);
FUNCTION_TEST_END();
while (iniValueNext(this) != NULL);
FUNCTION_TEST_RETURN_VOID();
}
/***********************************************************************************************************************************
@ -57,6 +236,7 @@ iniGetInternal(const Ini *const this, const String *const section, const String
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(this->store != NULL);
ASSERT(section != NULL);
ASSERT(key != NULL);
@ -131,6 +311,7 @@ iniSectionKeyList(const Ini *const this, const String *const section)
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(this->store != NULL);
ASSERT(section != NULL);
StringList *result = NULL;
@ -155,228 +336,3 @@ iniSectionKeyList(const Ini *const this, const String *const section)
FUNCTION_TEST_RETURN(STRING_LIST, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
iniParse(Ini *this, const String *content)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INI, this);
FUNCTION_TEST_PARAM(STRING, content);
FUNCTION_TEST_END();
ASSERT(this != NULL);
MEM_CONTEXT_OBJ_BEGIN(this)
{
kvFree(this->store);
this->store = kvNew();
if (content != NULL)
{
MEM_CONTEXT_TEMP_BEGIN()
{
// Track the current section
String *section = NULL;
// Split the content into lines and loop
StringList *lines = strLstNewSplitZ(content, "\n");
for (unsigned int lineIdx = 0; lineIdx < strLstSize(lines); lineIdx++)
{
// Get next line
const String *line = strTrim(strLstGet(lines, lineIdx));
const char *linePtr = strZ(line);
// Only interested in lines that are not blank or comments
if (strSize(line) > 0 && linePtr[0] != '#')
{
// Looks like this line is a section
if (linePtr[0] == '[')
{
// Make sure the section ends with ]
if (linePtr[strSize(line) - 1] != ']')
THROW_FMT(FormatError, "ini section should end with ] at line %u: %s", lineIdx + 1, linePtr);
// Assign section
section = strNewZN(linePtr + 1, strSize(line) - 2);
}
// Else it should be a key/value
else
{
if (section == NULL)
THROW_FMT(FormatError, "key/value found outside of section at line %u: %s", lineIdx + 1, linePtr);
// Find the =
const char *lineEqual = strstr(linePtr, "=");
if (lineEqual == NULL)
THROW_FMT(FormatError, "missing '=' in key/value at line %u: %s", lineIdx + 1, linePtr);
// Extract the key
String *key = strTrim(strNewZN(linePtr, (size_t)(lineEqual - linePtr)));
if (strSize(key) == 0)
THROW_FMT(FormatError, "key is zero-length at line %u: %s", lineIdx++, linePtr);
// Store the section/key/value
iniSet(this, section, key, strTrim(strNewZ(lineEqual + 1)));
}
}
}
}
MEM_CONTEXT_TEMP_END();
}
}
MEM_CONTEXT_OBJ_END();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
iniSet(Ini *this, const String *section, const String *key, const String *value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INI, this);
FUNCTION_TEST_PARAM(STRING, section);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(STRING, value);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(section != NULL);
ASSERT(key != NULL);
ASSERT(value != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
const Variant *sectionKey = VARSTR(section);
KeyValue *sectionKv = varKv(kvGet(this->store, sectionKey));
if (sectionKv == NULL)
sectionKv = kvPutKv(this->store, sectionKey);
kvAdd(sectionKv, VARSTR(key), VARSTR(value));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
iniLoad(
IoRead *const read, void (*callbackFunction)(void *data, const String *section, const String *key, const String *value),
void *const callbackData)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(IO_READ, read);
FUNCTION_LOG_PARAM(FUNCTIONP, callbackFunction);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_END();
ASSERT(read != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// Track the current section/key/value
String *const section = strNew();
String *const key = strNew();
String *const value = strNew();
// Keep track of the line number for error reporting
unsigned int lineIdx = 0;
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
ioReadOpen(read);
do
{
const String *line = ioReadLineParam(read, true);
const char *linePtr = strZ(line);
// Only interested in lines that are not blank
if (strSize(line) > 0)
{
// The line is a section. Since the value must be valid JSON this means that the value must never be an array.
if (linePtr[0] == '[' && linePtr[strSize(line) - 1] == ']')
{
strCatZN(strTrunc(section), linePtr + 1, strSize(line) - 2);
if (strEmpty(section))
THROW_FMT(FormatError, "invalid empty section at line %u: %s", lineIdx + 1, linePtr);
}
// Else it is a key/value
else
{
if (strEmpty(section))
THROW_FMT(FormatError, "key/value found outside of section at line %u: %s", lineIdx + 1, linePtr);
// Find the =
const char *lineEqual = strstr(linePtr, "=");
if (lineEqual == NULL)
THROW_FMT(FormatError, "missing '=' in key/value at line %u: %s", lineIdx + 1, linePtr);
// Extract the key/value. This may require some retries if the key includes an = character since this is
// also the separator. We know the value must be valid JSON so if it isn't then add the characters up to
// the next = to the key and try to parse the value as JSON again. If the value never becomes valid JSON
// then an error is thrown.
bool retry;
do
{
retry = false;
// Get key/value
strCatZN(strTrunc(key), linePtr, (size_t)(lineEqual - linePtr));
strCatZ(strTrunc(value), lineEqual + 1);
// Check that the value is valid JSON
TRY_BEGIN()
{
jsonValidate(value);
}
CATCH(JsonFormatError)
{
// If value is not valid JSON look for another =. If not found then nothing to retry.
lineEqual = strstr(lineEqual + 1, "=");
if (lineEqual == NULL)
{
THROW_FMT(
FormatError, "invalid JSON value at line %u '%s': %s", lineIdx + 1, linePtr,
errorMessage());
}
// Try again with = in new position
retry = true;
}
TRY_END();
}
while (retry);
// Key may not be zero-length
if (strSize(key) == 0)
THROW_FMT(FormatError, "key is zero-length at line %u: %s", lineIdx + 1, linePtr);
// Callback with the section/key/value
callbackFunction(callbackData, section, key, value);
}
}
lineIdx++;
MEM_CONTEXT_TEMP_RESET(1000);
}
while (!ioReadEof(read));
ioReadClose(read);
}
MEM_CONTEXT_TEMP_END();
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}

48
src/common/ini.h
View File

@ -14,10 +14,30 @@ typedef struct Ini Ini;
#include "common/type/object.h"
#include "common/type/variant.h"
/***********************************************************************************************************************************
Ini section/key/value
***********************************************************************************************************************************/
typedef struct IniValue
{
String *section;
String *key;
String *value;
} IniValue;
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN Ini *iniNew(void);
typedef struct IniNewParam
{
VAR_PARAM_HEADER;
bool strict; // Expect all values to be JSON and do not trim
bool store; // Store in KeyValue so functions can be used to query
} IniNewParam;
#define iniNewP(read, ...) \
iniNew(read, (IniNewParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN Ini *iniNew(IoRead *read, IniNewParam param);
/***********************************************************************************************************************************
Functions
@ -29,15 +49,15 @@ iniMove(Ini *const this, MemContext *const parentNew)
return objMove(this, parentNew);
}
// Parse ini from a string. Comments are ignored and additional whitespace around sections, keys, and values is trimmed. Should be
// used *only* to read user-generated config files, for code-generated info files see iniLoad().
FN_EXTERN void iniParse(Ini *this, const String *content);
// Check that the ini is valid
FN_EXTERN void iniValid(Ini *this);
// Set an ini value
FN_EXTERN void iniSet(Ini *this, const String *section, const String *key, const String *value);
// Get the next value in the ini file. Note that members of IniValue are reused between calls so the members of a previous call may
// change after the next call. Any members that need to be preserved should be copied before the next call.
FN_EXTERN const IniValue *iniValueNext(Ini *this);
/***********************************************************************************************************************************
Getters/Setters
Getters
***********************************************************************************************************************************/
// Get an ini value -- error if it does not exist
FN_EXTERN const String *iniGet(const Ini *this, const String *section, const String *key);
@ -60,16 +80,6 @@ iniFree(Ini *const this)
objFree(this);
}
/***********************************************************************************************************************************
Helper Functions
***********************************************************************************************************************************/
// Load an ini file and return data to a callback. Intended to read info files that were generated by code so do not have comments
// or extraneous spaces, and where all values are valid JSON. This allows syntax characters such as [, =, #, and whitespace to be
// used in keys.
FN_EXTERN void iniLoad(
IoRead *read, void (*callbackFunction)(void *data, const String *section, const String *key, const String *value),
void *callbackData);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
@ -77,5 +87,9 @@ Macros for function logging
Ini *
#define FUNCTION_LOG_INI_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "Ini", buffer, bufferSize)
#define FUNCTION_LOG_INI_VALUE_TYPE \
IniValue *
#define FUNCTION_LOG_INI_VALUE_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "IniValue", buffer, bufferSize)
#endif

21
src/config/parse.c
View File

@ -12,6 +12,7 @@ Command and Option Parse
#include "common/debug.h"
#include "common/error.h"
#include "common/ini.h"
#include "common/io/bufferRead.h"
#include "common/log.h"
#include "common/macro.h"
#include "common/memContext.h"
@ -1228,13 +1229,10 @@ cfgFileLoadPart(String **config, const Buffer *configPart)
if (configPart != NULL)
{
String *configPartStr = strNewBuf(configPart);
// Validate the file by parsing it as an Ini object. If the file is not properly formed, an error will occur.
if (strSize(configPartStr) > 0)
if (bufUsed(configPart) > 0)
{
Ini *configPartIni = iniNew();
iniParse(configPartIni, configPartStr);
iniValid(iniNewP(ioBufferReadNew(configPart)));
// Create the result config file
if (*config == NULL)
@ -1246,7 +1244,7 @@ cfgFileLoadPart(String **config, const Buffer *configPart)
strCat(*config, LF_STR);
// Add the config part to the result config file
strCat(*config, configPartStr);
strCat(*config, strNewBuf(configPart));
}
}
@ -1346,13 +1344,10 @@ cfgFileLoad( // NOTE: Pas
// Load *.conf files from the include directory
if (loadConfigInclude)
{
if (result != NULL)
{
// Validate the file by parsing it as an Ini object. If the file is not properly formed, an error will occur.
Ini *ini = iniNew();
iniParse(ini, result);
}
if (result != NULL)
iniValid(iniNewP(ioBufferReadNew(BUFSTR(result))));
const String *configIncludePath = NULL;
@ -1761,8 +1756,8 @@ configParse(const Storage *storage, unsigned int argListSize, const char *argLis
if (configString != NULL)
{
Ini *ini = iniNew();
iniParse(ini, configString);
const Ini *const ini = iniNewP(ioBufferReadNew(BUFSTR(configString)), .store = true);
// Get the stanza name
String *stanza = NULL;

240
src/info/info.c
View File

@ -25,7 +25,6 @@ Object types
struct Info
{
InfoPub pub; // Publicly accessible variables
MemContext *memContext; // Mem context
};
struct InfoSave
@ -93,24 +92,6 @@ BUFFER_STRDEF_STATIC(INFO_CHECKSUM_END_BUF, "}}");
} \
while (0)
/***********************************************************************************************************************************
Internal constructor
***********************************************************************************************************************************/
static Info *
infoNewInternal(void)
{
FUNCTION_TEST_VOID();
Info *this = OBJ_NEW_ALLOC();
*this = (Info)
{
.memContext = memContextCurrent(),
};
FUNCTION_TEST_RETURN(INFO, this);
}
/**********************************************************************************************************************************/
FN_EXTERN Info *
infoNew(const String *cipherPass)
@ -123,7 +104,8 @@ infoNew(const String *cipherPass)
OBJ_NEW_BEGIN(Info, .childQty = MEM_CONTEXT_QTY_MAX)
{
this = infoNewInternal();
this = OBJ_NEW_ALLOC();
*this = (Info){};
// Cipher used to encrypt/decrypt subsequent dependent files. Value may be NULL.
infoCipherPassSet(this, cipherPass);
@ -134,117 +116,14 @@ infoNew(const String *cipherPass)
FUNCTION_LOG_RETURN(INFO, this);
}
/***********************************************************************************************************************************
Load and validate the info file (or copy)
***********************************************************************************************************************************/
/**********************************************************************************************************************************/
#define INFO_SECTION_BACKREST "backrest"
#define INFO_KEY_CHECKSUM "backrest-checksum"
#define INFO_SECTION_CIPHER "cipher"
#define INFO_KEY_CIPHER_PASS "cipher-pass"
typedef struct InfoLoadData
{
MemContext *memContext; // Mem context to use for storing data in this structure
InfoLoadNewCallback *callbackFunction; // Callback function for child object
void *callbackData; // Callback data for child object
Info *info; // Info object
String *sectionLast; // The last section seen during load
IoFilter *checksumActual; // Checksum calculated from the file
const String *checksumExpected; // Checksum found in ini file
} InfoLoadData;
static void
infoLoadCallback(void *const data, const String *const section, const String *const key, const String *const value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(STRING, section);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(STRING, value);
FUNCTION_TEST_END();
FUNCTION_AUDIT_CALLBACK();
ASSERT(data != NULL);
ASSERT(section != NULL);
ASSERT(key != NULL);
ASSERT(value != NULL);
InfoLoadData *const loadData = (InfoLoadData *)data;
// Calculate checksum
if (!(strEqZ(section, INFO_SECTION_BACKREST) && strEqZ(key, INFO_KEY_CHECKSUM)))
{
if (loadData->sectionLast == NULL || !strEq(section, loadData->sectionLast))
{
if (loadData->sectionLast != NULL)
INFO_CHECKSUM_SECTION_NEXT(loadData->checksumActual);
INFO_CHECKSUM_SECTION(loadData->checksumActual, section);
MEM_CONTEXT_BEGIN(loadData->memContext)
{
loadData->sectionLast = strDup(section);
}
MEM_CONTEXT_END();
}
else
INFO_CHECKSUM_KEY_VALUE_NEXT(loadData->checksumActual);
INFO_CHECKSUM_KEY_VALUE(loadData->checksumActual, key, value);
}
// Process backrest section
if (strEqZ(section, INFO_SECTION_BACKREST))
{
// Validate format
if (strEqZ(key, INFO_KEY_FORMAT))
{
if (varUInt64(jsonToVar(value)) != REPOSITORY_FORMAT)
THROW_FMT(FormatError, "expected format %d but found %" PRIu64, REPOSITORY_FORMAT, varUInt64(jsonToVar(value)));
}
// Store pgBackRest version
else if (strEqZ(key, INFO_KEY_VERSION))
{
MEM_CONTEXT_BEGIN(loadData->info->memContext)
{
loadData->info->pub.backrestVersion = varStr(jsonToVar(value));
}
MEM_CONTEXT_END();
}
// Store checksum to be validated later
else if (strEqZ(key, INFO_KEY_CHECKSUM))
{
MEM_CONTEXT_BEGIN(loadData->memContext)
{
loadData->checksumExpected = varStr(jsonToVar(value));
}
MEM_CONTEXT_END();
}
}
// Process cipher section
else if (strEqZ(section, INFO_SECTION_CIPHER))
{
// No validation needed for cipher-pass, just store it
if (strEqZ(key, INFO_KEY_CIPHER_PASS))
{
MEM_CONTEXT_BEGIN(loadData->info->memContext)
{
loadData->info->pub.cipherPass = varStr(jsonToVar(value));
}
MEM_CONTEXT_END();
}
}
// Else pass to callback for processing
else
loadData->callbackFunction(loadData->callbackData, section, key, value);
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN Info *
infoNewLoad(IoRead *read, InfoLoadNewCallback *callbackFunction, void *callbackData)
infoNewLoad(IoRead *const read, InfoLoadNewCallback *const callbackFunction, void *const callbackData)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(IO_READ, read);
@ -262,25 +141,98 @@ infoNewLoad(IoRead *read, InfoLoadNewCallback *callbackFunction, void *callbackD
OBJ_NEW_BEGIN(Info, .childQty = MEM_CONTEXT_QTY_MAX)
{
this = infoNewInternal();
this = OBJ_NEW_ALLOC();
*this = (Info){};
MEM_CONTEXT_TEMP_BEGIN()
{
// Load and parse the info file
InfoLoadData data =
{
.memContext = MEM_CONTEXT_TEMP(),
.callbackFunction = callbackFunction,
.callbackData = callbackData,
.info = this,
.checksumActual = cryptoHashNew(hashTypeSha1),
};
String *const sectionLast = strNew(); // The last section seen during load
IoFilter *const checksumActualFilter = cryptoHashNew(hashTypeSha1); // Checksum calculated from the file
const String *checksumExpected = NULL; // Checksum found in ini file
INFO_CHECKSUM_BEGIN(data.checksumActual);
INFO_CHECKSUM_BEGIN(checksumActualFilter);
TRY_BEGIN()
{
iniLoad(read, infoLoadCallback, &data);
Ini *const ini = iniNewP(read, .strict = true);
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
const IniValue *value = iniValueNext(ini);
while (value != NULL)
{
// Calculate checksum
if (!(strEqZ(value->section, INFO_SECTION_BACKREST) && strEqZ(value->key, INFO_KEY_CHECKSUM)))
{
if (strEmpty(sectionLast) || !strEq(value->section, sectionLast))
{
if (!strEmpty(sectionLast))
INFO_CHECKSUM_SECTION_NEXT(checksumActualFilter);
INFO_CHECKSUM_SECTION(checksumActualFilter, value->section);
strCat(strTrunc(sectionLast), value->section);
}
else
INFO_CHECKSUM_KEY_VALUE_NEXT(checksumActualFilter);
INFO_CHECKSUM_KEY_VALUE(checksumActualFilter, value->key, value->value);
}
// Process backrest section
if (strEqZ(value->section, INFO_SECTION_BACKREST))
{
// Validate format
if (strEqZ(value->key, INFO_KEY_FORMAT))
{
if (varUInt64(jsonToVar(value->value)) != REPOSITORY_FORMAT)
{
THROW_FMT(
FormatError, "expected format %d but found %" PRIu64, REPOSITORY_FORMAT,
varUInt64(jsonToVar(value->value)));
}
}
// Store pgBackRest version
else if (strEqZ(value->key, INFO_KEY_VERSION))
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->pub.backrestVersion = varStr(jsonToVar(value->value));
}
MEM_CONTEXT_END();
}
// Store checksum to be validated later
else if (strEqZ(value->key, INFO_KEY_CHECKSUM))
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
checksumExpected = varStr(jsonToVar(value->value));
}
MEM_CONTEXT_END();
}
}
// Process cipher section
else if (strEqZ(value->section, INFO_SECTION_CIPHER))
{
// No validation needed for cipher-pass, just store it
if (strEqZ(value->key, INFO_KEY_CIPHER_PASS))
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->pub.cipherPass = varStr(jsonToVar(value->value));
}
MEM_CONTEXT_END();
}
}
// Else pass to callback for processing
else
callbackFunction(callbackData, value->section, value->key, value->value);
value = iniValueNext(ini);
MEM_CONTEXT_TEMP_RESET(1000);
}
}
MEM_CONTEXT_TEMP_END();
}
CATCH(CryptoError)
{
@ -288,19 +240,19 @@ infoNewLoad(IoRead *read, InfoLoadNewCallback *callbackFunction, void *callbackD
}
TRY_END();
INFO_CHECKSUM_END(data.checksumActual);
INFO_CHECKSUM_END(checksumActualFilter);
// Verify the checksum
const String *const checksumActual = strNewEncode(
encodingHex, pckReadBinP(pckReadNew(ioFilterResult(data.checksumActual))));
encodingHex, pckReadBinP(pckReadNew(ioFilterResult(checksumActualFilter))));
if (data.checksumExpected == NULL)
if (checksumExpected == NULL)
THROW_FMT(ChecksumError, "invalid checksum, actual '%s' but no checksum found", strZ(checksumActual));
else if (!strEq(data.checksumExpected, checksumActual))
else if (!strEq(checksumExpected, checksumActual))
{
THROW_FMT(
ChecksumError, "invalid checksum, actual '%s' but expected '%s'", strZ(checksumActual),
strZ(data.checksumExpected));
strZ(checksumExpected));
}
}
MEM_CONTEXT_TEMP_END();
@ -456,11 +408,11 @@ infoCipherPassSet(Info *this, const String *cipherPass)
FUNCTION_TEST_PARAM(STRING, cipherPass);
FUNCTION_TEST_END();
FUNCTION_AUDIT_IF(memContextCurrent() != this->memContext); // Do not audit calls from within the object
FUNCTION_AUDIT_IF(memContextCurrent() != objMemContext(this)); // Do not audit calls from within the object
ASSERT(this != NULL);
MEM_CONTEXT_BEGIN(this->memContext)
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->pub.cipherPass = strDup(cipherPass);
}

2
test/define.yaml
View File

@ -385,7 +385,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: ini
total: 4
total: 2
coverage:
- common/ini

82
test/src/common/harnessInfo.c
View File

@ -19,46 +19,9 @@ Harness for Loading Test Configurations
/***********************************************************************************************************************************
Add header and checksum to an info file
This prevents churn in headers and checksums in the unit tests. We purposefully do not use the checksum macros from the info module
This prevents churn in headers and checksums in the unit tests. We purposefully do not use the checksum macros from the info module
here as a cross-check of that code.
***********************************************************************************************************************************/
typedef struct HarnessInfoChecksumData
{
MemContext *memContext; // Mem context to use for storing data in this structure
String *sectionLast; // The last section seen during load
IoFilter *checksum; // Checksum calculated from the file
} HarnessInfoChecksumData;
static void
harnessInfoChecksumCallback(
void *const callbackData, const String *const section, const String *const key, const String *const value)
{
HarnessInfoChecksumData *data = (HarnessInfoChecksumData *)callbackData;
// Calculate checksum
if (data->sectionLast == NULL || !strEq(section, data->sectionLast))
{
if (data->sectionLast != NULL)
ioFilterProcessIn(data->checksum, BUFSTRDEF("},"));
ioFilterProcessIn(data->checksum, BUFSTRDEF("\""));
ioFilterProcessIn(data->checksum, BUFSTR(section));
ioFilterProcessIn(data->checksum, BUFSTRDEF("\":{"));
MEM_CONTEXT_BEGIN(data->memContext)
{
data->sectionLast = strDup(section);
}
MEM_CONTEXT_END();
}
else
ioFilterProcessIn(data->checksum, BUFSTRDEF(","));
ioFilterProcessIn(data->checksum, BUFSTR(jsonFromVar(VARSTR(key))));
ioFilterProcessIn(data->checksum, BUFSTRDEF(":"));
ioFilterProcessIn(data->checksum, BUFSTR(value));
}
Buffer *
harnessInfoChecksum(const String *info)
{
@ -72,12 +35,8 @@ harnessInfoChecksum(const String *info)
MEM_CONTEXT_TEMP_BEGIN()
{
// Initialize callback data
HarnessInfoChecksumData data =
{
.memContext = MEM_CONTEXT_TEMP(),
.checksum = cryptoHashNew(hashTypeSha1),
};
const String *sectionLast = NULL; // The last section seen during load
IoFilter *const checksum = cryptoHashNew(hashTypeSha1); // Checksum calculated from the file
// Create buffer with space for data, header, and checksum
result = bufNew(strSize(info) + 256);
@ -90,14 +49,39 @@ harnessInfoChecksum(const String *info)
bufCat(result, BUFSTR(info));
// Generate checksum by loading ini file
ioFilterProcessIn(data.checksum, BUFSTRDEF("{"));
iniLoad(ioBufferReadNew(result), harnessInfoChecksumCallback, &data);
ioFilterProcessIn(data.checksum, BUFSTRDEF("}}"));
ioFilterProcessIn(checksum, BUFSTRDEF("{"));
Ini *const ini = iniNewP(ioBufferReadNew(result), .strict = true);
const IniValue *value = iniValueNext(ini);
while (value != NULL)
{
if (sectionLast == NULL || !strEq(value->section, sectionLast))
{
if (sectionLast != NULL)
ioFilterProcessIn(checksum, BUFSTRDEF("},"));
ioFilterProcessIn(checksum, BUFSTRDEF("\""));
ioFilterProcessIn(checksum, BUFSTR(value->section));
ioFilterProcessIn(checksum, BUFSTRDEF("\":{"));
sectionLast = strDup(value->section);
}
else
ioFilterProcessIn(checksum, BUFSTRDEF(","));
ioFilterProcessIn(checksum, BUFSTR(jsonFromVar(VARSTR(value->key))));
ioFilterProcessIn(checksum, BUFSTRDEF(":"));
ioFilterProcessIn(checksum, BUFSTR(value->value));
value = iniValueNext(ini);
}
ioFilterProcessIn(checksum, BUFSTRDEF("}}"));
// Append checksum to buffer
bufCat(result, BUFSTRDEF("\n[backrest]\nbackrest-checksum="));
bufCat(
result, BUFSTR(jsonFromVar(VARSTR(strNewEncode(encodingHex, pckReadBinP(pckReadNew(ioFilterResult(data.checksum))))))));
bufCat(result, BUFSTR(jsonFromVar(VARSTR(strNewEncode(encodingHex, pckReadBinP(pckReadNew(ioFilterResult(checksum))))))));
bufCat(result, BUFSTRDEF("\n"));
bufMove(result, memContextPrior());

192
test/src/module/common/iniTest.c
View File

@ -8,10 +8,19 @@ Test Ini
/***********************************************************************************************************************************
Test callback to accumulate ini load results
***********************************************************************************************************************************/
static void
testIniLoadCallback(void *data, const String *section, const String *key, const String *value)
static String *
testIniNextValue(Ini *const ini)
{
strCatFmt((String *)data, "%s:%s:%s\n", strZ(section), strZ(key), strZ(value));
String *const result = strNew();
const IniValue *value = iniValueNext(ini);
while (value != NULL)
{
strCatFmt(result, "%s:%s:%s\n", strZ(value->section), strZ(value->key), strZ(value->value));
value = iniValueNext(ini);
}
return result;
}
/***********************************************************************************************************************************
@ -23,86 +32,53 @@ testRun(void)
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("iniLoad()"))
if (testBegin("iniNewP() strict, iniValid()"))
{
// Empty file
// -------------------------------------------------------------------------------------------------------------------------
const Buffer *iniBuf = bufNew(0);
String *result = strNew();
TEST_RESULT_VOID(iniLoad(ioBufferReadNew(iniBuf), testIniLoadCallback, result), "load ini");
TEST_RESULT_STR_Z(result, "", " check ini");
// Key outside of section
// -------------------------------------------------------------------------------------------------------------------------
iniBuf = BUFSTRZ(
"key=value\n");
TEST_TITLE("errors");
TEST_ERROR(
iniLoad(ioBufferReadNew(iniBuf), testIniLoadCallback, result), FormatError,
testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("key=value\n")), .strict = true)), FormatError,
"key/value found outside of section at line 1: key=value");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("empty section");
iniBuf = BUFSTRZ(
"\n[]\n");
TEST_ERROR(
iniLoad(ioBufferReadNew(iniBuf), testIniLoadCallback, result), FormatError,
testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("\n[]\n")), .strict = true)), FormatError,
"invalid empty section at line 2: []");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("invalid JSON value");
iniBuf = BUFSTRZ("[section]\nkey=value\n");
TEST_ERROR(
iniLoad(ioBufferReadNew(iniBuf), testIniLoadCallback, result), FormatError,
testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("[section]\nkey=value\n")), .strict = true)), FormatError,
"invalid JSON value at line 2 'key=value': invalid type at: value");
// Key outside of section
// -------------------------------------------------------------------------------------------------------------------------
iniBuf = BUFSTRZ(
"[section]\n"
"key");
TEST_ERROR(
iniLoad(ioBufferReadNew(iniBuf), testIniLoadCallback, result), FormatError,
testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("[section]\n""key")), .strict = true)), FormatError,
"missing '=' in key/value at line 2: key");
// Zero length key
// -------------------------------------------------------------------------------------------------------------------------
iniBuf = BUFSTRZ(
"[section]\n"
"=\"value\"");
TEST_ERROR(
iniLoad(ioBufferReadNew(iniBuf), testIniLoadCallback, result), FormatError,
testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("[section]\n=\"value\"")), .strict = true)), FormatError,
"key is zero-length at line 2: =\"value\"");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("empty ini");
TEST_RESULT_STR_Z(testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("")), .strict = true)), "", "check");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("one section");
iniBuf = BUFSTRZ(
const Buffer *iniBuf = BUFSTRZ(
"[section1]\n"
" key1 =\"value1\"\n"
"key2=\"value2\"\n"
"key=3==\"value3\"\n"
"==\"=\"");
result = strNew();
TEST_RESULT_VOID(iniLoad(ioBufferReadNew(iniBuf), testIniLoadCallback, result), "load ini");
TEST_RESULT_STR_Z(
result,
testIniNextValue(iniNewP(ioBufferReadNew(iniBuf), .strict = true)),
"section1: key1 :\"value1\"\n"
"section1:key2:\"value2\"\n"
"section1:key=3=:\"value3\"\n"
"section1:=:\"=\"\n",
" check ini");
"check");
// Two sections
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("two sections");
iniBuf = BUFSTRZ(
"[section1]\n"
"[key1=\"value1\"\n"
@ -111,93 +87,71 @@ testRun(void)
"[section2]\n"
"\n"
"#key2=\"value2\"");
result = strNew();
TEST_RESULT_VOID(iniLoad(ioBufferReadNew(iniBuf), testIniLoadCallback, result), "load ini");
TEST_RESULT_STR_Z(
result,
testIniNextValue(iniNewP(ioBufferReadNew(iniBuf), .strict = true)),
"section1:[key1:\"value1\"\n"
"section1:key2:\"value2\"\n"
"section2:#key2:\"value2\"\n",
" check ini");
}
// *****************************************************************************************************************************
if (testBegin("iniNew() and iniFree()"))
{
Ini *ini = NULL;
TEST_ASSIGN(ini, iniNew(), "new ini");
TEST_RESULT_PTR_NE(ini->store, NULL, "store is set");
TEST_RESULT_VOID(iniFree(ini), "free ini");
}
// *****************************************************************************************************************************
if (testBegin("iniSet(), iniGet(), and iniSectionKeyList()"))
{
Ini *ini = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(ini, iniNew(), "new ini");
TEST_RESULT_VOID(iniSet(ini, STRDEF("section1"), STRDEF("key1"), STRDEF("11")), "set section, key");
TEST_RESULT_VOID(iniSet(ini, STRDEF("section1"), STRDEF("key2"), STRDEF("1.234")), "set section, key");
TEST_RESULT_VOID(iniMove(ini, memContextPrior()), "move ini");
TEST_RESULT_VOID(iniMove(NULL, memContextPrior()), "move null ini");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_STR_Z(iniGet(ini, STRDEF("section1"), STRDEF("key1")), "11", "get section, key");
TEST_RESULT_STR_Z(iniGet(ini, STRDEF("section1"), STRDEF("key2")), "1.234", "get section, key");
TEST_ERROR(iniGet(ini, STRDEF("section2"), STRDEF("key2")), FormatError, "section 'section2', key 'key2' does not exist");
TEST_RESULT_INT(strLstSize(iniSectionKeyList(ini, STRDEF("bogus"))), 0, "get keys for missing section");
TEST_RESULT_STRLST_Z(iniSectionKeyList(ini, STRDEF("section1")), "key1\nkey2\n", "get keys for section");
TEST_RESULT_VOID(iniSet(ini, STRDEF("section2"), STRDEF("key2"), STRDEF("2")), "set section2, key");
TEST_RESULT_BOOL(iniSectionKeyIsList(ini, STRDEF("section1"), STRDEF("key1")), false, "single value is not list");
TEST_RESULT_VOID(iniSet(ini, STRDEF("section2"), STRDEF("key2"), STRDEF("7")), "set section2, key");
TEST_RESULT_BOOL(iniSectionKeyIsList(ini, STRDEF("section2"), STRDEF("key2")), true, "section2, key2 is a list");
TEST_RESULT_STRLST_Z(iniGetList(ini, STRDEF("section2"), STRDEF("key2")), "2\n7\n", "get list");
TEST_RESULT_PTR(iniGetList(ini, STRDEF("section2"), STRDEF("key-missing")), NULL, "get missing list");
TEST_RESULT_VOID(iniFree(ini), "free ini");
}
// *****************************************************************************************************************************
if (testBegin("iniParse()"))
{
Ini *ini = NULL;
const String *content = NULL;
"check");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(iniParse(iniNew(), NULL), "no content");
TEST_TITLE("validate");
TEST_RESULT_VOID(iniValid(iniNewP(ioBufferReadNew(iniBuf), .strict = true)), "ini valid");
TEST_ERROR(
iniParse(iniNew(), STRDEF("compress=y\n")), FormatError, "key/value found outside of section at line 1: compress=y");
TEST_ERROR(iniParse(iniNew(), STRDEF("[section\n")), FormatError, "ini section should end with ] at line 1: [section");
TEST_ERROR(iniParse(iniNew(), STRDEF("[section]\nkey")), FormatError, "missing '=' in key/value at line 2: key");
TEST_ERROR(iniParse(iniNew(), STRDEF("[section]\n =value")), FormatError, "key is zero-length at line 1: =value");
iniValid(iniNewP(ioBufferReadNew(BUFSTRDEF("key=value\n")), .strict = true)), FormatError,
"key/value found outside of section at line 1: key=value");
}
// *****************************************************************************************************************************
if (testBegin("iniNewP() loose"))
{
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("errors");
TEST_ERROR(
testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("compress=y\n")))), FormatError,
"key/value found outside of section at line 1: compress=y");
TEST_ERROR(
testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("[section\n")))), FormatError,
"ini section should end with ] at line 1: [section");
TEST_ERROR(
testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("[section]\nkey")))), FormatError,
"missing '=' in key/value at line 2: key");
TEST_ERROR(
testIniNextValue(iniNewP(ioBufferReadNew(BUFSTRDEF("[section]\n =value")))), FormatError,
"key is zero-length at line 2: =value");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(ini, iniNew(), "new ini");
TEST_TITLE("store and retrieve values");
content = STRDEF
const Buffer *iniBuf = BUFSTRDEF
(
"# Comment\n"
"[global] \n"
"compress=y \n"
" repeat = 1 \n"
"repeat=2\n"
"\n"
" [db]\n"
"pg1-path = /path/to/pg"
);
TEST_RESULT_VOID(iniParse(ini, content), "load ini");
Ini *ini = NULL;
TEST_ASSIGN(ini, iniNewP(ioBufferReadNew(iniBuf), .store = true), "new ini");
TEST_RESULT_STR_Z(iniGet(ini, STRDEF("global"), STRDEF("compress")), "y", "ini get");
TEST_RESULT_STR_Z(iniGet(ini, STRDEF("db"), STRDEF("pg1-path")), "/path/to/pg", "ini get");
TEST_ERROR(iniGet(ini, STRDEF("sec"), STRDEF("key")), FormatError, "section 'sec', key 'key' does not exist");
TEST_RESULT_STR_Z(iniGet(ini, STRDEF("global"), STRDEF("compress")), "y", "get compress");
TEST_RESULT_STR_Z(iniGet(ini, STRDEF("db"), STRDEF("pg1-path")), "/path/to/pg", "get pg1-path");
TEST_RESULT_BOOL(iniSectionKeyIsList(ini, STRDEF("global"), STRDEF("repeat")), true, "key is list");
TEST_RESULT_STRLST_Z(iniGetList(ini, STRDEF("global"), STRDEF("repeat")), "1\n2\n", "key list");
TEST_RESULT_PTR(iniGetList(ini, STRDEF("globalx"), STRDEF("repeat2")), NULL, "null key list");
TEST_RESULT_STRLST_Z(iniSectionKeyList(ini, STRDEF("global")), "compress\nrepeat\n", "section keys");
TEST_RESULT_STRLST_Z(iniSectionKeyList(ini, STRDEF("bogus")), NULL, "empty section keys");
TEST_RESULT_VOID(iniFree(ini), "ini free");
}
FUNCTION_HARNESS_RETURN_VOID();

28
test/src/module/performance/typeTest.c
View File

@ -43,18 +43,6 @@ testComparator(const void *item1, const void *item2)
return 0;
}
/***********************************************************************************************************************************
Test callback to count ini load results
***********************************************************************************************************************************/
static void
testIniLoadCountCallback(void *const data, const String *const section, const String *const key, const String *const value)
{
(*(unsigned int *)data)++;
(void)section;
(void)key;
(void)value;
}
/***********************************************************************************************************************************
Driver to test manifestNewBuild(). Generates files for a valid-looking PostgreSQL cluster that can be scaled to any size.
***********************************************************************************************************************************/
@ -220,7 +208,7 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("iniLoad()"))
if (testBegin("iniValueNext()"))
{
ASSERT(TEST_SCALE <= 10000);
@ -233,11 +221,19 @@ testRun(void)
TEST_LOG_FMT("ini size = %s, keys = %u", strZ(strSizeFormat(strSize(iniStr))), iniMax);
TimeMSec timeBegin = timeMSec();
unsigned int iniTotal = 0;
Ini *ini = iniNewP(ioBufferReadNew(BUFSTR(iniStr)), .strict = true);
TEST_RESULT_VOID(iniLoad(ioBufferReadNew(BUFSTR(iniStr)), testIniLoadCountCallback, &iniTotal), "parse ini");
unsigned int iniTotal = 0;
const IniValue *value = iniValueNext(ini);
while (value != NULL)
{
iniTotal++;
value = iniValueNext(ini);
}
TEST_RESULT_INT(iniTotal, iniMax, "check ini value total");
TEST_LOG_FMT("parse completed in %ums", (unsigned int)(timeMSec() - timeBegin));
TEST_RESULT_INT(iniTotal, iniMax, " check ini total");
}
// Build/load/save a larger manifest to test performance and memory usage. The default sizing is for a "typical" large cluster