db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Improve JSON handling.

Browse Source 
Previously read/writing JSON required parsing/render via a variant, which add many more memory allocations and loops.

Instead allow JSON to be read/written serially to improve performance and simplify the code. This also allows us to get rid of many String and Variant constant which are no longer required.

The goal is to be able to read/write very large (e.g. gigabyte manifest) JSON structures, which would not be practical with the current code.

Note that external JSON (GCS, S3, etc) is still handled using variants. Converting these will require more consideration about key ordering since it cannot be guaranteed as in our own formats.
This commit is contained in:
David Steele 2022-04-25 09:06:26 -04:00 committed by GitHub
parent 58f24568f5
commit 45c3f4d53c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 2944 additions and 1837 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
doc/xml/release.xml
View File

@ -22,6 +22,7 @@
<commit subject="Auto-select backup for restore command --type=lsn.">
<github-pull-request id="1706"/>
</commit>
<commit subject="Fix ordering of backup-lsn-stop field in command/restore unit test."/>
<release-item-contributor-list>
<release-item-contributor id="reid.thompson"/>
@ -142,6 +143,24 @@
<p>Improve small file support.</p>
</release-item>
<release-item>
<commit subject="Add test for protocol greeting when a field is missing."/>
<commit subject="Return stats as a JSON string rather than a KeyValue object."/>
<commit subject="Add size to info/manifest unit test."/>
<commit subject="Handle missing archive start/stop in info/info backup unit test."/>
<commit subject="Add JSON error when value does not parse in Ini object."/>
<commit subject="Improve JSON handling.">
<github-pull-request id="1721"/>
</commit>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="reid.thompson"/>
</release-item-contributor-list>
<p>Improve JSON handling.</p>
</release-item>
<release-item>
<commit subject="Set option-archive-copy flag in backup.manifest to false when offline."/>
<commit subject="Move cfgParseOptionalFilterDepend() and add comment block."/>

10
src/command/repo/create.c
View File

@ -34,11 +34,15 @@ cmdRepoCreate(void)
case STORAGE_GCS_TYPE:
{
const KeyValue *const kvContent = kvPut(kvNew(), GCS_JSON_NAME_VAR, VARSTR(cfgOptionStr(cfgOptRepoGcsBucket)));
storageGcsRequestP(
(StorageGcs *)storageDriver(storageRepoWrite()), HTTP_VERB_POST_STR, .noBucket = true,
.content = BUFSTR(jsonFromKv(kvContent)));
.content = BUFSTR(
jsonWriteResult(
jsonWriteObjectEnd(
jsonWriteStr(
jsonWriteKeyZ(
jsonWriteObjectBegin(
jsonWriteNewP()), GCS_JSON_NAME), cfgOptionStr(cfgOptRepoGcsBucket))))));
break;
}

4
src/command/repo/ls.c
View File

@ -52,7 +52,7 @@ storageListRenderCallback(void *data, const StorageInfo *info)
// Render in json
if (listData->json)
{
ioWriteStr(listData->write, jsonFromStr(info->name));
ioWriteStr(listData->write, jsonFromVar(VARSTR(info->name)));
ioWrite(listData->write, BUFSTRDEF(":{\"type\":\""));
switch (info->type)
@ -81,7 +81,7 @@ storageListRenderCallback(void *data, const StorageInfo *info)
}
if (info->type == storageTypeLink)
ioWriteStr(listData->write, strNewFmt(",\"destination\":%s", strZ(jsonFromStr(info->linkDestination))));
ioWriteStr(listData->write, strNewFmt(",\"destination\":%s", strZ(jsonFromVar(VARSTR(info->linkDestination)))));
ioWrite(listData->write, BRACER_BUF);
}

10
src/common/ini.c
View File

@ -306,9 +306,8 @@ iniSet(Ini *this, const String *section, const String *key, const String *value)
/**********************************************************************************************************************************/
void
iniLoad(
IoRead *read,
void (*callbackFunction)(void *data, const String *section, const String *key, const String *value, const Variant *valueVar),
void *callbackData)
IoRead *const read, void (*callbackFunction)(void *data, const String *section, const String *key, const String *value),
void *const callbackData)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(IO_READ, read);
@ -366,7 +365,6 @@ iniLoad(
// then an error is thrown.
String *key;
String *value;
Variant *valueVar = NULL;
bool retry;
@ -381,7 +379,7 @@ iniLoad(
// Check that the value is valid JSON
TRY_BEGIN()
{
valueVar = jsonToVar(value);
jsonValidate(value);
}
CATCH(JsonFormatError)
{
@ -407,7 +405,7 @@ iniLoad(
THROW_FMT(FormatError, "key is zero-length at line %u: %s", lineIdx + 1, linePtr);
// Callback with the section/key/value
callbackFunction(callbackData, section, key, value, valueVar);
callbackFunction(callbackData, section, key, value);
}
}

3
src/common/ini.h
View File

@ -73,8 +73,7 @@ Helper Functions
// or extraneous spaces, and where all values are valid JSON. This allows syntax characters such as [, =, #, and whitespace to be
// used in keys.
void iniLoad(
IoRead *read,
void (*callbackFunction)(void *data, const String *section, const String *key, const String *value, const Variant *valueVar),
IoRead *read, void (*callbackFunction)(void *data, const String *section, const String *key, const String *value),
void *callbackData);
/***********************************************************************************************************************************

15
src/common/stat.c
View File

@ -113,23 +113,22 @@ statToJson(void)
if (!lstEmpty(statLocalData.stat))
{
result = strNew();
MEM_CONTEXT_TEMP_BEGIN()
{
KeyValue *const kv = kvNew();
JsonWrite *const json = jsonWriteObjectBegin(jsonWriteNewP(.json = result));
for (unsigned int statIdx = 0; statIdx < lstSize(statLocalData.stat); statIdx++)
{
const Stat *const stat = lstGet(statLocalData.stat, statIdx);
KeyValue *const statKv = kvPutKv(kv, VARSTR(stat->key));
kvPut(statKv, VARSTRDEF("total"), VARUINT64(stat->total));
jsonWriteObjectBegin(jsonWriteKey(json, stat->key));
jsonWriteUInt64(jsonWriteKeyZ(json, "total"), stat->total);
jsonWriteObjectEnd(json);
}
MEM_CONTEXT_PRIOR_BEGIN()
{
result = jsonFromKv(kv);
}
MEM_CONTEXT_PRIOR_END();
jsonWriteObjectEnd(json);
}
MEM_CONTEXT_TEMP_END();
}

2493
src/common/type/json.c
View File

File diff suppressed because it is too large Load Diff

206
src/common/type/json.h
View File

@ -1,51 +1,199 @@
/***********************************************************************************************************************************
Convert JSON to/from KeyValue
JSON read/write
***********************************************************************************************************************************/
#ifndef COMMON_TYPE_JSON_H
#define COMMON_TYPE_JSON_H
#include "common/type/keyValue.h"
#include "common/type/stringId.h"
#include "common/type/variant.h"
/***********************************************************************************************************************************
Functions
JSON types
***********************************************************************************************************************************/
// Convert a json string to a bool
bool jsonToBool(const String *json);
typedef enum
{
jsonTypeBool = STRID5("bool", 0x63de20), // Boolean
jsonTypeNull = STRID5("null", 0x632ae0), // Null
jsonTypeNumber = STRID5("nmbr", 0x909ae0), // Number
jsonTypeString = STRID5("str", 0x4a930), // String
// Convert a json number to various integer types
int jsonToInt(const String *json);
int64_t jsonToInt64(const String *json);
unsigned int jsonToUInt(const String *json);
uint64_t jsonToUInt64(const String *json);
jsonTypeArrayBegin = STRID5("ary-b", 0x2de6410), // Array begin
jsonTypeArrayEnd = STRID5("ary-e", 0x5de6410), // Array end
jsonTypeObjectBegin = STRID5("obj-b", 0x2da84f0), // Object begin
jsonTypeObjectEnd = STRID5("obj-e", 0x5da84f0), // Object end
} JsonType;
// Convert a json object to a KeyValue
KeyValue *jsonToKv(const String *json);
/***********************************************************************************************************************************
Object types
***********************************************************************************************************************************/
typedef struct JsonRead JsonRead;
typedef struct JsonWrite JsonWrite;
// Convert a json string to a String
String *jsonToStr(const String *json);
/***********************************************************************************************************************************
Read Constructors
***********************************************************************************************************************************/
JsonRead *jsonReadNew(const String *string);
// Convert JSON to a variant
/***********************************************************************************************************************************
Read Functions
***********************************************************************************************************************************/
// Read next JSON type. This is based on an examination of the first character so there may be an error when the type is read, but
// the type will not change.
JsonType jsonReadTypeNext(JsonRead *this);
// Read array begin/end
void jsonReadArrayBegin(JsonRead *this);
void jsonReadArrayEnd(JsonRead *this);
// Read boolean
bool jsonReadBool(JsonRead *this);
// Read integer
int jsonReadInt(JsonRead *this);
int64_t jsonReadInt64(JsonRead *this);
unsigned int jsonReadUInt(JsonRead *this);
uint64_t jsonReadUInt64(JsonRead *this);
// Read key
String *jsonReadKey(JsonRead *this);
// Read an expected key
bool jsonReadKeyExpect(JsonRead *this, const String *key);
bool jsonReadKeyExpectStrId(JsonRead *this, StringId key);
bool jsonReadKeyExpectZ(JsonRead *this, const char *key);
// Read an required key
JsonRead *jsonReadKeyRequire(JsonRead *this, const String *key);
JsonRead *jsonReadKeyRequireStrId(JsonRead *this, StringId key);
JsonRead *jsonReadKeyRequireZ(JsonRead *this, const char *key);
// Read null
void jsonReadNull(JsonRead *this);
// Read object begin/end
void jsonReadObjectBegin(JsonRead *this);
void jsonReadObjectEnd(JsonRead *this);
// Skip value
void jsonReadSkip(JsonRead *this);
// Read string
String *jsonReadStr(JsonRead *this);
StringId jsonReadStrId(JsonRead *this);
StringList *jsonReadStrLst(JsonRead *this);
// Read variant
Variant *jsonReadVar(JsonRead *this);
/***********************************************************************************************************************************
Read Helper Functions
***********************************************************************************************************************************/
// Convert JSON to Variant
Variant *jsonToVar(const String *json);
// Convert a json array to a VariantList
VariantList *jsonToVarLst(const String *json);
// Validate JSON
void jsonValidate(const String *json);
// Convert a boolean to JSON
const String *jsonFromBool(bool value);
/***********************************************************************************************************************************
Read Destructor
***********************************************************************************************************************************/
__attribute__((always_inline)) static inline void
jsonReadFree(JsonRead *const this)
{
objFree(this);
}
// Convert various integer types to JSON
String *jsonFromInt(int number);
String *jsonFromInt64(int64_t number);
String *jsonFromUInt(unsigned int number);
String *jsonFromUInt64(uint64_t number);
/***********************************************************************************************************************************
Write Constructors
***********************************************************************************************************************************/
typedef struct JsonWriteNewParam
{
VAR_PARAM_HEADER;
String *json;
} JsonWriteNewParam;
// Convert KeyValue to JSON
String *jsonFromKv(const KeyValue *kv);
#define jsonWriteNewP(...) \
jsonWriteNew((JsonWriteNewParam){VAR_PARAM_INIT, __VA_ARGS__})
// Convert a String to JSON
String *jsonFromStr(const String *string);
JsonWrite *jsonWriteNew(JsonWriteNewParam param);
/***********************************************************************************************************************************
Write Functions
***********************************************************************************************************************************/
// Write array begin/end
JsonWrite *jsonWriteArrayBegin(JsonWrite *this);
JsonWrite *jsonWriteArrayEnd(JsonWrite *this);
// Write boolean
JsonWrite *jsonWriteBool(JsonWrite *this, bool value);
// Write integer
JsonWrite *jsonWriteInt(JsonWrite *this, int value);
JsonWrite *jsonWriteInt64(JsonWrite *this, int64_t value);
JsonWrite *jsonWriteUInt(JsonWrite *this, unsigned int value);
JsonWrite *jsonWriteUInt64(JsonWrite *this, uint64_t value);
// Write JSON
JsonWrite *jsonWriteJson(JsonWrite *this, const String *value);
// Write key
JsonWrite *jsonWriteKey(JsonWrite *this, const String *key);
JsonWrite *jsonWriteKeyStrId(JsonWrite *this, StringId key);
JsonWrite *jsonWriteKeyZ(JsonWrite *this, const char *key);
// Write null
JsonWrite *jsonWriteNull(JsonWrite *this);
// Write object begin/end
JsonWrite *jsonWriteObjectBegin(JsonWrite *this);
JsonWrite *jsonWriteObjectEnd(JsonWrite *this);
// Write string
JsonWrite *jsonWriteStr(JsonWrite *this, const String *value);
JsonWrite *jsonWriteStrFmt(JsonWrite *this, const char *format, ...) __attribute__((format(printf, 2, 3)));
JsonWrite *jsonWriteStrId(JsonWrite *this, StringId value);
JsonWrite *jsonWriteStrLst(JsonWrite *this, const StringList *value);
JsonWrite *jsonWriteZ(JsonWrite *this, const char *value);
// Write variant
JsonWrite *jsonWriteVar(JsonWrite *this, const Variant *value);
/***********************************************************************************************************************************
Write Getters/Setters
***********************************************************************************************************************************/
// Get JSON result
const String *jsonWriteResult(JsonWrite *this);
/***********************************************************************************************************************************
Write Destructor
***********************************************************************************************************************************/
__attribute__((always_inline)) static inline void
jsonWriteFree(JsonWrite *const this)
{
objFree(this);
}
/***********************************************************************************************************************************
Write Helper Functions
***********************************************************************************************************************************/
// Convert Variant to JSON
String *jsonFromVar(const Variant *var);
String *jsonFromVar(const Variant *value);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
String *jsonReadToLog(const JsonRead *this);
#define FUNCTION_LOG_JSON_READ_TYPE \
JsonRead *
#define FUNCTION_LOG_JSON_READ_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, jsonReadToLog, buffer, bufferSize)
String *jsonWriteToLog(const JsonWrite *this);
#define FUNCTION_LOG_JSON_WRITE_TYPE \
JsonWrite *
#define FUNCTION_LOG_JSON_WRITE_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, jsonWriteToLog, buffer, bufferSize)
#endif

98
src/info/info.c
View File

@ -19,19 +19,6 @@ Info Handler
#include "storage/helper.h"
#include "version.h"
/***********************************************************************************************************************************
Internal constants
***********************************************************************************************************************************/
#define INFO_SECTION_BACKREST "backrest"
STRING_STATIC(INFO_SECTION_BACKREST_STR, INFO_SECTION_BACKREST);
STRING_STATIC(INFO_SECTION_CIPHER_STR, "cipher");
STRING_STATIC(INFO_KEY_CIPHER_PASS_STR, "cipher-pass");
#define INFO_KEY_CHECKSUM "backrest-checksum"
STRING_STATIC(INFO_KEY_CHECKSUM_STR, INFO_KEY_CHECKSUM);
STRING_EXTERN(INFO_KEY_FORMAT_STR, INFO_KEY_FORMAT);
STRING_EXTERN(INFO_KEY_VERSION_STR, INFO_KEY_VERSION);
/***********************************************************************************************************************************
Object types
***********************************************************************************************************************************/
@ -84,7 +71,7 @@ BUFFER_STRDEF_STATIC(INFO_CHECKSUM_KEY_VALUE_END_BUF, ":");
#define INFO_CHECKSUM_KEY_VALUE(checksum, key, value) \
do \
{ \
ioFilterProcessIn(checksum, BUFSTR(jsonFromStr(key))); \
ioFilterProcessIn(checksum, BUFSTR(jsonFromVar(VARSTR(key)))); \
ioFilterProcessIn(checksum, INFO_CHECKSUM_KEY_VALUE_END_BUF); \
ioFilterProcessIn(checksum, BUFSTR(value)); \
} \
@ -150,6 +137,11 @@ infoNew(const String *cipherPass)
/***********************************************************************************************************************************
Load and validate the info file (or copy)
***********************************************************************************************************************************/
#define INFO_SECTION_BACKREST "backrest"
#define INFO_KEY_CHECKSUM "backrest-checksum"
#define INFO_SECTION_CIPHER "cipher"
#define INFO_KEY_CIPHER_PASS "cipher-pass"
typedef struct InfoLoadData
{
MemContext *memContext; // Mem context to use for storing data in this structure
@ -158,18 +150,17 @@ typedef struct InfoLoadData
Info *info; // Info object
String *sectionLast; // The last section seen during load
IoFilter *checksumActual; // Checksum calculated from the file
String *checksumExpected; // Checksum found in ini file
const String *checksumExpected; // Checksum found in ini file
} InfoLoadData;
static void
infoLoadCallback(void *data, const String *section, const String *key, const String *value, const Variant *valueVar)
infoLoadCallback(void *const data, const String *const section, const String *const key, const String *const value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(STRING, section);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(STRING, value);
FUNCTION_TEST_PARAM(VARIANT, valueVar);
FUNCTION_TEST_END();
ASSERT(data != NULL);
@ -177,10 +168,10 @@ infoLoadCallback(void *data, const String *section, const String *key, const Str
ASSERT(key != NULL);
ASSERT(value != NULL);
InfoLoadData *loadData = (InfoLoadData *)data;
InfoLoadData *const loadData = (InfoLoadData *)data;
// Calculate checksum
if (!(strEq(section, INFO_SECTION_BACKREST_STR) && strEq(key, INFO_KEY_CHECKSUM_STR)))
if (!(strEqZ(section, INFO_SECTION_BACKREST) && strEqZ(key, INFO_KEY_CHECKSUM)))
{
if (loadData->sectionLast == NULL || !strEq(section, loadData->sectionLast))
{
@ -202,53 +193,49 @@ infoLoadCallback(void *data, const String *section, const String *key, const Str
}
// Process backrest section
if (strEq(section, INFO_SECTION_BACKREST_STR))
if (strEqZ(section, INFO_SECTION_BACKREST))
{
ASSERT(valueVar != NULL);
// Validate format
if (strEq(key, INFO_KEY_FORMAT_STR))
if (strEqZ(key, INFO_KEY_FORMAT))
{
if (varUInt64(valueVar) != REPOSITORY_FORMAT)
THROW_FMT(FormatError, "expected format %d but found %" PRIu64, REPOSITORY_FORMAT, varUInt64(valueVar));
if (varUInt64(jsonToVar(value)) != REPOSITORY_FORMAT)
THROW_FMT(FormatError, "expected format %d but found %" PRIu64, REPOSITORY_FORMAT, varUInt64(jsonToVar(value)));
}
// Store pgBackRest version
else if (strEq(key, INFO_KEY_VERSION_STR))
else if (strEqZ(key, INFO_KEY_VERSION))
{
MEM_CONTEXT_BEGIN(loadData->info->memContext)
{
loadData->info->pub.backrestVersion = strDup(varStr(valueVar));
loadData->info->pub.backrestVersion = varStr(jsonToVar(value));
}
MEM_CONTEXT_END();
}
// Store checksum to be validated later
else if (strEq(key, INFO_KEY_CHECKSUM_STR))
else if (strEqZ(key, INFO_KEY_CHECKSUM))
{
MEM_CONTEXT_BEGIN(loadData->memContext)
{
loadData->checksumExpected = strDup(varStr(valueVar));
loadData->checksumExpected = varStr(jsonToVar(value));
}
MEM_CONTEXT_END();
}
}
// Process cipher section
else if (strEq(section, INFO_SECTION_CIPHER_STR))
else if (strEqZ(section, INFO_SECTION_CIPHER))
{
ASSERT(valueVar != NULL);
// No validation needed for cipher-pass, just store it
if (strEq(key, INFO_KEY_CIPHER_PASS_STR))
if (strEqZ(key, INFO_KEY_CIPHER_PASS))
{
MEM_CONTEXT_BEGIN(loadData->info->memContext)
{
loadData->info->pub.cipherPass = strDup(varStr(valueVar));
loadData->info->pub.cipherPass = varStr(jsonToVar(value));
}
MEM_CONTEXT_END();
}
}
// Else pass to callback for processing
else
loadData->callbackFunction(loadData->callbackData, section, key, valueVar);
loadData->callbackFunction(loadData->callbackData, section, key, value);
FUNCTION_TEST_RETURN_VOID();
}
@ -320,28 +307,31 @@ infoNewLoad(IoRead *read, InfoLoadNewCallback *callbackFunction, void *callbackD
/**********************************************************************************************************************************/
bool
infoSaveSection(InfoSave *infoSaveData, const String *section, const String *sectionNext)
infoSaveSection(InfoSave *const infoSaveData, const char *const section, const String *const sectionNext)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INFO_SAVE, infoSaveData);
FUNCTION_TEST_PARAM(STRING, section);
FUNCTION_TEST_PARAM(STRINGZ, section);
FUNCTION_TEST_PARAM(STRING, sectionNext);
FUNCTION_TEST_END();
ASSERT(infoSaveData != NULL);
ASSERT(section != NULL);
FUNCTION_TEST_RETURN(
BOOL,
(infoSaveData->sectionLast == NULL || strCmp(section, infoSaveData->sectionLast) > 0) &&
(sectionNext == NULL || strCmp(section, sectionNext) < 0));
(infoSaveData->sectionLast == NULL || strCmpZ(infoSaveData->sectionLast, section) < 0) &&
(sectionNext == NULL || strCmpZ(sectionNext, section) > 0));
}
/**********************************************************************************************************************************/
void
infoSaveValue(InfoSave *infoSaveData, const String *section, const String *key, const String *jsonValue)
infoSaveValue(InfoSave *const infoSaveData, const char *const section, const char *const key, const String *const jsonValue)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INFO_SAVE, infoSaveData);
FUNCTION_TEST_PARAM(STRING, section);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(STRINGZ, section);
FUNCTION_TEST_PARAM(STRINGZ, key);
FUNCTION_TEST_PARAM(STRING, jsonValue);
FUNCTION_TEST_END();
@ -354,7 +344,7 @@ infoSaveValue(InfoSave *infoSaveData, const String *section, const String *key,
ASSERT(strZ(jsonValue)[0] != '[');
// Save section
if (infoSaveData->sectionLast == NULL || !strEq(section, infoSaveData->sectionLast))
if (infoSaveData->sectionLast == NULL || !strEqZ(infoSaveData->sectionLast, section))
{
if (infoSaveData->sectionLast != NULL)
{
@ -362,15 +352,15 @@ infoSaveValue(InfoSave *infoSaveData, const String *section, const String *key,
ioWriteLine(infoSaveData->write, BUFSTRDEF(""));
}
INFO_CHECKSUM_SECTION(infoSaveData->checksum, section);
INFO_CHECKSUM_SECTION(infoSaveData->checksum, STR(section));
ioWrite(infoSaveData->write, BRACKETL_BUF);
ioWrite(infoSaveData->write, BUFSTR(section));
ioWrite(infoSaveData->write, BUFSTRZ(section));
ioWriteLine(infoSaveData->write, BRACKETR_BUF);
MEM_CONTEXT_BEGIN(infoSaveData->memContext)
{
infoSaveData->sectionLast = strDup(section);
infoSaveData->sectionLast = strNewZ(section);
}
MEM_CONTEXT_END();
}
@ -378,9 +368,9 @@ infoSaveValue(InfoSave *infoSaveData, const String *section, const String *key,
INFO_CHECKSUM_KEY_VALUE_NEXT(infoSaveData->checksum);
// Save key/value
INFO_CHECKSUM_KEY_VALUE(infoSaveData->checksum, key, jsonValue);
INFO_CHECKSUM_KEY_VALUE(infoSaveData->checksum, STR(key), jsonValue);
ioWrite(infoSaveData->write, BUFSTR(key));
ioWrite(infoSaveData->write, BUFSTRZ(key));
ioWrite(infoSaveData->write, EQ_BUF);
ioWriteLine(infoSaveData->write, BUFSTR(jsonValue));
@ -418,15 +408,15 @@ infoSave(Info *this, IoWrite *write, InfoSaveCallback *callbackFunction, void *c
INFO_CHECKSUM_BEGIN(data.checksum);
// Add version and format
callbackFunction(callbackData, INFO_SECTION_BACKREST_STR, &data);
infoSaveValue(&data, INFO_SECTION_BACKREST_STR, INFO_KEY_FORMAT_STR, jsonFromUInt(REPOSITORY_FORMAT));
infoSaveValue(&data, INFO_SECTION_BACKREST_STR, INFO_KEY_VERSION_STR, jsonFromStr(STRDEF(PROJECT_VERSION)));
callbackFunction(callbackData, STRDEF(INFO_SECTION_BACKREST), &data);
infoSaveValue(&data, INFO_SECTION_BACKREST, INFO_KEY_FORMAT, jsonFromVar(VARUINT(REPOSITORY_FORMAT)));
infoSaveValue(&data, INFO_SECTION_BACKREST, INFO_KEY_VERSION, jsonFromVar(VARSTRDEF(PROJECT_VERSION)));
// Add cipher passphrase if defined
if (infoCipherPass(this) != NULL)
{
callbackFunction(callbackData, INFO_SECTION_CIPHER_STR, &data);
infoSaveValue(&data, INFO_SECTION_CIPHER_STR, INFO_KEY_CIPHER_PASS_STR, jsonFromStr(infoCipherPass(this)));
callbackFunction(callbackData, STRDEF(INFO_SECTION_CIPHER), &data);
infoSaveValue(&data, INFO_SECTION_CIPHER, INFO_KEY_CIPHER_PASS, jsonFromVar(VARSTR(infoCipherPass(this))));
}
// Flush out any additional sections
@ -436,7 +426,7 @@ infoSave(Info *this, IoWrite *write, InfoSaveCallback *callbackFunction, void *c
INFO_CHECKSUM_END(data.checksum);
ioWrite(data.write, BUFSTRDEF("\n[" INFO_SECTION_BACKREST "]\n" INFO_KEY_CHECKSUM "="));
ioWriteLine(data.write, BUFSTR(jsonFromStr(pckReadStrP(pckReadNew(ioFilterResult(data.checksum))))));
ioWriteLine(data.write, BUFSTR(jsonFromVar(VARSTR(pckReadStrP(pckReadNew(ioFilterResult(data.checksum)))))));
// Close the file
ioWriteClose(data.write);

8
src/info/info.h
View File

@ -19,9 +19,7 @@ Constants
#define INFO_COPY_EXT ".copy"
#define INFO_KEY_FORMAT "backrest-format"
STRING_DECLARE(INFO_KEY_FORMAT_STR);
#define INFO_KEY_VERSION "backrest-version"
STRING_DECLARE(INFO_KEY_VERSION_STR);
/***********************************************************************************************************************************
Function types for loading and saving
@ -31,7 +29,7 @@ Function types for loading and saving
// start at 0 and be incremented on each call.
typedef bool InfoLoadCallback(void *data, unsigned int try);
typedef void InfoLoadNewCallback(void *data, const String *section, const String *key, const Variant *value);
typedef void InfoLoadNewCallback(void *data, const String *section, const String *key, const String *value);
typedef void InfoSaveCallback(void *data, const String *sectionNext, InfoSave *infoSaveData);
/***********************************************************************************************************************************
@ -74,10 +72,10 @@ Functions
void infoSave(Info *this, IoWrite *write, InfoSaveCallback *callbackFunction, void *callbackData);
// Check if the section should be saved
bool infoSaveSection(InfoSave *infoSaveData, const String *section, const String *sectionNext);
bool infoSaveSection(InfoSave *infoSaveData, const char *section, const String *sectionNext);
// Save a JSON formatted value and update checksum
void infoSaveValue(InfoSave *infoSaveData, const String *section, const String *key, const String *jsonValue);
void infoSaveValue(InfoSave *infoSaveData, const char *section, const char *key, const String *jsonValue);
/***********************************************************************************************************************************
Helper functions

246
src/info/infoBackup.c
View File

@ -28,32 +28,6 @@ Backup Info Handler
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define INFO_BACKUP_SECTION "backup"
#define INFO_BACKUP_SECTION_BACKUP_CURRENT INFO_BACKUP_SECTION ":current"
STRING_STATIC(INFO_BACKUP_SECTION_BACKUP_CURRENT_STR, INFO_BACKUP_SECTION_BACKUP_CURRENT);
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_ARCHIVE_START_VAR, "backup-archive-start");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_ARCHIVE_STOP_VAR, "backup-archive-stop");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE_VAR, "backup-info-repo-size");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE_DELTA_VAR, "backup-info-repo-size-delta");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_INFO_SIZE_VAR, "backup-info-size");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_INFO_SIZE_DELTA_VAR, "backup-info-size-delta");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_LSN_START_VAR, "backup-lsn-start");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_LSN_STOP_VAR, "backup-lsn-stop");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_PRIOR_VAR, "backup-prior");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_REFERENCE_VAR, "backup-reference");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_TIMESTAMP_START_VAR, "backup-timestamp-start");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_TIMESTAMP_STOP_VAR, "backup-timestamp-stop");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_TYPE_VAR, "backup-type");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_BACKUP_ERROR_VAR, "backup-error");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_OPT_ARCHIVE_CHECK_VAR, "option-archive-check");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_OPT_ARCHIVE_COPY_VAR, "option-archive-copy");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_OPT_BACKUP_STANDBY_VAR, "option-backup-standby");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_OPT_CHECKSUM_PAGE_VAR, "option-checksum-page");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_OPT_COMPRESS_VAR, "option-compress");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_OPT_HARDLINK_VAR, "option-hardlink");
VARIANT_STRDEF_STATIC(INFO_BACKUP_KEY_OPT_ONLINE_VAR, "option-online");
STRING_EXTERN(INFO_BACKUP_PATH_FILE_STR, INFO_BACKUP_PATH_FILE);
STRING_EXTERN(INFO_BACKUP_PATH_FILE_COPY_STR, INFO_BACKUP_PATH_FILE_COPY);
@ -118,14 +92,39 @@ infoBackupNew(unsigned int pgVersion, uint64_t pgSystemId, unsigned int pgCatalo
/***********************************************************************************************************************************
Create new object and load contents from a file
***********************************************************************************************************************************/
#define INFO_BACKUP_SECTION "backup"
#define INFO_BACKUP_SECTION_BACKUP_CURRENT INFO_BACKUP_SECTION ":current"
#define INFO_BACKUP_KEY_BACKUP_ARCHIVE_START "backup-archive-start"
#define INFO_BACKUP_KEY_BACKUP_ARCHIVE_STOP "backup-archive-stop"
#define INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE "backup-info-repo-size"
#define INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE_DELTA "backup-info-repo-size-delta"
#define INFO_BACKUP_KEY_BACKUP_INFO_SIZE "backup-info-size"
#define INFO_BACKUP_KEY_BACKUP_INFO_SIZE_DELTA "backup-info-size-delta"
#define INFO_BACKUP_KEY_BACKUP_LSN_START "backup-lsn-start"
#define INFO_BACKUP_KEY_BACKUP_LSN_STOP "backup-lsn-stop"
#define INFO_BACKUP_KEY_BACKUP_PRIOR STRID5("backup-prior", 0x93d3286e1558c220)
#define INFO_BACKUP_KEY_BACKUP_REFERENCE "backup-reference"
#define INFO_BACKUP_KEY_BACKUP_TIMESTAMP_START "backup-timestamp-start"
#define INFO_BACKUP_KEY_BACKUP_TIMESTAMP_STOP "backup-timestamp-stop"
#define INFO_BACKUP_KEY_BACKUP_TYPE STRID5("backup-type", 0x1619a6e1558c220)
#define INFO_BACKUP_KEY_BACKUP_ERROR STRID5("backup-error", 0x93e522ee1558c220)
#define INFO_BACKUP_KEY_OPT_ARCHIVE_CHECK "option-archive-check"
#define INFO_BACKUP_KEY_OPT_ARCHIVE_COPY "option-archive-copy"
#define INFO_BACKUP_KEY_OPT_BACKUP_STANDBY "option-backup-standby"
#define INFO_BACKUP_KEY_OPT_CHECKSUM_PAGE "option-checksum-page"
#define INFO_BACKUP_KEY_OPT_COMPRESS "option-compress"
#define INFO_BACKUP_KEY_OPT_HARDLINK "option-hardlink"
#define INFO_BACKUP_KEY_OPT_ONLINE "option-online"
static void
infoBackupLoadCallback(void *data, const String *section, const String *key, const Variant *value)
infoBackupLoadCallback(void *data, const String *section, const String *key, const String *value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(STRING, section);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(VARIANT, value);
FUNCTION_TEST_PARAM(STRING, value);
FUNCTION_TEST_END();
ASSERT(data != NULL);
@ -133,60 +132,80 @@ infoBackupLoadCallback(void *data, const String *section, const String *key, con
ASSERT(key != NULL);
ASSERT(value != NULL);
InfoBackup *infoBackup = (InfoBackup *)data;
InfoBackup *const infoBackup = (InfoBackup *)data;
// Process current backup list
if (strEq(section, INFO_BACKUP_SECTION_BACKUP_CURRENT_STR))
if (strEqZ(section, INFO_BACKUP_SECTION_BACKUP_CURRENT))
{
const KeyValue *backupKv = varKv(value);
MEM_CONTEXT_BEGIN(lstMemContext(infoBackup->pub.backup))
{
InfoBackupData infoBackupData =
JsonRead *const json = jsonReadNew(value);
jsonReadObjectBegin(json);
InfoBackupData info =
{
.backrestFormat = varUIntForce(kvGet(backupKv, VARSTR(INFO_KEY_FORMAT_STR))),
.backrestVersion = varStrForce(kvGet(backupKv, VARSTR(INFO_KEY_VERSION_STR))),
.backupInfoRepoSize = varUInt64(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE_VAR)),
.backupInfoRepoSizeDelta = varUInt64(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE_DELTA_VAR)),
.backupInfoSize = varUInt64(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_INFO_SIZE_VAR)),
.backupInfoSizeDelta = varUInt64(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_INFO_SIZE_DELTA_VAR)),
.backupLabel = strDup(key),
.backupPgId = cvtZToUInt(strZ(varStrForce(kvGet(backupKv, INFO_KEY_DB_ID_VAR)))),
// When reading timestamps, read as uint64 to ensure always positive value (guarantee no backups before 1970)
.backupTimestampStart = (time_t)varUInt64(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_START_VAR)),
.backupTimestampStop= (time_t)varUInt64(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_STOP_VAR)),
.backupType = (BackupType)strIdFromStr(varStr(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_TYPE_VAR))),
// Possible NULL values
.backupArchiveStart = strDup(varStr(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_ARCHIVE_START_VAR))),
.backupArchiveStop = strDup(varStr(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_ARCHIVE_STOP_VAR))),
.backupLsnStart = strDup(varStr(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_LSN_START_VAR))),
.backupLsnStop = strDup(varStr(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_LSN_STOP_VAR))),
.backupPrior = strDup(varStr(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_PRIOR_VAR))),
.backupReference =
kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_REFERENCE_VAR) != NULL ?
strLstNewVarLst(varVarLst(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_REFERENCE_VAR))) : NULL,
// Report errors detected during the backup. The key may not exist in older versions.
.backupError = varDup(kvGet(backupKv, INFO_BACKUP_KEY_BACKUP_ERROR_VAR)),
// Options
.optionArchiveCheck = varBool(kvGet(backupKv, INFO_BACKUP_KEY_OPT_ARCHIVE_CHECK_VAR)),
.optionArchiveCopy = varBool(kvGet(backupKv, INFO_BACKUP_KEY_OPT_ARCHIVE_COPY_VAR)),
.optionBackupStandby = varBool(kvGet(backupKv, INFO_BACKUP_KEY_OPT_BACKUP_STANDBY_VAR)),
.optionChecksumPage = varBool(kvGet(backupKv, INFO_BACKUP_KEY_OPT_CHECKSUM_PAGE_VAR)),
.optionCompress = varBool(kvGet(backupKv, INFO_BACKUP_KEY_OPT_COMPRESS_VAR)),
.optionHardlink = varBool(kvGet(backupKv, INFO_BACKUP_KEY_OPT_HARDLINK_VAR)),
.optionOnline = varBool(kvGet(backupKv, INFO_BACKUP_KEY_OPT_ONLINE_VAR)),
};
ASSERT(
infoBackupData.backupType == backupTypeFull || infoBackupData.backupType == backupTypeDiff ||
infoBackupData.backupType == backupTypeIncr);
// Format and version
info.backrestFormat = jsonReadUInt(jsonReadKeyRequireZ(json, INFO_KEY_FORMAT));
info.backrestVersion = jsonReadStr(jsonReadKeyRequireZ(json, INFO_KEY_VERSION));
// Archive start/stop
if (jsonReadKeyExpectZ(json, INFO_BACKUP_KEY_BACKUP_ARCHIVE_START))
info.backupArchiveStart = jsonReadStr(json);
if (jsonReadKeyExpectZ(json, INFO_BACKUP_KEY_BACKUP_ARCHIVE_STOP))
info.backupArchiveStop = jsonReadStr(json);
// Report errors detected during the backup. The key may not exist in older versions.
if (jsonReadKeyExpectStrId(json, INFO_BACKUP_KEY_BACKUP_ERROR))
info.backupError = varNewBool(jsonReadBool(json));
// Size info
info.backupInfoRepoSize = jsonReadUInt64(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE));
info.backupInfoRepoSizeDelta = jsonReadUInt64(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE_DELTA));
info.backupInfoSize = jsonReadUInt64(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_BACKUP_INFO_SIZE));
info.backupInfoSizeDelta = jsonReadUInt64(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_BACKUP_INFO_SIZE_DELTA));
// Lsn start/stop
if (jsonReadKeyExpectZ(json, INFO_BACKUP_KEY_BACKUP_LSN_START))
info.backupLsnStart = jsonReadStr(json);
if (jsonReadKeyExpectZ(json, INFO_BACKUP_KEY_BACKUP_LSN_STOP))
info.backupLsnStop = jsonReadStr(json);
// Prior backup
if (jsonReadKeyExpectStrId(json, INFO_BACKUP_KEY_BACKUP_PRIOR))
info.backupPrior = jsonReadStr(json);
// Reference
if (jsonReadKeyExpectZ(json, INFO_BACKUP_KEY_BACKUP_REFERENCE))
info.backupReference = jsonReadStrLst(json);
// Read timestamps as uint64 to ensure a positive value (guarantee no backups before 1970)
info.backupTimestampStart = (time_t)jsonReadUInt64(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_START));
info.backupTimestampStop = (time_t)jsonReadUInt64(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_STOP));
// backup type
info.backupType = (BackupType)jsonReadStrId(jsonReadKeyRequireStrId(json, INFO_BACKUP_KEY_BACKUP_TYPE));
ASSERT(info.backupType == backupTypeFull || info.backupType == backupTypeDiff || info.backupType == backupTypeIncr);
// Database id
info.backupPgId = jsonReadUInt(jsonReadKeyRequireZ(json, INFO_KEY_DB_ID));
// Options
info.optionArchiveCheck = jsonReadBool(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_OPT_ARCHIVE_CHECK));
info.optionArchiveCopy = jsonReadBool(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_OPT_ARCHIVE_COPY));
info.optionBackupStandby = jsonReadBool(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_OPT_BACKUP_STANDBY));
info.optionChecksumPage = jsonReadBool(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_OPT_CHECKSUM_PAGE));
info.optionCompress = jsonReadBool(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_OPT_COMPRESS));
info.optionHardlink = jsonReadBool(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_OPT_HARDLINK));
info.optionOnline = jsonReadBool(jsonReadKeyRequireZ(json, INFO_BACKUP_KEY_OPT_ONLINE));
// Add the backup data to the list
lstAdd(infoBackup->pub.backup, &infoBackupData);
lstAdd(infoBackup->pub.backup, &info);
}
MEM_CONTEXT_END();
}
@ -220,7 +239,7 @@ infoBackupNewLoad(IoRead *read)
Save to file
***********************************************************************************************************************************/
static void
infoBackupSaveCallback(void *data, const String *sectionNext, InfoSave *infoSaveData)
infoBackupSaveCallback(void *const data, const String *const sectionNext, InfoSave *const infoSaveData)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
@ -231,63 +250,62 @@ infoBackupSaveCallback(void *data, const String *sectionNext, InfoSave *infoSave
ASSERT(data != NULL);
ASSERT(infoSaveData != NULL);
InfoBackup *infoBackup = (InfoBackup *)data;
InfoBackup *const infoBackup = (InfoBackup *)data;
if (infoSaveSection(infoSaveData, INFO_BACKUP_SECTION_BACKUP_CURRENT_STR, sectionNext))
if (infoSaveSection(infoSaveData, INFO_BACKUP_SECTION_BACKUP_CURRENT, sectionNext))
{
// Set the backup current section
for (unsigned int backupIdx = 0; backupIdx < infoBackupDataTotal(infoBackup); backupIdx++)
{
InfoBackupData backupData = infoBackupData(infoBackup, backupIdx);
const InfoBackupData backupData = infoBackupData(infoBackup, backupIdx);
JsonWrite *const json = jsonWriteObjectBegin(jsonWriteNewP());
KeyValue *backupDataKv = kvNew();
kvPut(backupDataKv, VARSTR(INFO_KEY_FORMAT_STR), VARUINT(backupData.backrestFormat));
kvPut(backupDataKv, VARSTR(INFO_KEY_VERSION_STR), VARSTR(backupData.backrestVersion));
jsonWriteUInt(jsonWriteKeyZ(json, INFO_KEY_FORMAT), backupData.backrestFormat);
jsonWriteStr(jsonWriteKeyZ(json, INFO_KEY_VERSION), backupData.backrestVersion);
kvPut(backupDataKv, INFO_KEY_DB_ID_VAR, VARUINT(backupData.backupPgId));
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_ARCHIVE_START_VAR, VARSTR(backupData.backupArchiveStart));
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_ARCHIVE_STOP_VAR, VARSTR(backupData.backupArchiveStop));
if (backupData.backupLsnStart != NULL)
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_LSN_START_VAR, VARSTR(backupData.backupLsnStart));
if (backupData.backupLsnStop != NULL)
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_LSN_STOP_VAR, VARSTR(backupData.backupLsnStop));
if (backupData.backupPrior != NULL)
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_PRIOR_VAR, VARSTR(backupData.backupPrior));
if (backupData.backupReference != NULL)
{
kvPut(
backupDataKv, INFO_BACKUP_KEY_BACKUP_REFERENCE_VAR, varNewVarLst(varLstNewStrLst(backupData.backupReference)));
}
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE_VAR, VARUINT64(backupData.backupInfoRepoSize));
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE_DELTA_VAR, VARUINT64(backupData.backupInfoRepoSizeDelta));
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_INFO_SIZE_VAR, VARUINT64(backupData.backupInfoSize));
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_INFO_SIZE_DELTA_VAR, VARUINT64(backupData.backupInfoSizeDelta));
// When storing time_t treat as signed int to avoid casting
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_START_VAR, VARINT64(backupData.backupTimestampStart));
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_STOP_VAR, VARINT64(backupData.backupTimestampStop));
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_TYPE_VAR, VARSTR(strIdToStr(backupData.backupType)));
jsonWriteStr(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_ARCHIVE_START), backupData.backupArchiveStart);
jsonWriteStr(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_ARCHIVE_STOP), backupData.backupArchiveStop);
// Do not save backup-error if it was not loaded. This prevents backups that were added before the backup-error flag
// was introduced from being saved with an incorrect value.
if (backupData.backupError != NULL)
kvPut(backupDataKv, INFO_BACKUP_KEY_BACKUP_ERROR_VAR, backupData.backupError);
jsonWriteBool(jsonWriteKeyStrId(json, INFO_BACKUP_KEY_BACKUP_ERROR), varBool(backupData.backupError));
kvPut(backupDataKv, INFO_BACKUP_KEY_OPT_ARCHIVE_CHECK_VAR, VARBOOL(backupData.optionArchiveCheck));
kvPut(backupDataKv, INFO_BACKUP_KEY_OPT_ARCHIVE_COPY_VAR, VARBOOL(backupData.optionArchiveCopy));
kvPut(backupDataKv, INFO_BACKUP_KEY_OPT_BACKUP_STANDBY_VAR, VARBOOL(backupData.optionBackupStandby));
kvPut(backupDataKv, INFO_BACKUP_KEY_OPT_CHECKSUM_PAGE_VAR, VARBOOL(backupData.optionChecksumPage));
kvPut(backupDataKv, INFO_BACKUP_KEY_OPT_COMPRESS_VAR, VARBOOL(backupData.optionCompress));
kvPut(backupDataKv, INFO_BACKUP_KEY_OPT_HARDLINK_VAR, VARBOOL(backupData.optionHardlink));
kvPut(backupDataKv, INFO_BACKUP_KEY_OPT_ONLINE_VAR, VARBOOL(backupData.optionOnline));
jsonWriteUInt64(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE), backupData.backupInfoRepoSize);
jsonWriteUInt64(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_INFO_REPO_SIZE_DELTA), backupData.backupInfoRepoSizeDelta);
jsonWriteUInt64(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_INFO_SIZE), backupData.backupInfoSize);
jsonWriteUInt64(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_INFO_SIZE_DELTA), backupData.backupInfoSizeDelta);
if (backupData.backupLsnStart != NULL)
jsonWriteStr(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_LSN_START), backupData.backupLsnStart);
if (backupData.backupLsnStop != NULL)
jsonWriteStr(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_LSN_STOP), backupData.backupLsnStop);
if (backupData.backupPrior != NULL)
jsonWriteStr(jsonWriteKeyStrId(json, INFO_BACKUP_KEY_BACKUP_PRIOR), backupData.backupPrior);
if (backupData.backupReference != NULL)
jsonWriteStrLst(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_REFERENCE), backupData.backupReference);
// When storing time_t treat as signed int to avoid casting
jsonWriteInt64(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_START), backupData.backupTimestampStart);
jsonWriteInt64(jsonWriteKeyZ(json, INFO_BACKUP_KEY_BACKUP_TIMESTAMP_STOP), backupData.backupTimestampStop);
jsonWriteStrId(jsonWriteKeyStrId(json, INFO_BACKUP_KEY_BACKUP_TYPE), backupData.backupType);
jsonWriteUInt(jsonWriteKeyZ(json, INFO_KEY_DB_ID), backupData.backupPgId);
jsonWriteBool(jsonWriteKeyZ(json, INFO_BACKUP_KEY_OPT_ARCHIVE_CHECK), backupData.optionArchiveCheck);
jsonWriteBool(jsonWriteKeyZ(json, INFO_BACKUP_KEY_OPT_ARCHIVE_COPY), backupData.optionArchiveCopy);
jsonWriteBool(jsonWriteKeyZ(json, INFO_BACKUP_KEY_OPT_BACKUP_STANDBY), backupData.optionBackupStandby);
jsonWriteBool(jsonWriteKeyZ(json, INFO_BACKUP_KEY_OPT_CHECKSUM_PAGE), backupData.optionChecksumPage);
jsonWriteBool(jsonWriteKeyZ(json, INFO_BACKUP_KEY_OPT_COMPRESS), backupData.optionCompress);
jsonWriteBool(jsonWriteKeyZ(json, INFO_BACKUP_KEY_OPT_HARDLINK), backupData.optionHardlink);
jsonWriteBool(jsonWriteKeyZ(json, INFO_BACKUP_KEY_OPT_ONLINE), backupData.optionOnline);
infoSaveValue(
infoSaveData, INFO_BACKUP_SECTION_BACKUP_CURRENT_STR, backupData.backupLabel, jsonFromKv(backupDataKv));
infoSaveData, INFO_BACKUP_SECTION_BACKUP_CURRENT, strZ(backupData.backupLabel),
jsonWriteResult(jsonWriteObjectEnd(json)));
}
}

113
src/info/infoPg.c
View File

@ -22,19 +22,6 @@ PostgreSQL Info Handler
#include "postgres/version.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Internal constants
***********************************************************************************************************************************/
STRING_STATIC(INFO_SECTION_DB_STR, "db");
STRING_STATIC(INFO_SECTION_DB_HISTORY_STR, "db:history");
STRING_STATIC(INFO_KEY_DB_ID_STR, INFO_KEY_DB_ID);
VARIANT_STRDEF_EXTERN(INFO_KEY_DB_ID_VAR, INFO_KEY_DB_ID);
VARIANT_STRDEF_STATIC(INFO_KEY_DB_CATALOG_VERSION_VAR, "db-catalog-version");
VARIANT_STRDEF_STATIC(INFO_KEY_DB_CONTROL_VERSION_VAR, "db-control-version");
VARIANT_STRDEF_STATIC(INFO_KEY_DB_SYSTEM_ID_VAR, "db-system-id");
VARIANT_STRDEF_STATIC(INFO_KEY_DB_VERSION_VAR, "db-version");
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -93,6 +80,14 @@ infoPgNew(InfoPgType type, const String *cipherPassSub)
}
/**********************************************************************************************************************************/
#define INFO_SECTION_DB "db"
#define INFO_SECTION_DB_HISTORY "db:history"
#define INFO_KEY_DB_CATALOG_VERSION "db-catalog-version"
#define INFO_KEY_DB_CONTROL_VERSION "db-control-version"
#define INFO_KEY_DB_SYSTEM_ID "db-system-id"
#define INFO_KEY_DB_VERSION "db-version"
typedef struct InfoPgLoadData
{
InfoLoadNewCallback *callbackFunction; // Callback function for child object
@ -102,13 +97,13 @@ typedef struct InfoPgLoadData
} InfoPgLoadData;
static void
infoPgLoadCallback(void *data, const String *section, const String *key, const Variant *value)
infoPgLoadCallback(void *const data, const String *const section, const String *const key, const String *const value)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(STRING, section);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_TEST_PARAM(VARIANT, value);
FUNCTION_TEST_PARAM(STRING, value);
FUNCTION_TEST_END();
ASSERT(data != NULL);
@ -116,31 +111,32 @@ infoPgLoadCallback(void *data, const String *section, const String *key, const V
ASSERT(key != NULL);
ASSERT(value != NULL);
InfoPgLoadData *loadData = (InfoPgLoadData *)data;
InfoPgLoadData *const loadData = data;
// Process db section
if (strEq(section, INFO_SECTION_DB_STR))
if (strEqZ(section, INFO_SECTION_DB))
{
if (strEq(key, INFO_KEY_DB_ID_STR))
loadData->currentId = varUIntForce(value);
if (strEqZ(key, INFO_KEY_DB_ID))
loadData->currentId = varUIntForce(jsonToVar(value));
}
// Process db:history section
else if (strEq(section, INFO_SECTION_DB_HISTORY_STR))
else if (strEqZ(section, INFO_SECTION_DB_HISTORY))
{
// Get db values that are common to all info files
const KeyValue *pgDataKv = varKv(value);
InfoPgData infoPgData = {.id = cvtZToUInt(strZ(key))};
InfoPgData infoPgData =
{
.id = cvtZToUInt(strZ(key)),
.version = pgVersionFromStr(varStr(kvGet(pgDataKv, INFO_KEY_DB_VERSION_VAR))),
.catalogVersion =
loadData->infoPg->type == infoPgBackup ? varUIntForce(kvGet(pgDataKv, INFO_KEY_DB_CATALOG_VERSION_VAR)) : 0,
JsonRead *const json = jsonReadNew(value);
jsonReadObjectBegin(json);
// This is different in archive.info due to a typo that can't be fixed without a format version bump
.systemId = varUInt64Force(
kvGet(pgDataKv, loadData->infoPg->type == infoPgArchive ? INFO_KEY_DB_ID_VAR : INFO_KEY_DB_SYSTEM_ID_VAR)),
};
// Catalog version
if (loadData->infoPg->type == infoPgBackup)
infoPgData.catalogVersion = jsonReadUInt(jsonReadKeyRequireZ(json, INFO_KEY_DB_CATALOG_VERSION));
// System id
infoPgData.systemId = jsonReadUInt64(
jsonReadKeyRequireZ(json, loadData->infoPg->type == infoPgArchive ? INFO_KEY_DB_ID : INFO_KEY_DB_SYSTEM_ID));
// PostgreSQL version
infoPgData.version = pgVersionFromStr(jsonReadStr(jsonReadKeyRequireZ(json, INFO_KEY_DB_VERSION)));
// Insert at beginning of list so the history is reverse ordered
lstInsert(loadData->infoPg->pub.history, 0, &infoPgData);
@ -274,7 +270,7 @@ typedef struct InfoPgSaveData
} InfoPgSaveData;
static void
infoPgSaveCallback(void *data, const String *sectionNext, InfoSave *infoSaveData)
infoPgSaveCallback(void *const data, const String *const sectionNext, InfoSave *const infoSaveData)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
@ -285,56 +281,57 @@ infoPgSaveCallback(void *data, const String *sectionNext, InfoSave *infoSaveData
ASSERT(data != NULL);
ASSERT(infoSaveData != NULL);
InfoPgSaveData *saveData = (InfoPgSaveData *)data;
InfoPgSaveData *const saveData = (InfoPgSaveData *)data;
if (infoSaveSection(infoSaveData, INFO_SECTION_DB_STR, sectionNext))
if (infoSaveSection(infoSaveData, INFO_SECTION_DB, sectionNext))
{
if (saveData->callbackFunction != NULL)
saveData->callbackFunction(saveData->callbackData, INFO_SECTION_DB_STR, infoSaveData);
saveData->callbackFunction(saveData->callbackData, STRDEF(INFO_SECTION_DB), infoSaveData);
InfoPgData pgData = infoPgDataCurrent(saveData->infoPg);
// These need to be saved because older pgBackRest versions expect them
if (saveData->infoPg->type == infoPgBackup)
{
infoSaveValue(infoSaveData, INFO_SECTION_DB, INFO_KEY_DB_CATALOG_VERSION, jsonFromVar(VARUINT(pgData.catalogVersion)));
infoSaveValue(
infoSaveData, INFO_SECTION_DB_STR, varStr(INFO_KEY_DB_CATALOG_VERSION_VAR), jsonFromUInt(pgData.catalogVersion));
infoSaveValue(
infoSaveData, INFO_SECTION_DB_STR, varStr(INFO_KEY_DB_CONTROL_VERSION_VAR),
jsonFromUInt(pgControlVersion(pgData.version)));
infoSaveData, INFO_SECTION_DB, INFO_KEY_DB_CONTROL_VERSION, jsonFromVar(VARUINT(pgControlVersion(pgData.version))));
}
infoSaveValue(infoSaveData, INFO_SECTION_DB_STR, varStr(INFO_KEY_DB_ID_VAR), jsonFromUInt(pgData.id));
infoSaveValue(infoSaveData, INFO_SECTION_DB_STR, varStr(INFO_KEY_DB_SYSTEM_ID_VAR), jsonFromUInt64(pgData.systemId));
infoSaveValue(
infoSaveData, INFO_SECTION_DB_STR, varStr(INFO_KEY_DB_VERSION_VAR), jsonFromStr(pgVersionToStr(pgData.version)));
infoSaveValue(infoSaveData, INFO_SECTION_DB, INFO_KEY_DB_ID, jsonFromVar(VARUINT(pgData.id)));
infoSaveValue(infoSaveData, INFO_SECTION_DB, INFO_KEY_DB_SYSTEM_ID, jsonFromVar(VARUINT64(pgData.systemId)));
infoSaveValue(infoSaveData, INFO_SECTION_DB, INFO_KEY_DB_VERSION, jsonFromVar(VARSTR(pgVersionToStr(pgData.version))));
}
if (infoSaveSection(infoSaveData, INFO_SECTION_DB_HISTORY_STR, sectionNext))
if (infoSaveSection(infoSaveData, INFO_SECTION_DB_HISTORY, sectionNext))
{
if (saveData->callbackFunction != NULL)
saveData->callbackFunction(saveData->callbackData, INFO_SECTION_DB_HISTORY_STR, infoSaveData);
saveData->callbackFunction(saveData->callbackData, STRDEF(INFO_SECTION_DB_HISTORY), infoSaveData);
// Set the db history section in reverse so oldest history is first instead of last to be consistent with load
for (unsigned int pgDataIdx = infoPgDataTotal(saveData->infoPg) - 1; (int)pgDataIdx >= 0; pgDataIdx--)
{
InfoPgData pgData = infoPgData(saveData->infoPg, pgDataIdx);
KeyValue *pgDataKv = kvNew();
kvPut(pgDataKv, INFO_KEY_DB_VERSION_VAR, VARSTR(pgVersionToStr(pgData.version)));
const InfoPgData pgData = infoPgData(saveData->infoPg, pgDataIdx);
JsonWrite *const json = jsonWriteObjectBegin(jsonWriteNewP());
// These need to be saved because older pgBackRest versions expect them
if (saveData->infoPg->type == infoPgBackup)
{
kvPut(pgDataKv, INFO_KEY_DB_SYSTEM_ID_VAR, VARUINT64(pgData.systemId));
// These need to be saved because older pgBackRest versions expect them
kvPut(pgDataKv, INFO_KEY_DB_CATALOG_VERSION_VAR, VARUINT(pgData.catalogVersion));
kvPut(pgDataKv, INFO_KEY_DB_CONTROL_VERSION_VAR, VARUINT(pgControlVersion(pgData.version)));
jsonWriteUInt(jsonWriteKeyZ(json, INFO_KEY_DB_CATALOG_VERSION), pgData.catalogVersion);
jsonWriteUInt(jsonWriteKeyZ(json, INFO_KEY_DB_CONTROL_VERSION), pgControlVersion(pgData.version));
}
else
kvPut(pgDataKv, INFO_KEY_DB_ID_VAR, VARUINT64(pgData.systemId));
infoSaveValue(infoSaveData, INFO_SECTION_DB_HISTORY_STR, varStrForce(VARUINT(pgData.id)), jsonFromKv(pgDataKv));
if (saveData->infoPg->type == infoPgArchive)
jsonWriteUInt64(jsonWriteKeyZ(json, INFO_KEY_DB_ID), pgData.systemId);
if (saveData->infoPg->type == infoPgBackup)
jsonWriteUInt64(jsonWriteKeyZ(json, INFO_KEY_DB_SYSTEM_ID), pgData.systemId);
jsonWriteStr(jsonWriteKeyZ(json, INFO_KEY_DB_VERSION), pgVersionToStr(pgData.version));
infoSaveValue(
infoSaveData, INFO_SECTION_DB_HISTORY, strZ(varStrForce(VARUINT(pgData.id))),
jsonWriteResult(jsonWriteObjectEnd(json)));
}
}

1
src/info/infoPg.h
View File

@ -20,7 +20,6 @@ typedef struct InfoPg InfoPg;
Constants
***********************************************************************************************************************************/
#define INFO_KEY_DB_ID "db-id"
VARIANT_DECLARE(INFO_KEY_DB_ID_VAR);
/***********************************************************************************************************************************
Information about the PostgreSQL cluster

824
src/info/manifest.c
View File

File diff suppressed because it is too large Load Diff

45
src/protocol/client.c
View File

@ -12,13 +12,6 @@ Protocol Client
#include "protocol/server.h"
#include "version.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
STRING_EXTERN(PROTOCOL_GREETING_NAME_STR, PROTOCOL_GREETING_NAME);
STRING_EXTERN(PROTOCOL_GREETING_SERVICE_STR, PROTOCOL_GREETING_SERVICE);
STRING_EXTERN(PROTOCOL_GREETING_VERSION_STR, PROTOCOL_GREETING_VERSION);
/***********************************************************************************************************************************
Client state enum
***********************************************************************************************************************************/
@ -117,38 +110,42 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
// Read, parse, and check the protocol greeting
MEM_CONTEXT_TEMP_BEGIN()
{
String *greeting = ioReadLine(this->pub.read);
KeyValue *greetingKv = jsonToKv(greeting);
JsonRead *const greeting = jsonReadNew(ioReadLine(this->pub.read));
const String *expected[] =
jsonReadObjectBegin(greeting);
const struct
{
PROTOCOL_GREETING_NAME_STR, STRDEF(PROJECT_NAME),
PROTOCOL_GREETING_SERVICE_STR, service,
PROTOCOL_GREETING_VERSION_STR, STRDEF(PROJECT_VERSION),
const StringId key;
const char *const value;
} expected[] =
{
{.key = PROTOCOL_GREETING_NAME, .value = PROJECT_NAME},
{.key = PROTOCOL_GREETING_SERVICE, .value = strZ(service)},
{.key = PROTOCOL_GREETING_VERSION, .value = PROJECT_VERSION},
};
for (unsigned int expectedIdx = 0; expectedIdx < LENGTH_OF(expected) / 2; expectedIdx++)
for (unsigned int expectedIdx = 0; expectedIdx < LENGTH_OF(expected); expectedIdx++)
{
const String *expectedKey = expected[expectedIdx * 2];
const String *expectedValue = expected[expectedIdx * 2 + 1];
if (!jsonReadKeyExpectStrId(greeting, expected[expectedIdx].key))
THROW_FMT(ProtocolError, "unable to find greeting key '%s'", strZ(strIdToStr(expected[expectedIdx].key)));
const Variant *actualValue = kvGet(greetingKv, VARSTR(expectedKey));
if (jsonReadTypeNext(greeting) != jsonTypeString)
THROW_FMT(ProtocolError, "greeting key '%s' must be string type", strZ(strIdToStr(expected[expectedIdx].key)));
if (actualValue == NULL)
THROW_FMT(ProtocolError, "unable to find greeting key '%s'", strZ(expectedKey));
const String *const actualValue = jsonReadStr(greeting);
if (varType(actualValue) != varTypeString)
THROW_FMT(ProtocolError, "greeting key '%s' must be string type", strZ(expectedKey));
if (!strEq(varStr(actualValue), expectedValue))
if (!strEqZ(actualValue, expected[expectedIdx].value))
{
THROW_FMT(
ProtocolError,
"expected value '%s' for greeting key '%s' but got '%s'\n"
"HINT: is the same version of " PROJECT_NAME " installed on the local and remote host?",
strZ(expectedValue), strZ(expectedKey), strZ(varStr(actualValue)));
expected[expectedIdx].value, strZ(strIdToStr(expected[expectedIdx].key)), strZ(actualValue));
}
}
jsonReadObjectEnd(greeting);
}
MEM_CONTEXT_TEMP_END();

9
src/protocol/client.h
View File

@ -35,12 +35,9 @@ typedef struct ProtocolClient ProtocolClient;
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define PROTOCOL_GREETING_NAME "name"
STRING_DECLARE(PROTOCOL_GREETING_NAME_STR);
#define PROTOCOL_GREETING_SERVICE "service"
STRING_DECLARE(PROTOCOL_GREETING_SERVICE_STR);
#define PROTOCOL_GREETING_VERSION "version"
STRING_DECLARE(PROTOCOL_GREETING_VERSION_STR);
#define PROTOCOL_GREETING_NAME STRID5("name", 0x2b42e0)
#define PROTOCOL_GREETING_SERVICE STRID5("service", 0x1469b48b30)
#define PROTOCOL_GREETING_VERSION STRID5("version", 0x39e99c8b60)
#define PROTOCOL_COMMAND_CONFIG STRID5("config", 0xe9339e30)
#define PROTOCOL_COMMAND_EXIT STRID5("exit", 0xa27050)

11
src/protocol/server.c
View File

@ -56,12 +56,13 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
// Send the protocol greeting
MEM_CONTEXT_TEMP_BEGIN()
{
KeyValue *greetingKv = kvNew();
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_NAME_STR), VARSTRZ(PROJECT_NAME));
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_SERVICE_STR), VARSTR(service));
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_VERSION_STR), VARSTRZ(PROJECT_VERSION));
JsonWrite *const json = jsonWriteObjectBegin(jsonWriteNewP());
ioWriteStrLine(this->write, jsonFromKv(greetingKv));
jsonWriteZ(jsonWriteKeyStrId(json, PROTOCOL_GREETING_NAME), PROJECT_NAME);
jsonWriteStr(jsonWriteKeyStrId(json, PROTOCOL_GREETING_SERVICE), service);
jsonWriteZ(jsonWriteKeyStrId(json, PROTOCOL_GREETING_VERSION), PROJECT_VERSION);
ioWriteStrLine(this->write, jsonWriteResult(jsonWriteObjectEnd(json)));
ioWriteFlush(this->write);
}
MEM_CONTEXT_TEMP_END();

8
src/storage/gcs/storage.c
View File

@ -122,7 +122,7 @@ storageGcsAuthToken(HttpRequest *request, time_t timeBegin)
StorageGcsAuthTokenResult result = {0};
// Get the response
KeyValue *kvResponse = jsonToKv(strNewBuf(httpResponseContent(httpRequestResponse(request, true))));
KeyValue *kvResponse = varKv(jsonToVar(strNewBuf(httpResponseContent(httpRequestResponse(request, true)))));
// Check for an error
const String *error = varStr(kvGet(kvResponse, GCS_JSON_ERROR_VAR));
@ -597,7 +597,7 @@ storageGcsListInternal(
else
response = storageGcsRequestP(this, HTTP_VERB_GET_STR, .query = query);
KeyValue *content = jsonToKv(strNewBuf(httpResponseContent(response)));
KeyValue *content = varKv(jsonToVar(strNewBuf(httpResponseContent(response))));
// If next page token exists then send an async request to get more data
const String *nextPageToken = varStr(kvGet(content, GCS_JSON_NEXT_PAGE_TOKEN_VAR));
@ -716,7 +716,7 @@ storageGcsInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageInt
if (result.level >= storageInfoLevelBasic && result.exists)
{
result.type = storageTypeFile;
storageGcsInfoFile(&result, jsonToKv(strNewBuf(httpResponseContent(httpResponse))));
storageGcsInfoFile(&result, varKv(jsonToVar(strNewBuf(httpResponseContent(httpResponse)))));
}
FUNCTION_LOG_RETURN(STORAGE_INFO, result);
@ -965,7 +965,7 @@ storageGcsNew(
// Read data from file for service keys
case storageGcsKeyTypeService:
{
KeyValue *kvKey = jsonToKv(strNewBuf(storageGetP(storageNewReadP(storagePosixNewP(FSLASH_STR), key))));
KeyValue *kvKey = varKv(jsonToVar(strNewBuf(storageGetP(storageNewReadP(storagePosixNewP(FSLASH_STR), key)))));
driver->credential = varStr(kvGet(kvKey, GCS_JSON_CLIENT_EMAIL_VAR));
driver->privateKey = varStr(kvGet(kvKey, GCS_JSON_PRIVATE_KEY_VAR));
const String *const uri = varStr(kvGet(kvKey, GCS_JSON_TOKEN_URI_VAR));

2
src/storage/gcs/write.c
View File

@ -81,7 +81,7 @@ storageWriteGcsVerify(StorageWriteGcs *this, HttpResponse *response)
FUNCTION_LOG_PARAM(HTTP_RESPONSE, response);
FUNCTION_LOG_END();
KeyValue *content = jsonToKv(strNewBuf(httpResponseContent(response)));
KeyValue *content = varKv(jsonToVar(strNewBuf(httpResponseContent(response))));
// Check the md5 hash
const String *md5base64 = varStr(kvGet(content, GCS_JSON_MD5_HASH_VAR));

2
src/storage/s3/storage.c
View File

@ -346,7 +346,7 @@ storageS3AuthAuto(StorageS3 *const this, HttpHeader *const header)
httpRequestError(request, response);
// Get credentials from the JSON response
KeyValue *credential = jsonToKv(strNewBuf(httpResponseContent(response)));
KeyValue *credential = varKv(jsonToVar(strNewBuf(httpResponseContent(response))));
MEM_CONTEXT_BEGIN(THIS_MEM_CONTEXT())
{

2
test/define.yaml
View File

@ -216,7 +216,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-json
total: 11
total: 2
coverage:
- common/type/json

16
test/src/common/harnessInfo.c
View File

@ -31,10 +31,9 @@ typedef struct HarnessInfoChecksumData
static void
harnessInfoChecksumCallback(
void *callbackData, const String *section, const String *key, const String *value, const Variant *valueVar)
void *const callbackData, const String *const section, const String *const key, const String *const value)
{
HarnessInfoChecksumData *data = (HarnessInfoChecksumData *)callbackData;
(void)valueVar;
// Calculate checksum
if (data->sectionLast == NULL || !strEq(section, data->sectionLast))
@ -55,7 +54,7 @@ harnessInfoChecksumCallback(
else
ioFilterProcessIn(data->checksum, BUFSTRDEF(","));
ioFilterProcessIn(data->checksum, BUFSTR(jsonFromStr(key)));
ioFilterProcessIn(data->checksum, BUFSTR(jsonFromVar(VARSTR(key))));
ioFilterProcessIn(data->checksum, BUFSTRDEF(":"));
ioFilterProcessIn(data->checksum, BUFSTR(value));
}
@ -84,9 +83,9 @@ harnessInfoChecksum(const String *info)
result = bufNew(strSize(info) + 256);
bufCat(result, BUFSTRDEF("[backrest]\nbackrest-format="));
bufCat(result, BUFSTR(jsonFromUInt(REPOSITORY_FORMAT)));
bufCat(result, BUFSTR(jsonFromVar(VARUINT(REPOSITORY_FORMAT))));
bufCat(result, BUFSTRDEF("\nbackrest-version="));
bufCat(result, BUFSTR(jsonFromStr(STRDEF(PROJECT_VERSION))));
bufCat(result, BUFSTR(jsonFromVar(VARSTRDEF(PROJECT_VERSION))));
bufCat(result, BUFSTRDEF("\n\n"));
bufCat(result, BUFSTR(info));
@ -97,7 +96,7 @@ harnessInfoChecksum(const String *info)
// Append checksum to buffer
bufCat(result, BUFSTRDEF("\n[backrest]\nbackrest-checksum="));
bufCat(result, BUFSTR(jsonFromStr(pckReadStrP(pckReadNew(ioFilterResult(data.checksum))))));
bufCat(result, BUFSTR(jsonFromVar(VARSTR(pckReadStrP(pckReadNew(ioFilterResult(data.checksum)))))));
bufCat(result, BUFSTRDEF("\n"));
bufMove(result, memContextPrior());
@ -123,8 +122,9 @@ harnessInfoChecksumZ(const char *info)
Test callback that logs the results to a string
***********************************************************************************************************************************/
void
harnessInfoLoadNewCallback(void *callbackData, const String *section, const String *key, const Variant *value)
harnessInfoLoadNewCallback(
void *const callbackData, const String *const section, const String *const key, const String *const value)
{
if (callbackData != NULL)
strCatFmt((String *)callbackData, "[%s] %s=%s\n", strZ(section), strZ(key), strZ(jsonFromVar(value)));
strCatFmt((String *)callbackData, "[%s] %s=%s\n", strZ(section), strZ(key), strZ(value));
}

2
test/src/common/harnessInfo.h
View File

@ -24,4 +24,4 @@ Functions
Buffer *harnessInfoChecksum(const String *info);
Buffer *harnessInfoChecksumZ(const char *info);
void harnessInfoLoadNewCallback(void *callbackData, const String *section, const String *key, const Variant *value);
void harnessInfoLoadNewCallback(void *callbackData, const String *section, const String *key, const String *value);

4
test/src/common/harnessServer.c
View File

@ -82,7 +82,7 @@ hrnServerScriptCommand(IoWrite *write, HrnServerCmd cmd, const Variant *data)
ASSERT(write != NULL);
ioWriteStrLine(write, jsonFromUInt(cmd));
ioWriteStrLine(write, jsonFromVar(VARUINT(cmd)));
ioWriteStrLine(write, jsonFromVar(data));
ioWriteFlush(write);
@ -268,7 +268,7 @@ void hrnServerRun(IoRead *read, HrnServerProtocol protocol, HrnServerRunParam pa
do
{
HrnServerCmd cmd = jsonToUInt(ioReadLine(read));
HrnServerCmd cmd = varUIntForce(jsonToVar(ioReadLine(read)));
const Variant *data = jsonToVar(ioReadLine(read));
switch (cmd)

16
test/src/module/command/backupTest.c
View File

@ -255,14 +255,14 @@ testBackupValidate(const Storage *storage, const String *path)
{
const String *section = strSubN(line, 1, strSize(line) - 2);
if (strEq(section, INFO_SECTION_BACKREST_STR) ||
strEq(section, MANIFEST_SECTION_BACKUP_STR) ||
strEq(section, MANIFEST_SECTION_BACKUP_DB_STR) ||
strEq(section, MANIFEST_SECTION_BACKUP_OPTION_STR) ||
strEq(section, MANIFEST_SECTION_DB_STR) ||
strEq(section, MANIFEST_SECTION_TARGET_FILE_DEFAULT_STR) ||
strEq(section, MANIFEST_SECTION_TARGET_LINK_DEFAULT_STR) ||
strEq(section, MANIFEST_SECTION_TARGET_PATH_DEFAULT_STR))
if (strEqZ(section, INFO_SECTION_BACKREST) ||
strEqZ(section, MANIFEST_SECTION_BACKUP) ||
strEqZ(section, MANIFEST_SECTION_BACKUP_DB) ||
strEqZ(section, MANIFEST_SECTION_BACKUP_OPTION) ||
strEqZ(section, MANIFEST_SECTION_DB) ||
strEqZ(section, MANIFEST_SECTION_TARGET_FILE_DEFAULT) ||
strEqZ(section, MANIFEST_SECTION_TARGET_LINK_DEFAULT) ||
strEqZ(section, MANIFEST_SECTION_TARGET_PATH_DEFAULT))
{
bSkipSection = true;
}

5
test/src/module/common/iniTest.c
View File

@ -9,9 +9,8 @@ Test Ini
Test callback to accumulate ini load results
***********************************************************************************************************************************/
static void
testIniLoadCallback(void *data, const String *section, const String *key, const String *value, const Variant *valueVar)
testIniLoadCallback(void *data, const String *section, const String *key, const String *value)
{
ASSERT(strEq(value, jsonFromVar(valueVar)));
strCatFmt((String *)data, "%s:%s:%s\n", strZ(section), strZ(key), strZ(value));
}
@ -50,7 +49,7 @@ testRun(void)
TEST_ERROR(
iniLoad(ioBufferReadNew(iniBuf), testIniLoadCallback, result), FormatError,
"invalid JSON value at line 2 'key=value': invalid type at 'value'");
"invalid JSON value at line 2 'key=value': invalid type at: value");
// Key outside of section
// -------------------------------------------------------------------------------------------------------------------------

576
test/src/module/common/typeJsonTest.c
View File

@ -1,5 +1,5 @@
/***********************************************************************************************************************************
Test Convert JSON to/from KeyValue
Test JSON read/write
***********************************************************************************************************************************/
/***********************************************************************************************************************************
@ -11,303 +11,297 @@ testRun(void)
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("jsonToBool() and jsonToBoolInternal()"))
if (testBegin("JsonRead"))
{
TEST_ERROR(jsonToBool(STRDEF("z")), JsonFormatError, "expected boolean at 'z'");
TEST_ERROR(jsonToBool(STRDEF("falsex")), JsonFormatError, "unexpected characters after boolean at 'x'");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on no data");
TEST_RESULT_BOOL(jsonToBool(STRDEF("true")), true, "bool is true");
TEST_RESULT_BOOL(jsonToBool(STRDEF("false")), false, "bool is false");
}
// *****************************************************************************************************************************
if (testBegin("jsonToInt(), jsonToInt64(), jsonToUInt(), jsonToUInt64() and jsonToIntInternal()"))
{
TEST_ERROR(jsonToUInt(STRDEF("-")), JsonFormatError, "found '-' with no integer at '-'");
TEST_ERROR(jsonToUInt(STRDEF(" 555555555 A")), JsonFormatError, "unexpected characters after number at 'A'");
TEST_RESULT_INT(jsonToInt(STRDEF("-555555555 ")), -555555555, "integer");
TEST_RESULT_INT(jsonToInt64(STRDEF("-555555555555 ")), -555555555555, "integer");
TEST_RESULT_UINT(jsonToUInt(STRDEF("\t555555555\n\r")), 555555555, "unsigned integer");
TEST_RESULT_UINT(jsonToUInt64(STRDEF(" 555555555555 ")), 555555555555, "unsigned integer");
}
// *****************************************************************************************************************************
if (testBegin("jsonToStr() and jsonToStrInternal()"))
{
TEST_ERROR(jsonToStr(STRDEF("\"\\j\"")), JsonFormatError, "invalid escape character 'j'");
TEST_ERROR(jsonToStr(STRDEF("\"\\uffff\"")), JsonFormatError, "unable to decode 'ffff'");
TEST_ERROR(jsonToStr(STRDEF("\"runonstring")), JsonFormatError, "expected '\"' but found null delimiter");
TEST_ERROR(jsonToStr(STRDEF("\"normal\"L")), JsonFormatError, "unexpected characters after string at 'L'");
TEST_ERROR(jsonToStr(STRDEF("nullz")), JsonFormatError, "unexpected characters after string at 'z'");
TEST_RESULT_STR(jsonToStr(STRDEF("null")), NULL, "null string");
TEST_RESULT_STR_Z(jsonToStr(STRDEF(" \"test\"")), "test", "simple string");
TEST_RESULT_STR_Z(jsonToStr(STRDEF("\"te\\\"st\" ")), "te\"st", "string with quote");
TEST_RESULT_STR_Z(jsonToStr(STRDEF("\"\\\"\\\\\\/\\b\\n\\r\\t\\f\\u0026\"")), "\"\\/\b\n\r\t\f&", "string with escapes");
}
// *****************************************************************************************************************************
if (testBegin("jsonToKv() and jsonToKvInternal()"))
{
TEST_ERROR(jsonToKv(STRDEF("[")), JsonFormatError, "expected '{' at '['");
TEST_ERROR(jsonToKv(STRDEF("{\"key1\"= 747}")), JsonFormatError, "expected ':' at '= 747}'");
TEST_ERROR(jsonToKv(STRDEF("{\"key1\" : 747'")), JsonFormatError, "expected '}' at '''");
TEST_ERROR(jsonToKv(STRDEF("{key1")), JsonFormatError, "expected '\"' at 'key1'");
TEST_ERROR(jsonToKv(STRDEF("{}BOGUS")), JsonFormatError, "unexpected characters after object at 'BOGUS'");
KeyValue *kv = NULL;
TEST_ASSIGN(kv, jsonToKv(STRDEF("{\"key1\": 747, \"key2\":\"value2\",\"key3\"\t:\t[\t] }")), "object");
TEST_RESULT_UINT(varLstSize(kvKeyList(kv)), 3, "check key total");
TEST_RESULT_UINT(varUInt64(kvGet(kv, VARSTRDEF("key1"))), 747, "check object uint");
TEST_RESULT_STR_Z(varStr(kvGet(kv, VARSTRDEF("key2"))), "value2", "check object str");
TEST_RESULT_UINT(varLstSize(varVarLst(kvGet(kv, VARSTRDEF("key3")))), 0, "check empty array");
TEST_ASSIGN(kv, jsonToKv(STRDEF("\t{\n} ")), "empty object");
TEST_RESULT_UINT(varLstSize(kvKeyList(kv)), 0, "check key total");
}
// *****************************************************************************************************************************
if (testBegin("jsonToVar()"))
{
TEST_ERROR(jsonToVar(strNew()), JsonFormatError, "expected data");
TEST_ERROR(jsonToVar(STRDEF(" \t\r\n ")), JsonFormatError, "expected data");
TEST_ERROR(jsonToVar(STRDEF("z")), JsonFormatError, "invalid type at 'z'");
TEST_ERROR(jsonToVar(STRDEF("3 =")), JsonFormatError, "unexpected characters after JSON at '='");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_STR_Z(varStr(jsonToVar(STRDEF(" \"test\""))), "test", "simple string");
TEST_RESULT_STR_Z(varStr(jsonToVar(STRDEF("\"te\\\"st\" "))), "te\"st", "string with quote");
TEST_RESULT_STR_Z(varStr(jsonToVar(STRDEF("\"\\\"\\\\\\/\\b\\n\\r\\t\\f\""))), "\"\\/\b\n\r\t\f", "string with escapes");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(jsonToVar(STRDEF("ton")), JsonFormatError, "expected boolean at 'ton'");
TEST_RESULT_BOOL(varBool(jsonToVar(STRDEF(" true"))), true, "boolean true");
TEST_RESULT_BOOL(varBool(jsonToVar(STRDEF("false "))), false, "boolean false");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(jsonToVar(STRDEF("not")), JsonFormatError, "expected null at 'not'");
TEST_RESULT_PTR(jsonToVar(STRDEF("null")), NULL, "null value");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(jsonToVar(STRDEF("[1, \"test\", false")), JsonFormatError, "expected ']' at ''");
VariantList *valueList = NULL;
TEST_ASSIGN(valueList, varVarLst(jsonToVar(STRDEF("[1, \"test\", false]"))), "array");
TEST_RESULT_UINT(varLstSize(valueList), 3, "check array size");
TEST_RESULT_UINT(varUInt64(varLstGet(valueList, 0)), 1, "check array int");
TEST_RESULT_STR_Z(varStr(varLstGet(valueList, 1)), "test", "check array str");
TEST_RESULT_BOOL(varBool(varLstGet(valueList, 2)), false, "check array bool");
TEST_ASSIGN(valueList, varVarLst(jsonToVar(STRDEF("[ ]"))), "empty array");
TEST_RESULT_UINT(varLstSize(valueList), 0, "check array size");
// -------------------------------------------------------------------------------------------------------------------------
KeyValue *kv = NULL;
TEST_ASSIGN(kv, varKv(jsonToVar(STRDEF("\t{\n} "))), "empty object");
TEST_RESULT_UINT(varLstSize(kvKeyList(kv)), 0, "check key total");
}
// *****************************************************************************************************************************
if (testBegin("jsonToVarLst() and jsonToArrayInternal()"))
{
TEST_ERROR(jsonToVarLst(STRDEF("{")), JsonFormatError, "expected '[' at '{'");
TEST_ERROR(jsonToVarLst(STRDEF("[")), JsonFormatError, "expected data");
TEST_ERROR(jsonToVarLst(STRDEF(" [] ZZZ")), JsonFormatError, "unexpected characters after array at 'ZZZ'");
TEST_RESULT_STRLST_Z(strLstNewVarLst(jsonToVarLst(STRDEF("[\"e1\", \"e2\"]"))), "e1\ne2\n", "json list");
}
// *****************************************************************************************************************************
if (testBegin("jsonFromBool()"))
{
TEST_RESULT_STR_Z(jsonFromBool(true), "true", "json bool true");
TEST_RESULT_STR_Z(jsonFromBool(false), "false", "json bool true");
}
// *****************************************************************************************************************************
if (testBegin("jsonFromInt(), jsonFromInt64(), jsonFromUInt() and jsonFromUInt64()"))
{
TEST_RESULT_STR_Z(jsonFromInt(-2147483648), "-2147483648", "json int");
TEST_RESULT_STR_Z(jsonFromInt64(-9223372036854775807L), "-9223372036854775807", "json int64");
TEST_RESULT_STR_Z(jsonFromUInt(4294967295), "4294967295", "json uint");
TEST_RESULT_STR_Z(jsonFromUInt64(18446744073709551615UL), "18446744073709551615", "json uint64");
}
// *****************************************************************************************************************************
if (testBegin("jsonFromStr() and jsonFromStrInternal()"))
{
TEST_RESULT_STR_Z(jsonFromStr(NULL), "null", "null string");
TEST_RESULT_STR_Z(jsonFromStr(STRDEF("simple string")), "\"simple string\"", "simple string");
TEST_RESULT_STR_Z(jsonFromStr(STRDEF("\"\\/\b\n\r\t\f")), "\"\\\"\\\\/\\b\\n\\r\\t\\f\"", "string escapes");
}
// *****************************************************************************************************************************
if (testBegin("jsonFromKv(), jsonFromKvInternal()"))
{
KeyValue *keyValue = kvNew();
String *json = NULL;
TEST_ASSIGN(json, jsonFromKv(keyValue), "kvToJson - empty");
TEST_RESULT_STR_Z(json, "{}", " empty curly brackets");
kvPut(keyValue, varNewStrZ("backup"), varNewVarLst(varLstNew()));
TEST_ASSIGN(json, jsonFromKv(keyValue), "kvToJson - kv with empty array");
TEST_RESULT_STR_Z(json, "{\"backup\":[]}", " kv with empty array brackets");
kvPutKv(keyValue, VARSTRDEF("archive"));
kvPutKv(keyValue, VARSTRDEF("empty"));
kvPut(keyValue, varNewStrZ("bool1"), varNewBool(true));
kvPut(keyValue, varNewStrZ("bool2"), varNewBool(false));
kvPut(keyValue, varNewStrZ("null-str"), varNewStr(NULL));
kvPut(keyValue, varNewStrZ("checknull"), (Variant *)NULL);
VariantList *dbList = varLstNew();
Variant *dbInfo = varNewKv(kvNew());
kvPut(varKv(dbInfo), VARSTRDEF("id"), varNewInt(1));
kvPut(varKv(dbInfo), VARSTRDEF("version"), VARSTRDEF("9.4"));
varLstAdd(dbList, dbInfo);
varLstAdd(dbList, NULL);
kvPut(keyValue, varNewStrZ("db"), varNewVarLst(dbList));
kvPut(keyValue, varNewStrZ("null-list"), varNewVarLst(NULL));
TEST_ASSIGN(json, jsonFromKv(keyValue), "kvToJson - kv with empty array, kv, varList with values");
TEST_RESULT_STR_Z(
json,
"{"
"\"archive\":{},"
"\"backup\":[],"
"\"bool1\":true,"
"\"bool2\":false,"
"\"checknull\":null,"
"\"db\":["
"{"
"\"id\":1,"
"\"version\":\"9.4\""
"},"
"null"
"],"
"\"empty\":{},"
"\"null-list\":null,"
"\"null-str\":null"
"}",
" kv with empty array, kv, varList with values");
TEST_ASSIGN(
keyValue,
varKv(
jsonToVar(
STRDEF(
"{\"backup-info-size-delta\":1982702,\"backup-prior\":\"20161219-212741F_20161219-212803I\","
"\"backup-reference\":[\"20161219-212741F\",\"20161219-212741F_20161219-212803I\"],"
"\"checksum-page-error\":[1,[4,6]],\"backup-timestamp-start\":1482182951}"))),
"multiple values with array");
TEST_ASSIGN(json, jsonFromKv(keyValue), " kvToJson - sorted");
TEST_RESULT_STR_Z(
json,
"{\"backup-info-size-delta\":1982702,\"backup-prior\":\"20161219-212741F_20161219-212803I\","
"\"backup-reference\":[\"20161219-212741F\",\"20161219-212741F_20161219-212803I\"],"
"\"backup-timestamp-start\":1482182951,\"checksum-page-error\":[1,[4,6]]}",
" check string no pretty print");
}
// *****************************************************************************************************************************
if (testBegin("jsonFromVar()"))
{
TEST_ERROR(jsonFromVar(varNewInt(1)), JsonFormatError, "variant type is invalid");
String *json = NULL;
Variant *keyValue = NULL;
TEST_ASSIGN(keyValue, varNewKv(kvNew()), "build new kv");
kvPut(varKv(keyValue), varNewStrZ("backup-info-size-delta"), varNewInt(1982702));
kvPut(varKv(keyValue), varNewStrZ("backup-prior"), varNewStrZ("20161219-212741F_20161219-212803I"));
Variant *listVar = NULL;
TEST_ASSIGN(listVar, varNewVarLst(varLstNew()), " new string array to kv");
varLstAdd(varVarLst(listVar), varNewStrZ("20161219-212741F"));
varLstAdd(varVarLst(listVar), varNewStrZ("20161219-212741F_20161219-212803I"));
varLstAdd(varVarLst(listVar), varNewStrZ(NULL));
kvPut(varKv(keyValue), varNewStrZ("backup-reference"), listVar);
kvPut(varKv(keyValue), varNewStrZ("backup-timestamp-start"), varNewInt(1482182951));
Variant *listVar2 = NULL;
TEST_ASSIGN(listVar2, varNewVarLst(varLstNew()), " new int array to kv");
varLstAdd(varVarLst(listVar2), varNewInt(1));
kvPut(varKv(keyValue), varNewStrZ("checksum-page-error"), listVar2);
// Embed a keyValue section to test recursion
const Variant *sectionKey = VARSTRDEF("section");
KeyValue *sectionKv = kvPutKv(varKv(keyValue), sectionKey);
kvPut(sectionKv, VARSTRDEF("key1"), VARSTRDEF("value1"));
kvPut(sectionKv, VARSTRDEF("key2"), (Variant *)NULL);
kvPut(sectionKv, VARSTRDEF("key3"), VARSTRDEF("value2"));
kvAdd(sectionKv, VARSTRDEF("escape"), VARSTRDEF("\"\\/\b\n\r\t\f"));
TEST_ASSIGN(json, jsonFromVar(keyValue), "KeyValue");
TEST_RESULT_STR_Z(
json,
"{\"backup-info-size-delta\":1982702,\"backup-prior\":\"20161219-212741F_20161219-212803I\","
"\"backup-reference\":[\"20161219-212741F\",\"20161219-212741F_20161219-212803I\",null],"
"\"backup-timestamp-start\":1482182951,\"checksum-page-error\":[1],"
"\"section\":{\"escape\":\"\\\"\\\\/\\b\\n\\r\\t\\f\",\"key1\":\"value1\",\"key2\":null,\"key3\":\"value2\"}}",
" sorted json string result, no pretty print");
TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF(""))), JsonFormatError, "expected data");
//--------------------------------------------------------------------------------------------------------------------------
Variant *varListOuter = NULL;
TEST_TITLE("error on invalid type");
TEST_ASSIGN(json, jsonFromVar(varNewVarLst(varLstNew())), "VariantList");
TEST_RESULT_STR_Z(json, "[]", " empty list no pretty print");
TEST_ASSIGN(varListOuter, varNewVarLst(varLstNew()), "new variant list with keyValues");
varLstAdd(varVarLst(varListOuter), varNewStrZ("ASTRING"));
varLstAdd(varVarLst(varListOuter), varNewInt64(9223372036854775807LL));
varLstAdd(varVarLst(varListOuter), varNewInt(2147483647));
varLstAdd(varVarLst(varListOuter), varNewBool(true));
varLstAdd(varVarLst(varListOuter), varNewVarLst(NULL));
varLstAdd(varVarLst(varListOuter), varNewVarLst(varLstNew()));
varLstAdd(varVarLst(varListOuter), NULL);
varLstAdd(varVarLst(varListOuter), keyValue);
TEST_ASSIGN(json, jsonFromVar(varListOuter), "VariantList");
TEST_RESULT_STR_Z(
json,
"[\"ASTRING\",9223372036854775807,2147483647,true,null,[],null,{\"backup-info-size-delta\":1982702,"
"\"backup-prior\":\"20161219-212741F_20161219-212803I\","
"\"backup-reference\":[\"20161219-212741F\",\"20161219-212741F_20161219-212803I\",null],"
"\"backup-timestamp-start\":1482182951,\"checksum-page-error\":[1],"
"\"section\":{\"escape\":\"\\\"\\\\/\\b\\n\\r\\t\\f\",\"key1\":\"value1\",\"key2\":null,\"key3\":\"value2\"}}]",
" sorted json string result no pretty print");
Variant *keyValue2 = varDup(keyValue);
varLstAdd(varVarLst(varListOuter), keyValue2);
TEST_ASSIGN(json, jsonFromVar(varListOuter), "VariantList - multiple elements");
TEST_RESULT_STR_Z(
json,
"[\"ASTRING\",9223372036854775807,2147483647,true,null,[],null,{\"backup-info-size-delta\":1982702,"
"\"backup-prior\":\"20161219-212741F_20161219-212803I\","
"\"backup-reference\":[\"20161219-212741F\",\"20161219-212741F_20161219-212803I\",null],"
"\"backup-timestamp-start\":1482182951,\"checksum-page-error\":[1],"
"\"section\":{\"escape\":\"\\\"\\\\/\\b\\n\\r\\t\\f\",\"key1\":\"value1\",\"key2\":null,\"key3\":\"value2\"}},"
"{\"backup-info-size-delta\":1982702,\"backup-prior\":\"20161219-212741F_20161219-212803I\","
"\"backup-reference\":[\"20161219-212741F\",\"20161219-212741F_20161219-212803I\",null],"
"\"backup-timestamp-start\":1482182951,\"checksum-page-error\":[1],"
"\"section\":{\"escape\":\"\\\"\\\\/\\b\\n\\r\\t\\f\",\"key1\":\"value1\",\"key2\":null,\"key3\":\"value2\"}}]",
" sorted json string result no pretty print");
VariantList *varList = varLstNew();
varLstAdd(varList, varNewUInt(32));
varLstAdd(varList, varNewUInt64(10000000000));
TEST_RESULT_STR_Z(jsonFromVar(varNewVarLst(varList)), "[32,10000000000]", "list various types");
TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF("x"))), JsonFormatError, "invalid type at: x");
//--------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_STR_Z(jsonFromVar(NULL), "null", "null variant");
TEST_RESULT_STR_Z(jsonFromVar(varNewBool(true)), "true", "bool variant");
TEST_RESULT_STR_Z(jsonFromVar(varNewUInt(66)), "66", "uint variant");
TEST_RESULT_STR_Z(jsonFromVar(varNewUInt64(10000000001)), "10000000001", "uint64 variant");
TEST_RESULT_STR_Z(jsonFromVar(varNewStrZ("test \" string")), "\"test \\\" string\"", "string variant");
TEST_TITLE("error on missing comma");
JsonRead *read = NULL;
TEST_ASSIGN(read, jsonReadNew(STRDEF("\r[\ttrue true]")), "new read");
TEST_RESULT_VOID(jsonReadArrayBegin(read), "array begin");
TEST_RESULT_BOOL(jsonReadBool(read), true, "bool true");
jsonReadConsumeWhiteSpace(read);
TEST_ERROR(jsonReadBool(read), JsonFormatError, "missing comma at: true]");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on mismatched container end");
TEST_ASSIGN(read, jsonReadNew(STRDEF("\r{ ] ")), "new read");
TEST_RESULT_VOID(jsonReadObjectBegin(read), "object begin");
jsonReadConsumeWhiteSpace(read);
TEST_ERROR(jsonReadObjectEnd(read), JsonFormatError, "expected } but found ] at: ] ");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on no digits in number");
TEST_ERROR(jsonReadNumber(jsonReadNew(STRDEF("a"))), JsonFormatError, "no digits found at: a");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on number larger than available buffer");
TEST_ERROR(
jsonReadNumber(jsonReadNew(STRDEF("9999999999999999999999999999999999999999999999999999999999999999"))),
JsonFormatError, "invalid number at: 9999999999999999999999999999999999999999999999999999999999999999");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on invalid ascii encoding");
TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF("\"\\u1111"))), JsonFormatError, "unable to decode at: \\u1111");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on invalid escape");
TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF("\"\\xy"))), JsonFormatError, "invalid escape character at: \\xy");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on unterminated string");
TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF("\""))), JsonFormatError, "expected '\"' but found null delimiter");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on invalid bool");
TEST_ERROR(jsonReadBool(jsonReadNew(STRDEF("fase"))), JsonFormatError, "expected boolean at: fase");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on invalid integers");
TEST_ERROR(
jsonReadInt(jsonReadNew(strNewFmt("%" PRId64, (int64_t)INT_MIN - 1))), JsonFormatError,
"-2147483649 is out of range for int");
TEST_ERROR(
jsonReadInt(jsonReadNew(strNewFmt("%" PRId64, (int64_t)INT_MAX + 1))), JsonFormatError,
"2147483648 is out of range for int");
TEST_ERROR(
jsonReadInt64(jsonReadNew(strNewFmt("%" PRIu64, (uint64_t)INT64_MAX + 1))), JsonFormatError,
"9223372036854775808 is out of range for int64");
TEST_ERROR(jsonReadUInt(jsonReadNew(STRDEF("-1"))), JsonFormatError, "-1 is out of range for uint");
TEST_ERROR(
jsonReadUInt(jsonReadNew(strNewFmt("%" PRIu64, (uint64_t)UINT_MAX + 1))), JsonFormatError,
"4294967296 is out of range for uint");
TEST_ERROR(jsonReadUInt64(jsonReadNew(STRDEF("-1"))), JsonFormatError, "-1 is out of range for uint64");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on invalid null");
TEST_ERROR(jsonReadNull(jsonReadNew(STRDEF("nil"))), JsonFormatError, "expected null at: nil");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on escape characters in key");
TEST_ASSIGN(read, jsonReadNew(STRDEF("{\"\\\"")), "new read");
TEST_RESULT_VOID(jsonReadObjectBegin(read), "object begin");
jsonReadConsumeWhiteSpace(read);
TEST_ERROR(jsonReadKey(read), JsonFormatError, "escape character not allowed in key at: \\\"");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on unterminated key");
TEST_ASSIGN(read, jsonReadNew(STRDEF("{\"")), "new read");
TEST_RESULT_VOID(jsonReadObjectBegin(read), "object begin");
jsonReadConsumeWhiteSpace(read);
TEST_ERROR(jsonReadKey(read), JsonFormatError, "expected '\"' but found null delimiter");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on missing : after key");
TEST_ASSIGN(read, jsonReadNew(STRDEF("\r{ \"key1\"xxx")), "new read");
TEST_RESULT_VOID(jsonReadObjectBegin(read), "object begin");
TEST_ERROR(jsonReadKey(read), JsonFormatError, "expected : after key 'key1' at: xxx");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("scalar");
TEST_ASSIGN(read, jsonReadNew(STRDEF("\rtrue\t")), "new read");
TEST_RESULT_BOOL(jsonReadBool(read), true, "bool true");
TEST_ERROR(jsonReadBool(read), FormatError, "JSON read is complete");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("complex");
#define TEST_VARIANT "[-1,true,\"zxv\",{\"key1\":null,\"key2\":777}]"
const String *json = STRDEF(
"\t\r\n " // Leading whitespace
"[\n"
" {"
" \"key1\"\t: \"\\u0076\\na\\tlu e\\f\\b\\/\\\\\\\"\\r1\","
" \"key2\"\t: 777,"
" \"key3\"\t: 777,"
" \"key4\"\t: 9223372036854775807"
" },"
" \"abc\\/\","
" \"def\","
" null,"
" [\"a\", \"b\"],"
" 123,"
" 18446744073709551615,"
" -1,"
" -9223372036854775807,"
" [true, {\"key1\":null,\"key1\":\"value1\"}],"
" true,"
" " TEST_VARIANT ","
" false\n"
"]"
" "); // Extra whitespace at the end
TEST_ASSIGN(read, jsonReadNew(strNewFmt("%s]", strZ(json))), "new read");
TEST_RESULT_VOID(jsonReadArrayBegin(read), "array begin");
jsonReadConsumeWhiteSpace(read);
TEST_ERROR_FMT(jsonReadArrayEnd(read), JsonFormatError, "expected ] but found { at: %s", read->json);
TEST_RESULT_VOID(jsonReadObjectBegin(read), "object begin");
TEST_RESULT_BOOL(jsonReadKeyExpect(read, STRDEF("key0")), false, "expect key0");
TEST_RESULT_STR_Z(jsonReadKey(read), "key1", "key1");
jsonReadConsumeWhiteSpace(read);
TEST_ERROR_FMT(jsonReadKey(read), AssertError, "key has already been read at: %s", read->json);
TEST_RESULT_STR_Z(jsonReadStr(read), "v\na\tlu e\f\b/\\\"\r1", "str");
TEST_ERROR_FMT(jsonReadStr(read), AssertError, "key has not been read at: %s", read->json);
TEST_ERROR(jsonReadKeyRequire(read, STRDEF("key1")), JsonFormatError, "required key 'key1' not found");
TEST_RESULT_VOID(jsonReadKeyRequire(read, STRDEF("key2")), "key2");
jsonReadConsumeWhiteSpace(read);
TEST_ERROR_FMT(jsonReadStr(read), JsonFormatError, "expected 'str' but found 'nmbr' at: %s", read->json);
TEST_RESULT_INT(jsonReadInt(read), 777, "int");
TEST_RESULT_BOOL(jsonReadKeyExpectZ(read, "key4"), true, "expect key4");
TEST_RESULT_INT(jsonReadInt64(read), INT64_MAX, "int");
TEST_RESULT_BOOL(jsonReadKeyExpectStrId(read, STRID5("key5", 0xee4ab0)), false, "do not expect key5");
TEST_ERROR(jsonReadKeyRequireStrId(read, STRID5("key5", 0xee4ab0)), JsonFormatError, "required key 'key5' not found");
TEST_ERROR(jsonReadKeyRequireZ(read, "key5"), JsonFormatError, "required key 'key5' not found");
TEST_RESULT_VOID(jsonReadObjectEnd(read), "object end");
TEST_RESULT_STR_Z(jsonReadStr(read), "abc/", "str");
TEST_RESULT_STR_Z(strIdToStr(jsonReadStrId(read)), "def", "strid");
TEST_RESULT_STR_Z(jsonReadStr(read), NULL, "str null");
TEST_RESULT_STRLST_Z(jsonReadStrLst(read), "a\nb\n", "str list");
TEST_RESULT_UINT(jsonReadUInt(read), 123, "uint");
TEST_RESULT_UINT(jsonReadUInt64(read), 18446744073709551615U, "uint64");
TEST_RESULT_INT(jsonReadInt(read), -1, "int");
TEST_RESULT_INT(jsonReadInt64(read), -9223372036854775807L, "int64");
TEST_RESULT_VOID(jsonReadSkip(read), "skip");
TEST_RESULT_BOOL(jsonReadBool(read), true, "bool true");
TEST_RESULT_STR_Z(jsonFromVar(jsonReadVar(read)), TEST_VARIANT, "var");
TEST_RESULT_BOOL(jsonReadBool(read), false, "bool false");
TEST_RESULT_VOID(jsonReadArrayEnd(read), "array end");
TEST_ERROR(jsonReadArrayEnd(read), FormatError, "JSON read is complete");
TEST_RESULT_VOID(jsonReadFree(read), "free");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("variant");
TEST_RESULT_STR_Z(varStr(jsonToVar(STRDEF("\"test\""))), "test", "var");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("validate");
TEST_RESULT_VOID(jsonValidate(json), "valid");
TEST_ERROR(jsonValidate(STRDEF("\"\\u000")), JsonFormatError, "unable to decode at: \\u000");
TEST_ERROR(jsonValidate(STRDEF("\"")), JsonFormatError, "expected '\"' but found null delimiter");
TEST_ERROR(jsonValidate(STRDEF("{\"key\"x")), JsonFormatError, "expected : after key at: x");
TEST_ERROR(jsonValidate(strNewFmt("%s]", strZ(json))), JsonFormatError, "characters after JSON at: ]");
}
// *****************************************************************************************************************************
if (testBegin("JsonWrite"))
{
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("complex");
JsonWrite *write = NULL;
TEST_ASSIGN(write, jsonWriteNewP(), "new write");
TEST_RESULT_VOID(jsonWriteArrayBegin(write), "array begin");
TEST_RESULT_VOID(jsonWriteBool(write, true), "bool true");
TEST_RESULT_VOID(jsonWriteInt(write, 55), "int 55");
TEST_RESULT_VOID(jsonWriteInt64(write, INT64_MIN), "int64 min");
TEST_RESULT_VOID(jsonWriteStr(write, STRDEF("two\nlines")), "str with two lines");
TEST_RESULT_VOID(jsonWriteObjectBegin(write), "object begin");
TEST_ERROR(jsonWriteBool(write, false), JsonFormatError, "key has not been written");
TEST_ERROR(
jsonWriteKeyZ(write, "1234567890123456789012345678901234567890123456789012345678901234567"), AssertError,
"key '1234567890123456789012345678901234567890123456789012345678901234567' must not be longer than 66 bytes");
TEST_RESULT_VOID(jsonWriteKey(write, STRDEF("flag")), "key 'flag'");
TEST_ERROR(jsonWriteKey(write, STRDEF("flag")), JsonFormatError, "key has already been written");
TEST_RESULT_VOID(jsonWriteBool(write, false), "bool false");
TEST_ERROR(jsonWriteKey(write, STRDEF("flag")), JsonFormatError, "key 'flag' is not after prior key 'flag'");
TEST_RESULT_VOID(jsonWriteKeyStrId(write, STRID5("key5", 0xee4ab0)), "key 'key5'");
TEST_RESULT_VOID(jsonWriteStrFmt(write, "%d", 898), "str fmt");
TEST_RESULT_VOID(jsonWriteKeyZ(write, "val"), "key 'val'");
TEST_RESULT_VOID(jsonWriteUInt64(write, UINT64_MAX), "uint64 max");
TEST_RESULT_VOID(jsonWriteObjectEnd(write), "object end");
StringList *list = strLstNew();
strLstAddZ(list, "a");
strLstAddZ(list, "\n\"a\\\r\t\bbc");
TEST_RESULT_VOID(jsonWriteStrLst(write, list), "write str list");
TEST_RESULT_VOID(jsonWriteUInt(write, 66), "write int 66");
TEST_RESULT_VOID(jsonWriteJson(write, NULL), "null json");
TEST_RESULT_VOID(jsonWriteJson(write, STRDEF("{}")), "json");
TEST_RESULT_VOID(jsonWriteZ(write, NULL), "null z");
TEST_RESULT_VOID(jsonWriteZ(write, "a string\f"), "z");
TEST_RESULT_VOID(jsonWriteStrId(write, strIdFromZ("hilly")), "strid");
TEST_RESULT_VOID(jsonWriteVar(write, varNewKv(NULL)), "null kv");
TEST_RESULT_VOID(jsonWriteStr(write, NULL), "null str");
TEST_RESULT_VOID(jsonWriteArrayEnd(write), "array end");
TEST_ERROR(jsonWriteUInt(write, 66), JsonFormatError, "JSON write is complete");
TEST_RESULT_STR_Z(
jsonWriteResult(write),
"["
"true,"
"55,"
"-9223372036854775808,"
"\"two\\nlines\","
"{"
"\"flag\":false,"
"\"key5\":\"898\","
"\"val\":18446744073709551615"
"},"
"[\"a\",\"\\n\\\"a\\\\\\r\\t\\bbc\"],"
"66,"
"null,"
"{},"
"null,"
"\"a string\\f\","
"\"hilly\","
"null,"
"null"
"]",
"json result");
TEST_RESULT_VOID(jsonWriteFree(write), "free");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("variant");
TEST_RESULT_STR_Z(jsonFromVar(varNewStr(NULL)), "null", "null str");
TEST_RESULT_STR_Z(jsonFromVar(varNewVarLst(NULL)), "null", "null var lst");
TEST_RESULT_STR_Z(jsonFromVar(varNewUInt(47)), "47", "uint");
TEST_RESULT_STR_Z(jsonFromVar(varNewInt(-99)), "-99", "int");
}
FUNCTION_HARNESS_RETURN_VOID();

12
test/src/module/info/infoPgTest.c
View File

@ -15,14 +15,14 @@ testInfoBackupSaveCallback(void *data, const String *sectionNext, InfoSave *info
{
(void)data;
if (infoSaveSection(infoSaveData, STRDEF("backup:current"), sectionNext))
infoSaveValue(infoSaveData, STRDEF("backup:current"), STRDEF("20161219-212741F"), STRDEF("{}"));
if (infoSaveSection(infoSaveData, "backup:current", sectionNext))
infoSaveValue(infoSaveData, "backup:current", "20161219-212741F", STRDEF("{}"));
if (infoSaveSection(infoSaveData, STRDEF("db:backup"), sectionNext))
infoSaveValue(infoSaveData, STRDEF("db:backup"), STRDEF("key"), STRDEF("\"value\""));
if (infoSaveSection(infoSaveData, "db:backup", sectionNext))
infoSaveValue(infoSaveData, "db:backup", "key", STRDEF("\"value\""));
if (infoSaveSection(infoSaveData, STRDEF("later"), sectionNext))
infoSaveValue(infoSaveData, STRDEF("later"), STRDEF("key"), STRDEF("\"value\""));
if (infoSaveSection(infoSaveData, "later", sectionNext))
infoSaveValue(infoSaveData, "later", "key", STRDEF("\"value\""));
}
/***********************************************************************************************************************************

8
test/src/module/info/infoTest.c
View File

@ -60,11 +60,11 @@ Test save callbacks
static void
testInfoSaveCallback(void *data, const String *sectionNext, InfoSave *infoSaveData)
{
if (infoSaveSection(infoSaveData, STRDEF("c"), sectionNext))
infoSaveValue(infoSaveData, STRDEF("c"), STRDEF("key"), (String *)data);
if (infoSaveSection(infoSaveData, "c", sectionNext))
infoSaveValue(infoSaveData, "c", "key", (String *)data);
if (infoSaveSection(infoSaveData, STRDEF("d"), sectionNext))
infoSaveValue(infoSaveData, STRDEF("d"), STRDEF("key"), (String *)data);
if (infoSaveSection(infoSaveData, "d", sectionNext))
infoSaveValue(infoSaveData, "d", "key", (String *)data);
}
/***********************************************************************************************************************************

3
test/src/module/performance/typeTest.c
View File

@ -47,13 +47,12 @@ testComparator(const void *item1, const void *item2)
Test callback to count ini load results
***********************************************************************************************************************************/
static void
testIniLoadCountCallback(void *data, const String *section, const String *key, const String *value, const Variant *valueVar)
testIniLoadCountCallback(void *const data, const String *const section, const String *const key, const String *const value)
{
(*(unsigned int *)data)++;
(void)section;
(void)key;
(void)value;
(void)valueVar;
}
/***********************************************************************************************************************************

4
test/src/module/protocol/protocolTest.c
View File

@ -559,13 +559,13 @@ testRun(void)
TEST_ERROR(
protocolClientNew(STRDEF("test client"), STRDEF("test"), HRN_FORK_PARENT_READ(0), HRN_FORK_PARENT_WRITE(0)),
JsonFormatError, "expected '{' at 'bogus greeting'");
JsonFormatError, "invalid type at: bogus greeting");
TEST_ERROR(
protocolClientNew(STRDEF("test client"), STRDEF("test"), HRN_FORK_PARENT_READ(0), HRN_FORK_PARENT_WRITE(0)),
ProtocolError, "greeting key 'name' must be string type");
TEST_ERROR(
protocolClientNew(STRDEF("test client"), STRDEF("test"), HRN_FORK_PARENT_READ(0), HRN_FORK_PARENT_WRITE(0)),
ProtocolError, "unable to find greeting key 'name'");
ProtocolError, "greeting key 'name' must be string type");
TEST_ERROR(
protocolClientNew(STRDEF("test client"), STRDEF("test"), HRN_FORK_PARENT_READ(0), HRN_FORK_PARENT_WRITE(0)),
ProtocolError, "unable to find greeting key 'name'");

6
test/src/module/storage/gcsTest.c
View File

@ -368,7 +368,11 @@ testRun(void)
storageGcsRequestP(
(StorageGcs *)storageDriver(storage), HTTP_VERB_POST_STR, .noBucket = true,
.content = BUFSTR(jsonFromKv(kvPut(kvNew(), GCS_JSON_NAME_VAR, VARSTRDEF("bucket")))));
.content = BUFSTR(
jsonWriteResult(
jsonWriteObjectEnd(
jsonWriteStr(
jsonWriteKeyZ(jsonWriteObjectBegin(jsonWriteNewP()), GCS_JSON_NAME), STRDEF("bucket"))))));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("auth error");