db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-09-16 09:06:18 +02:00
Code Issues Releases Activity

Add pack type.

Browse Source 
The pack type is an architecture-independent format for serializing data compactly, inspired by ProtocolBuffers and Avro.

Also add ioReadSmall(), which is optimized for small binary reads, similar to ioReadLineParam().
This commit is contained in:
David Steele
2020-12-09 12:05:14 -05:00
committed by GitHub
parent 87996558d2
commit 8361a97482
12 changed files with 2660 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
doc/xml/release.xml
View File

@@ -15,6 +15,14 @@
<release date="XXXX-XX-XX" version="2.32dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-development-list>
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>Add <id>pack</id> type.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>

1
src/Makefile.in
View File

@@ -108,6 +108,7 @@ SRCS = \
common/type/keyValue.c \
common/type/list.c \
common/type/mcv.c \
common/type/pack.c \
common/type/string.c \
common/type/stringList.c \
common/type/variant.c \

66
src/common/io/read.c
View File

@@ -212,6 +212,72 @@ ioRead(IoRead *this, Buffer *buffer)
FUNCTION_LOG_RETURN(SIZE, outputRemains - bufRemains(buffer));
}
/**********************************************************************************************************************************/
size_t
ioReadSmall(IoRead *this, Buffer *buffer)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_READ, this);
FUNCTION_TEST_PARAM(BUFFER, buffer);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(buffer != NULL);
ASSERT(this->opened && !this->closed);
// Allocate the internal output buffer if it has not already been allocated
if (this->output == NULL)
{
MEM_CONTEXT_BEGIN(this->memContext)
{
this->output = bufNew(ioBufferSize());
}
MEM_CONTEXT_END();
}
// Store size of remaining portion of buffer to calculate total read at the end
size_t outputRemains = bufRemains(buffer);
do
{
// Internal output buffer remains taking into account the position
size_t outputInternalRemains = bufUsed(this->output) - this->outputPos;
// Use any data in the internal output buffer
if (outputInternalRemains > 0)
{
// Determine how much data should be copied
size_t size = outputInternalRemains > bufRemains(buffer) ? bufRemains(buffer) : outputInternalRemains;
// Copy data to the output buffer
bufCatSub(buffer, this->output, this->outputPos, size);
this->outputPos += size;
}
// If more data is required
if (!bufFull(buffer))
{
// If the data required is the same size as the internal output buffer then just read into the external buffer
if (bufRemains(buffer) >= bufSize(this->output))
{
ioReadInternal(this, buffer, true);
}
// Else read as much data as is available. If is not enough we will try again later.
else
{
// Clear the internal output buffer since all data was copied already
bufUsedZero(this->output);
this->outputPos = 0;
ioReadInternal(this, this->output, false);
}
}
}
while (!bufFull(buffer));
FUNCTION_TEST_RETURN(outputRemains - bufRemains(buffer));
}
/***********************************************************************************************************************************
The entire string to search for must fit within a single buffer.
***********************************************************************************************************************************/

3
src/common/io/read.h
View File

@@ -29,6 +29,9 @@ bool ioReadOpen(IoRead *this);
// Read data from IO and process filters
size_t ioRead(IoRead *this, Buffer *buffer);
// Same as ioRead() but optimized for small reads (intended for making repetitive reads that are smaller than ioBufferSize())
size_t ioReadSmall(IoRead *this, Buffer *buffer);
// Read linefeed-terminated string
String *ioReadLine(IoRead *this);

7
src/common/type/convert.h
View File

@@ -37,9 +37,10 @@ size_t cvtInt64ToZ(int64_t value, char *buffer, size_t bufferSize);
int64_t cvtZToInt64(const char *value);
int64_t cvtZToInt64Base(const char *value, int base);
// Convert int32/64 to uint32/64 using zigzag encoding and vice versa. Zigzag encoding places the sign bit in the least significant
// bit so that -1 is encoded as 1, 1 as 2, etc. This moves as many bits as possible into the low order bits which is good for other
// types of encoding, e.g. base-128.
// Convert int32/64 to uint32/64 using zigzag encoding and vice versa. Zigzag encoding places the sign in the least significant bit
// so that signed and unsigned values alternate, e.g. 0 = 0, -1 = 1, 1 = 2, -2 = 3, 2 = 4, -3 = 5, 3 = 6, etc. This moves as many
// bits as possible into the low order bits which is good for other types of encoding, e.g. base-128. See
// http://neurocline.github.io/dev/2015/09/17/zig-zag-encoding.html for details.
__attribute__((always_inline)) static inline uint32_t
cvtInt32ToZigZag(int32_t value)
{

1609
src/common/type/pack.c Normal file
View File

File diff suppressed because it is too large Load Diff

500
src/common/type/pack.h Normal file
View File

@@ -0,0 +1,500 @@
/***********************************************************************************************************************************
Pack Type
The pack type encodes binary data compactly while still maintaining structure and strict typing. The idea is based on Thrift,
ProtocolBuffers, and Avro, compared here: https://medium.com/better-programming/use-binary-encoding-instead-of-json-dec745ec09b6.
The pack type has been further optimized to balance between purely in-memory structures and those intended to be passed via a
protocol or saved in a file.
Integers are stored with base-128 varint encoding which is equivalent to network byte order, i.e., the endianness of the sending and
receiving host don't matter.
The overall idea is similar to JSON but IDs are used instead of names, typing is more granular, and the representation is far more
compact. A pack can readily be converted to JSON but the reverse is not as precise due to loose typing in JSON. A pack is a stream
format, i.e. it is intended to be read in order from beginning to end.
Fields in a pack are identified by IDs. A field ID is stored as a delta from the previous ID, which is very efficient, but means
that reading from the middle is generally not practical. The size of the gap between field IDs is important -- a gap of 1 never
incurs extra cost, but depending on the field type larger gaps may require additional bytes to store the field ID delta.
NULLs are not stored in a pack and are therefore not typed. A NULL is essentially just a gap in the field IDs. Fields that are
frequently NULL are best stored at the end of an object. When using .defaultWrite = false in write functions a NULL will be written
(by making a gap in the IDs) if the value matches the default. When using read functions the default will always be returned
when the field is NULL (i.e. missing). The standard default is the C default for that type (e.g. bool = false, int = 0) but can be
changed with the .defaultValue parameter. For example, pckWriteBoolP(write, false, .defaultWrite = true) will write a 0 with an ID
into the pack, but pckWriteBoolP(write, false) will not write to the pack, it will simply skip the ID. Note that
pckWriteStrP(packWrite, NULL, .defaultWrite = true) is not valid since there is no way to explcitly write a NULL.
A pack is an object by default. Objects can store fields, objects, or arrays. Objects and arrays will be referred to collectively as
containers. Fields contain data to be stored, e.g. integers, strings, etc.
Here is a simple example of a pack:
PackWrite *write = pckWriteNew(buffer);
pckWriteU64P(write, 77);
pckWriteBoolP(write, false, .defaultWrite = true);
pckWriteI32P(write, -1, .defaultValue = -1);
pckWriteStringP(write, STRDEF("sample"));
pckWriteEndP();
A string representation of this pack is `1:uint64:77,2:bool:false,4:str:sample`. The boolean was stored even though it was the
default because a write was explcitly requested. The int32 field was not stored because the value matched the expicitly set default.
Note that there is a gap in the ID stream, which represents the NULL/default value.
This pack can be read with:
PackRead *read = pckReadNew(buffer);
pckReadU64P(read);
pckReadBoolP(read);
pckReadI32P(read, .defaultValue = -1);
pckReadStringP(read);
pckReadEndP();
Note that defaults are not stored in the pack so any defaults that were applied when writing (by setting .defaulWrite and
optionally .defaultValue) must be applied again when reading (by optionally setting .defaultValue).
If we don't care about the NULL/default, another way to read is:
PackRead *read = pckReadNew(buffer);
pckReadU64P(read);
pckReadBoolP(read);
pckReadStringP(read, .id = 4);
pckReadEndP();
By default each read/write advances the field ID by one. If an ID is specified it must be unique and increasing, because it will
advance the field ID beyond the value specified. An error will occur if an ID is attempted to be read/written but the field ID has
advanced beyond it.
An array can be read with:
pckReadArrayBeginP(read);
while (pckReadNext(read))
{
// Read array element
}
pckReadArrayEndP(read);
Note that any container (i.e. array or object) resets the field ID to one so there is no need for the caller to maintain a
cumulative field ID. At the end of a container the numbering will continue from wherever the outer container left off.
***********************************************************************************************************************************/
#ifndef COMMON_TYPE_PACK_H
#define COMMON_TYPE_PACK_H
/***********************************************************************************************************************************
Minimum number of extra bytes to allocate for packs that are growing or are likely to grow
***********************************************************************************************************************************/
#ifndef PACK_EXTRA_MIN
#define PACK_EXTRA_MIN 128
#endif
/***********************************************************************************************************************************
Object types
***********************************************************************************************************************************/
#define PACK_READ_TYPE PackRead
#define PACK_READ_PREFIX pckRead
typedef struct PackRead PackRead;
#define PACK_WRITE_TYPE PackWrite
#define PACK_WRITE_PREFIX pckWrite
typedef struct PackWrite PackWrite;
#include "common/io/read.h"
#include "common/io/write.h"
#include "common/type/string.h"
/***********************************************************************************************************************************
Pack data type
***********************************************************************************************************************************/
typedef enum
{
pckTypeUnknown = 0,
pckTypeArray,
pckTypeBin,
pckTypeBool,
pckTypeI32,
pckTypeI64,
pckTypeObj,
pckTypePtr,
pckTypeStr,
pckTypeTime,
pckTypeU32,
pckTypeU64,
} PackType;
/***********************************************************************************************************************************
Read Constructors
***********************************************************************************************************************************/
PackRead *pckReadNew(IoRead *read);
PackRead *pckReadNewBuf(const Buffer *buffer);
/***********************************************************************************************************************************
Read Functions
***********************************************************************************************************************************/
typedef struct PackIdParam
{
VAR_PARAM_HEADER;
unsigned int id;
} PackIdParam;
// Read next field. This is useful when the type of the next field is unknown, i.e. a completely dynamic data structure, or for
// debugging. If you just need to know if the field exists or not, then use pckReadNullP().
bool pckReadNext(PackRead *this);
// Current field id. Set after a call to pckReadNext().
unsigned int pckReadId(PackRead *this);
// Current field type. Set after a call to pckReadNext().
PackType pckReadType(PackRead *this);
// Is the field NULL?
#define pckReadNullP(this, ...) \
pckReadNull(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__})
bool pckReadNull(PackRead *this, PackIdParam param);
// Read array begin/end
#define pckReadArrayBeginP(this, ...) \
pckReadArrayBegin(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__})
void pckReadArrayBegin(PackRead *this, PackIdParam param);
#define pckReadArrayEndP(this) \
pckReadArrayEnd(this)
void pckReadArrayEnd(PackRead *this);
// Read binary
typedef struct PckReadBinParam
{
VAR_PARAM_HEADER;
unsigned int id;
} PckReadBinParam;
#define pckReadBinP(this, ...) \
pckReadBin(this, (PckReadBinParam){VAR_PARAM_INIT, __VA_ARGS__})
Buffer *pckReadBin(PackRead *this, PckReadBinParam param);
// Read boolean
typedef struct PckReadBoolParam
{
VAR_PARAM_HEADER;
unsigned int id;
uint32_t defaultValue;
} PckReadBoolParam;
#define pckReadBoolP(this, ...) \
pckReadBool(this, (PckReadBoolParam){VAR_PARAM_INIT, __VA_ARGS__})
bool pckReadBool(PackRead *this, PckReadBoolParam param);
// Read 32-bit signed integer
typedef struct PckReadInt32Param
{
VAR_PARAM_HEADER;
unsigned int id;
int32_t defaultValue;
} PckReadInt32Param;
#define pckReadI32P(this, ...) \
pckReadI32(this, (PckReadInt32Param){VAR_PARAM_INIT, __VA_ARGS__})
int32_t pckReadI32(PackRead *this, PckReadInt32Param param);
// Read 64-bit signed integer
typedef struct PckReadInt64Param
{
VAR_PARAM_HEADER;
unsigned int id;
int64_t defaultValue;
} PckReadInt64Param;
#define pckReadI64P(this, ...) \
pckReadI64(this, (PckReadInt64Param){VAR_PARAM_INIT, __VA_ARGS__})
int64_t pckReadI64(PackRead *this, PckReadInt64Param param);
// Read object begin/end
#define pckReadObjBeginP(this, ...) \
pckReadObjBegin(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__})
void pckReadObjBegin(PackRead *this, PackIdParam param);
#define pckReadObjEndP(this) \
pckReadObjEnd(this)
void pckReadObjEnd(PackRead *this);
// Read pointer. Use with extreme caution. Pointers cannot be sent to another host -- they must only be used locally.
typedef struct PckReadPtrParam
{
VAR_PARAM_HEADER;
unsigned int id;
} PckReadPtrParam;
#define pckReadPtrP(this, ...) \
pckReadPtr(this, (PckReadPtrParam){VAR_PARAM_INIT, __VA_ARGS__})
void *pckReadPtr(PackRead *this, PckReadPtrParam param);
// Read string
typedef struct PckReadStrParam
{
VAR_PARAM_HEADER;
unsigned int id;
const String *defaultValue;
} PckReadStrParam;
#define pckReadStrP(this, ...) \
pckReadStr(this, (PckReadStrParam){VAR_PARAM_INIT, __VA_ARGS__})
String *pckReadStr(PackRead *this, PckReadStrParam param);
// Read time
typedef struct PckReadTimeParam
{
VAR_PARAM_HEADER;
unsigned int id;
time_t defaultValue;
} PckReadTimeParam;
#define pckReadTimeP(this, ...) \
pckReadTime(this, (PckReadTimeParam){VAR_PARAM_INIT, __VA_ARGS__})
time_t pckReadTime(PackRead *this, PckReadTimeParam param);
// Read 32-bit unsigned integer
typedef struct PckReadUInt32Param
{
VAR_PARAM_HEADER;
unsigned int id;
uint32_t defaultValue;
} PckReadUInt32Param;
#define pckReadU32P(this, ...) \
pckReadU32(this, (PckReadUInt32Param){VAR_PARAM_INIT, __VA_ARGS__})
uint32_t pckReadU32(PackRead *this, PckReadUInt32Param param);
// Read 64-bit unsigned integer
typedef struct PckReadUInt64Param
{
VAR_PARAM_HEADER;
unsigned int id;
uint64_t defaultValue;
} PckReadUInt64Param;
#define pckReadU64P(this, ...) \
pckReadU64(this, (PckReadUInt64Param){VAR_PARAM_INIT, __VA_ARGS__})
uint64_t pckReadU64(PackRead *this, PckReadUInt64Param param);
// Read end
#define pckReadEndP(this) \
pckReadEnd(this)
void pckReadEnd(PackRead *this);
/***********************************************************************************************************************************
Read Destructor
***********************************************************************************************************************************/
void pckReadFree(PackRead *this);
/***********************************************************************************************************************************
Write Constructors
***********************************************************************************************************************************/
PackWrite *pckWriteNew(IoWrite *write);
PackWrite *pckWriteNewBuf(Buffer *buffer);
/***********************************************************************************************************************************
Write Functions
***********************************************************************************************************************************/
// Write array begin/end
#define pckWriteArrayBeginP(this, ...) \
pckWriteArrayBegin(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteArrayBegin(PackWrite *this, PackIdParam param);
#define pckWriteArrayEndP(this) \
pckWriteArrayEnd(this)
PackWrite *pckWriteArrayEnd(PackWrite *this);
// Write binary
typedef struct PckWriteBinParam
{
VAR_PARAM_HEADER;
unsigned int id;
} PckWriteBinParam;
#define pckWriteBinP(this, value, ...) \
pckWriteBin(this, value, (PckWriteBinParam){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteBin(PackWrite *this, const Buffer *value, PckWriteBinParam param);
// Write boolean
typedef struct PckWriteBoolParam
{
VAR_PARAM_HEADER;
bool defaultWrite;
unsigned int id;
uint32_t defaultValue;
} PckWriteBoolParam;
#define pckWriteBoolP(this, value, ...) \
pckWriteBool(this, value, (PckWriteBoolParam){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteBool(PackWrite *this, bool value, PckWriteBoolParam param);
// Write 32-bit signed integer
typedef struct PckWriteInt32Param
{
VAR_PARAM_HEADER;
bool defaultWrite;
unsigned int id;
int32_t defaultValue;
} PckWriteInt32Param;
#define pckWriteI32P(this, value, ...) \
pckWriteI32(this, value, (PckWriteInt32Param){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteI32(PackWrite *this, int32_t value, PckWriteInt32Param param);
// Write 64-bit signed integer
typedef struct PckWriteInt64Param
{
VAR_PARAM_HEADER;
bool defaultWrite;
unsigned int id;
int64_t defaultValue;
} PckWriteInt64Param;
#define pckWriteI64P(this, value, ...) \
pckWriteI64(this, value, (PckWriteInt64Param){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteI64(PackWrite *this, int64_t value, PckWriteInt64Param param);
// Write null
#define pckWriteNullP(this) \
pckWriteNull(this)
PackWrite *pckWriteNull(PackWrite *this);
// Write object begin/end
#define pckWriteObjBeginP(this, ...) \
pckWriteObjBegin(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteObjBegin(PackWrite *this, PackIdParam param);
#define pckWriteObjEndP(this) \
pckWriteObjEnd(this)
PackWrite *pckWriteObjEnd(PackWrite *this);
// Write pointer. Use with extreme caution. Pointers cannot be sent to another host -- they must only be used locally.
typedef struct PckWritePtrParam
{
VAR_PARAM_HEADER;
bool defaultWrite;
unsigned int id;
} PckWritePtrParam;
#define pckWritePtrP(this, value, ...) \
pckWritePtr(this, value, (PckWritePtrParam){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWritePtr(PackWrite *this, const void *value, PckWritePtrParam param);
// Write string
typedef struct PckWriteStrParam
{
VAR_PARAM_HEADER;
bool defaultWrite;
unsigned int id;
const String *defaultValue;
} PckWriteStrParam;
#define pckWriteStrP(this, value, ...) \
pckWriteStr(this, value, (PckWriteStrParam){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteStr(PackWrite *this, const String *value, PckWriteStrParam param);
// Write time
typedef struct PckWriteTimeParam
{
VAR_PARAM_HEADER;
bool defaultWrite;
unsigned int id;
time_t defaultValue;
} PckWriteTimeParam;
#define pckWriteTimeP(this, value, ...) \
pckWriteTime(this, value, (PckWriteTimeParam){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteTime(PackWrite *this, time_t value, PckWriteTimeParam param);
// Write 32-bit unsigned integer
typedef struct PckWriteUInt32Param
{
VAR_PARAM_HEADER;
bool defaultWrite;
unsigned int id;
uint32_t defaultValue;
} PckWriteUInt32Param;
#define pckWriteU32P(this, value, ...) \
pckWriteU32(this, value, (PckWriteUInt32Param){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteU32(PackWrite *this, uint32_t value, PckWriteUInt32Param param);
// Write 64-bit unsigned integer
typedef struct PckWriteUInt64Param
{
VAR_PARAM_HEADER;
bool defaultWrite;
unsigned int id;
uint64_t defaultValue;
} PckWriteUInt64Param;
#define pckWriteU64P(this, value, ...) \
pckWriteU64(this, value, (PckWriteUInt64Param){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWriteU64(PackWrite *this, uint64_t value, PckWriteUInt64Param param);
// Write end
#define pckWriteEndP(this) \
pckWriteEnd(this)
PackWrite *pckWriteEnd(PackWrite *this);
/***********************************************************************************************************************************
Write Destructor
***********************************************************************************************************************************/
void pckWriteFree(PackWrite *this);
/***********************************************************************************************************************************
Helper Functions
***********************************************************************************************************************************/
const String *pckTypeToStr(PackType type);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
String *pckReadToLog(const PackRead *this);
#define FUNCTION_LOG_PACK_READ_TYPE \
PackRead *
#define FUNCTION_LOG_PACK_READ_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, pckReadToLog, buffer, bufferSize)
String *pckWriteToLog(const PackWrite *this);
#define FUNCTION_LOG_PACK_WRITE_TYPE \
PackWrite *
#define FUNCTION_LOG_PACK_WRITE_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, pckWriteToLog, buffer, bufferSize)
#endif

7
test/define.yaml
View File

@@ -184,6 +184,13 @@ unit:
common/type/variant: full
common/type/variantList: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-pack
total: 1
coverage:
common/type/pack: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-mcv
total: 1

122
test/src/common/harnessPack.c Normal file
View File

@@ -0,0 +1,122 @@
/***********************************************************************************************************************************
Harness for Loading Test Configurations
***********************************************************************************************************************************/
#include "common/assert.h"
#include "common/type/convert.h"
#include "common/type/pack.h"
#include "common/type/stringz.h"
#include "common/harnessDebug.h"
#include "common/harnessPack.h"
/**********************************************************************************************************************************/
String *hrnPackBufToStr(const Buffer *buffer)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(BUFFER, buffer);
FUNCTION_HARNESS_END();
FUNCTION_HARNESS_RESULT(STRING, hrnPackToStr(pckReadNewBuf(buffer)));
}
/**********************************************************************************************************************************/
String *hrnPackToStr(PackRead *read)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, read);
FUNCTION_HARNESS_END();
String *result = strNew("");
bool first = true;
while (pckReadNext(read))
{
if (!first)
strCatZ(result, ", ");
PackType type = pckReadType(read);
unsigned int id = pckReadId(read);
strCatFmt(result, "%u:%s:", id, strZ(pckTypeToStr(type)));
switch (type)
{
case pckTypeUnknown:
THROW_FMT(AssertError, "invalid type %s", strZ(pckTypeToStr(type)));
case pckTypeArray:
{
pckReadArrayBeginP(read, .id = id);
strCatFmt(result, "[%s]", strZ(hrnPackToStr(read)));
pckReadArrayEndP(read);
break;
}
case pckTypeBool:
{
strCatZ(result, cvtBoolToConstZ(pckReadBoolP(read, .id = id)));
break;
}
case pckTypeBin:
{
strCatFmt(result, "%s", strZ(bufHex(pckReadBinP(read, .id = id))));
break;
}
case pckTypeI32:
{
strCatFmt(result, "%d", pckReadI32P(read, .id = id));
break;
}
case pckTypeI64:
{
strCatFmt(result, "%" PRId64, pckReadI64P(read, .id = id));
break;
}
case pckTypeObj:
{
pckReadObjBeginP(read, .id = id);
strCatFmt(result, "{%s}", strZ(hrnPackToStr(read)));
pckReadObjEndP(read);
break;
}
case pckTypePtr:
{
strCatFmt(result, "%p", pckReadPtrP(read, .id = id));
break;
}
case pckTypeStr:
{
strCatFmt(result, "%s", strZ(pckReadStrP(read, .id = id)));
break;
}
case pckTypeTime:
{
strCatFmt(result, "%" PRId64, (int64_t)pckReadTimeP(read, .id = id));
break;
}
case pckTypeU32:
{
strCatFmt(result, "%u", pckReadU32P(read, .id = id));
break;
}
case pckTypeU64:
{
strCatFmt(result, "%" PRIu64, pckReadU64P(read, .id = id));
break;
}
}
first = false;
}
FUNCTION_HARNESS_RESULT(STRING, result);
}

13
test/src/common/harnessPack.h Normal file
View File

@@ -0,0 +1,13 @@
/***********************************************************************************************************************************
Harness for Testing Packs
***********************************************************************************************************************************/
#include "common/type/buffer.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Convert a pack to a string
String *hrnPackToStr(PackRead *read);
// Convert a pack buffer to a string
String *hrnPackBufToStr(const Buffer *buffer);

16
test/src/module/common/ioTest.c
View File

@@ -398,13 +398,15 @@ testRun(void)
// Mixed line and buffer read
// -------------------------------------------------------------------------------------------------------------------------
ioBufferSizeSet(5);
read = ioBufferReadNew(BUFSTRDEF("AAA123\n1234\n\n12\nBDDDEFF"));
read = ioBufferReadNew(BUFSTRDEF("AAAAAA123\n1234\n\n12\nBDDDEFF"));
ioReadOpen(read);
buffer = bufNew(3);
buffer = bufNew(6);
// Start with a buffer read
TEST_RESULT_UINT(ioRead(read, buffer), 3, "read buffer");
TEST_RESULT_STR_Z(strNewBuf(buffer), "AAA", " check buffer");
// Start with a small read
TEST_RESULT_UINT(ioReadSmall(read, buffer), 6, "read buffer");
TEST_RESULT_STR_Z(strNewBuf(buffer), "AAAAAA", " check buffer");
bufUsedSet(buffer, 3);
bufLimitSet(buffer, 3);
// Do line reads of various lengths
TEST_RESULT_STR_Z(ioReadLine(read), "123", "read line");
@@ -415,12 +417,12 @@ testRun(void)
// Read what was left in the line buffer
TEST_RESULT_UINT(ioRead(read, buffer), 0, "read buffer");
bufUsedSet(buffer, 2);
TEST_RESULT_UINT(ioRead(read, buffer), 1, "read buffer");
TEST_RESULT_UINT(ioReadSmall(read, buffer), 1, "read buffer");
TEST_RESULT_STR_Z(strNewBuf(buffer), "AAB", " check buffer");
bufUsedSet(buffer, 0);
// Now do a full buffer read from the input
TEST_RESULT_UINT(ioRead(read, buffer), 3, "read buffer");
TEST_RESULT_UINT(ioReadSmall(read, buffer), 3, "read buffer");
TEST_RESULT_STR_Z(strNewBuf(buffer), "DDD", " check buffer");
// Read line doesn't work without a linefeed

318
test/src/module/common/typePackTest.c Normal file
View File

@@ -0,0 +1,318 @@
/***********************************************************************************************************************************
Test Pack Type
***********************************************************************************************************************************/
#include "common/io/bufferRead.h"
#include "common/io/bufferWrite.h"
#include "common/harnessPack.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("PackWrite and PackRead"))
{
TEST_TITLE("write pack");
Buffer *pack = bufNew(0);
IoWrite *write = ioBufferWriteNew(pack);
ioWriteOpen(write);
ioBufferSizeSet(3);
PackWrite *packWrite = NULL;
TEST_ASSIGN(packWrite, pckWriteNew(write), "new write");
TEST_RESULT_STR_Z(pckWriteToLog(packWrite), "{depth: 1, idLast: 0}", "log");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 0750), "write mode");
TEST_RESULT_STR_Z(pckWriteToLog(packWrite), "{depth: 1, idLast: 1}", "log");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 1911246845), "write timestamp");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 0xFFFFFFFFFFFFFFFF, .id = 7), "write max u64");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 1, .id = 10), "write 1");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 77), "write 77");
TEST_RESULT_VOID(pckWriteU32P(packWrite, 127, .id = 12), "write 127");
TEST_RESULT_VOID(pckWriteI64P(packWrite, -1, .id = 13), "write -1");
TEST_RESULT_VOID(pckWriteI32P(packWrite, -1), "write -1");
TEST_RESULT_VOID(pckWriteBoolP(packWrite, true), "write true");
TEST_RESULT_VOID(pckWriteBoolP(packWrite, false, .id = 20, .defaultWrite = true), "write false");
TEST_RESULT_VOID(pckWriteObjBeginP(packWrite, .id = 28), "write obj begin");
TEST_RESULT_VOID(pckWriteBoolP(packWrite, true), "write true");
TEST_RESULT_VOID(pckWriteBoolP(packWrite, false, .defaultWrite = true), "write false");
TEST_RESULT_VOID(pckWriteObjEndP(packWrite), "write obj end");
TEST_RESULT_VOID(pckWriteArrayBeginP(packWrite, .id = 37), "write array begin");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 0, .defaultWrite = true), "write 0");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 1), "write 1");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 2), "write 2");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 3), "write 3");
TEST_RESULT_VOID(pckWriteArrayEndP(packWrite), "write array end");
TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("sample"), .id = 38), "write string");
TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("enoughtoincreasebuffer")), "write string");
TEST_RESULT_VOID(pckWriteStrP(packWrite, EMPTY_STR), "write zero-length string");
TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("small"), .id = 41), "write string");
TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("")), "write zero-length string");
TEST_RESULT_VOID(pckWriteStrP(packWrite, NULL, .id = 43), "write NULL string");
TEST_RESULT_VOID(pckWriteStrP(packWrite, NULL), "write NULL string");
TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("")), "write zero-length string");
TEST_RESULT_VOID(pckWriteU32P(packWrite, 0), "write default 0");
TEST_RESULT_VOID(pckWriteU32P(packWrite, 0, .defaultValue = 1), "write 0");
TEST_RESULT_VOID(pckWriteArrayBeginP(packWrite), "write array begin");
TEST_RESULT_VOID(pckWriteObjBeginP(packWrite), "write obj begin");
TEST_RESULT_VOID(pckWriteI32P(packWrite, 555), "write 555");
TEST_RESULT_VOID(pckWriteI32P(packWrite, 777, .id = 3), "write 777");
TEST_RESULT_VOID(pckWriteI64P(packWrite, 0), "write 0");
TEST_RESULT_VOID(pckWriteI64P(packWrite, 1), "write 1");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 0), "write 0");
TEST_RESULT_VOID(pckWriteU64P(packWrite, 1), "write 1");
TEST_RESULT_VOID(pckWriteObjEndP(packWrite), "write obj end");
TEST_RESULT_VOID(pckWriteNull(packWrite), "write null");
TEST_RESULT_VOID(
pckWriteStrP(packWrite, STRDEF("A"), .defaultValue = STRDEF("")), "write A");
TEST_RESULT_VOID(pckWriteTimeP(packWrite, 0), "write null");
TEST_RESULT_VOID(pckWriteTimeP(packWrite, 33), "write 33");
TEST_RESULT_VOID(pckWriteTimeP(packWrite, 66, .id = 6), "write 66");
TEST_RESULT_VOID(pckWriteI32P(packWrite, 1, .defaultValue = 1), "write default 1");
TEST_RESULT_VOID(pckWriteBoolP(packWrite, false), "write default false");
TEST_RESULT_VOID(pckWriteArrayEndP(packWrite), "write array end");
const unsigned char bin[] = {0x05, 0x04, 0x03, 0x02, 0x01, 0x00};
TEST_RESULT_VOID(pckWriteBinP(packWrite, BUF(bin, sizeof(bin))), "write bin");
TEST_RESULT_VOID(pckWriteBinP(packWrite, NULL), "write bin NULL default");
TEST_RESULT_VOID(pckWriteBinP(packWrite, bufNew(0)), "write bin zero length");
TEST_RESULT_VOID(pckWriteEndP(packWrite), "end");
TEST_RESULT_VOID(pckWriteFree(packWrite), "free");
ioWriteClose(write);
TEST_RESULT_STR_Z(
hrnPackBufToStr(pack),
"1:u64:488"
", 2:u64:1911246845"
", 7:u64:18446744073709551615"
", 10:u64:1"
", 11:u64:77"
", 12:u32:127"
", 13:i64:-1"
", 14:i32:-1"
", 15:bool:true"
", 20:bool:false"
", 28:obj:"
"{"
"1:bool:true"
", 2:bool:false"
"}"
", 37:array:"
"["
"1:u64:0"
", 2:u64:1"
", 3:u64:2"
", 4:u64:3"
"]"
", 38:str:sample"
", 39:str:enoughtoincreasebuffer"
", 40:str:"
", 41:str:small"
", 42:str:"
", 45:str:"
", 47:u32:0"
", 48:array:"
"["
"1:obj:"
"{"
"1:i32:555"
", 3:i32:777"
", 5:i64:1"
", 7:u64:1"
"}"
", 3:str:A"
", 5:time:33"
", 6:time:66"
"]"
", 49:bin:050403020100"
", 51:bin:",
"check pack string");
TEST_RESULT_STR_Z(
bufHex(pack),
"b8e803" // 1, u64, 750
"b8fd9fad8f07" // 2, u64, 1911246845
"bc01ffffffffffffffffff01" // 7, u64, 0xFFFFFFFFFFFFFFFF
"b601" // 10, u64, 1
"b84d" // 11, u64, 77
"a87f" // 12, u32, 127
"54" // 13, i64, -1
"44" // 14, i32, -1
"38" // 15, bool, true
"3401" // 20, bool, false
"67" // 28, obj begin
"38" // 1, bool
"30" // 2, bool
"00" // obj end
"1801" // 37, array begin
"b0" // 1, u64, 0
"b4" // 2, u64, 1
"b802" // 3, u64, 2
"b803" // 4, u64, 3
"00" // array end
"880673616d706c65" // 38, str, sample
"8816656e6f756768746f696e637265617365627566666572" // 39, str, enoughtoincreasebuffer
"80" // 40, str, zero length
"8805736d616c6c" // 41, str, small
"80" // 42, str, zero length
"82" // 45, str, zero length
"a1" // 47, u32, 0
"10" // 48, array begin
"60" // 1, obj begin
"48d608" // 1, i32, 555
"49920c" // 3, i32, 777
"5902" // 5, i64, 1
"b5" // 7, u64, 1
"00" // obj end
"890141" // 3, str, A
"9942" // 5, time, 33
"988401" // 6, time, 66
"00" // array end
"2806050403020100" // 49, bin, 0x050403020100
"21" // 51, bin, zero length
"00", // end
"check pack hex");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("read pack");
IoRead *read = ioBufferReadNew(pack);
ioReadOpen(read);
PackRead *packRead = NULL;
TEST_ASSIGN(packRead, pckReadNew(read), "new read");
TEST_RESULT_UINT(pckReadU64P(packRead), 0750, "read mode");
TEST_RESULT_UINT(pckReadU64P(packRead), 1911246845, "read timestamp");
TEST_ERROR(pckReadU64P(packRead, .id = 2), FormatError, "field 2 was already read");
TEST_ERROR(pckReadU32P(packRead, .id = 7), FormatError, "field 7 is type 'u64' but expected 'u32'");
TEST_RESULT_UINT(pckReadU64P(packRead, .id = 7), 0xFFFFFFFFFFFFFFFF, "read max u64");
TEST_RESULT_BOOL(pckReadNullP(packRead, .id = 9), true, "field 9 is null");
TEST_RESULT_UINT(pckReadU64P(packRead, .id = 9), 0, "field 9 default is 0");
TEST_RESULT_BOOL(pckReadNullP(packRead, .id = 10), false, "field 10 is not null");
TEST_RESULT_UINT(pckReadU64P(packRead, .id = 10), 1, "read 1");
TEST_RESULT_UINT(pckReadU32P(packRead, .id = 12), 127, "read 127 (skip field 11)");
TEST_RESULT_INT(pckReadI64P(packRead), -1, "read -1");
TEST_RESULT_INT(pckReadI32P(packRead, .id = 14), -1, "read -1");
TEST_RESULT_BOOL(pckReadBoolP(packRead, .id = 15), true, "read true");
TEST_RESULT_BOOL(pckReadBoolP(packRead, .id = 20), false, "read false");
TEST_ERROR(pckReadObjEndP(packRead), FormatError, "not in object");
TEST_RESULT_VOID(pckReadObjBeginP(packRead, .id = 28), "read object begin");
TEST_ERROR(pckReadArrayEndP(packRead), FormatError, "not in array");
TEST_RESULT_BOOL(pckReadBoolP(packRead), true, "read true");
TEST_RESULT_BOOL(pckReadBoolP(packRead), false, "read false");
TEST_RESULT_BOOL(pckReadNullP(packRead), true, "field 3 is null");
TEST_RESULT_BOOL(pckReadBoolP(packRead), false, "field 3 default is false");
TEST_RESULT_BOOL(pckReadNullP(packRead, .id = 4), true, "field 4 is null");
TEST_RESULT_BOOL(pckReadBoolP(packRead), false, "read default false");
TEST_RESULT_VOID(pckReadObjEndP(packRead), "read object end");
TEST_ERROR(pckReadArrayEndP(packRead), FormatError, "not in array");
TEST_RESULT_BOOL(pckReadNext(packRead), true, "read next tag which should be an array");
TEST_RESULT_UINT(pckReadId(packRead), 37, "check array id");
TEST_RESULT_VOID(pckReadArrayBeginP(packRead, .id = pckReadId(packRead)), "read array begin");
TEST_ERROR(pckReadObjEndP(packRead), FormatError, "not in object");
unsigned int value = 0;
while (pckReadNext(packRead))
{
TEST_RESULT_UINT(pckReadU64P(packRead, .id = pckReadId(packRead)), value, "read %u", value);
value++;
}
TEST_RESULT_VOID(pckReadArrayEndP(packRead), "read array end");
TEST_RESULT_STR_Z(pckReadStrP(packRead, .id = 39), "enoughtoincreasebuffer", "read string (skipped prior)");
TEST_RESULT_STR_Z(pckReadStrP(packRead, .id = 41), "small", "read string (skipped prior)");
TEST_RESULT_STR_Z(pckReadStrP(packRead), "", "zero length (skipped prior)");
TEST_RESULT_STR(pckReadStrP(packRead, .id = 43), NULL, "read NULL string");
TEST_RESULT_STR(pckReadStrP(packRead), NULL, "read NULL string");
TEST_RESULT_STR_Z(pckReadStrP(packRead), "", "read empty string");
TEST_RESULT_UINT(pckReadU32P(packRead), 0, "read default 0");
TEST_RESULT_UINT(pckReadU32P(packRead, .id = 47), 0, "read 0");
TEST_RESULT_VOID(pckReadArrayBeginP(packRead), "read array begin");
TEST_RESULT_VOID(pckReadObjBeginP(packRead), "read object begin");
TEST_RESULT_INT(pckReadI32P(packRead), 555, "read 0");
TEST_RESULT_INT(pckReadI32P(packRead, .id = 3), 777, "read 0");
TEST_RESULT_INT(pckReadI64P(packRead, .defaultValue = 44), 44, "read default 44");
TEST_RESULT_INT(pckReadI64P(packRead, .defaultValue = 44), 1, "read 1");
TEST_RESULT_UINT(pckReadU64P(packRead, .defaultValue = 55), 55, "read default 55");
TEST_RESULT_UINT(pckReadU64P(packRead, .defaultValue = 55), 1, "read 1");
TEST_RESULT_VOID(pckReadObjEndP(packRead), "read object end");
TEST_RESULT_STR_Z(pckReadStrP(packRead, .id = 3), "A", "read A");
TEST_RESULT_INT(pckReadTimeP(packRead, .defaultValue = 99), 99, "read default 99");
TEST_RESULT_INT(pckReadTimeP(packRead, .id = 5, .defaultValue = 44), 33, "read 33");
TEST_RESULT_INT(pckReadI32P(packRead, .id = 7, .defaultValue = 1), 1, "read default 1");
TEST_RESULT_VOID(pckReadArrayEndP(packRead), "read array end");
TEST_RESULT_STR_Z(bufHex(pckReadBinP(packRead)), "050403020100", "read bin");
TEST_RESULT_PTR(pckReadBinP(packRead), NULL, "read bin null");
TEST_RESULT_UINT(bufSize(pckReadBinP(packRead)), 0, "read bin zero length");
TEST_RESULT_BOOL(pckReadNullP(packRead, .id = 999), true, "field 999 is null");
TEST_RESULT_UINT(pckReadU64P(packRead, .id = 999), 0, "field 999 default is 0");
TEST_RESULT_VOID(pckReadEndP(packRead), "end");
TEST_RESULT_VOID(pckReadFree(packRead), "free");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("EOF on short buffer");
TEST_ASSIGN(packRead, pckReadNewBuf(BUFSTRDEF("\255")), "new read");
TEST_ERROR(pckReadUInt64Internal(packRead), FormatError, "unexpected EOF");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on invalid uint64");
TEST_ASSIGN(packRead, pckReadNewBuf(BUFSTRDEF("\255\255\255\255\255\255\255\255\255\255")), "new read");
TEST_ERROR(pckReadUInt64Internal(packRead), FormatError, "unterminated base-128 integer");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("pack/unpack pointer");
pack = bufNew(0);
TEST_ASSIGN(packWrite, pckWriteNewBuf(pack), "new write");
TEST_RESULT_VOID(pckWritePtrP(packWrite, NULL), "write default pointer");
TEST_RESULT_VOID(pckWritePtrP(packWrite, "sample"), "write pointer");
TEST_RESULT_VOID(pckWriteEndP(packWrite), "write end");
TEST_ASSIGN(packRead, pckReadNewBuf(pack), "new read");
TEST_RESULT_Z(pckReadPtrP(packRead), NULL, "read default pointer");
TEST_RESULT_Z(pckReadPtrP(packRead, .id = 2), "sample", "read pointer");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("pack/unpack write internal buffer empty");
pack = bufNew(0);
write = ioBufferWriteNew(pack);
ioWriteOpen(write);
// Make internal buffer small enough that it will never be used
ioBufferSizeSet(0);
TEST_ASSIGN(packWrite, pckWriteNew(write), "new write");
TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("test")), "write string longer than internal buffer");
TEST_RESULT_VOID(pckWriteEndP(packWrite), "end with internal buffer empty");
TEST_ASSIGN(packRead, pckReadNewBuf(pack), "new read");
TEST_RESULT_STR_Z(pckReadStrP(packRead), "test", "read string");
}
FUNCTION_HARNESS_RESULT_VOID();
}