From 8361a97482c9b481863cf0d7300936a7d3dbe9df Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 9 Dec 2020 12:05:14 -0500 Subject: [PATCH] Add pack type. The pack type is an architecture-independent format for serializing data compactly, inspired by ProtocolBuffers and Avro. Also add ioReadSmall(), which is optimized for small binary reads, similar to ioReadLineParam(). --- doc/xml/release.xml | 8 + src/Makefile.in | 1 + src/common/io/read.c | 66 + src/common/io/read.h | 3 + src/common/type/convert.h | 7 +- src/common/type/pack.c | 1609 +++++++++++++++++++++++++ src/common/type/pack.h | 500 ++++++++ test/define.yaml | 7 + test/src/common/harnessPack.c | 122 ++ test/src/common/harnessPack.h | 13 + test/src/module/common/ioTest.c | 16 +- test/src/module/common/typePackTest.c | 318 +++++ 12 files changed, 2660 insertions(+), 10 deletions(-) create mode 100644 src/common/type/pack.c create mode 100644 src/common/type/pack.h create mode 100644 test/src/common/harnessPack.c create mode 100644 test/src/common/harnessPack.h create mode 100644 test/src/module/common/typePackTest.c diff --git a/doc/xml/release.xml b/doc/xml/release.xml index eda1718cb..f1e102cfc 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -15,6 +15,14 @@ + + + + + +

Add pack type.

+ + diff --git a/src/Makefile.in b/src/Makefile.in index 16f95cb5f..b2a6b0bd6 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -108,6 +108,7 @@ SRCS = \ common/type/keyValue.c \ common/type/list.c \ common/type/mcv.c \ + common/type/pack.c \ common/type/string.c \ common/type/stringList.c \ common/type/variant.c \ diff --git a/src/common/io/read.c b/src/common/io/read.c index d560be10c..5a36af4bd 100644 --- a/src/common/io/read.c +++ b/src/common/io/read.c @@ -212,6 +212,72 @@ ioRead(IoRead *this, Buffer *buffer) FUNCTION_LOG_RETURN(SIZE, outputRemains - bufRemains(buffer)); } +/**********************************************************************************************************************************/ +size_t +ioReadSmall(IoRead *this, Buffer *buffer) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(IO_READ, this); + FUNCTION_TEST_PARAM(BUFFER, buffer); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(buffer != NULL); + ASSERT(this->opened && !this->closed); + + // Allocate the internal output buffer if it has not already been allocated + if (this->output == NULL) + { + MEM_CONTEXT_BEGIN(this->memContext) + { + this->output = bufNew(ioBufferSize()); + } + MEM_CONTEXT_END(); + } + + // Store size of remaining portion of buffer to calculate total read at the end + size_t outputRemains = bufRemains(buffer); + + do + { + // Internal output buffer remains taking into account the position + size_t outputInternalRemains = bufUsed(this->output) - this->outputPos; + + // Use any data in the internal output buffer + if (outputInternalRemains > 0) + { + // Determine how much data should be copied + size_t size = outputInternalRemains > bufRemains(buffer) ? bufRemains(buffer) : outputInternalRemains; + + // Copy data to the output buffer + bufCatSub(buffer, this->output, this->outputPos, size); + this->outputPos += size; + } + + // If more data is required + if (!bufFull(buffer)) + { + // If the data required is the same size as the internal output buffer then just read into the external buffer + if (bufRemains(buffer) >= bufSize(this->output)) + { + ioReadInternal(this, buffer, true); + } + // Else read as much data as is available. If is not enough we will try again later. + else + { + // Clear the internal output buffer since all data was copied already + bufUsedZero(this->output); + this->outputPos = 0; + + ioReadInternal(this, this->output, false); + } + } + } + while (!bufFull(buffer)); + + FUNCTION_TEST_RETURN(outputRemains - bufRemains(buffer)); +} + /*********************************************************************************************************************************** The entire string to search for must fit within a single buffer. ***********************************************************************************************************************************/ diff --git a/src/common/io/read.h b/src/common/io/read.h index c484610e4..0523c2e77 100644 --- a/src/common/io/read.h +++ b/src/common/io/read.h @@ -29,6 +29,9 @@ bool ioReadOpen(IoRead *this); // Read data from IO and process filters size_t ioRead(IoRead *this, Buffer *buffer); +// Same as ioRead() but optimized for small reads (intended for making repetitive reads that are smaller than ioBufferSize()) +size_t ioReadSmall(IoRead *this, Buffer *buffer); + // Read linefeed-terminated string String *ioReadLine(IoRead *this); diff --git a/src/common/type/convert.h b/src/common/type/convert.h index e479bfddf..0010d7d05 100644 --- a/src/common/type/convert.h +++ b/src/common/type/convert.h @@ -37,9 +37,10 @@ size_t cvtInt64ToZ(int64_t value, char *buffer, size_t bufferSize); int64_t cvtZToInt64(const char *value); int64_t cvtZToInt64Base(const char *value, int base); -// Convert int32/64 to uint32/64 using zigzag encoding and vice versa. Zigzag encoding places the sign bit in the least significant -// bit so that -1 is encoded as 1, 1 as 2, etc. This moves as many bits as possible into the low order bits which is good for other -// types of encoding, e.g. base-128. +// Convert int32/64 to uint32/64 using zigzag encoding and vice versa. Zigzag encoding places the sign in the least significant bit +// so that signed and unsigned values alternate, e.g. 0 = 0, -1 = 1, 1 = 2, -2 = 3, 2 = 4, -3 = 5, 3 = 6, etc. This moves as many +// bits as possible into the low order bits which is good for other types of encoding, e.g. base-128. See +// http://neurocline.github.io/dev/2015/09/17/zig-zag-encoding.html for details. __attribute__((always_inline)) static inline uint32_t cvtInt32ToZigZag(int32_t value) { diff --git a/src/common/type/pack.c b/src/common/type/pack.c new file mode 100644 index 000000000..12a106b04 --- /dev/null +++ b/src/common/type/pack.c @@ -0,0 +1,1609 @@ +/*********************************************************************************************************************************** +Pack Type + +Each pack field begins with a one byte tag. The four high order bits of the tag contain the field type (PackType). The four lower +order bits vary by type. + +When the "more ID delta" indicator is set then the tag will be followed by a base-128 encoded integer with the higher order ID delta +bits. The ID delta represents the delta from the ID of the previous field. When the "more value indicator" then the tag (and the ID +delta, if any) will be followed by a base-128 encoded integer with the high order value bits, i.e. the bits that were not stored +directly in the tag. + +For integer types the value is the integer being stored but for string and binary types the value is 1 if the size is greater than 0 +and 0 if the size is 0. When the size is greater than 0 the tag is immediately followed by (or after the delta ID if "more ID delta" +is set) the base-128 encoded size and then by the string/binary bytes. For string and binary types the value bit indicates if there +is data, not the length of the data, which is why the length is stored immediately following the tag when the value bit is set. This +prevents storing an additional byte when the string/binary length is zero. + +The following are definitions for the pack tag field and examples of how it is interpretted. + +Integer types (packTypeData[type].valueMultiBit) when an unsigned value is <= 1 or a signed value is >= -1 and <= 0: + 3 - more value indicator bit set to 0 + 2 - value low order bit + 1 - more ID delta indicator bit + 0 - ID delta low order bit + + Example: b704 + b = unsigned int 64 type + 7 = tag byte low bits: 0 1 1 1 meaning: + "value low order bit" - the value of the u64 field is 1 + "more ID delta indicator bit" - there exists a gap (i.e. NULLs are not stored so there is a gap between the stored IDs) + "ID delta low order bit" - gaps are interpretted as the currently stored ID minus previously stored ID minus 1, therefore if + the previously store ID is 1 and the ID of this u64 field is 11 then a gap of 10 exists. 10 is represented internally as + 9 since there is always at least a gap of 1 which never needs to be recorded (it is a given). 9 in bit format is + 1 0 0 1 - the low-order bit is 1 so the "ID delta low order bit" is set. + 04 = since the low order bit of the internal ID delta was already set in bit 0 of the tag byte, then the remain bits are shifted + right by one and represented in this second byte as 4. To get the ID delta for 04, shift the 4 back to the left one and then + add back the "ID delta low order bit" to give a binary representation of 1 0 0 1 = 9. Add back the 1 which is never + recorded and the ID gap is 10. + +Integer types (packTypeData[type].valueMultiBit) when an unsigned value is > 1 or a signed value is < -1 or > 0: + 3 - more value indicator bit set to 1 + 2 - more ID delta indicator bit +0-1 - ID delta low order bits + + Example: 5e021f + 5 = signed int 64 type + e = tag byte low bits: 1 1 1 0 meaning: + "more value indicator bit set to 1" - the actual value is < -1 or > 0 + "more ID delta indicator bit" - there exists a gap (i.e. NULLs are not stored so there is a gap between the stored IDs) + "ID delta low order bits" - here the bit 1 is set to 1 and bit 0 is not so the ID delta has the second low order bit set but + not the first + 02 = since bit 0 and bit 1 of the tag byte are accounted for then the 02 is the result of shifting the ID delta right by 2. + Shifting the 2 back to the left by 2 and adding back the second low order bit as 1 and the first low order bit as 0 then + the bit representation would be 1 0 1 0 which is ten (10) so the gap between the IDs is 11. + 1f = signed, zigzag representation of -16 (the actual value) + +String, binary types, and boolean (packTypeData[type].valueSingleBit): + 3 - value bit + 2 - more ID delta indicator bit +0-1 - ID delta low order bits + Note: binary type is interpretted the same way as string type + + Example: 8c090673616d706c65 + 8 = string type + c = tag byte low bits: 1 1 0 0 meaning: + "value bit" - there is data + "more ID delta indicator bit" - there exists a gap (i.e. NULLs are not stored so there is a gap between the stored IDs) + 09 = since neither "ID delta low order bits" is set in the tag, they are both 0, so shifting 9 left by 2, the 2 low order bits + are now 0 so the result is 0x24 = 36 in decimal. Add back the 1 which is never recorded and the ID gap is 37. + 06 = the length of the string is 6 bytes + 73616d706c65 = the 6 bytes of the string value ("sample") + + Example: 30 + 3 = boolean type + 0 = "value bit" 0 means the value is false + Note that if the boolean had been pack written with .defaultWrite = false there would have been a gap instead of the 30. + +Array and object types: + 3 - more ID delta indicator bit +0-2 - ID delta low order bits + Note: arrays and objects are merely containers for the other pack types. + + Example: 1801 (container begin) + 1 = array type + 8 = "more ID delta indicator bit" - there exists a gap (i.e. NULLs are not stored so there is a gap between the stored IDs) + 01 = since there are three "ID delta low order bits", the 01 will be shifted left by 3 with zeros, resulting in 8. Add back + the 1 which is never recorded and the ID gap is 9. + ... + 00 = container end - the array/object container end will occur when a 0 byte (00) is encountered that is not part of a pack + field within the array/object + +***********************************************************************************************************************************/ +#include "build.auto.h" + +#include + +#include "common/debug.h" +#include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" +#include "common/io/io.h" +#include "common/io/read.h" +#include "common/io/write.h" +#include "common/type/convert.h" +#include "common/type/object.h" +#include "common/type/pack.h" + +/*********************************************************************************************************************************** +Constants +***********************************************************************************************************************************/ +#define PACK_UINT64_SIZE_MAX 10 + +/*********************************************************************************************************************************** +Type data +***********************************************************************************************************************************/ +typedef struct PackTypeData +{ + PackType type; // Data type + bool valueSingleBit; // Can the value be stored in a single bit (e.g. bool) + bool valueMultiBit; // Can the value require multiple bits (e.g. integer) + bool size; // Does the type require a size (e.g. string) + const String *const name; // Type name used in error messages +} PackTypeData; + +static const PackTypeData packTypeData[] = +{ + { + .type = pckTypeUnknown, + .name = STRDEF("unknown"), + }, + { + .type = pckTypeArray, + .name = STRDEF("array"), + }, + { + .type = pckTypeBin, + .valueSingleBit = true, + .size = true, + .name = STRDEF("bin"), + }, + { + .type = pckTypeBool, + .valueSingleBit = true, + .name = STRDEF("bool"), + }, + { + .type = pckTypeI32, + .valueMultiBit = true, + .name = STRDEF("i32"), + }, + { + .type = pckTypeI64, + .valueMultiBit = true, + .name = STRDEF("i64"), + }, + { + .type = pckTypeObj, + .name = STRDEF("obj"), + }, + { + .type = pckTypePtr, + .valueMultiBit = true, + .name = STRDEF("ptr"), + }, + { + .type = pckTypeStr, + .valueSingleBit = true, + .size = true, + .name = STRDEF("str"), + }, + { + .type = pckTypeTime, + .valueMultiBit = true, + .name = STRDEF("time"), + }, + { + .type = pckTypeU32, + .valueMultiBit = true, + .name = STRDEF("u32"), + }, + { + .type = pckTypeU64, + .valueMultiBit = true, + .name = STRDEF("u64"), + }, +}; + +/*********************************************************************************************************************************** +Object types +***********************************************************************************************************************************/ +typedef struct PackTagStack +{ + PackType type; + unsigned int idLast; + unsigned int nullTotal; +} PackTagStack; + +struct PackRead +{ + MemContext *memContext; // Mem context + IoRead *read; // Read pack from + Buffer *buffer; // Buffer to contain read data + const uint8_t *bufferPtr; // Pointer to buffer + size_t bufferPos; // Position in the buffer + size_t bufferUsed; // Amount of data in the buffer + + unsigned int tagNextId; // Next tag id + PackType tagNextType; // Next tag type + uint64_t tagNextValue; // Next tag value + + List *tagStack; // Stack of object/array tags + PackTagStack *tagStackTop; // Top tag on the stack +}; + +OBJECT_DEFINE_FREE(PACK_READ); + +struct PackWrite +{ + MemContext *memContext; // Mem context + IoWrite *write; // Write pack to + Buffer *buffer; // Buffer to contain write data + + List *tagStack; // Stack of object/array tags + PackTagStack *tagStackTop; // Top tag on the stack +}; + +OBJECT_DEFINE_FREE(PACK_WRITE); + +/**********************************************************************************************************************************/ +// Helper to create common data +static PackRead * +pckReadNewInternal(void) +{ + FUNCTION_TEST_VOID(); + + PackRead *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("PackRead") + { + this = memNew(sizeof(PackRead)); + + *this = (PackRead) + { + .memContext = MEM_CONTEXT_NEW(), + .tagStack = lstNewP(sizeof(PackTagStack)), + }; + + this->tagStackTop = lstAdd(this->tagStack, &(PackTagStack){.type = pckTypeObj}); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_TEST_RETURN(this); +} + +PackRead * +pckReadNew(IoRead *read) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(IO_READ, read); + FUNCTION_TEST_END(); + + ASSERT(read != NULL); + + PackRead *this = pckReadNewInternal(); + this->read = read; + this->buffer = bufNew(ioBufferSize()); + this->bufferPtr = bufPtr(this->buffer); + + FUNCTION_TEST_RETURN(this); +} + +PackRead * +pckReadNewBuf(const Buffer *buffer) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BUFFER, buffer); + FUNCTION_TEST_END(); + + ASSERT(buffer != NULL); + + PackRead *this = pckReadNewInternal(); + this->bufferPtr = bufPtrConst(buffer); + this->bufferUsed = bufUsed(buffer); + + FUNCTION_TEST_RETURN(this); +} + +/*********************************************************************************************************************************** +Read bytes from the buffer + +IMPORTANT NOTE: To avoid having dyamically created return buffers the current buffer position (this->bufferPos) is stored in the +object. Therefore this function should not be used as a parameter in other function calls since the value of this->bufferPos will +change. +***********************************************************************************************************************************/ +static size_t +pckReadBuffer(PackRead *this, size_t size) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(SIZE, size); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + size_t remaining = this->bufferUsed - this->bufferPos; + + if (remaining < size) + { + if (this->read != NULL) + { + // Nothing can be remaining since each read fetches exactly the number of bytes required + ASSERT(remaining == 0); + bufUsedZero(this->buffer); + + // Limit the buffer for the next read so we don't read past the end of the pack + bufLimitSet(this->buffer, size < bufSizeAlloc(this->buffer) ? size : bufSizeAlloc(this->buffer)); + + // Read bytes + ioReadSmall(this->read, this->buffer); + this->bufferPos = 0; + this->bufferUsed = bufUsed(this->buffer); + remaining = this->bufferUsed; + } + + if (remaining < 1) + THROW(FormatError, "unexpected EOF"); + + FUNCTION_TEST_RETURN(remaining < size ? remaining : size); + } + + FUNCTION_TEST_RETURN(size); +} + +/*********************************************************************************************************************************** +Unpack an unsigned 64-bit integer from base-128 varint encoding +***********************************************************************************************************************************/ +static uint64_t +pckReadUInt64Internal(PackRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + uint64_t result = 0; + uint8_t byte; + + // Convert bytes from varint-128 encoding to a uint64 + for (unsigned int bufferIdx = 0; bufferIdx < PACK_UINT64_SIZE_MAX; bufferIdx++) + { + // Get the next encoded byte + pckReadBuffer(this, 1); + byte = this->bufferPtr[this->bufferPos]; + + // Shift the lower order 7 encoded bits into the uint64 in reverse order + result |= (uint64_t)(byte & 0x7f) << (7 * bufferIdx); + + // Increment buffer position to indicate that the byte has been processed + this->bufferPos++; + + // Done if the high order bit is not set to indicate more data + if (byte < 0x80) + break; + } + + // By this point all bytes should have been read so error if this is not the case. This could be due to a coding error or + // corrupton in the data stream. + if (byte >= 0x80) + THROW(FormatError, "unterminated base-128 integer"); + + FUNCTION_TEST_RETURN(result); +} + +/*********************************************************************************************************************************** +Read next field tag +***********************************************************************************************************************************/ +static bool +pckReadTagNext(PackRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + bool result = false; + + // Read the tag byte + pckReadBuffer(this, 1); + unsigned int tag = this->bufferPtr[this->bufferPos]; + this->bufferPos++; + + // If the current container is complete (e.g. object) + if (tag == 0) + { + this->tagNextId = UINT_MAX; + } + // Else a regular tag + else + { + // Read field type (e.g. int64, string) + this->tagNextType = tag >> 4; + + // If the value can contain multiple bits (e.g. integer) + if (packTypeData[this->tagNextType].valueMultiBit) + { + // If the value is stored following the tag (value > 1 bit) + if (tag & 0x8) + { + // Read low order bits of the field ID delta + this->tagNextId = tag & 0x3; + + // Read high order bits of the field ID delta when specified + if (tag & 0x4) + this->tagNextId |= (unsigned int)pckReadUInt64Internal(this) << 2; + + // Read value + this->tagNextValue = pckReadUInt64Internal(this); + } + // Else the value is stored in the tag (value == 1 bit) + else + { + // Read low order bit of the field ID delta + this->tagNextId = tag & 0x1; + + // Read high order bits of the field ID delta when specified + if (tag & 0x2) + this->tagNextId |= (unsigned int)pckReadUInt64Internal(this) << 1; + + // Read value + this->tagNextValue = (tag >> 2) & 0x3; + } + } + // Else the value is a single bit (e.g. boolean) + else if (packTypeData[this->tagNextType].valueSingleBit) + { + // Read low order bits of the field ID delta + this->tagNextId = tag & 0x3; + + // Read high order bits of the field ID delta when specified + if (tag & 0x4) + this->tagNextId |= (unsigned int)pckReadUInt64Internal(this) << 2; + + // Read value + this->tagNextValue = (tag >> 3) & 0x1; + } + // Else the value is multiple tags (e.g. container) + else + { + // Read low order bits of the field ID delta + this->tagNextId = tag & 0x7; + + // Read high order bits of the field ID delta when specified + if (tag & 0x8) + this->tagNextId |= (unsigned int)pckReadUInt64Internal(this) << 3; + + // Value length is variable so is stored after the tag + this->tagNextValue = 0; + } + + // Increment the next tag id + this->tagNextId += this->tagStackTop->idLast + 1; + + // Tag was found + result = true; + } + + FUNCTION_TEST_RETURN(result); +} + +/*********************************************************************************************************************************** +Read field tag + +Some tags and data may be skipped based on the value of the id parameter. +***********************************************************************************************************************************/ +static uint64_t +pckReadTag(PackRead *this, unsigned int *id, PackType type, bool peek) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM_P(UINT, id); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_PARAM(BOOL, peek); // Look at the next tag without advancing the field id + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(id != NULL); + ASSERT((peek && type == pckTypeUnknown) || (!peek && type != pckTypeUnknown)); + + // Increment the id by one if no id was specified + if (*id == 0) + { + *id = this->tagStackTop->idLast + 1; + } + // Else check that the id has been incremented + else if (*id <= this->tagStackTop->idLast) + THROW_FMT(FormatError, "field %u was already read", *id); + + // Search for the requested id + do + { + // Get the next tag if it has not been read yet + if (this->tagNextId == 0) + pckReadTagNext(this); + + // Return if the id does not exist + if (*id < this->tagNextId) + { + break; + } + // Else the id exists + else if (*id == this->tagNextId) + { + // When not peeking the next tag (just to see what it is) then error if the type is not as specified + if (!peek) + { + if (this->tagNextType != type) + { + THROW_FMT( + FormatError, "field %u is type '%s' but expected '%s'", this->tagNextId, + strZ(packTypeData[this->tagNextType].name), strZ(packTypeData[type].name)); + } + + this->tagStackTop->idLast = this->tagNextId; + this->tagNextId = 0; + } + + break; + } + + // Read data for the field being skipped if this is not the field requested + if (packTypeData[this->tagNextType].size && this->tagNextValue != 0) + { + size_t sizeExpected = (size_t)pckReadUInt64Internal(this); + + while (sizeExpected != 0) + { + size_t sizeRead = pckReadBuffer(this, sizeExpected); + sizeExpected -= sizeRead; + this->bufferPos += sizeRead; + } + } + + // Increment the last id to the id just read + this->tagStackTop->idLast = this->tagNextId; + + // Read tag on the next iteration + this->tagNextId = 0; + } + while (1); + + FUNCTION_TEST_RETURN(this->tagNextValue); +} + +/**********************************************************************************************************************************/ +bool +pckReadNext(PackRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(pckReadTagNext(this)); +} + +/**********************************************************************************************************************************/ +unsigned int +pckReadId(PackRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(this->tagNextId); +} + +/**********************************************************************************************************************************/ +// Internal version of pckReadNull() that does not require a PackIdParam struct. Some functions already have an id variable so +// assigning that to a PackIdParam struct and then copying it back is wasteful. +static inline bool +pckReadNullInternal(PackRead *this, unsigned int *id) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM_P(UINT, id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(id != NULL); + + pckReadTag(this, id, pckTypeUnknown, true); + + FUNCTION_TEST_RETURN(*id < this->tagNextId); +} + +bool +pckReadNull(PackRead *this, PackIdParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(pckReadNullInternal(this, ¶m.id)); +} + +/*********************************************************************************************************************************** +Helper function to determine whether a default should be returned when the field is NULL (missing) +***********************************************************************************************************************************/ +static inline bool +pckReadDefaultNull(PackRead *this, unsigned int *id) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM_P(UINT, id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(id != NULL); + + // If the field is NULL then set idLast (to avoid rechecking the same id on the next call) and return true + if (pckReadNullInternal(this, id)) + { + this->tagStackTop->idLast = *id; + FUNCTION_TEST_RETURN(true); + } + + // The field is not NULL + FUNCTION_TEST_RETURN(false); +} + +/**********************************************************************************************************************************/ +PackType +pckReadType(PackRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + FUNCTION_TEST_RETURN(this->tagNextType); +} + +/**********************************************************************************************************************************/ +void +pckReadArrayBegin(PackRead *this, PackIdParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + // Read array begin + pckReadTag(this, ¶m.id, pckTypeArray, false); + + // Add array to the tag stack so IDs can be tracked separately from the parent container + this->tagStackTop = lstAdd(this->tagStack, &(PackTagStack){.type = pckTypeArray}); + + FUNCTION_TEST_RETURN_VOID(); +} + +void +pckReadArrayEnd(PackRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (lstSize(this->tagStack) == 1 || this->tagStackTop->type != pckTypeArray) + THROW(FormatError, "not in array"); + + // Make sure we are at the end of the array + unsigned int id = UINT_MAX - 1; + pckReadTag(this, &id, pckTypeUnknown, true); + + // Pop array off the stack + lstRemoveLast(this->tagStack); + this->tagStackTop = lstGetLast(this->tagStack); + + // Reset tagNextId to keep reading + this->tagNextId = 0; + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +Buffer * +pckReadBin(PackRead *this, PckReadBinParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (pckReadDefaultNull(this, ¶m.id)) + FUNCTION_TEST_RETURN(NULL); + + Buffer *result = NULL; + + // If buffer size > 0 + if (pckReadTag(this, ¶m.id, pckTypeBin, false)) + { + // Get the buffer size + result = bufNew((size_t)pckReadUInt64Internal(this)); + + // Read the buffer out in chunks + while (bufUsed(result) < bufSize(result)) + { + size_t size = pckReadBuffer(this, bufRemains(result)); + bufCatC(result, this->bufferPtr, this->bufferPos, size); + this->bufferPos += size; + } + } + // Else return a zero-sized buffer + else + result = bufNew(0); + + FUNCTION_TEST_RETURN(result); +} + +/**********************************************************************************************************************************/ +bool +pckReadBool(PackRead *this, PckReadBoolParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(BOOL, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (pckReadDefaultNull(this, ¶m.id)) + FUNCTION_TEST_RETURN(param.defaultValue); + + FUNCTION_TEST_RETURN(pckReadTag(this, ¶m.id, pckTypeBool, false)); +} + +/**********************************************************************************************************************************/ +int32_t +pckReadI32(PackRead *this, PckReadInt32Param param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(INT, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (pckReadDefaultNull(this, ¶m.id)) + FUNCTION_TEST_RETURN(param.defaultValue); + + FUNCTION_TEST_RETURN(cvtInt32FromZigZag((uint32_t)pckReadTag(this, ¶m.id, pckTypeI32, false))); +} + +/**********************************************************************************************************************************/ +int64_t +pckReadI64(PackRead *this, PckReadInt64Param param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(INT64, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (pckReadDefaultNull(this, ¶m.id)) + FUNCTION_TEST_RETURN(param.defaultValue); + + FUNCTION_TEST_RETURN(cvtInt64FromZigZag(pckReadTag(this, ¶m.id, pckTypeI64, false))); +} + +/**********************************************************************************************************************************/ +void +pckReadObjBegin(PackRead *this, PackIdParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + // Read object begin + pckReadTag(this, ¶m.id, pckTypeObj, false); + + // Add object to the tag stack so IDs can be tracked separately from the parent container + this->tagStackTop = lstAdd(this->tagStack, &(PackTagStack){.type = pckTypeObj}); + + FUNCTION_TEST_RETURN_VOID(); +} + +void +pckReadObjEnd(PackRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (lstSize(this->tagStack) == 1 || ((PackTagStack *)lstGetLast(this->tagStack))->type != pckTypeObj) + THROW(FormatError, "not in object"); + + // Make sure we are at the end of the object + unsigned id = UINT_MAX - 1; + pckReadTag(this, &id, pckTypeUnknown, true); + + // Pop object off the stack + lstRemoveLast(this->tagStack); + this->tagStackTop = lstGetLast(this->tagStack); + + // Reset tagNextId to keep reading + this->tagNextId = 0; + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +void * +pckReadPtr(PackRead *this, PckReadPtrParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (pckReadDefaultNull(this, ¶m.id)) + FUNCTION_TEST_RETURN(NULL); + + FUNCTION_TEST_RETURN((void *)(uintptr_t)pckReadTag(this, ¶m.id, pckTypePtr, false)); +} + +/**********************************************************************************************************************************/ +String * +pckReadStr(PackRead *this, PckReadStrParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(STRING, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (pckReadDefaultNull(this, ¶m.id)) + FUNCTION_TEST_RETURN(strDup(param.defaultValue)); + + String *result = NULL; + + // If string size > 0 + if (pckReadTag(this, ¶m.id, pckTypeStr, false)) + { + // Read the string size + size_t sizeExpected = (size_t)pckReadUInt64Internal(this); + + // Read the string out in chunks + result = strNew(""); + + while (strSize(result) != sizeExpected) + { + size_t sizeRead = pckReadBuffer(this, sizeExpected - strSize(result)); + strCatZN(result, (char *)this->bufferPtr + this->bufferPos, sizeRead); + this->bufferPos += sizeRead; + } + } + // Else return an empty string + else + result = strNew(""); + + FUNCTION_TEST_RETURN(result); +} + +/**********************************************************************************************************************************/ +time_t +pckReadTime(PackRead *this, PckReadTimeParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(TIME, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (pckReadDefaultNull(this, ¶m.id)) + FUNCTION_TEST_RETURN(param.defaultValue); + + FUNCTION_TEST_RETURN((time_t)cvtInt64FromZigZag(pckReadTag(this, ¶m.id, pckTypeTime, false))); +} + +/**********************************************************************************************************************************/ +uint32_t +pckReadU32(PackRead *this, PckReadUInt32Param param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(UINT32, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (pckReadDefaultNull(this, ¶m.id)) + FUNCTION_TEST_RETURN(param.defaultValue); + + FUNCTION_TEST_RETURN((uint32_t)pckReadTag(this, ¶m.id, pckTypeU32, false)); +} + +/**********************************************************************************************************************************/ +uint64_t +pckReadU64(PackRead *this, PckReadUInt64Param param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(UINT64, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (pckReadDefaultNull(this, ¶m.id)) + FUNCTION_TEST_RETURN(param.defaultValue); + + FUNCTION_TEST_RETURN(pckReadTag(this, ¶m.id, pckTypeU64, false)); +} + +/**********************************************************************************************************************************/ +void +pckReadEnd(PackRead *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_READ, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + // Read object end markers + while (lstSize(this->tagStack) > 0) + { + // Make sure we are at the end of the container + unsigned int id = UINT_MAX - 1; + pckReadTag(this, &id, pckTypeUnknown, true); + + // Remove from stack + lstRemoveLast(this->tagStack); + } + + this->tagStackTop = NULL; + + FUNCTION_TEST_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +String * +pckReadToLog(const PackRead *this) +{ + return strNewFmt( + "{depth: %u, idLast: %u, tagNextId: %u, tagNextType: %u, tagNextValue %" PRIu64 "}", lstSize(this->tagStack), + this->tagStackTop != NULL ? this->tagStackTop->idLast : 0, this->tagNextId, this->tagNextType, this->tagNextValue); +} + +/**********************************************************************************************************************************/ +// Helper to create common data +static PackWrite * +pckWriteNewInternal(void) +{ + FUNCTION_TEST_VOID(); + + PackWrite *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("PackWrite") + { + this = memNew(sizeof(PackWrite)); + + *this = (PackWrite) + { + .memContext = MEM_CONTEXT_NEW(), + .tagStack = lstNewP(sizeof(PackTagStack)), + }; + + this->tagStackTop = lstAdd(this->tagStack, &(PackTagStack){.type = pckTypeObj}); + } + MEM_CONTEXT_NEW_END(); + + FUNCTION_TEST_RETURN(this); +} + +PackWrite * +pckWriteNew(IoWrite *write) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(IO_WRITE, write); + FUNCTION_TEST_END(); + + ASSERT(write != NULL); + + PackWrite *this = pckWriteNewInternal(); + this->write = write; + this->buffer = bufNew(ioBufferSize()); + + FUNCTION_TEST_RETURN(this); +} + +PackWrite * +pckWriteNewBuf(Buffer *buffer) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BUFFER, buffer); + FUNCTION_TEST_END(); + + ASSERT(buffer != NULL); + + PackWrite *this = pckWriteNewInternal(); + + MEM_CONTEXT_BEGIN(this->memContext) + { + this->buffer = buffer; + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN(this); +} + +/*********************************************************************************************************************************** +Write to io or buffer +***********************************************************************************************************************************/ +static void +pckWriteBuffer(PackWrite *this, const Buffer *buffer) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(BUFFER, buffer); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + // If writing directly to a buffer + if (this->write == NULL) + { + // Add space in the buffer to write and add extra space so future writes won't always need to resize the buffer + if (bufRemains(this->buffer) < bufUsed(buffer)) + bufResize(this->buffer, (bufSizeAlloc(this->buffer) + bufUsed(buffer)) + PACK_EXTRA_MIN); + + // Write to the buffer + bufCat(this->buffer, buffer); + } + // Else writing to io + else + { + // If there's enough space to write to the internal buffer then do that + if (bufRemains(this->buffer) >= bufUsed(buffer)) + bufCat(this->buffer, buffer); + else + { + // Flush the internal buffer if it has data + if (bufUsed(this->buffer) > 0) + { + ioWrite(this->write, this->buffer); + bufUsedZero(this->buffer); + } + + // If there's enough space to write to the internal buffer then do that + if (bufRemains(this->buffer) >= bufUsed(buffer)) + { + bufCat(this->buffer, buffer); + } + // Else write directly to io + else + ioWrite(this->write, buffer); + } + } + + FUNCTION_TEST_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Pack an unsigned 64-bit integer to base-128 varint encoding +***********************************************************************************************************************************/ +static void +pckWriteUInt64Internal(PackWrite *this, uint64_t value) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(UINT64, value); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + unsigned char buffer[PACK_UINT64_SIZE_MAX]; + size_t size = 0; + + // Convert uint64 to varint-128 encloding. Keep writing out bytes while the remaining value is greater than 7 bits. + while (value >= 0x80) + { + // Encode the lower order 7 bits, adding the continuation bit to indicate there is more data + buffer[size] = (unsigned char)value | 0x80; + + // Shift the value to remove bits that have been encoded + value >>= 7; + + // Keep track of size so we know how many bytes to write out + size++; + } + + // Encode the last 7 bits of value + buffer[size] = (unsigned char)value; + + // Write encoded bytes to the buffer + pckWriteBuffer(this, BUF(buffer, size + 1)); + + FUNCTION_TEST_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Write field tag +***********************************************************************************************************************************/ +static void +pckWriteTag(PackWrite *this, PackType type, unsigned int id, uint64_t value) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_PARAM(UINT, id); + FUNCTION_TEST_PARAM(UINT64, value); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + // If id is not specified then add one to previous tag (and include all NULLs) + if (id == 0) + { + id = this->tagStackTop->idLast + this->tagStackTop->nullTotal + 1; + } + // Else the id must be greater than the last one + else + CHECK(id > this->tagStackTop->idLast); + + // Clear NULLs now that field id has been calculated + this->tagStackTop->nullTotal = 0; + + // Calculate field ID delta + unsigned int tagId = id - this->tagStackTop->idLast - 1; + + // Write field type (e.g. int64, string) + uint64_t tag = type << 4; + + // If the value can contain multiple bits (e.g. integer) + if (packTypeData[type].valueMultiBit) + { + // If the value is stored in the tag (value == 1 bit) + if (value < 2) + { + // Write low order bit of the value + tag |= (value & 0x1) << 2; + value >>= 1; + + // Write low order bit of the field ID delta + tag |= tagId & 0x1; + tagId >>= 1; + + // Set bit to indicate that high order bits of the field ID delta are be written after the tag + if (tagId > 0) + tag |= 0x2; + } + // Else the value is stored following the tag (value > 1 bit) + else + { + // Set bit to indicate that the value is written after the tag + tag |= 0x8; + + // Write low order bits of the field ID delta + tag |= tagId & 0x3; + tagId >>= 2; + + // Set bit to indicate that high order bits of the field ID delta are be written after the tag + if (tagId > 0) + tag |= 0x4; + } + } + // Else the value is a single bit (e.g. boolean) + else if (packTypeData[type].valueSingleBit) + { + // Write value + tag |= (value & 0x1) << 3; + value >>= 1; + + // Write low order bits of the field ID delta + tag |= tagId & 0x3; + tagId >>= 2; + + // Set bit to indicate that high order bits of the field ID delta are be written after the tag + if (tagId > 0) + tag |= 0x4; + } + else + { + // No value expected + ASSERT(value == 0); + + // Write low order bits of the field ID delta + tag |= tagId & 0x7; + tagId >>= 3; + + // Set bit to indicate that high order bits of the field ID delta must be written after the tag + if (tagId > 0) + tag |= 0x8; + } + + // Write tag + uint8_t tagByte = (uint8_t)tag; + pckWriteBuffer(this, BUF(&tagByte, 1)); + + // Write low order bits of the field ID delta + if (tagId > 0) + pckWriteUInt64Internal(this, tagId); + + // Write low order bits of the value + if (value > 0) + pckWriteUInt64Internal(this, value); + + // Set last field id + this->tagStackTop->idLast = id; + + FUNCTION_TEST_RETURN_VOID(); +} + +/*********************************************************************************************************************************** +Write a default as NULL (missing) +***********************************************************************************************************************************/ +static inline bool +pckWriteDefaultNull(PackWrite *this, bool defaultWrite, bool defaultEqual) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(BOOL, defaultWrite); + FUNCTION_TEST_PARAM(BOOL, defaultEqual); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + // Write a NULL if not forcing the default to be written and the value passed equals the default + if (!defaultWrite && defaultEqual) + { + this->tagStackTop->nullTotal++; + FUNCTION_TEST_RETURN(true); + } + + // Let the caller know that it should write the value + FUNCTION_TEST_RETURN(false); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteNull(PackWrite *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_END(); + + this->tagStackTop->nullTotal++; + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteArrayBegin(PackWrite *this, PackIdParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + // Write the array tag + pckWriteTag(this, pckTypeArray, param.id, 0); + + // Add array to the tag stack so IDs can be tracked separately from the parent container + this->tagStackTop = lstAdd(this->tagStack, &(PackTagStack){.type = pckTypeArray}); + + FUNCTION_TEST_RETURN(this); +} + +PackWrite * +pckWriteArrayEnd(PackWrite *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(lstSize(this->tagStack) != 1); + ASSERT(((PackTagStack *)lstGetLast(this->tagStack))->type == pckTypeArray); + + // Write end of array tag + pckWriteUInt64Internal(this, 0); + + // Pop array off the stack to revert to ID tracking for the prior container + lstRemoveLast(this->tagStack); + this->tagStackTop = lstGetLast(this->tagStack); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteBin(PackWrite *this, const Buffer *value, PckWriteBinParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(BUFFER, value); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (!pckWriteDefaultNull(this, false, value == NULL)) + { + ASSERT(value != NULL); + + // Write buffer size if > 0 + pckWriteTag(this, pckTypeBin, param.id, bufUsed(value) > 0); + + // Write buffer data if size > 0 + if (bufUsed(value) > 0) + { + pckWriteUInt64Internal(this, bufUsed(value)); + pckWriteBuffer(this, value); + } + } + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteBool(PackWrite *this, bool value, PckWriteBoolParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(BOOL, value); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(BOOL, param.defaultWrite); + FUNCTION_TEST_PARAM(BOOL, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (!pckWriteDefaultNull(this, param.defaultWrite, value == param.defaultValue)) + pckWriteTag(this, pckTypeBool, param.id, value); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteI32(PackWrite *this, int32_t value, PckWriteInt32Param param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(INT, value); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(BOOL, param.defaultWrite); + FUNCTION_TEST_PARAM(INT, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (!pckWriteDefaultNull(this, param.defaultWrite, value == param.defaultValue)) + pckWriteTag(this, pckTypeI32, param.id, cvtInt32ToZigZag(value)); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteI64(PackWrite *this, int64_t value, PckWriteInt64Param param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(INT64, value); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(BOOL, param.defaultWrite); + FUNCTION_TEST_PARAM(INT64, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (!pckWriteDefaultNull(this, param.defaultWrite, value == param.defaultValue)) + pckWriteTag(this, pckTypeI64, param.id, cvtInt64ToZigZag(value)); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteObjBegin(PackWrite *this, PackIdParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + // Write the object tag + pckWriteTag(this, pckTypeObj, param.id, 0); + + // Add object to the tag stack so IDs can be tracked separately from the parent container + this->tagStackTop = lstAdd(this->tagStack, &(PackTagStack){.type = pckTypeObj}); + + FUNCTION_TEST_RETURN(this); +} + +PackWrite * +pckWriteObjEnd(PackWrite *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(lstSize(this->tagStack) != 1); + ASSERT(((PackTagStack *)lstGetLast(this->tagStack))->type == pckTypeObj); + + // Write end of object tag + pckWriteUInt64Internal(this, 0); + + // Pop object off the stack to revert to ID tracking for the prior container + lstRemoveLast(this->tagStack); + this->tagStackTop = lstGetLast(this->tagStack); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWritePtr(PackWrite *this, const void *value, PckWritePtrParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM_P(VOID, value); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(BOOL, param.defaultWrite); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (!pckWriteDefaultNull(this, param.defaultWrite, value == NULL)) + pckWriteTag(this, pckTypePtr, param.id, (uintptr_t)value); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteStr(PackWrite *this, const String *value, PckWriteStrParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(STRING, value); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(BOOL, param.defaultWrite); + FUNCTION_TEST_PARAM(STRING, param.defaultValue); + FUNCTION_TEST_END(); + + if (!pckWriteDefaultNull(this, param.defaultWrite, strEq(value, param.defaultValue))) + { + ASSERT(value != NULL); + + // Write string size if > 0 + pckWriteTag(this, pckTypeStr, param.id, strSize(value) > 0); + + // Write string data if size > 0 + if (strSize(value) > 0) + { + pckWriteUInt64Internal(this, strSize(value)); + pckWriteBuffer(this, BUF(strZ(value), strSize(value))); + } + } + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteTime(PackWrite *this, time_t value, PckWriteTimeParam param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(TIME, value); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(BOOL, param.defaultWrite); + FUNCTION_TEST_PARAM(TIME, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (!pckWriteDefaultNull(this, param.defaultWrite, value == param.defaultValue)) + pckWriteTag(this, pckTypeTime, param.id, cvtInt64ToZigZag(value)); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteU32(PackWrite *this, uint32_t value, PckWriteUInt32Param param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(UINT32, value); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(BOOL, param.defaultWrite); + FUNCTION_TEST_PARAM(UINT32, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (!pckWriteDefaultNull(this, param.defaultWrite, value == param.defaultValue)) + pckWriteTag(this, pckTypeU32, param.id, value); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteU64(PackWrite *this, uint64_t value, PckWriteUInt64Param param) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_PARAM(UINT64, value); + FUNCTION_TEST_PARAM(UINT, param.id); + FUNCTION_TEST_PARAM(BOOL, param.defaultWrite); + FUNCTION_TEST_PARAM(UINT64, param.defaultValue); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + + if (!pckWriteDefaultNull(this, param.defaultWrite, value == param.defaultValue)) + pckWriteTag(this, pckTypeU64, param.id, value); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +PackWrite * +pckWriteEnd(PackWrite *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(PACK_WRITE, this); + FUNCTION_TEST_END(); + + ASSERT(this != NULL); + ASSERT(lstSize(this->tagStack) == 1); + + pckWriteUInt64Internal(this, 0); + this->tagStackTop = NULL; + + // If writing to io flush the internal buffer + if (this->write != NULL) + { + if (bufUsed(this->buffer) > 0) + ioWrite(this->write, this->buffer); + } + // Else resize the external buffer to trim off extra space added during processing + else + bufResize(this->buffer, bufUsed(this->buffer)); + + FUNCTION_TEST_RETURN(this); +} + +/**********************************************************************************************************************************/ +String * +pckWriteToLog(const PackWrite *this) +{ + return strNewFmt("{depth: %u, idLast: %u}", lstSize(this->tagStack), this->tagStackTop == NULL ? 0 : this->tagStackTop->idLast); +} + +/**********************************************************************************************************************************/ +const String * +pckTypeToStr(PackType type) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_END(); + + ASSERT(type < sizeof(packTypeData) / sizeof(PackTypeData)); + + FUNCTION_TEST_RETURN(packTypeData[type].name); +} diff --git a/src/common/type/pack.h b/src/common/type/pack.h new file mode 100644 index 000000000..b4c339dd3 --- /dev/null +++ b/src/common/type/pack.h @@ -0,0 +1,500 @@ +/*********************************************************************************************************************************** +Pack Type + +The pack type encodes binary data compactly while still maintaining structure and strict typing. The idea is based on Thrift, +ProtocolBuffers, and Avro, compared here: https://medium.com/better-programming/use-binary-encoding-instead-of-json-dec745ec09b6. +The pack type has been further optimized to balance between purely in-memory structures and those intended to be passed via a +protocol or saved in a file. + +Integers are stored with base-128 varint encoding which is equivalent to network byte order, i.e., the endianness of the sending and +receiving host don't matter. + +The overall idea is similar to JSON but IDs are used instead of names, typing is more granular, and the representation is far more +compact. A pack can readily be converted to JSON but the reverse is not as precise due to loose typing in JSON. A pack is a stream +format, i.e. it is intended to be read in order from beginning to end. + +Fields in a pack are identified by IDs. A field ID is stored as a delta from the previous ID, which is very efficient, but means +that reading from the middle is generally not practical. The size of the gap between field IDs is important -- a gap of 1 never +incurs extra cost, but depending on the field type larger gaps may require additional bytes to store the field ID delta. + +NULLs are not stored in a pack and are therefore not typed. A NULL is essentially just a gap in the field IDs. Fields that are +frequently NULL are best stored at the end of an object. When using .defaultWrite = false in write functions a NULL will be written +(by making a gap in the IDs) if the value matches the default. When using read functions the default will always be returned +when the field is NULL (i.e. missing). The standard default is the C default for that type (e.g. bool = false, int = 0) but can be +changed with the .defaultValue parameter. For example, pckWriteBoolP(write, false, .defaultWrite = true) will write a 0 with an ID +into the pack, but pckWriteBoolP(write, false) will not write to the pack, it will simply skip the ID. Note that +pckWriteStrP(packWrite, NULL, .defaultWrite = true) is not valid since there is no way to explcitly write a NULL. + +A pack is an object by default. Objects can store fields, objects, or arrays. Objects and arrays will be referred to collectively as +containers. Fields contain data to be stored, e.g. integers, strings, etc. + +Here is a simple example of a pack: + +PackWrite *write = pckWriteNew(buffer); +pckWriteU64P(write, 77); +pckWriteBoolP(write, false, .defaultWrite = true); +pckWriteI32P(write, -1, .defaultValue = -1); +pckWriteStringP(write, STRDEF("sample")); +pckWriteEndP(); + +A string representation of this pack is `1:uint64:77,2:bool:false,4:str:sample`. The boolean was stored even though it was the +default because a write was explcitly requested. The int32 field was not stored because the value matched the expicitly set default. +Note that there is a gap in the ID stream, which represents the NULL/default value. + +This pack can be read with: + +PackRead *read = pckReadNew(buffer); +pckReadU64P(read); +pckReadBoolP(read); +pckReadI32P(read, .defaultValue = -1); +pckReadStringP(read); +pckReadEndP(); + +Note that defaults are not stored in the pack so any defaults that were applied when writing (by setting .defaulWrite and +optionally .defaultValue) must be applied again when reading (by optionally setting .defaultValue). + +If we don't care about the NULL/default, another way to read is: + +PackRead *read = pckReadNew(buffer); +pckReadU64P(read); +pckReadBoolP(read); +pckReadStringP(read, .id = 4); +pckReadEndP(); + +By default each read/write advances the field ID by one. If an ID is specified it must be unique and increasing, because it will +advance the field ID beyond the value specified. An error will occur if an ID is attempted to be read/written but the field ID has +advanced beyond it. + +An array can be read with: + +pckReadArrayBeginP(read); + +while (pckReadNext(read)) +{ + // Read array element +} + +pckReadArrayEndP(read); + +Note that any container (i.e. array or object) resets the field ID to one so there is no need for the caller to maintain a +cumulative field ID. At the end of a container the numbering will continue from wherever the outer container left off. +***********************************************************************************************************************************/ +#ifndef COMMON_TYPE_PACK_H +#define COMMON_TYPE_PACK_H + +/*********************************************************************************************************************************** +Minimum number of extra bytes to allocate for packs that are growing or are likely to grow +***********************************************************************************************************************************/ +#ifndef PACK_EXTRA_MIN + #define PACK_EXTRA_MIN 128 +#endif + +/*********************************************************************************************************************************** +Object types +***********************************************************************************************************************************/ +#define PACK_READ_TYPE PackRead +#define PACK_READ_PREFIX pckRead + +typedef struct PackRead PackRead; + +#define PACK_WRITE_TYPE PackWrite +#define PACK_WRITE_PREFIX pckWrite + +typedef struct PackWrite PackWrite; + +#include "common/io/read.h" +#include "common/io/write.h" +#include "common/type/string.h" + +/*********************************************************************************************************************************** +Pack data type +***********************************************************************************************************************************/ +typedef enum +{ + pckTypeUnknown = 0, + pckTypeArray, + pckTypeBin, + pckTypeBool, + pckTypeI32, + pckTypeI64, + pckTypeObj, + pckTypePtr, + pckTypeStr, + pckTypeTime, + pckTypeU32, + pckTypeU64, +} PackType; + +/*********************************************************************************************************************************** +Read Constructors +***********************************************************************************************************************************/ +PackRead *pckReadNew(IoRead *read); +PackRead *pckReadNewBuf(const Buffer *buffer); + +/*********************************************************************************************************************************** +Read Functions +***********************************************************************************************************************************/ +typedef struct PackIdParam +{ + VAR_PARAM_HEADER; + unsigned int id; +} PackIdParam; + +// Read next field. This is useful when the type of the next field is unknown, i.e. a completely dynamic data structure, or for +// debugging. If you just need to know if the field exists or not, then use pckReadNullP(). +bool pckReadNext(PackRead *this); + +// Current field id. Set after a call to pckReadNext(). +unsigned int pckReadId(PackRead *this); + +// Current field type. Set after a call to pckReadNext(). +PackType pckReadType(PackRead *this); + +// Is the field NULL? +#define pckReadNullP(this, ...) \ + pckReadNull(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__}) + +bool pckReadNull(PackRead *this, PackIdParam param); + +// Read array begin/end +#define pckReadArrayBeginP(this, ...) \ + pckReadArrayBegin(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__}) + +void pckReadArrayBegin(PackRead *this, PackIdParam param); + +#define pckReadArrayEndP(this) \ + pckReadArrayEnd(this) + +void pckReadArrayEnd(PackRead *this); + +// Read binary +typedef struct PckReadBinParam +{ + VAR_PARAM_HEADER; + unsigned int id; +} PckReadBinParam; + +#define pckReadBinP(this, ...) \ + pckReadBin(this, (PckReadBinParam){VAR_PARAM_INIT, __VA_ARGS__}) + +Buffer *pckReadBin(PackRead *this, PckReadBinParam param); + +// Read boolean +typedef struct PckReadBoolParam +{ + VAR_PARAM_HEADER; + unsigned int id; + uint32_t defaultValue; +} PckReadBoolParam; + +#define pckReadBoolP(this, ...) \ + pckReadBool(this, (PckReadBoolParam){VAR_PARAM_INIT, __VA_ARGS__}) + +bool pckReadBool(PackRead *this, PckReadBoolParam param); + +// Read 32-bit signed integer +typedef struct PckReadInt32Param +{ + VAR_PARAM_HEADER; + unsigned int id; + int32_t defaultValue; +} PckReadInt32Param; + +#define pckReadI32P(this, ...) \ + pckReadI32(this, (PckReadInt32Param){VAR_PARAM_INIT, __VA_ARGS__}) + +int32_t pckReadI32(PackRead *this, PckReadInt32Param param); + +// Read 64-bit signed integer +typedef struct PckReadInt64Param +{ + VAR_PARAM_HEADER; + unsigned int id; + int64_t defaultValue; +} PckReadInt64Param; + +#define pckReadI64P(this, ...) \ + pckReadI64(this, (PckReadInt64Param){VAR_PARAM_INIT, __VA_ARGS__}) + +int64_t pckReadI64(PackRead *this, PckReadInt64Param param); + +// Read object begin/end +#define pckReadObjBeginP(this, ...) \ + pckReadObjBegin(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__}) + +void pckReadObjBegin(PackRead *this, PackIdParam param); + +#define pckReadObjEndP(this) \ + pckReadObjEnd(this) + +void pckReadObjEnd(PackRead *this); + +// Read pointer. Use with extreme caution. Pointers cannot be sent to another host -- they must only be used locally. +typedef struct PckReadPtrParam +{ + VAR_PARAM_HEADER; + unsigned int id; +} PckReadPtrParam; + +#define pckReadPtrP(this, ...) \ + pckReadPtr(this, (PckReadPtrParam){VAR_PARAM_INIT, __VA_ARGS__}) + +void *pckReadPtr(PackRead *this, PckReadPtrParam param); + +// Read string +typedef struct PckReadStrParam +{ + VAR_PARAM_HEADER; + unsigned int id; + const String *defaultValue; +} PckReadStrParam; + +#define pckReadStrP(this, ...) \ + pckReadStr(this, (PckReadStrParam){VAR_PARAM_INIT, __VA_ARGS__}) + +String *pckReadStr(PackRead *this, PckReadStrParam param); + +// Read time +typedef struct PckReadTimeParam +{ + VAR_PARAM_HEADER; + unsigned int id; + time_t defaultValue; +} PckReadTimeParam; + +#define pckReadTimeP(this, ...) \ + pckReadTime(this, (PckReadTimeParam){VAR_PARAM_INIT, __VA_ARGS__}) + +time_t pckReadTime(PackRead *this, PckReadTimeParam param); + +// Read 32-bit unsigned integer +typedef struct PckReadUInt32Param +{ + VAR_PARAM_HEADER; + unsigned int id; + uint32_t defaultValue; +} PckReadUInt32Param; + +#define pckReadU32P(this, ...) \ + pckReadU32(this, (PckReadUInt32Param){VAR_PARAM_INIT, __VA_ARGS__}) + +uint32_t pckReadU32(PackRead *this, PckReadUInt32Param param); + +// Read 64-bit unsigned integer +typedef struct PckReadUInt64Param +{ + VAR_PARAM_HEADER; + unsigned int id; + uint64_t defaultValue; +} PckReadUInt64Param; + +#define pckReadU64P(this, ...) \ + pckReadU64(this, (PckReadUInt64Param){VAR_PARAM_INIT, __VA_ARGS__}) + +uint64_t pckReadU64(PackRead *this, PckReadUInt64Param param); + +// Read end +#define pckReadEndP(this) \ + pckReadEnd(this) + +void pckReadEnd(PackRead *this); + +/*********************************************************************************************************************************** +Read Destructor +***********************************************************************************************************************************/ +void pckReadFree(PackRead *this); + +/*********************************************************************************************************************************** +Write Constructors +***********************************************************************************************************************************/ +PackWrite *pckWriteNew(IoWrite *write); +PackWrite *pckWriteNewBuf(Buffer *buffer); + +/*********************************************************************************************************************************** +Write Functions +***********************************************************************************************************************************/ +// Write array begin/end +#define pckWriteArrayBeginP(this, ...) \ + pckWriteArrayBegin(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteArrayBegin(PackWrite *this, PackIdParam param); + +#define pckWriteArrayEndP(this) \ + pckWriteArrayEnd(this) + +PackWrite *pckWriteArrayEnd(PackWrite *this); + +// Write binary +typedef struct PckWriteBinParam +{ + VAR_PARAM_HEADER; + unsigned int id; +} PckWriteBinParam; + +#define pckWriteBinP(this, value, ...) \ + pckWriteBin(this, value, (PckWriteBinParam){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteBin(PackWrite *this, const Buffer *value, PckWriteBinParam param); + +// Write boolean +typedef struct PckWriteBoolParam +{ + VAR_PARAM_HEADER; + bool defaultWrite; + unsigned int id; + uint32_t defaultValue; +} PckWriteBoolParam; + +#define pckWriteBoolP(this, value, ...) \ + pckWriteBool(this, value, (PckWriteBoolParam){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteBool(PackWrite *this, bool value, PckWriteBoolParam param); + +// Write 32-bit signed integer +typedef struct PckWriteInt32Param +{ + VAR_PARAM_HEADER; + bool defaultWrite; + unsigned int id; + int32_t defaultValue; +} PckWriteInt32Param; + +#define pckWriteI32P(this, value, ...) \ + pckWriteI32(this, value, (PckWriteInt32Param){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteI32(PackWrite *this, int32_t value, PckWriteInt32Param param); + +// Write 64-bit signed integer +typedef struct PckWriteInt64Param +{ + VAR_PARAM_HEADER; + bool defaultWrite; + unsigned int id; + int64_t defaultValue; +} PckWriteInt64Param; + +#define pckWriteI64P(this, value, ...) \ + pckWriteI64(this, value, (PckWriteInt64Param){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteI64(PackWrite *this, int64_t value, PckWriteInt64Param param); + +// Write null +#define pckWriteNullP(this) \ + pckWriteNull(this) + +PackWrite *pckWriteNull(PackWrite *this); + +// Write object begin/end +#define pckWriteObjBeginP(this, ...) \ + pckWriteObjBegin(this, (PackIdParam){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteObjBegin(PackWrite *this, PackIdParam param); + +#define pckWriteObjEndP(this) \ + pckWriteObjEnd(this) + +PackWrite *pckWriteObjEnd(PackWrite *this); + +// Write pointer. Use with extreme caution. Pointers cannot be sent to another host -- they must only be used locally. +typedef struct PckWritePtrParam +{ + VAR_PARAM_HEADER; + bool defaultWrite; + unsigned int id; +} PckWritePtrParam; + +#define pckWritePtrP(this, value, ...) \ + pckWritePtr(this, value, (PckWritePtrParam){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWritePtr(PackWrite *this, const void *value, PckWritePtrParam param); + +// Write string +typedef struct PckWriteStrParam +{ + VAR_PARAM_HEADER; + bool defaultWrite; + unsigned int id; + const String *defaultValue; +} PckWriteStrParam; + +#define pckWriteStrP(this, value, ...) \ + pckWriteStr(this, value, (PckWriteStrParam){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteStr(PackWrite *this, const String *value, PckWriteStrParam param); + +// Write time +typedef struct PckWriteTimeParam +{ + VAR_PARAM_HEADER; + bool defaultWrite; + unsigned int id; + time_t defaultValue; +} PckWriteTimeParam; + +#define pckWriteTimeP(this, value, ...) \ + pckWriteTime(this, value, (PckWriteTimeParam){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteTime(PackWrite *this, time_t value, PckWriteTimeParam param); + +// Write 32-bit unsigned integer +typedef struct PckWriteUInt32Param +{ + VAR_PARAM_HEADER; + bool defaultWrite; + unsigned int id; + uint32_t defaultValue; +} PckWriteUInt32Param; + +#define pckWriteU32P(this, value, ...) \ + pckWriteU32(this, value, (PckWriteUInt32Param){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteU32(PackWrite *this, uint32_t value, PckWriteUInt32Param param); + +// Write 64-bit unsigned integer +typedef struct PckWriteUInt64Param +{ + VAR_PARAM_HEADER; + bool defaultWrite; + unsigned int id; + uint64_t defaultValue; +} PckWriteUInt64Param; + +#define pckWriteU64P(this, value, ...) \ + pckWriteU64(this, value, (PckWriteUInt64Param){VAR_PARAM_INIT, __VA_ARGS__}) + +PackWrite *pckWriteU64(PackWrite *this, uint64_t value, PckWriteUInt64Param param); + +// Write end +#define pckWriteEndP(this) \ + pckWriteEnd(this) + +PackWrite *pckWriteEnd(PackWrite *this); + +/*********************************************************************************************************************************** +Write Destructor +***********************************************************************************************************************************/ +void pckWriteFree(PackWrite *this); + +/*********************************************************************************************************************************** +Helper Functions +***********************************************************************************************************************************/ +const String *pckTypeToStr(PackType type); + +/*********************************************************************************************************************************** +Macros for function logging +***********************************************************************************************************************************/ +String *pckReadToLog(const PackRead *this); + +#define FUNCTION_LOG_PACK_READ_TYPE \ + PackRead * +#define FUNCTION_LOG_PACK_READ_FORMAT(value, buffer, bufferSize) \ + FUNCTION_LOG_STRING_OBJECT_FORMAT(value, pckReadToLog, buffer, bufferSize) + +String *pckWriteToLog(const PackWrite *this); + +#define FUNCTION_LOG_PACK_WRITE_TYPE \ + PackWrite * +#define FUNCTION_LOG_PACK_WRITE_FORMAT(value, buffer, bufferSize) \ + FUNCTION_LOG_STRING_OBJECT_FORMAT(value, pckWriteToLog, buffer, bufferSize) + +#endif diff --git a/test/define.yaml b/test/define.yaml index d947629b2..bc3190992 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -184,6 +184,13 @@ unit: common/type/variant: full common/type/variantList: full + # ---------------------------------------------------------------------------------------------------------------------------- + - name: type-pack + total: 1 + + coverage: + common/type/pack: full + # ---------------------------------------------------------------------------------------------------------------------------- - name: type-mcv total: 1 diff --git a/test/src/common/harnessPack.c b/test/src/common/harnessPack.c new file mode 100644 index 000000000..f4bf6c332 --- /dev/null +++ b/test/src/common/harnessPack.c @@ -0,0 +1,122 @@ +/*********************************************************************************************************************************** +Harness for Loading Test Configurations +***********************************************************************************************************************************/ +#include "common/assert.h" +#include "common/type/convert.h" +#include "common/type/pack.h" +#include "common/type/stringz.h" + +#include "common/harnessDebug.h" +#include "common/harnessPack.h" + +/**********************************************************************************************************************************/ +String *hrnPackBufToStr(const Buffer *buffer) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(BUFFER, buffer); + FUNCTION_HARNESS_END(); + + FUNCTION_HARNESS_RESULT(STRING, hrnPackToStr(pckReadNewBuf(buffer))); +} + +/**********************************************************************************************************************************/ +String *hrnPackToStr(PackRead *read) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(PACK_READ, read); + FUNCTION_HARNESS_END(); + + String *result = strNew(""); + bool first = true; + + while (pckReadNext(read)) + { + if (!first) + strCatZ(result, ", "); + + PackType type = pckReadType(read); + unsigned int id = pckReadId(read); + + strCatFmt(result, "%u:%s:", id, strZ(pckTypeToStr(type))); + + switch (type) + { + case pckTypeUnknown: + THROW_FMT(AssertError, "invalid type %s", strZ(pckTypeToStr(type))); + + case pckTypeArray: + { + pckReadArrayBeginP(read, .id = id); + strCatFmt(result, "[%s]", strZ(hrnPackToStr(read))); + pckReadArrayEndP(read); + break; + } + + case pckTypeBool: + { + strCatZ(result, cvtBoolToConstZ(pckReadBoolP(read, .id = id))); + break; + } + + case pckTypeBin: + { + strCatFmt(result, "%s", strZ(bufHex(pckReadBinP(read, .id = id)))); + break; + } + + case pckTypeI32: + { + strCatFmt(result, "%d", pckReadI32P(read, .id = id)); + break; + } + + case pckTypeI64: + { + strCatFmt(result, "%" PRId64, pckReadI64P(read, .id = id)); + break; + } + + case pckTypeObj: + { + pckReadObjBeginP(read, .id = id); + strCatFmt(result, "{%s}", strZ(hrnPackToStr(read))); + pckReadObjEndP(read); + break; + } + + case pckTypePtr: + { + strCatFmt(result, "%p", pckReadPtrP(read, .id = id)); + break; + } + + case pckTypeStr: + { + strCatFmt(result, "%s", strZ(pckReadStrP(read, .id = id))); + break; + } + + case pckTypeTime: + { + strCatFmt(result, "%" PRId64, (int64_t)pckReadTimeP(read, .id = id)); + break; + } + + case pckTypeU32: + { + strCatFmt(result, "%u", pckReadU32P(read, .id = id)); + break; + } + + case pckTypeU64: + { + strCatFmt(result, "%" PRIu64, pckReadU64P(read, .id = id)); + break; + } + } + + first = false; + } + + FUNCTION_HARNESS_RESULT(STRING, result); +} diff --git a/test/src/common/harnessPack.h b/test/src/common/harnessPack.h new file mode 100644 index 000000000..2052237c6 --- /dev/null +++ b/test/src/common/harnessPack.h @@ -0,0 +1,13 @@ +/*********************************************************************************************************************************** +Harness for Testing Packs +***********************************************************************************************************************************/ +#include "common/type/buffer.h" + +/*********************************************************************************************************************************** +Functions +***********************************************************************************************************************************/ +// Convert a pack to a string +String *hrnPackToStr(PackRead *read); + +// Convert a pack buffer to a string +String *hrnPackBufToStr(const Buffer *buffer); diff --git a/test/src/module/common/ioTest.c b/test/src/module/common/ioTest.c index c4158e713..08c99df12 100644 --- a/test/src/module/common/ioTest.c +++ b/test/src/module/common/ioTest.c @@ -398,13 +398,15 @@ testRun(void) // Mixed line and buffer read // ------------------------------------------------------------------------------------------------------------------------- ioBufferSizeSet(5); - read = ioBufferReadNew(BUFSTRDEF("AAA123\n1234\n\n12\nBDDDEFF")); + read = ioBufferReadNew(BUFSTRDEF("AAAAAA123\n1234\n\n12\nBDDDEFF")); ioReadOpen(read); - buffer = bufNew(3); + buffer = bufNew(6); - // Start with a buffer read - TEST_RESULT_UINT(ioRead(read, buffer), 3, "read buffer"); - TEST_RESULT_STR_Z(strNewBuf(buffer), "AAA", " check buffer"); + // Start with a small read + TEST_RESULT_UINT(ioReadSmall(read, buffer), 6, "read buffer"); + TEST_RESULT_STR_Z(strNewBuf(buffer), "AAAAAA", " check buffer"); + bufUsedSet(buffer, 3); + bufLimitSet(buffer, 3); // Do line reads of various lengths TEST_RESULT_STR_Z(ioReadLine(read), "123", "read line"); @@ -415,12 +417,12 @@ testRun(void) // Read what was left in the line buffer TEST_RESULT_UINT(ioRead(read, buffer), 0, "read buffer"); bufUsedSet(buffer, 2); - TEST_RESULT_UINT(ioRead(read, buffer), 1, "read buffer"); + TEST_RESULT_UINT(ioReadSmall(read, buffer), 1, "read buffer"); TEST_RESULT_STR_Z(strNewBuf(buffer), "AAB", " check buffer"); bufUsedSet(buffer, 0); // Now do a full buffer read from the input - TEST_RESULT_UINT(ioRead(read, buffer), 3, "read buffer"); + TEST_RESULT_UINT(ioReadSmall(read, buffer), 3, "read buffer"); TEST_RESULT_STR_Z(strNewBuf(buffer), "DDD", " check buffer"); // Read line doesn't work without a linefeed diff --git a/test/src/module/common/typePackTest.c b/test/src/module/common/typePackTest.c new file mode 100644 index 000000000..bb58be6e3 --- /dev/null +++ b/test/src/module/common/typePackTest.c @@ -0,0 +1,318 @@ +/*********************************************************************************************************************************** +Test Pack Type +***********************************************************************************************************************************/ +#include "common/io/bufferRead.h" +#include "common/io/bufferWrite.h" + +#include "common/harnessPack.h" + +/*********************************************************************************************************************************** +Test Run +***********************************************************************************************************************************/ +void +testRun(void) +{ + FUNCTION_HARNESS_VOID(); + + // ***************************************************************************************************************************** + if (testBegin("PackWrite and PackRead")) + { + TEST_TITLE("write pack"); + + Buffer *pack = bufNew(0); + + IoWrite *write = ioBufferWriteNew(pack); + ioWriteOpen(write); + + ioBufferSizeSet(3); + + PackWrite *packWrite = NULL; + TEST_ASSIGN(packWrite, pckWriteNew(write), "new write"); + + TEST_RESULT_STR_Z(pckWriteToLog(packWrite), "{depth: 1, idLast: 0}", "log"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 0750), "write mode"); + TEST_RESULT_STR_Z(pckWriteToLog(packWrite), "{depth: 1, idLast: 1}", "log"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 1911246845), "write timestamp"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 0xFFFFFFFFFFFFFFFF, .id = 7), "write max u64"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 1, .id = 10), "write 1"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 77), "write 77"); + TEST_RESULT_VOID(pckWriteU32P(packWrite, 127, .id = 12), "write 127"); + TEST_RESULT_VOID(pckWriteI64P(packWrite, -1, .id = 13), "write -1"); + TEST_RESULT_VOID(pckWriteI32P(packWrite, -1), "write -1"); + TEST_RESULT_VOID(pckWriteBoolP(packWrite, true), "write true"); + TEST_RESULT_VOID(pckWriteBoolP(packWrite, false, .id = 20, .defaultWrite = true), "write false"); + TEST_RESULT_VOID(pckWriteObjBeginP(packWrite, .id = 28), "write obj begin"); + TEST_RESULT_VOID(pckWriteBoolP(packWrite, true), "write true"); + TEST_RESULT_VOID(pckWriteBoolP(packWrite, false, .defaultWrite = true), "write false"); + TEST_RESULT_VOID(pckWriteObjEndP(packWrite), "write obj end"); + TEST_RESULT_VOID(pckWriteArrayBeginP(packWrite, .id = 37), "write array begin"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 0, .defaultWrite = true), "write 0"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 1), "write 1"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 2), "write 2"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 3), "write 3"); + TEST_RESULT_VOID(pckWriteArrayEndP(packWrite), "write array end"); + TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("sample"), .id = 38), "write string"); + TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("enoughtoincreasebuffer")), "write string"); + TEST_RESULT_VOID(pckWriteStrP(packWrite, EMPTY_STR), "write zero-length string"); + TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("small"), .id = 41), "write string"); + TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("")), "write zero-length string"); + TEST_RESULT_VOID(pckWriteStrP(packWrite, NULL, .id = 43), "write NULL string"); + TEST_RESULT_VOID(pckWriteStrP(packWrite, NULL), "write NULL string"); + TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("")), "write zero-length string"); + TEST_RESULT_VOID(pckWriteU32P(packWrite, 0), "write default 0"); + TEST_RESULT_VOID(pckWriteU32P(packWrite, 0, .defaultValue = 1), "write 0"); + TEST_RESULT_VOID(pckWriteArrayBeginP(packWrite), "write array begin"); + TEST_RESULT_VOID(pckWriteObjBeginP(packWrite), "write obj begin"); + TEST_RESULT_VOID(pckWriteI32P(packWrite, 555), "write 555"); + TEST_RESULT_VOID(pckWriteI32P(packWrite, 777, .id = 3), "write 777"); + TEST_RESULT_VOID(pckWriteI64P(packWrite, 0), "write 0"); + TEST_RESULT_VOID(pckWriteI64P(packWrite, 1), "write 1"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 0), "write 0"); + TEST_RESULT_VOID(pckWriteU64P(packWrite, 1), "write 1"); + TEST_RESULT_VOID(pckWriteObjEndP(packWrite), "write obj end"); + TEST_RESULT_VOID(pckWriteNull(packWrite), "write null"); + TEST_RESULT_VOID( + pckWriteStrP(packWrite, STRDEF("A"), .defaultValue = STRDEF("")), "write A"); + TEST_RESULT_VOID(pckWriteTimeP(packWrite, 0), "write null"); + TEST_RESULT_VOID(pckWriteTimeP(packWrite, 33), "write 33"); + TEST_RESULT_VOID(pckWriteTimeP(packWrite, 66, .id = 6), "write 66"); + TEST_RESULT_VOID(pckWriteI32P(packWrite, 1, .defaultValue = 1), "write default 1"); + TEST_RESULT_VOID(pckWriteBoolP(packWrite, false), "write default false"); + TEST_RESULT_VOID(pckWriteArrayEndP(packWrite), "write array end"); + + const unsigned char bin[] = {0x05, 0x04, 0x03, 0x02, 0x01, 0x00}; + TEST_RESULT_VOID(pckWriteBinP(packWrite, BUF(bin, sizeof(bin))), "write bin"); + TEST_RESULT_VOID(pckWriteBinP(packWrite, NULL), "write bin NULL default"); + TEST_RESULT_VOID(pckWriteBinP(packWrite, bufNew(0)), "write bin zero length"); + + TEST_RESULT_VOID(pckWriteEndP(packWrite), "end"); + TEST_RESULT_VOID(pckWriteFree(packWrite), "free"); + + ioWriteClose(write); + + TEST_RESULT_STR_Z( + hrnPackBufToStr(pack), + "1:u64:488" + ", 2:u64:1911246845" + ", 7:u64:18446744073709551615" + ", 10:u64:1" + ", 11:u64:77" + ", 12:u32:127" + ", 13:i64:-1" + ", 14:i32:-1" + ", 15:bool:true" + ", 20:bool:false" + ", 28:obj:" + "{" + "1:bool:true" + ", 2:bool:false" + "}" + ", 37:array:" + "[" + "1:u64:0" + ", 2:u64:1" + ", 3:u64:2" + ", 4:u64:3" + "]" + ", 38:str:sample" + ", 39:str:enoughtoincreasebuffer" + ", 40:str:" + ", 41:str:small" + ", 42:str:" + ", 45:str:" + ", 47:u32:0" + ", 48:array:" + "[" + "1:obj:" + "{" + "1:i32:555" + ", 3:i32:777" + ", 5:i64:1" + ", 7:u64:1" + "}" + ", 3:str:A" + ", 5:time:33" + ", 6:time:66" + "]" + ", 49:bin:050403020100" + ", 51:bin:", + "check pack string"); + + TEST_RESULT_STR_Z( + bufHex(pack), + "b8e803" // 1, u64, 750 + "b8fd9fad8f07" // 2, u64, 1911246845 + "bc01ffffffffffffffffff01" // 7, u64, 0xFFFFFFFFFFFFFFFF + "b601" // 10, u64, 1 + "b84d" // 11, u64, 77 + "a87f" // 12, u32, 127 + "54" // 13, i64, -1 + "44" // 14, i32, -1 + "38" // 15, bool, true + "3401" // 20, bool, false + "67" // 28, obj begin + "38" // 1, bool + "30" // 2, bool + "00" // obj end + "1801" // 37, array begin + "b0" // 1, u64, 0 + "b4" // 2, u64, 1 + "b802" // 3, u64, 2 + "b803" // 4, u64, 3 + "00" // array end + "880673616d706c65" // 38, str, sample + "8816656e6f756768746f696e637265617365627566666572" // 39, str, enoughtoincreasebuffer + "80" // 40, str, zero length + "8805736d616c6c" // 41, str, small + "80" // 42, str, zero length + "82" // 45, str, zero length + "a1" // 47, u32, 0 + "10" // 48, array begin + "60" // 1, obj begin + "48d608" // 1, i32, 555 + "49920c" // 3, i32, 777 + "5902" // 5, i64, 1 + "b5" // 7, u64, 1 + "00" // obj end + "890141" // 3, str, A + "9942" // 5, time, 33 + "988401" // 6, time, 66 + "00" // array end + "2806050403020100" // 49, bin, 0x050403020100 + "21" // 51, bin, zero length + "00", // end + "check pack hex"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("read pack"); + + IoRead *read = ioBufferReadNew(pack); + ioReadOpen(read); + + PackRead *packRead = NULL; + TEST_ASSIGN(packRead, pckReadNew(read), "new read"); + + TEST_RESULT_UINT(pckReadU64P(packRead), 0750, "read mode"); + TEST_RESULT_UINT(pckReadU64P(packRead), 1911246845, "read timestamp"); + TEST_ERROR(pckReadU64P(packRead, .id = 2), FormatError, "field 2 was already read"); + TEST_ERROR(pckReadU32P(packRead, .id = 7), FormatError, "field 7 is type 'u64' but expected 'u32'"); + TEST_RESULT_UINT(pckReadU64P(packRead, .id = 7), 0xFFFFFFFFFFFFFFFF, "read max u64"); + TEST_RESULT_BOOL(pckReadNullP(packRead, .id = 9), true, "field 9 is null"); + TEST_RESULT_UINT(pckReadU64P(packRead, .id = 9), 0, "field 9 default is 0"); + TEST_RESULT_BOOL(pckReadNullP(packRead, .id = 10), false, "field 10 is not null"); + TEST_RESULT_UINT(pckReadU64P(packRead, .id = 10), 1, "read 1"); + TEST_RESULT_UINT(pckReadU32P(packRead, .id = 12), 127, "read 127 (skip field 11)"); + TEST_RESULT_INT(pckReadI64P(packRead), -1, "read -1"); + TEST_RESULT_INT(pckReadI32P(packRead, .id = 14), -1, "read -1"); + TEST_RESULT_BOOL(pckReadBoolP(packRead, .id = 15), true, "read true"); + TEST_RESULT_BOOL(pckReadBoolP(packRead, .id = 20), false, "read false"); + + TEST_ERROR(pckReadObjEndP(packRead), FormatError, "not in object"); + TEST_RESULT_VOID(pckReadObjBeginP(packRead, .id = 28), "read object begin"); + TEST_ERROR(pckReadArrayEndP(packRead), FormatError, "not in array"); + TEST_RESULT_BOOL(pckReadBoolP(packRead), true, "read true"); + TEST_RESULT_BOOL(pckReadBoolP(packRead), false, "read false"); + TEST_RESULT_BOOL(pckReadNullP(packRead), true, "field 3 is null"); + TEST_RESULT_BOOL(pckReadBoolP(packRead), false, "field 3 default is false"); + TEST_RESULT_BOOL(pckReadNullP(packRead, .id = 4), true, "field 4 is null"); + TEST_RESULT_BOOL(pckReadBoolP(packRead), false, "read default false"); + TEST_RESULT_VOID(pckReadObjEndP(packRead), "read object end"); + + TEST_ERROR(pckReadArrayEndP(packRead), FormatError, "not in array"); + TEST_RESULT_BOOL(pckReadNext(packRead), true, "read next tag which should be an array"); + TEST_RESULT_UINT(pckReadId(packRead), 37, "check array id"); + TEST_RESULT_VOID(pckReadArrayBeginP(packRead, .id = pckReadId(packRead)), "read array begin"); + + TEST_ERROR(pckReadObjEndP(packRead), FormatError, "not in object"); + + unsigned int value = 0; + + while (pckReadNext(packRead)) + { + TEST_RESULT_UINT(pckReadU64P(packRead, .id = pckReadId(packRead)), value, "read %u", value); + value++; + } + + TEST_RESULT_VOID(pckReadArrayEndP(packRead), "read array end"); + + TEST_RESULT_STR_Z(pckReadStrP(packRead, .id = 39), "enoughtoincreasebuffer", "read string (skipped prior)"); + TEST_RESULT_STR_Z(pckReadStrP(packRead, .id = 41), "small", "read string (skipped prior)"); + TEST_RESULT_STR_Z(pckReadStrP(packRead), "", "zero length (skipped prior)"); + TEST_RESULT_STR(pckReadStrP(packRead, .id = 43), NULL, "read NULL string"); + TEST_RESULT_STR(pckReadStrP(packRead), NULL, "read NULL string"); + TEST_RESULT_STR_Z(pckReadStrP(packRead), "", "read empty string"); + + TEST_RESULT_UINT(pckReadU32P(packRead), 0, "read default 0"); + TEST_RESULT_UINT(pckReadU32P(packRead, .id = 47), 0, "read 0"); + + TEST_RESULT_VOID(pckReadArrayBeginP(packRead), "read array begin"); + TEST_RESULT_VOID(pckReadObjBeginP(packRead), "read object begin"); + TEST_RESULT_INT(pckReadI32P(packRead), 555, "read 0"); + TEST_RESULT_INT(pckReadI32P(packRead, .id = 3), 777, "read 0"); + TEST_RESULT_INT(pckReadI64P(packRead, .defaultValue = 44), 44, "read default 44"); + TEST_RESULT_INT(pckReadI64P(packRead, .defaultValue = 44), 1, "read 1"); + TEST_RESULT_UINT(pckReadU64P(packRead, .defaultValue = 55), 55, "read default 55"); + TEST_RESULT_UINT(pckReadU64P(packRead, .defaultValue = 55), 1, "read 1"); + TEST_RESULT_VOID(pckReadObjEndP(packRead), "read object end"); + TEST_RESULT_STR_Z(pckReadStrP(packRead, .id = 3), "A", "read A"); + TEST_RESULT_INT(pckReadTimeP(packRead, .defaultValue = 99), 99, "read default 99"); + TEST_RESULT_INT(pckReadTimeP(packRead, .id = 5, .defaultValue = 44), 33, "read 33"); + TEST_RESULT_INT(pckReadI32P(packRead, .id = 7, .defaultValue = 1), 1, "read default 1"); + TEST_RESULT_VOID(pckReadArrayEndP(packRead), "read array end"); + + TEST_RESULT_STR_Z(bufHex(pckReadBinP(packRead)), "050403020100", "read bin"); + TEST_RESULT_PTR(pckReadBinP(packRead), NULL, "read bin null"); + TEST_RESULT_UINT(bufSize(pckReadBinP(packRead)), 0, "read bin zero length"); + + TEST_RESULT_BOOL(pckReadNullP(packRead, .id = 999), true, "field 999 is null"); + TEST_RESULT_UINT(pckReadU64P(packRead, .id = 999), 0, "field 999 default is 0"); + + TEST_RESULT_VOID(pckReadEndP(packRead), "end"); + TEST_RESULT_VOID(pckReadFree(packRead), "free"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("EOF on short buffer"); + + TEST_ASSIGN(packRead, pckReadNewBuf(BUFSTRDEF("\255")), "new read"); + TEST_ERROR(pckReadUInt64Internal(packRead), FormatError, "unexpected EOF"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on invalid uint64"); + + TEST_ASSIGN(packRead, pckReadNewBuf(BUFSTRDEF("\255\255\255\255\255\255\255\255\255\255")), "new read"); + TEST_ERROR(pckReadUInt64Internal(packRead), FormatError, "unterminated base-128 integer"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("pack/unpack pointer"); + + pack = bufNew(0); + + TEST_ASSIGN(packWrite, pckWriteNewBuf(pack), "new write"); + TEST_RESULT_VOID(pckWritePtrP(packWrite, NULL), "write default pointer"); + TEST_RESULT_VOID(pckWritePtrP(packWrite, "sample"), "write pointer"); + TEST_RESULT_VOID(pckWriteEndP(packWrite), "write end"); + + TEST_ASSIGN(packRead, pckReadNewBuf(pack), "new read"); + TEST_RESULT_Z(pckReadPtrP(packRead), NULL, "read default pointer"); + TEST_RESULT_Z(pckReadPtrP(packRead, .id = 2), "sample", "read pointer"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("pack/unpack write internal buffer empty"); + + pack = bufNew(0); + write = ioBufferWriteNew(pack); + ioWriteOpen(write); + + // Make internal buffer small enough that it will never be used + ioBufferSizeSet(0); + + TEST_ASSIGN(packWrite, pckWriteNew(write), "new write"); + TEST_RESULT_VOID(pckWriteStrP(packWrite, STRDEF("test")), "write string longer than internal buffer"); + TEST_RESULT_VOID(pckWriteEndP(packWrite), "end with internal buffer empty"); + + TEST_ASSIGN(packRead, pckReadNewBuf(pack), "new read"); + TEST_RESULT_STR_Z(pckReadStrP(packRead), "test", "read string"); + } + + FUNCTION_HARNESS_RESULT_VOID(); +}