From 12b3be1d8ed38beea67c4d4de582af9039c5bd4b Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 8 Nov 2018 16:48:33 -0500 Subject: [PATCH] Limit usable Buffer size without changing allocated size. Allow buffers to report a lower size than their allocated size. This means a larger buffer can be used to do the work of a smaller buffer without having to create a new buffer and concatenate. This is useful for blocking I/O where the buffer may be too large for the amount of data that is available to read. --- doc/xml/release.xml | 4 ++ src/common/type/buffer.c | 72 ++++++++++++++++++++----- src/common/type/buffer.h | 2 + test/expect/mock-archive-001.log | 4 +- test/expect/mock-stanza-001.log | 2 +- test/src/module/common/typeBufferTest.c | 18 ++++++- 6 files changed, 84 insertions(+), 18 deletions(-) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 03cb6e25d..aceaf2ed8 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -90,6 +90,10 @@

Change infoArchiveCheckPg() to display the version as a string (e.g. 9.4) instead of the integer representation (e.g. 90400) when throwing an error.

+ +

Limit usable Buffer size without changing allocated size.

+ +

Construct Wait object in milliseconds instead of fractional seconds.

 diff --git a/src/common/type/buffer.c b/src/common/type/buffer.c index a0e21cbc8..6b4faa7bb 100644 --- a/src/common/type/buffer.c +++ b/src/common/type/buffer.c @@ -1,5 +1,5 @@ /*********************************************************************************************************************************** -String Handler +Buffer Handler ***********************************************************************************************************************************/ #include #include @@ -15,6 +15,8 @@ struct Buffer { MemContext *memContext; size_t size; // Actual size of buffer + bool limitSet; // Has a limit been set? + size_t limit; // Limited reported size of the buffer to make it appear smaller size_t used; // Amount of buffer used unsigned char *buffer; // Buffer allocation }; @@ -83,8 +85,8 @@ bufNewStr(const String *string) // Create object and copy string Buffer *this = bufNew(strSize(string)); - memcpy(this->buffer, strPtr(string), this->size); - this->used = this->size; + memcpy(this->buffer, strPtr(string), bufSize(this)); + this->used = bufSize(this); FUNCTION_TEST_RESULT(BUFFER, this); } @@ -103,8 +105,8 @@ bufNewZ(const char *string) // Create a new buffer and then copy the string into it. Buffer *this = bufNew(strlen(string)); - memcpy(this->buffer, string, this->size); - this->used = this->size; + memcpy(this->buffer, string, bufSize(this)); + this->used = bufSize(this); FUNCTION_TEST_RESULT(BUFFER, this); } @@ -146,7 +148,7 @@ bufCatC(Buffer *this, const unsigned char *cat, size_t catOffset, size_t catSize if (catSize > 0) { - if (this->used + catSize > this->size) + if (this->used + catSize > bufSize(this)) bufResize(this, this->used + catSize); // Just here to silence nonnull warnings from clang static analyzer @@ -221,7 +223,7 @@ bufHex(const Buffer *this) String *result = strNew(""); - for (unsigned int bufferIdx = 0; bufferIdx < this->size; bufferIdx++) + for (unsigned int bufferIdx = 0; bufferIdx < bufSize(this); bufferIdx++) strCatFmt(result, "%02x", this->buffer[bufferIdx]); FUNCTION_TEST_RESULT(STRING, result); @@ -294,6 +296,9 @@ bufResize(Buffer *this, size_t size) if (this->used > this->size) this->used = this->size; + + if (this->limitSet && this->limit > this->size) + this->limit = this->size; } FUNCTION_TEST_RESULT(BUFFER, this); @@ -311,7 +316,41 @@ bufFull(const Buffer *this) FUNCTION_TEST_ASSERT(this != NULL); FUNCTION_TEST_END(); - FUNCTION_TEST_RESULT(BOOL, this->used == this->size); + FUNCTION_TEST_RESULT(BOOL, this->used == bufSize(this)); +} + +/*********************************************************************************************************************************** +Set and clear buffer limits +***********************************************************************************************************************************/ +void +bufLimitClear(Buffer *this) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BUFFER, this); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_END(); + + this->limitSet = false; + + FUNCTION_TEST_RESULT_VOID(); +} + +void +bufLimitSet(Buffer *this, size_t limit) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(BUFFER, this); + FUNCTION_TEST_PARAM(SIZE, limit); + + FUNCTION_TEST_ASSERT(this != NULL); + FUNCTION_TEST_ASSERT(limit <= this->size); + FUNCTION_TEST_END(); + + this->limit = limit; + this->limitSet = true; + + FUNCTION_TEST_RESULT_VOID(); } /*********************************************************************************************************************************** @@ -341,7 +380,7 @@ bufRemains(const Buffer *this) FUNCTION_TEST_ASSERT(this != NULL); FUNCTION_TEST_END(); - FUNCTION_TEST_RESULT(SIZE, this->size - this->used); + FUNCTION_TEST_RESULT(SIZE, bufSize(this) - this->used); } /*********************************************************************************************************************************** @@ -371,7 +410,7 @@ bufSize(const Buffer *this) FUNCTION_TEST_ASSERT(this != NULL); FUNCTION_TEST_END(); - FUNCTION_TEST_RESULT(SIZE, this->size); + FUNCTION_TEST_RESULT(SIZE, this->limitSet ? this->limit : this->size); } /*********************************************************************************************************************************** @@ -400,7 +439,7 @@ bufUsedInc(Buffer *this, size_t inc) FUNCTION_TEST_PARAM(SIZE, inc); FUNCTION_TEST_ASSERT(this != NULL); - FUNCTION_TEST_ASSERT(this->used + inc <= this->size); + FUNCTION_TEST_ASSERT(this->used + inc <= bufSize(this)); FUNCTION_TEST_END(); this->used += inc; @@ -416,7 +455,7 @@ bufUsedSet(Buffer *this, size_t used) FUNCTION_TEST_PARAM(SIZE, used); FUNCTION_TEST_ASSERT(this != NULL); - FUNCTION_TEST_ASSERT(used <= this->size); + FUNCTION_TEST_ASSERT(used <= bufSize(this)); FUNCTION_TEST_END(); this->used = used; @@ -444,7 +483,14 @@ Render as string for logging String * bufToLog(const Buffer *this) { - return strNewFmt("{used: %zu, size: %zu}", this->used, this->size); + String *result = strNewFmt("{used: %zu, size: %zu, limit: ", this->used, this->size); + + if (this->limitSet) + strCatFmt(result, "%zu}", this->limit); + else + strCat(result, "}"); + + return result; } /*********************************************************************************************************************************** diff --git a/src/common/type/buffer.h b/src/common/type/buffer.h index e368e9dcd..47e1b4b5a 100644 --- a/src/common/type/buffer.h +++ b/src/common/type/buffer.h @@ -29,6 +29,8 @@ Buffer *bufMove(Buffer *this, MemContext *parentNew); Buffer *bufResize(Buffer *this, size_t size); bool bufFull(const Buffer *this); +void bufLimitClear(Buffer *this); +void bufLimitSet(Buffer *this, size_t limit); unsigned char *bufPtr(const Buffer *this); size_t bufRemains(const Buffer *this); unsigned char *bufRemainsPtr(const Buffer *this); diff --git a/test/expect/mock-archive-001.log b/test/expect/mock-archive-001.log index 13fe7790f..18cdb2c58 100644 --- a/test/expect/mock-archive-001.log +++ b/test/expect/mock-archive-001.log @@ -127,7 +127,7 @@ P00 DEBUG: command/control/control::lockStopTest: => void P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"700000007000000070000000"}) P00 DEBUG: postgres/interface::pgControlFromFile: (pgPath: {"[TEST_PATH]/db-master/db/base"}) P00 DEBUG: storage/storage::storageGet: (file: {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false}, param.exactSize: 512) -P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512} +P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: } P00 DEBUG: postgres/interface::pgControlFromFile: => {version: 90400, systemId: 1000000000000000094, walSegmentSize: 16777216, pageChecksum: true} P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"[TEST_PATH]/db-master/repo"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *)) P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix} @@ -169,7 +169,7 @@ P00 DEBUG: command/control/control::lockStopTest: => void P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000001"}) P00 DEBUG: postgres/interface::pgControlFromFile: (pgPath: {"[TEST_PATH]/db-master/db/base"}) P00 DEBUG: storage/storage::storageGet: (file: {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false}, param.exactSize: 512) -P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512} +P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: } P00 DEBUG: postgres/interface::pgControlFromFile: => {version: 90400, systemId: 1000000000000000094, walSegmentSize: 16777216, pageChecksum: true} P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"[TEST_PATH]/db-master/repo"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *)) P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix} diff --git a/test/expect/mock-stanza-001.log b/test/expect/mock-stanza-001.log index 427c606ad..3a38f2c0d 100644 --- a/test/expect/mock-stanza-001.log +++ b/test/expect/mock-stanza-001.log @@ -544,7 +544,7 @@ P00 DEBUG: command/control/control::lockStopTest: => void P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000002"}) P00 DEBUG: postgres/interface::pgControlFromFile: (pgPath: {"[TEST_PATH]/db-master/db/base"}) P00 DEBUG: storage/storage::storageGet: (file: {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false}, param.exactSize: 512) -P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512} +P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: } P00 DEBUG: postgres/interface::pgControlFromFile: => {version: 90300, systemId: 1000000000000000093, walSegmentSize: 16777216, pageChecksum: true} P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"[TEST_PATH]/db-master/repo"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *)) P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix} diff --git a/test/src/module/common/typeBufferTest.c b/test/src/module/common/typeBufferTest.c index be8c3bc7d..4daeb4eab 100644 --- a/test/src/module/common/typeBufferTest.c +++ b/test/src/module/common/typeBufferTest.c @@ -45,7 +45,7 @@ testRun(void) } // ***************************************************************************************************************************** - if (testBegin("bufResize(), bufFull(), bufRemains*(), and bufUsed*()")) + if (testBegin("bufResize(), bufFull(), bufLimit*(), bufRemains*(), and bufUsed*()")) { Buffer *buffer = NULL; unsigned char *bufferPtr = NULL; @@ -94,10 +94,20 @@ testRun(void) TEST_RESULT_INT(sameTotal, 128, "original bytes match"); + // Use limits to change size reporting + TEST_RESULT_VOID(bufLimitSet(buffer, 64), "set limit"); + TEST_RESULT_INT(bufSize(buffer), 64, " check limited size"); + TEST_RESULT_VOID(bufLimitClear(buffer), " clear limit"); + TEST_RESULT_INT(bufSize(buffer), 128, " check unlimited size"); + // Resize to zero buffer TEST_RESULT_VOID(bufUsedZero(buffer), "set used to 0"); TEST_RESULT_INT(bufUsed(buffer), 0, "check used is zero"); + TEST_RESULT_VOID(bufLimitSet(buffer, 64), "set limit to make sure it gets reduced with the resize"); + TEST_RESULT_VOID(bufResize(buffer, 32), "decrease size to 32"); + TEST_RESULT_INT(bufSize(buffer), 32, "check size"); + TEST_RESULT_VOID(bufLimitSet(buffer, 0), "set limit so that it won't need to be resized"); TEST_RESULT_VOID(bufResize(buffer, 0), "decrease size to zero"); TEST_RESULT_INT(bufSize(buffer), 0, "check size"); TEST_RESULT_VOID(bufResize(buffer, 0), "decrease size to zero again"); @@ -138,7 +148,11 @@ testRun(void) // ***************************************************************************************************************************** if (testBegin("bufToLog()")) { - TEST_RESULT_STR(strPtr(bufToLog(bufNew(100))), "{used: 0, size: 100}", "buf to log"); + Buffer *buffer = bufNew(100); + TEST_RESULT_STR(strPtr(bufToLog(buffer)), "{used: 0, size: 100, limit: }", "buf to log"); + + bufLimitSet(buffer, 50); + TEST_RESULT_STR(strPtr(bufToLog(buffer)), "{used: 0, size: 100, limit: 50}", "buf to log"); } FUNCTION_HARNESS_RESULT_VOID();