db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Limit usable Buffer size without changing allocated size.

Browse Source 
Allow buffers to report a lower size than their allocated size. This means a larger buffer can be used to do the work of a smaller buffer without having to create a new buffer and concatenate.

This is useful for blocking I/O where the buffer may be too large for the amount of data that is available to read.
This commit is contained in:
David Steele
2018-11-08 16:48:33 -05:00
parent edb2c6eb26
commit 12b3be1d8e
6 changed files with 84 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/xml/release.xml
+4
View File
@@ -90,6 +90,10 @@
<p>Change <code>infoArchiveCheckPg()</code> to display the <postgres/> version as a string (e.g. 9.4) instead of the integer representation (e.g. 90400) when throwing an error.</p>
</release-item>
<release-item>
<p>Limit usable <code>Buffer</code> size without changing allocated size.</p>
</release-item>
<release-item>
<p>Construct <code>Wait</code> object in milliseconds instead of fractional seconds.</p>
</release-item>
src/common/type/buffer.c
+59 -13
View File
@@ -1,5 +1,5 @@
/***********************************************************************************************************************************
String Handler
Buffer Handler
***********************************************************************************************************************************/
#include <stdio.h>
#include <string.h>
@@ -15,6 +15,8 @@ struct Buffer
{
MemContext *memContext;
size_t size; // Actual size of buffer
bool limitSet; // Has a limit been set?
size_t limit; // Limited reported size of the buffer to make it appear smaller
size_t used; // Amount of buffer used
unsigned char *buffer; // Buffer allocation
};
@@ -83,8 +85,8 @@ bufNewStr(const String *string)
// Create object and copy string
Buffer *this = bufNew(strSize(string));
memcpy(this->buffer, strPtr(string), this->size);
this->used = this->size;
memcpy(this->buffer, strPtr(string), bufSize(this));
this->used = bufSize(this);
FUNCTION_TEST_RESULT(BUFFER, this);
}
@@ -103,8 +105,8 @@ bufNewZ(const char *string)
// Create a new buffer and then copy the string into it.
Buffer *this = bufNew(strlen(string));
memcpy(this->buffer, string, this->size);
this->used = this->size;
memcpy(this->buffer, string, bufSize(this));
this->used = bufSize(this);
FUNCTION_TEST_RESULT(BUFFER, this);
}
@@ -146,7 +148,7 @@ bufCatC(Buffer *this, const unsigned char *cat, size_t catOffset, size_t catSize
if (catSize > 0)
{
if (this->used + catSize > this->size)
if (this->used + catSize > bufSize(this))
bufResize(this, this->used + catSize);
// Just here to silence nonnull warnings from clang static analyzer
@@ -221,7 +223,7 @@ bufHex(const Buffer *this)
String *result = strNew("");
for (unsigned int bufferIdx = 0; bufferIdx < this->size; bufferIdx++)
for (unsigned int bufferIdx = 0; bufferIdx < bufSize(this); bufferIdx++)
strCatFmt(result, "%02x", this->buffer[bufferIdx]);
FUNCTION_TEST_RESULT(STRING, result);
@@ -294,6 +296,9 @@ bufResize(Buffer *this, size_t size)
if (this->used > this->size)
this->used = this->size;
if (this->limitSet && this->limit > this->size)
this->limit = this->size;
}
FUNCTION_TEST_RESULT(BUFFER, this);
@@ -311,7 +316,41 @@ bufFull(const Buffer *this)
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->used == this->size);
FUNCTION_TEST_RESULT(BOOL, this->used == bufSize(this));
}
/***********************************************************************************************************************************
Set and clear buffer limits
***********************************************************************************************************************************/
void
bufLimitClear(Buffer *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BUFFER, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
this->limitSet = false;
FUNCTION_TEST_RESULT_VOID();
}
void
bufLimitSet(Buffer *this, size_t limit)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BUFFER, this);
FUNCTION_TEST_PARAM(SIZE, limit);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(limit <= this->size);
FUNCTION_TEST_END();
this->limit = limit;
this->limitSet = true;
FUNCTION_TEST_RESULT_VOID();
}
/***********************************************************************************************************************************
@@ -341,7 +380,7 @@ bufRemains(const Buffer *this)
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(SIZE, this->size - this->used);
FUNCTION_TEST_RESULT(SIZE, bufSize(this) - this->used);
}
/***********************************************************************************************************************************
@@ -371,7 +410,7 @@ bufSize(const Buffer *this)
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(SIZE, this->size);
FUNCTION_TEST_RESULT(SIZE, this->limitSet ? this->limit : this->size);
}
/***********************************************************************************************************************************
@@ -400,7 +439,7 @@ bufUsedInc(Buffer *this, size_t inc)
FUNCTION_TEST_PARAM(SIZE, inc);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->used + inc <= this->size);
FUNCTION_TEST_ASSERT(this->used + inc <= bufSize(this));
FUNCTION_TEST_END();
this->used += inc;
@@ -416,7 +455,7 @@ bufUsedSet(Buffer *this, size_t used)
FUNCTION_TEST_PARAM(SIZE, used);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(used <= this->size);
FUNCTION_TEST_ASSERT(used <= bufSize(this));
FUNCTION_TEST_END();
this->used = used;
@@ -444,7 +483,14 @@ Render as string for logging
String *
bufToLog(const Buffer *this)
{
return strNewFmt("{used: %zu, size: %zu}", this->used, this->size);
String *result = strNewFmt("{used: %zu, size: %zu, limit: ", this->used, this->size);
if (this->limitSet)
strCatFmt(result, "%zu}", this->limit);
else
strCat(result, "<off>}");
return result;
}
/***********************************************************************************************************************************
src/common/type/buffer.h
+2
View File
@@ -29,6 +29,8 @@ Buffer *bufMove(Buffer *this, MemContext *parentNew);
Buffer *bufResize(Buffer *this, size_t size);
bool bufFull(const Buffer *this);
void bufLimitClear(Buffer *this);
void bufLimitSet(Buffer *this, size_t limit);
unsigned char *bufPtr(const Buffer *this);
size_t bufRemains(const Buffer *this);
unsigned char *bufRemainsPtr(const Buffer *this);
test/expect/mock-archive-001.log
+2 -2
View File
@@ -127,7 +127,7 @@ P00 DEBUG: command/control/control::lockStopTest: => void
P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"700000007000000070000000"})
P00 DEBUG: postgres/interface::pgControlFromFile: (pgPath: {"[TEST_PATH]/db-master/db/base"})
P00 DEBUG: storage/storage::storageGet: (file: {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false}, param.exactSize: 512)
P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512}
P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: <off>}
P00 DEBUG: postgres/interface::pgControlFromFile: => {version: 90400, systemId: 1000000000000000094, walSegmentSize: 16777216, pageChecksum: true}
P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"[TEST_PATH]/db-master/repo"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *))
P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix}
@@ -169,7 +169,7 @@ P00 DEBUG: command/control/control::lockStopTest: => void
P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000001"})
P00 DEBUG: postgres/interface::pgControlFromFile: (pgPath: {"[TEST_PATH]/db-master/db/base"})
P00 DEBUG: storage/storage::storageGet: (file: {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false}, param.exactSize: 512)
P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512}
P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: <off>}
P00 DEBUG: postgres/interface::pgControlFromFile: => {version: 90400, systemId: 1000000000000000094, walSegmentSize: 16777216, pageChecksum: true}
P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"[TEST_PATH]/db-master/repo"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *))
P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix}
test/expect/mock-stanza-001.log
+1 -1
View File
@@ -544,7 +544,7 @@ P00 DEBUG: command/control/control::lockStopTest: => void
P00 DEBUG: command/archive/get/file::archiveGetCheck: (archiveFile: {"000000010000000100000002"})
P00 DEBUG: postgres/interface::pgControlFromFile: (pgPath: {"[TEST_PATH]/db-master/db/base"})
P00 DEBUG: storage/storage::storageGet: (file: {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false}, param.exactSize: 512)
P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512}
P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: <off>}
P00 DEBUG: postgres/interface::pgControlFromFile: => {version: 90300, systemId: 1000000000000000093, walSegmentSize: 16777216, pageChecksum: true}
P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: (path: {"[TEST_PATH]/db-master/repo"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *))
P00 DEBUG: storage/driver/posix/storage::storageDriverPosixNew: => {StorageDriverPosix}
test/src/module/common/typeBufferTest.c
+16 -2
View File
@@ -45,7 +45,7 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("bufResize(), bufFull(), bufRemains*(), and bufUsed*()"))
if (testBegin("bufResize(), bufFull(), bufLimit*(), bufRemains*(), and bufUsed*()"))
{
Buffer *buffer = NULL;
unsigned char *bufferPtr = NULL;
@@ -94,10 +94,20 @@ testRun(void)
TEST_RESULT_INT(sameTotal, 128, "original bytes match");
// Use limits to change size reporting
TEST_RESULT_VOID(bufLimitSet(buffer, 64), "set limit");
TEST_RESULT_INT(bufSize(buffer), 64, " check limited size");
TEST_RESULT_VOID(bufLimitClear(buffer), " clear limit");
TEST_RESULT_INT(bufSize(buffer), 128, " check unlimited size");
// Resize to zero buffer
TEST_RESULT_VOID(bufUsedZero(buffer), "set used to 0");
TEST_RESULT_INT(bufUsed(buffer), 0, "check used is zero");
TEST_RESULT_VOID(bufLimitSet(buffer, 64), "set limit to make sure it gets reduced with the resize");
TEST_RESULT_VOID(bufResize(buffer, 32), "decrease size to 32");
TEST_RESULT_INT(bufSize(buffer), 32, "check size");
TEST_RESULT_VOID(bufLimitSet(buffer, 0), "set limit so that it won't need to be resized");
TEST_RESULT_VOID(bufResize(buffer, 0), "decrease size to zero");
TEST_RESULT_INT(bufSize(buffer), 0, "check size");
TEST_RESULT_VOID(bufResize(buffer, 0), "decrease size to zero again");
@@ -138,7 +148,11 @@ testRun(void)
// *****************************************************************************************************************************
if (testBegin("bufToLog()"))
{
TEST_RESULT_STR(strPtr(bufToLog(bufNew(100))), "{used: 0, size: 100}", "buf to log");
Buffer *buffer = bufNew(100);
TEST_RESULT_STR(strPtr(bufToLog(buffer)), "{used: 0, size: 100, limit: <off>}", "buf to log");
bufLimitSet(buffer, 50);
TEST_RESULT_STR(strPtr(bufToLog(buffer)), "{used: 0, size: 100, limit: 50}", "buf to log");
}
FUNCTION_HARNESS_RESULT_VOID();