mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-04-25 12:04:48 +02:00
storageFileRead() accepts a buffer for output rather than creating one.
This is more efficient overall and allows the caller to specify how many bytes will be read on each call. Reads are appended if the buffer already contains data but the buffer size will never increase. Allow Buffer object "used size" to be different than "allocated size". Add functions to manage used size and remaining size and update automatically when possible.
This commit is contained in:
David Steele
parent
0acf705416
commit
5dc8a2ec08
@ -33,6 +33,14 @@
|
||||
<p>Add <code>uint64</code> variant type and supporting conversion functions.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<p>Allow <code>Buffer</code> object <quote>used size</quote> to be different than <quote>allocated size</quote>. Add functions to manage used size and remaining size and update automatically when possible.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<p><code>storageFileRead()</code> accepts a buffer for output rather than creating one. This is more efficient overall and allows the caller to specify how many bytes will be read on each call. Reads are appended if the buffer already contains data but the buffer size will never increase.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<release-item-contributor-list>
|
||||
<release-item-contributor id="shang.cynthia"/>
|
||||
|
||||
@ -14,8 +14,9 @@ Contains information about the buffer
|
||||
struct Buffer
|
||||
{
|
||||
MemContext *memContext;
|
||||
size_t size;
|
||||
unsigned char *buffer;
|
||||
size_t size; // Actual size of buffer
|
||||
size_t used; // Amount of buffer used
|
||||
unsigned char *buffer; // Buffer allocation
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -36,6 +37,7 @@ bufNew(size_t size)
|
||||
this = memNew(sizeof(Buffer));
|
||||
this->memContext = MEM_CONTEXT_NEW();
|
||||
this->size = size;
|
||||
this->used = 0;
|
||||
|
||||
// Allocate buffer
|
||||
if (size > 0)
|
||||
@ -62,6 +64,7 @@ bufNewC(size_t size, const void *buffer)
|
||||
// Create object and copy data
|
||||
Buffer *this = bufNew(size);
|
||||
memcpy(this->buffer, buffer, this->size);
|
||||
this->used = this->size;
|
||||
|
||||
FUNCTION_TEST_RESULT(BUFFER, this);
|
||||
}
|
||||
@ -78,11 +81,10 @@ bufNewStr(const String *string)
|
||||
FUNCTION_TEST_ASSERT(string != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
// Create object
|
||||
// Create object and copy string
|
||||
Buffer *this = bufNew(strSize(string));
|
||||
|
||||
// Copy the data
|
||||
memcpy(this->buffer, strPtr(string), this->size);
|
||||
this->used = this->size;
|
||||
|
||||
FUNCTION_TEST_RESULT(BUFFER, this);
|
||||
}
|
||||
@ -100,16 +102,16 @@ bufCat(Buffer *this, const Buffer *cat)
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
if (cat != NULL && cat->size > 0)
|
||||
if (cat != NULL && cat->used > 0)
|
||||
{
|
||||
size_t sizeOld = this->size;
|
||||
|
||||
bufResize(this, sizeOld + cat->size);
|
||||
if (this->used + cat->used > this->size)
|
||||
bufResize(this, this->used + cat->used);
|
||||
|
||||
// Just here to silence nonnull warnings from clang static analyzer
|
||||
ASSERT_DEBUG(this->buffer != NULL);
|
||||
|
||||
memcpy(this->buffer + sizeOld, cat->buffer, cat->size);
|
||||
memcpy(this->buffer + this->used, cat->buffer, cat->used);
|
||||
this->used = this->used + cat->used;
|
||||
}
|
||||
|
||||
FUNCTION_TEST_RESULT(BUFFER, this);
|
||||
@ -131,8 +133,8 @@ bufEq(const Buffer *this, const Buffer *compare)
|
||||
|
||||
bool result = false;
|
||||
|
||||
if (this->size == compare->size)
|
||||
result = memcmp(this->buffer, compare->buffer, compare->size) == 0;
|
||||
if (this->used == compare->used)
|
||||
result = memcmp(this->buffer, compare->buffer, compare->used) == 0;
|
||||
|
||||
FUNCTION_TEST_RESULT(BOOL, result);
|
||||
}
|
||||
@ -156,21 +158,6 @@ bufMove(Buffer *this, MemContext *parentNew)
|
||||
FUNCTION_TEST_RESULT(BUFFER, this);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Return buffer ptr
|
||||
***********************************************************************************************************************************/
|
||||
unsigned char *
|
||||
bufPtr(const Buffer *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(UCHARP, this->buffer);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Resize the buffer
|
||||
***********************************************************************************************************************************/
|
||||
@ -216,13 +203,76 @@ bufResize(Buffer *this, size_t size)
|
||||
|
||||
this->size = size;
|
||||
}
|
||||
|
||||
if (this->used > this->size)
|
||||
this->used = this->size;
|
||||
}
|
||||
|
||||
FUNCTION_TEST_RESULT(BUFFER, this);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Return buffer size
|
||||
Is the buffer full?
|
||||
***********************************************************************************************************************************/
|
||||
bool
|
||||
bufFull(const Buffer *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(BOOL, this->used == this->size);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Return buffer ptr
|
||||
***********************************************************************************************************************************/
|
||||
unsigned char *
|
||||
bufPtr(const Buffer *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(UCHARP, this->buffer);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get remaining space in the buffer
|
||||
***********************************************************************************************************************************/
|
||||
size_t
|
||||
bufRemains(const Buffer *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(SIZE, this->size - this->used);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Return pointer to remaining space in the buffer (after used space)
|
||||
***********************************************************************************************************************************/
|
||||
unsigned char *
|
||||
bufRemainsPtr(const Buffer *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(UCHARP, this->buffer + this->used);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get buffer size
|
||||
***********************************************************************************************************************************/
|
||||
size_t
|
||||
bufSize(const Buffer *this)
|
||||
@ -236,6 +286,70 @@ bufSize(const Buffer *this)
|
||||
FUNCTION_TEST_RESULT(SIZE, this->size);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get/set the amount of the buffer actually used
|
||||
|
||||
Tracks how much of the buffer has actually been used. This will be updated automatically when possible but if the buffer is
|
||||
modified by using bufPtr() then the user is reponsible for updating the used size.
|
||||
***********************************************************************************************************************************/
|
||||
size_t
|
||||
bufUsed(const Buffer *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(SIZE, this->used);
|
||||
}
|
||||
|
||||
void
|
||||
bufUsedInc(Buffer *this, size_t inc)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, this);
|
||||
FUNCTION_TEST_PARAM(SIZE, inc);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_ASSERT(this->used + inc <= this->size);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
this->used += inc;
|
||||
|
||||
FUNCTION_TEST_RESULT_VOID();
|
||||
}
|
||||
|
||||
void
|
||||
bufUsedSet(Buffer *this, size_t used)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, this);
|
||||
FUNCTION_TEST_PARAM(SIZE, used);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_ASSERT(used <= this->size);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
this->used = used;
|
||||
|
||||
FUNCTION_TEST_RESULT_VOID();
|
||||
}
|
||||
|
||||
void
|
||||
bufUsedZero(Buffer *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(BUFFER, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
this->used = 0;
|
||||
|
||||
FUNCTION_TEST_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Convert to a zero-terminated string for logging
|
||||
***********************************************************************************************************************************/
|
||||
@ -251,7 +365,7 @@ bufToLog(const Buffer *this, char *buffer, size_t bufferSize)
|
||||
if (this == NULL)
|
||||
string = strNew("null");
|
||||
else
|
||||
string = strNewFmt("{size: %zu}", this->size);
|
||||
string = strNewFmt("{used: %zu, size: %zu}", this->used, this->size);
|
||||
|
||||
result = (size_t)snprintf(buffer, bufferSize, "%s", strPtr(string));
|
||||
}
|
||||
|
||||
@ -18,12 +18,22 @@ Functions
|
||||
Buffer *bufNew(size_t size);
|
||||
Buffer *bufNewC(size_t size, const void *buffer);
|
||||
Buffer *bufNewStr(const String *string);
|
||||
|
||||
Buffer *bufCat(Buffer *this, const Buffer *cat);
|
||||
bool bufEq(const Buffer *this, const Buffer *compare);
|
||||
Buffer *bufMove(Buffer *this, MemContext *parentNew);
|
||||
Buffer *bufResize(Buffer *this, size_t size);
|
||||
size_t bufSize(const Buffer *this);
|
||||
|
||||
bool bufFull(const Buffer *this);
|
||||
unsigned char *bufPtr(const Buffer *this);
|
||||
size_t bufRemains(const Buffer *this);
|
||||
unsigned char *bufRemainsPtr(const Buffer *this);
|
||||
size_t bufSize(const Buffer *this);
|
||||
size_t bufUsed(const Buffer *this);
|
||||
void bufUsedInc(Buffer *this, size_t inc);
|
||||
void bufUsedSet(Buffer *this, size_t used);
|
||||
void bufUsedZero(Buffer *this);
|
||||
|
||||
void bufFree(Buffer *this);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
|
||||
@ -9,6 +9,12 @@ PostgreSQL Info
|
||||
#include "postgres/version.h"
|
||||
#include "storage/helper.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Set control data size. The control file is actually 8192 bytes but only the first 512 bytes are used to prevent torn pages even on
|
||||
really old storage with 512-byte sectors.
|
||||
***********************************************************************************************************************************/
|
||||
#define PG_CONTROL_DATA_SIZE 512
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Map control/catalog version to PostgreSQL version
|
||||
***********************************************************************************************************************************/
|
||||
@ -72,15 +78,13 @@ pgControlInfo(const String *pgPath)
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
// Open control file for read
|
||||
StorageFileRead *controlRead = storageNewReadNP(
|
||||
storageLocal(), strNewFmt("%s/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, strPtr(pgPath)));
|
||||
storageFileReadOpen(controlRead);
|
||||
// Read control file
|
||||
PgControlFile *control = (PgControlFile *)bufPtr(
|
||||
storageGetP(
|
||||
storageNewReadNP(storageLocal(), strNewFmt("%s/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, strPtr(pgPath))),
|
||||
.exactSize = PG_CONTROL_DATA_SIZE));
|
||||
|
||||
// Read contents
|
||||
PgControlFile *control = (PgControlFile *)bufPtr(storageFileRead(controlRead));
|
||||
|
||||
// Copy to result structure and get PostgreSQL version
|
||||
// Get PostgreSQL version
|
||||
result.systemId = control->systemId;
|
||||
result.controlVersion = control->controlVersion;
|
||||
result.catalogVersion = control->catalogVersion;
|
||||
|
||||
@ -20,7 +20,6 @@ struct StorageFileReadPosix
|
||||
|
||||
String *name;
|
||||
bool ignoreMissing;
|
||||
size_t bufferSize;
|
||||
|
||||
int handle;
|
||||
bool eof;
|
||||
@ -31,15 +30,13 @@ struct StorageFileReadPosix
|
||||
Create a new file
|
||||
***********************************************************************************************************************************/
|
||||
StorageFileReadPosix *
|
||||
storageFileReadPosixNew(const String *name, bool ignoreMissing, size_t bufferSize)
|
||||
storageFileReadPosixNew(const String *name, bool ignoreMissing)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
||||
FUNCTION_DEBUG_PARAM(STRING, name);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, ignoreMissing);
|
||||
FUNCTION_DEBUG_PARAM(BOOL, bufferSize);
|
||||
|
||||
FUNCTION_TEST_ASSERT(name != NULL);
|
||||
FUNCTION_TEST_ASSERT(bufferSize > 0);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
StorageFileReadPosix *this = NULL;
|
||||
@ -51,7 +48,6 @@ storageFileReadPosixNew(const String *name, bool ignoreMissing, size_t bufferSiz
|
||||
this->memContext = MEM_CONTEXT_NEW();
|
||||
this->name = strDup(name);
|
||||
this->ignoreMissing = ignoreMissing;
|
||||
this->bufferSize = bufferSize;
|
||||
|
||||
this->handle = -1;
|
||||
}
|
||||
@ -91,47 +87,39 @@ storageFileReadPosixOpen(StorageFileReadPosix *this)
|
||||
/***********************************************************************************************************************************
|
||||
Read from a file
|
||||
***********************************************************************************************************************************/
|
||||
Buffer *
|
||||
storageFileReadPosix(StorageFileReadPosix *this)
|
||||
void
|
||||
storageFileReadPosix(StorageFileReadPosix *this, Buffer *buffer)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_FILE_READ_POSIX, this);
|
||||
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_ASSERT(this->handle != -1);
|
||||
FUNCTION_DEBUG_ASSERT(this != NULL && this->handle != -1);
|
||||
FUNCTION_DEBUG_ASSERT(buffer != NULL && !bufFull(buffer));
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
Buffer *result = NULL;
|
||||
|
||||
ASSERT_DEBUG(this != NULL);
|
||||
|
||||
// Read if EOF has not been reached
|
||||
if (!this->eof)
|
||||
{
|
||||
result = bufNew(this->bufferSize);
|
||||
|
||||
// Read and handle errors
|
||||
ssize_t actualBytes = read(this->handle, bufPtr(result), this->bufferSize);
|
||||
size_t expectedBytes = bufRemains(buffer);
|
||||
ssize_t actualBytes = read(this->handle, bufRemainsPtr(buffer), expectedBytes);
|
||||
|
||||
// Error occurred during write
|
||||
if (actualBytes == -1)
|
||||
THROW_SYS_ERROR_FMT(FileReadError, "unable to read '%s'", strPtr(this->name));
|
||||
|
||||
// If no data was read then free the buffer and mark the file as EOF
|
||||
if (actualBytes == 0)
|
||||
{
|
||||
// Update amount of buffer used and total size
|
||||
bufUsedInc(buffer, (size_t)actualBytes);
|
||||
this->size += (size_t)actualBytes;
|
||||
|
||||
// If less data than expected was read then EOF. The file may not actually be EOF but we are not concerned with files that
|
||||
// are growing. Just read up to the point where the file is being extended.
|
||||
if ((size_t)actualBytes != expectedBytes)
|
||||
this->eof = true;
|
||||
bufFree(result);
|
||||
result = NULL;
|
||||
}
|
||||
else
|
||||
{
|
||||
bufResize(result, (size_t)actualBytes);
|
||||
this->size += (size_t)actualBytes;
|
||||
}
|
||||
}
|
||||
|
||||
FUNCTION_DEBUG_RESULT(BUFFER, result);
|
||||
FUNCTION_DEBUG_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -158,6 +146,21 @@ storageFileReadPosixClose(StorageFileReadPosix *this)
|
||||
FUNCTION_DEBUG_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Has file reached EOF?
|
||||
***********************************************************************************************************************************/
|
||||
bool
|
||||
storageFileReadPosixEof(StorageFileReadPosix *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_FILE_READ_POSIX, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(BOOL, this->eof);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Should a missing file be ignored?
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -15,18 +15,19 @@ typedef struct StorageFileReadPosix StorageFileReadPosix;
|
||||
/***********************************************************************************************************************************
|
||||
Constructor
|
||||
***********************************************************************************************************************************/
|
||||
StorageFileReadPosix *storageFileReadPosixNew(const String *name, bool ignoreMissing, size_t bufferSize);
|
||||
StorageFileReadPosix *storageFileReadPosixNew(const String *name, bool ignoreMissing);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Functions
|
||||
***********************************************************************************************************************************/
|
||||
bool storageFileReadPosixOpen(StorageFileReadPosix *this);
|
||||
Buffer *storageFileReadPosix(StorageFileReadPosix *this);
|
||||
void storageFileReadPosix(StorageFileReadPosix *this, Buffer *buffer);
|
||||
void storageFileReadPosixClose(StorageFileReadPosix *this);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Getters
|
||||
***********************************************************************************************************************************/
|
||||
bool storageFileReadPosixEof(StorageFileReadPosix *this);
|
||||
bool storageFileReadPosixIgnoreMissing(StorageFileReadPosix *this);
|
||||
const String *storageFileReadPosixName(StorageFileReadPosix *this);
|
||||
size_t storageFileReadPosixSize(StorageFileReadPosix *this);
|
||||
|
||||
@ -138,7 +138,7 @@ storageFileWritePosix(StorageFileWritePosix *this, const Buffer *buffer)
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
// Write the data
|
||||
if (write(this->handle, bufPtr(buffer), bufSize(buffer)) != (ssize_t)bufSize(buffer))
|
||||
if (write(this->handle, bufPtr(buffer), bufUsed(buffer)) != (ssize_t)bufUsed(buffer))
|
||||
THROW_SYS_ERROR_FMT(FileWriteError, "unable to write '%s'", strPtr(this->name));
|
||||
|
||||
FUNCTION_DEBUG_RESULT_VOID();
|
||||
|
||||
@ -14,6 +14,7 @@ struct StorageFileRead
|
||||
{
|
||||
MemContext *memContext;
|
||||
StorageFileReadPosix *fileDriver;
|
||||
size_t bufferSize;
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -38,8 +39,8 @@ storageFileReadNew(const String *name, bool ignoreMissing, size_t bufferSize)
|
||||
this = memNew(sizeof(StorageFileRead));
|
||||
this->memContext = memContextCurrent();
|
||||
|
||||
// Call driver function
|
||||
this->fileDriver = storageFileReadPosixNew(name, ignoreMissing, bufferSize);
|
||||
this->fileDriver = storageFileReadPosixNew(name, ignoreMissing);
|
||||
this->bufferSize = bufferSize;
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
@ -64,8 +65,8 @@ storageFileReadOpen(StorageFileRead *this)
|
||||
/***********************************************************************************************************************************
|
||||
Read data from the file
|
||||
***********************************************************************************************************************************/
|
||||
Buffer *
|
||||
storageFileRead(StorageFileRead *this)
|
||||
void
|
||||
storageFileRead(StorageFileRead *this, Buffer *buffer)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelTrace);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_FILE_READ, this);
|
||||
@ -73,7 +74,9 @@ storageFileRead(StorageFileRead *this)
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT(BUFFER, storageFileReadPosix(this->fileDriver));
|
||||
storageFileReadPosix(this->fileDriver, buffer);
|
||||
|
||||
FUNCTION_DEBUG_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -112,6 +115,21 @@ storageFileReadClose(StorageFileRead *this)
|
||||
FUNCTION_DEBUG_RESULT_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get buffer size
|
||||
***********************************************************************************************************************************/
|
||||
size_t
|
||||
storageFileReadBufferSize(const StorageFileRead *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_FILE_READ, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(SIZE, this->bufferSize);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get file driver
|
||||
***********************************************************************************************************************************/
|
||||
@ -127,6 +145,21 @@ storageFileReadFileDriver(const StorageFileRead *this)
|
||||
FUNCTION_TEST_RESULT(STORAGE_FILE_READ_POSIX, this->fileDriver);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Has file reached EOF?
|
||||
***********************************************************************************************************************************/
|
||||
bool
|
||||
storageFileReadEof(const StorageFileRead *this)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_FILE_READ, this);
|
||||
|
||||
FUNCTION_TEST_ASSERT(this != NULL);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
FUNCTION_TEST_RESULT(BOOL, storageFileReadPosixEof(this->fileDriver));
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Should a missing file be ignored?
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -22,7 +22,7 @@ StorageFileRead *storageFileReadNew(const String *name, bool ignoreMissing, size
|
||||
Functions
|
||||
***********************************************************************************************************************************/
|
||||
bool storageFileReadOpen(StorageFileRead *this);
|
||||
Buffer *storageFileRead(StorageFileRead *this);
|
||||
void storageFileRead(StorageFileRead *this, Buffer *buffer);
|
||||
void storageFileReadClose(StorageFileRead *this);
|
||||
|
||||
StorageFileRead *storageFileReadMove(StorageFileRead *this, MemContext *parentNew);
|
||||
@ -30,7 +30,9 @@ StorageFileRead *storageFileReadMove(StorageFileRead *this, MemContext *parentNe
|
||||
/***********************************************************************************************************************************
|
||||
Getters
|
||||
***********************************************************************************************************************************/
|
||||
size_t storageFileReadBufferSize(const StorageFileRead *this);
|
||||
StorageFileReadPosix *storageFileReadFileDriver(const StorageFileRead *this);
|
||||
bool storageFileReadEof(const StorageFileRead *this);
|
||||
bool storageFileReadIgnoreMissing(const StorageFileRead *this);
|
||||
const String *storageFileReadName(const StorageFileRead *this);
|
||||
size_t storageFileReadSize(const StorageFileRead *this);
|
||||
|
||||
@ -87,14 +87,15 @@ storageCopy(StorageFileRead *source, StorageFileWrite *destination)
|
||||
storageFileWriteOpen(destination);
|
||||
|
||||
// Copy data from source to destination
|
||||
Buffer *read = NULL;
|
||||
Buffer *read = bufNew(storageFileReadBufferSize(source));
|
||||
|
||||
do
|
||||
{
|
||||
read = storageFileRead(source);
|
||||
storageFileRead(source, read);
|
||||
storageFileWrite(destination, read);
|
||||
bufUsedZero(read);
|
||||
}
|
||||
while (read != NULL);
|
||||
while (!storageFileReadEof(source));
|
||||
|
||||
// Close the source and destination files
|
||||
storageFileReadClose(source);
|
||||
@ -151,10 +152,11 @@ storageExists(const Storage *this, const String *pathExp, StorageExistsParam par
|
||||
Read from storage into a buffer
|
||||
***********************************************************************************************************************************/
|
||||
Buffer *
|
||||
storageGet(StorageFileRead *file)
|
||||
storageGet(StorageFileRead *file, StorageGetParam param)
|
||||
{
|
||||
FUNCTION_DEBUG_BEGIN(logLevelDebug);
|
||||
FUNCTION_DEBUG_PARAM(STORAGE_FILE_READ, file);
|
||||
FUNCTION_DEBUG_PARAM(SIZE, param.exactSize);
|
||||
|
||||
FUNCTION_TEST_ASSERT(file != NULL);
|
||||
FUNCTION_DEBUG_END();
|
||||
@ -166,20 +168,38 @@ storageGet(StorageFileRead *file)
|
||||
{
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
result = bufNew(0);
|
||||
Buffer *read = NULL;
|
||||
|
||||
do
|
||||
// If exact read
|
||||
if (param.exactSize > 0)
|
||||
{
|
||||
// Read data
|
||||
read = storageFileRead(file);
|
||||
result = bufNew(param.exactSize);
|
||||
storageFileRead(file, result);
|
||||
|
||||
// Add to result and free read buffer
|
||||
bufCat(result, read);
|
||||
bufFree(read);
|
||||
// If an exact read make sure the size is as expected
|
||||
if (bufUsed(result) != param.exactSize)
|
||||
{
|
||||
THROW_FMT(
|
||||
FileReadError, "unable to read %zu byte(s) from '%s'", param.exactSize, strPtr(storageFileReadName(file)));
|
||||
}
|
||||
}
|
||||
while (read != NULL);
|
||||
// Else read entire file
|
||||
else
|
||||
{
|
||||
result = bufNew(0);
|
||||
Buffer *read = bufNew(storageFileReadBufferSize(file));
|
||||
|
||||
do
|
||||
{
|
||||
// Read data
|
||||
storageFileRead(file, read);
|
||||
|
||||
// Add to result and free read buffer
|
||||
bufCat(result, read);
|
||||
bufUsedZero(read);
|
||||
}
|
||||
while (!storageFileReadEof(file));
|
||||
}
|
||||
|
||||
// Move buffer to parent context on success
|
||||
bufMove(result, MEM_CONTEXT_OLD());
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
@ -302,14 +322,12 @@ storageNewRead(const Storage *this, const String *fileExp, StorageNewReadParam p
|
||||
|
||||
StorageFileRead *result = NULL;
|
||||
|
||||
MEM_CONTEXT_NEW_BEGIN("StorageFileRead")
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
String *fileName = storagePathNP(this, fileExp);
|
||||
|
||||
// Create the file
|
||||
result = storageFileReadMove(storageFileReadNew(fileName, param.ignoreMissing, this->bufferSize), MEM_CONTEXT_OLD());
|
||||
result = storageFileReadMove(
|
||||
storageFileReadNew(storagePathNP(this, fileExp), param.ignoreMissing, this->bufferSize), MEM_CONTEXT_OLD());
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_DEBUG_RESULT(STORAGE_FILE_READ, result);
|
||||
}
|
||||
@ -338,12 +356,9 @@ storageNewWrite(const Storage *this, const String *fileExp, StorageNewWriteParam
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
String *fileName = storagePathNP(this, fileExp);
|
||||
|
||||
// Create the file
|
||||
result = storageFileWriteMove(
|
||||
storageFileWriteNew(
|
||||
fileName, param.modeFile != 0 ? param.modeFile : this->modeFile,
|
||||
storagePathNP(this, fileExp), param.modeFile != 0 ? param.modeFile : this->modeFile,
|
||||
param.modePath != 0 ? param.modePath : this->modePath, param.noCreatePath, param.noSyncFile, param.noSyncPath,
|
||||
param.noAtomic),
|
||||
MEM_CONTEXT_OLD());
|
||||
|
||||
@ -81,10 +81,17 @@ bool storageExists(const Storage *this, const String *pathExp, StorageExistsPara
|
||||
/***********************************************************************************************************************************
|
||||
storageGet
|
||||
***********************************************************************************************************************************/
|
||||
#define storageGetNP(file) \
|
||||
storageGet(file)
|
||||
typedef struct StorageGetParam
|
||||
{
|
||||
size_t exactSize;
|
||||
} StorageGetParam;
|
||||
|
||||
Buffer *storageGet(StorageFileRead *file);
|
||||
#define storageGetP(file, ...) \
|
||||
storageGet(file, (StorageGetParam){__VA_ARGS__})
|
||||
#define storageGetNP(file) \
|
||||
storageGet(file, (StorageGetParam){0})
|
||||
|
||||
Buffer *storageGet(StorageFileRead *file, StorageGetParam param);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
storageInfo
|
||||
|
||||
@ -510,6 +510,10 @@ sub end
|
||||
# Combine with prior run if there was one
|
||||
if ($self->{oStorageTest}->exists($strLCovFile))
|
||||
{
|
||||
my $strCoverage = ${$self->{oStorageTest}->get($strLCovOutTmp)};
|
||||
$strCoverage =~ s/^SF\:.*$/SF:$strModulePath\.c/mg;
|
||||
$self->{oStorageTest}->put($strLCovOutTmp, $strCoverage);
|
||||
|
||||
executeTest(
|
||||
'docker exec -i -u ' . TEST_USER . " ${strImage} " .
|
||||
"${strLCovExe} --add-tracefile=${strLCovOutTmp} --add-tracefile=${strLCovFile} --o=${strLCovOutTmp}");
|
||||
|
||||
@ -116,7 +116,10 @@ testRun()
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
String *controlFile = strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL);
|
||||
PgControlFile control = {.systemId = 0xFACEFACE, .controlVersion = 1002, .catalogVersion = 201707211};
|
||||
storagePutNP(storageNewWriteNP(storageTest, controlFile), bufNewC(sizeof(PgControlFile), &control));
|
||||
Buffer *controlBuffer = bufNew(512);
|
||||
memcpy(bufPtr(controlBuffer), &control, sizeof(PgControlFile));
|
||||
bufUsedSet(controlBuffer, bufSize(controlBuffer));
|
||||
storagePutNP(storageNewWriteNP(storageTest, controlFile), controlBuffer);
|
||||
|
||||
storagePathCreateNP(storageTest, strNewFmt("%s/db/pg_wal", testPath()));
|
||||
|
||||
|
||||
@ -42,7 +42,7 @@ testRun()
|
||||
}
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("bufResize()"))
|
||||
if (testBegin("bufResize(), bufFull(), bufRemains*(), and bufUsed*()"))
|
||||
{
|
||||
Buffer *buffer = NULL;
|
||||
unsigned char *bufferPtr = NULL;
|
||||
@ -51,6 +51,8 @@ testRun()
|
||||
TEST_RESULT_INT(bufSize(buffer), 0, "check size");
|
||||
TEST_RESULT_PTR(bufResize(buffer, 256), buffer, "resize buffer");
|
||||
TEST_RESULT_INT(bufSize(buffer), 256, "check size");
|
||||
TEST_RESULT_VOID(bufUsedSet(buffer, 256), "set used size");
|
||||
TEST_RESULT_BOOL(bufFull(buffer), true, "check buffer full");
|
||||
|
||||
// Load data
|
||||
TEST_ASSIGN(bufferPtr, bufPtr(buffer), "buffer pointer");
|
||||
@ -62,6 +64,12 @@ testRun()
|
||||
TEST_ASSIGN(bufferPtr, bufPtr(bufResize(buffer, 512)), "increase buffer size");
|
||||
TEST_ASSIGN(bufferPtr, bufPtr(bufResize(buffer, 512)), "set to same size");
|
||||
TEST_RESULT_INT(bufSize(buffer), 512, "check size");
|
||||
TEST_RESULT_INT(bufUsed(buffer), 256, "check used size");
|
||||
TEST_RESULT_BOOL(bufFull(buffer), false, "check buffer not full");
|
||||
TEST_RESULT_INT(bufRemains(buffer), 256, "check remaining buffer space");
|
||||
TEST_RESULT_PTR(bufRemainsPtr(buffer), bufPtr(buffer) + 256, "check remaining buffer space pointer");
|
||||
TEST_RESULT_VOID(bufUsedInc(buffer, 256), "set used size");
|
||||
TEST_RESULT_BOOL(bufFull(buffer), true, "check buffer full");
|
||||
|
||||
// Test that no bytes have changed in the original data
|
||||
unsigned int sameTotal = 0;
|
||||
@ -84,9 +92,12 @@ testRun()
|
||||
TEST_RESULT_INT(sameTotal, 128, "original bytes match");
|
||||
|
||||
// Resize to zero buffer
|
||||
TEST_ASSIGN(bufferPtr, bufPtr(bufResize(buffer, 0)), "decrease to zero");
|
||||
TEST_RESULT_VOID(bufUsedZero(buffer), "set used to 0");
|
||||
TEST_RESULT_INT(bufUsed(buffer), 0, "check used is zero");
|
||||
|
||||
TEST_RESULT_VOID(bufResize(buffer, 0), "decrease size to zero");
|
||||
TEST_RESULT_INT(bufSize(buffer), 0, "check size");
|
||||
TEST_ASSIGN(bufferPtr, bufPtr(bufResize(buffer, 0)), "decrease to zero again");
|
||||
TEST_RESULT_VOID(bufResize(buffer, 0), "decrease size to zero again");
|
||||
TEST_RESULT_INT(bufSize(buffer), 0, "check size");
|
||||
}
|
||||
|
||||
@ -104,6 +115,10 @@ testRun()
|
||||
TEST_RESULT_STR(strPtr(strNewBuf(bufCat(bufNewStr(strNew("123")), NULL))), "123", "cat null buffer");
|
||||
TEST_RESULT_STR(strPtr(strNewBuf(bufCat(bufNewStr(strNew("123")), bufNew(0)))), "123", "cat empty buffer");
|
||||
TEST_RESULT_STR(strPtr(strNewBuf(bufCat(bufNewStr(strNew("123")), bufNewStr(strNew("ABC"))))), "123ABC", "cat buffer");
|
||||
|
||||
Buffer *buffer = NULL;
|
||||
TEST_ASSIGN(buffer, bufNew(2), "new buffer with space");
|
||||
TEST_RESULT_STR(strPtr(strNewBuf(bufCat(buffer, bufNewStr(strNew("AB"))))), "AB", "cat buffer with space");
|
||||
}
|
||||
|
||||
FUNCTION_HARNESS_RESULT_VOID();
|
||||
|
||||
@ -49,7 +49,10 @@ testRun()
|
||||
{
|
||||
String *controlFile = strNew(PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL);
|
||||
PgControlFile control = {.systemId = 0xFACEFACE, .controlVersion = 833, .catalogVersion = 200711281};
|
||||
storagePutNP(storageNewWriteNP(storageTest, controlFile), bufNewC(sizeof(PgControlFile), &control));
|
||||
Buffer *controlBuffer = bufNew(512);
|
||||
memcpy(bufPtr(controlBuffer), &control, sizeof(PgControlFile));
|
||||
bufUsedSet(controlBuffer, bufSize(controlBuffer));
|
||||
storagePutNP(storageNewWriteNP(storageTest, controlFile), controlBuffer);
|
||||
|
||||
PgControlInfo info = {0};
|
||||
TEST_ASSIGN(info, pgControlInfo(strNew(testPath())), "get control info");
|
||||
|
||||
@ -95,6 +95,7 @@ testRun()
|
||||
TEST_RESULT_BOOL(storageFileReadOpen(file), false, " missing file ignored");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
Buffer *outBuffer = bufNew(2);
|
||||
Buffer *expectedBuffer = bufNewStr(strNew("TESTFILE\n"));
|
||||
TEST_RESULT_VOID(storagePutNP(storageNewWriteNP(storageTest, fileName), expectedBuffer), "write test file");
|
||||
|
||||
@ -105,7 +106,7 @@ testRun()
|
||||
close(file->fileDriver->handle);
|
||||
|
||||
TEST_ERROR_FMT(
|
||||
storageFileRead(file), FileReadError,
|
||||
storageFileRead(file, outBuffer), FileReadError,
|
||||
"unable to read '%s': [9] Bad file descriptor", strPtr(fileName));
|
||||
TEST_ERROR_FMT(
|
||||
storageFileReadClose(file), FileCloseError,
|
||||
@ -127,19 +128,33 @@ testRun()
|
||||
TEST_RESULT_STR(strPtr(storageFileReadName(file)), strPtr(fileName), " check file name");
|
||||
TEST_RESULT_INT(storageFileReadSize(file), 0, " check size");
|
||||
|
||||
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
|
||||
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
|
||||
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
|
||||
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
|
||||
TEST_RESULT_VOID(storageFileRead(file, outBuffer), " load data");
|
||||
bufCat(buffer, outBuffer);
|
||||
bufUsedZero(outBuffer);
|
||||
TEST_RESULT_VOID(storageFileRead(file, outBuffer), " load data");
|
||||
bufCat(buffer, outBuffer);
|
||||
bufUsedZero(outBuffer);
|
||||
TEST_RESULT_VOID(storageFileRead(file, outBuffer), " load data");
|
||||
bufCat(buffer, outBuffer);
|
||||
bufUsedZero(outBuffer);
|
||||
TEST_RESULT_VOID(storageFileRead(file, outBuffer), " load data");
|
||||
bufCat(buffer, outBuffer);
|
||||
bufUsedZero(outBuffer);
|
||||
TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), false, " check file contents (not all loaded yet)");
|
||||
TEST_RESULT_INT(storageFileReadSize(file), 8, " check size");
|
||||
|
||||
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
|
||||
TEST_RESULT_VOID(storageFileRead(file, outBuffer), " load data");
|
||||
bufCat(buffer, outBuffer);
|
||||
bufUsedZero(outBuffer);
|
||||
|
||||
TEST_RESULT_VOID(storageFileRead(file, outBuffer), " no data to load");
|
||||
TEST_RESULT_INT(bufUsed(outBuffer), 0, " buffer is empty");
|
||||
|
||||
TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), true, " check file contents (all loaded)");
|
||||
TEST_RESULT_INT(storageFileReadSize(file), 9, " check size");
|
||||
|
||||
TEST_RESULT_PTR(storageFileRead(file), NULL, " eof");
|
||||
TEST_RESULT_PTR(storageFileRead(file), NULL, " still eof");
|
||||
TEST_RESULT_BOOL(storageFileReadEof(file), true, " eof");
|
||||
TEST_RESULT_BOOL(storageFileReadEof(file), true, " still eof");
|
||||
|
||||
TEST_RESULT_VOID(storageFileReadClose(file), " close file");
|
||||
TEST_RESULT_VOID(storageFileReadClose(file), " close again");
|
||||
|
||||
@ -549,6 +549,16 @@ testRun()
|
||||
TEST_RESULT_INT(bufSize(buffer), 9, "check size");
|
||||
TEST_RESULT_BOOL(memcmp(bufPtr(buffer), "TESTFILE\n", bufSize(buffer)) == 0, true, "check content");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_ASSIGN(
|
||||
buffer, storageGetP(storageNewReadNP(storageTest, strNewFmt("%s/test.txt", testPath())), .exactSize = 4), "get exact");
|
||||
TEST_RESULT_INT(bufSize(buffer), 4, "check size");
|
||||
TEST_RESULT_BOOL(memcmp(bufPtr(buffer), "TEST", bufSize(buffer)) == 0, true, "check content");
|
||||
|
||||
TEST_ERROR_FMT(
|
||||
storageGetP(storageNewReadNP(storageTest, strNewFmt("%s/test.txt", testPath())), .exactSize = 64), FileReadError,
|
||||
"unable to read 64 byte(s) from '%s/test.txt'", testPath());
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
const Storage *storage = storageTest;
|
||||
((Storage *)storage)->bufferSize = 2;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user