db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

Filters can now produce output that differs from input.

Browse Source 
This allows filters such as compression, encryption, etc. to be implemented.
This commit is contained in:
David Steele 2018-08-14 14:21:53 -04:00
parent a45d1b4a60
commit e3ff6b209d
27 changed files with 1320 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/xml/release.xml
View File

@ -92,7 +92,11 @@
</release-item>
<release-item>
<p>Abstract IO layer out of the storage layer. This allows the routines to be used for IO objects that do not have a storage representation. Implement buffer read and write IO objects. Implement filters that do not modify the buffer and update <code>cryptoHash</code> to use the new interface.</p>
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>Abstract IO layer out of the storage layer. This allows the routines to be used for IO objects that do not have a storage representation. Implement buffer read and write IO objects. Implement filters and update <code>cryptoHash</code> to use the new interface. Implement size and buffer filters.</p>
</release-item>
<release-item>

2
libc/Makefile.PL
View File

@ -50,8 +50,10 @@ my @stryCFile =
'common/ini.c',
'common/io/bufferRead.c',
'common/io/bufferWrite.c',
'common/io/filter/buffer.c',
'common/io/filter/filter.c',
'common/io/filter/group.c',
'common/io/filter/size.c',
'common/io/handle.c',
'common/io/io.c',
'common/io/read.c',

12
src/Makefile
View File

@ -67,8 +67,10 @@ SRCS = \
common/fork.c \
common/io/bufferRead.c \
common/io/bufferWrite.c \
common/io/filter/buffer.c \
common/io/filter/filter.c \
common/io/filter/group.c \
common/io/filter/size.c \
common/io/handle.c \
common/io/io.c \
common/io/read.c \
@ -172,12 +174,18 @@ common/io/bufferRead.o: common/io/bufferRead.c common/assert.h common/debug.h co
common/io/bufferWrite.o: common/io/bufferWrite.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/bufferWrite.h common/io/filter/filter.h common/io/filter/group.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h
$(CC) $(CFLAGS) -c common/io/bufferWrite.c -o common/io/bufferWrite.o
common/io/filter/buffer.o: common/io/filter/buffer.c common/debug.h common/error.auto.h common/error.h common/io/filter/buffer.h common/io/filter/filter.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h
$(CC) $(CFLAGS) -c common/io/filter/buffer.c -o common/io/filter/buffer.o
common/io/filter/filter.o: common/io/filter/filter.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h
$(CC) $(CFLAGS) -c common/io/filter/filter.c -o common/io/filter/filter.o
common/io/filter/group.o: common/io/filter/group.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/variant.h common/type/variantList.h
common/io/filter/group.o: common/io/filter/group.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/buffer.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/variant.h common/type/variantList.h
$(CC) $(CFLAGS) -c common/io/filter/group.c -o common/io/filter/group.o
common/io/filter/size.o: common/io/filter/size.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/size.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h
$(CC) $(CFLAGS) -c common/io/filter/size.c -o common/io/filter/size.o
common/io/handle.o: common/io/handle.c common/debug.h common/error.auto.h common/error.h common/io/handle.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/string.h
$(CC) $(CFLAGS) -c common/io/handle.c -o common/io/handle.o
@ -187,7 +195,7 @@ common/io/io.o: common/io/io.c common/assert.h common/debug.h common/error.auto.
common/io/read.o: common/io/read.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h
$(CC) $(CFLAGS) -c common/io/read.c -o common/io/read.o
common/io/write.o: common/io/write.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h
common/io/write.o: common/io/write.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h
$(CC) $(CFLAGS) -c common/io/write.c -o common/io/write.o
common/lock.o: common/lock.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handle.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h

101
src/common/io/bufferRead.c
View File

@ -8,21 +8,49 @@ Buffer IO Read
#include "common/memContext.h"
/***********************************************************************************************************************************
Buffer read object
Object type
***********************************************************************************************************************************/
struct IoBufferRead
{
MemContext *memContext;
IoRead *io;
const Buffer *read;
size_t readPos;
bool eof;
MemContext *memContext; // Object memory context
IoRead *io; // IoRead interface
const Buffer *read; // Buffer to read data from
size_t readPos; // Current position in the read buffer
bool eof; // Has the end of the buffer been reached?
};
/***********************************************************************************************************************************
Read from the buffer
New object
***********************************************************************************************************************************/
static size_t
IoBufferRead *
ioBufferReadNew(const Buffer *buffer)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
FUNCTION_TEST_ASSERT(buffer != NULL);
FUNCTION_DEBUG_END();
IoBufferRead *this = NULL;
MEM_CONTEXT_NEW_BEGIN("IoBufferRead")
{
this = memNew(sizeof(IoBufferRead));
this->memContext = memContextCurrent();
this->read = buffer;
this->io = ioReadNew(this, NULL, (IoReadProcess)ioBufferRead, NULL, (IoReadEof)ioBufferReadEof);
}
MEM_CONTEXT_NEW_END();
FUNCTION_DEBUG_RESULT(IO_BUFFER_READ, this);
}
/***********************************************************************************************************************************
Read data from the buffer
***********************************************************************************************************************************/
size_t
ioBufferRead(IoBufferRead *this, Buffer *buffer)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
@ -53,48 +81,6 @@ ioBufferRead(IoBufferRead *this, Buffer *buffer)
FUNCTION_DEBUG_RESULT(SIZE, actualBytes);
}
/***********************************************************************************************************************************
Have all bytes been read from the buffer?
***********************************************************************************************************************************/
static bool
ioBufferReadEof(IoBufferRead *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_BUFFER_READ, this);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_END();
FUNCTION_DEBUG_RESULT(BOOL, this->eof);
}
/***********************************************************************************************************************************
Create object
***********************************************************************************************************************************/
IoBufferRead *
ioBufferReadNew(const Buffer *buffer)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
FUNCTION_TEST_ASSERT(buffer != NULL);
FUNCTION_DEBUG_END();
IoBufferRead *this = NULL;
MEM_CONTEXT_NEW_BEGIN("IoBufferRead")
{
this = memNew(sizeof(IoBufferRead));
this->memContext = memContextCurrent();
this->read = buffer;
this->io = ioReadNew(this, NULL, (IoReadProcess)ioBufferRead, NULL, (IoReadEof)ioBufferReadEof);
}
MEM_CONTEXT_NEW_END();
FUNCTION_DEBUG_RESULT(IO_BUFFER_READ, this);
}
/***********************************************************************************************************************************
Move the object to a new context
***********************************************************************************************************************************/
@ -114,6 +100,21 @@ ioBufferReadMove(IoBufferRead *this, MemContext *parentNew)
FUNCTION_TEST_RESULT(IO_BUFFER_READ, this);
}
/***********************************************************************************************************************************
Have all bytes been read from the buffer?
***********************************************************************************************************************************/
bool
ioBufferReadEof(const IoBufferRead *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_BUFFER_READ, this);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_END();
FUNCTION_DEBUG_RESULT(BOOL, this->eof);
}
/***********************************************************************************************************************************
Get io interface
***********************************************************************************************************************************/

6
src/common/io/bufferRead.h
View File

@ -1,11 +1,13 @@
/***********************************************************************************************************************************
Buffer IO Read
Read from a Buffer object using the IoWrite interface.
***********************************************************************************************************************************/
#ifndef COMMON_IO_BUFFERREAD_H
#define COMMON_IO_BUFFERREAD_H
/***********************************************************************************************************************************
Buffer read object
Object type
***********************************************************************************************************************************/
typedef struct IoBufferRead IoBufferRead;
@ -19,11 +21,13 @@ IoBufferRead *ioBufferReadNew(const Buffer *buffer);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
size_t ioBufferRead(IoBufferRead *this, Buffer *buffer);
IoBufferRead *ioBufferReadMove(IoBufferRead *this, MemContext *parentNew);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool ioBufferReadEof(const IoBufferRead *this);
IoRead *ioBufferReadIo(const IoBufferRead *this);
/***********************************************************************************************************************************

46
src/common/io/bufferWrite.c
View File

@ -12,32 +12,13 @@ Buffer write object
***********************************************************************************************************************************/
struct IoBufferWrite
{
MemContext *memContext;
IoWrite *io;
Buffer *write;
MemContext *memContext; // Object memory context
IoWrite *io; // IoWrite interface
Buffer *write; // Buffer to write into
};
/***********************************************************************************************************************************
Write to the buffer
***********************************************************************************************************************************/
static void
ioBufferWrite(IoBufferWrite *this, Buffer *buffer)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_BUFFER_WRITE, this);
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_ASSERT(buffer != NULL);
FUNCTION_DEBUG_END();
bufCat(this->write, buffer);
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Create object
New object
***********************************************************************************************************************************/
IoBufferWrite *
ioBufferWriteNew(Buffer *buffer)
@ -63,6 +44,25 @@ ioBufferWriteNew(Buffer *buffer)
FUNCTION_DEBUG_RESULT(IO_BUFFER_WRITE, this);
}
/***********************************************************************************************************************************
Write to the buffer
***********************************************************************************************************************************/
void
ioBufferWrite(IoBufferWrite *this, Buffer *buffer)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_BUFFER_WRITE, this);
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_ASSERT(buffer != NULL);
FUNCTION_DEBUG_END();
bufCat(this->write, buffer);
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Move the object to a new context
***********************************************************************************************************************************/

5
src/common/io/bufferWrite.h
View File

@ -1,11 +1,13 @@
/***********************************************************************************************************************************
Buffer IO Write
Write to a Buffer object using the IoWrite interface.
***********************************************************************************************************************************/
#ifndef COMMON_IO_BUFFERWRITE_H
#define COMMON_IO_BUFFERWRITE_H
/***********************************************************************************************************************************
Buffer write object
Object type
***********************************************************************************************************************************/
typedef struct IoBufferWrite IoBufferWrite;
@ -19,6 +21,7 @@ IoBufferWrite *ioBufferWriteNew(Buffer *buffer);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void ioBufferWrite(IoBufferWrite *this, Buffer *buffer);
IoBufferWrite *ioBufferWriteMove(IoBufferWrite *this, MemContext *parentNew);
/***********************************************************************************************************************************

164
src/common/io/filter/buffer.c Normal file
View File

@ -0,0 +1,164 @@
/***********************************************************************************************************************************
IO Buffer Filter
***********************************************************************************************************************************/
#include <stdio.h>
#include "common/debug.h"
#include "common/io/filter/buffer.h"
#include "common/log.h"
#include "common/memContext.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define BUFFER_FILTER_TYPE "buffer"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct IoBuffer
{
MemContext *memContext; // Mem context of filter
IoFilter *filter; // Filter interface
size_t inputPos; // Position in input buffer
bool inputSame; // Is the same input required again?
};
/***********************************************************************************************************************************
New object
***********************************************************************************************************************************/
IoBuffer *
ioBufferNew(void)
{
FUNCTION_DEBUG_VOID(logLevelTrace);
IoBuffer *this = NULL;
MEM_CONTEXT_NEW_BEGIN("IoBuffer")
{
this = memNew(sizeof(IoBuffer));
this->memContext = memContextCurrent();
this->filter = ioFilterNew(
strNew(BUFFER_FILTER_TYPE), this, NULL, (IoFilterInputSame)ioBufferInputSame, NULL,
(IoFilterProcessInOut)ioBufferProcess, NULL);
}
MEM_CONTEXT_NEW_END();
FUNCTION_DEBUG_RESULT(IO_BUFFER, this);
}
/***********************************************************************************************************************************
Move data from the input buffer to the output buffer
***********************************************************************************************************************************/
void
ioBufferProcess(IoBuffer *this, const Buffer *input, Buffer *output)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_BUFFER, this);
FUNCTION_DEBUG_PARAM(BUFFER, input);
FUNCTION_DEBUG_PARAM(BUFFER, output);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_ASSERT(input != NULL);
FUNCTION_DEBUG_ASSERT(output != NULL);
FUNCTION_DEBUG_END();
// Determine how much data needs to be copied and reduce if there is not enough space in the output
size_t copySize = bufUsed(input) - this->inputPos;
if (copySize > bufRemains(output))
copySize = bufRemains(output);
// Copy data to the output buffer
bufCatSub(output, input, this->inputPos, copySize);
// If all data was copied then reset inputPos and allow new input
if (this->inputPos + copySize == bufUsed(input))
{
this->inputSame = false;
this->inputPos = 0;
}
// Else update inputPos and indicate that the same input should be passed again
else
{
this->inputSame = true;
this->inputPos += copySize;
}
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Is the same input required again?
If the remaining space in the output buffer is smaller than the used space in the input buffer then the same input must be provided
again.
***********************************************************************************************************************************/
bool
ioBufferInputSame(const IoBuffer *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_BUFFER, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->inputSame);
}
/***********************************************************************************************************************************
Get filter interface
***********************************************************************************************************************************/
IoFilter *
ioBufferFilter(const IoBuffer *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_BUFFER, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(IO_FILTER, this->filter);
}
/***********************************************************************************************************************************
Convert to a zero-terminated string for logging
***********************************************************************************************************************************/
size_t
ioBufferToLog(const IoBuffer *this, char *buffer, size_t bufferSize)
{
size_t result = 0;
MEM_CONTEXT_TEMP_BEGIN()
{
String *string = NULL;
if (this == NULL)
string = strNew("null");
else
string = strNewFmt("{inputSame: %s, inputPos: %zu}", cvtBoolToConstZ(this->inputSame), this->inputPos);
result = (size_t)snprintf(buffer, bufferSize, "%s", strPtr(string));
}
MEM_CONTEXT_TEMP_END();
return result;
}
/***********************************************************************************************************************************
Free the filter group
***********************************************************************************************************************************/
void
ioBufferFree(IoBuffer *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_BUFFER, this);
FUNCTION_DEBUG_END();
if (this != NULL)
memContextFree(this->memContext);
FUNCTION_DEBUG_RESULT_VOID();
}

49
src/common/io/filter/buffer.h Normal file
View File

@ -0,0 +1,49 @@
/***********************************************************************************************************************************
IO Buffer Filter
Move data from the input buffer to the output buffer without overflowing the output buffer. Automatically used as the last filter
in a FilterGroup if the last filter is not already an InOut filter, so there is no reason to add it manually to a FilterGroup.
***********************************************************************************************************************************/
#ifndef COMMON_IO_FILTER_BUFFER_H
#define COMMON_IO_FILTER_BUFFER_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct IoBuffer IoBuffer;
#include "common/io/filter/filter.h"
#include "common/type/buffer.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
IoBuffer *ioBufferNew(void);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void ioBufferProcess(IoBuffer *this, const Buffer *input, Buffer *output);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
IoFilter *ioBufferFilter(const IoBuffer *this);
bool ioBufferInputSame(const IoBuffer *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void ioBufferFree(IoBuffer *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
size_t ioBufferToLog(const IoBuffer *this, char *buffer, size_t bufferSize);
#define FUNCTION_DEBUG_IO_BUFFER_TYPE \
IoBuffer *
#define FUNCTION_DEBUG_IO_BUFFER_FORMAT(value, buffer, bufferSize) \
ioBufferToLog(value, buffer, bufferSize)
#endif

120
src/common/io/filter/filter.c
View File

@ -1,5 +1,5 @@
/***********************************************************************************************************************************
IO Filter
IO Filter Interface
***********************************************************************************************************************************/
#include "common/assert.h"
#include "common/debug.h"
@ -8,42 +8,59 @@ IO Filter
#include "common/memContext.h"
/***********************************************************************************************************************************
Filter structure
Object type
***********************************************************************************************************************************/
struct IoFilter
{
MemContext *memContext; // Mem context of filter
const String *type; // Filter type
void *data; // Filter data
IoFilterDone done; // Done processing?
IoFilterInputSame inputSame; // Does the filter need the same input again?
IoFilterProcessIn processIn; // Process in function
IoFilterProcessInOut processInOut; // Process in/out function
IoFilterResult result; // Result function
};
/***********************************************************************************************************************************
Create a new filter
New object
Allocations will be in the memory context of the caller.
***********************************************************************************************************************************/
IoFilter *
ioFilterNew(const String *type, void *data, IoFilterProcessIn processIn, IoFilterResult result)
ioFilterNew(
const String *type, void *data, IoFilterDone done, IoFilterInputSame inputSame, IoFilterProcessIn processIn,
IoFilterProcessInOut processInOut, IoFilterResult result)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(STRING, type);
FUNCTION_DEBUG_PARAM(VOIDP, data);
FUNCTION_DEBUG_PARAM(FUNCTIONP, done);
FUNCTION_DEBUG_PARAM(FUNCTIONP, inputSame);
FUNCTION_DEBUG_PARAM(FUNCTIONP, processIn);
FUNCTION_DEBUG_PARAM(FUNCTIONP, processInOut);
FUNCTION_DEBUG_PARAM(FUNCTIONP, result);
FUNCTION_DEBUG_ASSERT(type != NULL);
FUNCTION_DEBUG_ASSERT(data != NULL);
FUNCTION_DEBUG_ASSERT(processIn != NULL);
FUNCTION_DEBUG_ASSERT(result != NULL);
FUNCTION_TEST_ASSERT(type != NULL);
FUNCTION_TEST_ASSERT(data != NULL);
// One of processIn or processInOut must be set
FUNCTION_TEST_ASSERT(processIn != NULL || processInOut != NULL);
// But not both of them
FUNCTION_TEST_ASSERT(!(processIn != NULL && processInOut != NULL));
// If the filter does not produce output then it should produce a result
FUNCTION_TEST_ASSERT(processIn == NULL || (result != NULL && done == NULL && inputSame == NULL));
// Filters that produce output will not always be able to dump all their output and will need to get the same input again
FUNCTION_TEST_ASSERT(processInOut == NULL || inputSame != NULL);
FUNCTION_DEBUG_END();
IoFilter *this = memNew(sizeof(IoFilter));
this->memContext = memContextCurrent();
this->type = type;
this->data = data;
this->done = done;
this->inputSame = inputSame;
this->processIn = processIn;
this->processInOut = processInOut;
this->result = result;
FUNCTION_DEBUG_RESULT(IO_FILTER, this);
@ -60,6 +77,7 @@ ioFilterProcessIn(IoFilter *this, const Buffer *input)
FUNCTION_TEST_PARAM(BUFFER, input);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->processIn != NULL);
FUNCTION_TEST_END();
this->processIn(this->data, input);
@ -68,22 +86,28 @@ ioFilterProcessIn(IoFilter *this, const Buffer *input)
}
/***********************************************************************************************************************************
Close the filter and return any results
Filter input and produce output
***********************************************************************************************************************************/
const Variant *
ioFilterResult(IoFilter *this)
void
ioFilterProcessInOut(IoFilter *this, const Buffer *input, Buffer *output)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER, this);
FUNCTION_TEST_PARAM(BUFFER, input);
FUNCTION_TEST_PARAM(BUFFER, output);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(output != NULL);
FUNCTION_TEST_ASSERT(this->processInOut != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(VARIANT, this->result ? this->result(this->data) : NULL);
this->processInOut(this->data, input, output);
FUNCTION_TEST_RESULT_VOID();
}
/***********************************************************************************************************************************
Move the file object to a new context
Move the object to a new context
***********************************************************************************************************************************/
IoFilter *
ioFilterMove(IoFilter *this, MemContext *parentNew)
@ -101,11 +125,79 @@ ioFilterMove(IoFilter *this, MemContext *parentNew)
FUNCTION_TEST_RESULT(IO_FILTER, this);
}
/***********************************************************************************************************************************
Is the filter done?
If done is not defined by the filter then check inputSame. If inputSame is true then the filter is not done.
***********************************************************************************************************************************/
bool
ioFilterDone(const IoFilter *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->done != NULL ? this->done(this->data) : !ioFilterInputSame(this));
}
/***********************************************************************************************************************************
Does the filter need the same input again?
If the filter cannot get all its output into the output buffer then it may need access to the same input again.
***********************************************************************************************************************************/
bool
ioFilterInputSame(const IoFilter *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->inputSame != NULL ? this->inputSame(this->data) : false);
}
/***********************************************************************************************************************************
Does filter produce output?
All InOut filters produce output.
***********************************************************************************************************************************/
bool
ioFilterOutput(const IoFilter *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->processInOut != NULL);
}
/***********************************************************************************************************************************
Get filter result
***********************************************************************************************************************************/
const Variant *
ioFilterResult(const IoFilter *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(VARIANT, this->result ? this->result(this->data) : NULL);
}
/***********************************************************************************************************************************
Get filter type
This name identifies the filter and is used when pulling results from the filter group.
***********************************************************************************************************************************/
const String *
ioFilterType(IoFilter *this)
ioFilterType(const IoFilter *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER, this);

39
src/common/io/filter/filter.h
View File

@ -1,11 +1,25 @@
/***********************************************************************************************************************************
IO Filter
IO Filter Interface
Two types of filters are implemented using this interface: In and InOut.
In filters accept input and produce a result, but do not modify the input. An example is the IoSize filter which counts all bytes
that pass through it.
InOut filters accept input and produce output (and perhaps a result). Because the input/output buffers may not be the same size the
filter must be prepared to accept the same input again (by implementing IoFilterInputSame) if the output buffer is too small to
accept all processed data. If the filter holds state even when inputSame is false then it may also implement IoFilterDone to
indicate that the filter should be flushed (by passing NULL inputs) after all input has been processed. InOut filters should strive
to fill the output buffer as much as possible, i.e., if the output buffer is not full after processing then inputSame should be
false. An example is the IoBuffer filter which buffers data between unequally sized input/output buffers.
Each filter has a type that allows it to be identified in the filter list.
***********************************************************************************************************************************/
#ifndef IO_FILTER_FILTER_H
#define IO_FILTER_FILTER_H
#ifndef COMMON_IO_FILTER_FILTER_H
#define COMMON_IO_FILTER_FILTER_H
/***********************************************************************************************************************************
Storage file read object
Object type
***********************************************************************************************************************************/
typedef struct IoFilter IoFilter;
@ -15,25 +29,34 @@ typedef struct IoFilter IoFilter;
/***********************************************************************************************************************************
Function pointer types
***********************************************************************************************************************************/
typedef bool (*IoFilterDone)(const void *data);
typedef bool (*IoFilterInputSame)(const void *data);
typedef void (*IoFilterProcessIn)(void *data, const Buffer *);
typedef Variant *(*IoFilterResult)(void *data);
typedef void (*IoFilterProcessInOut)(void *data, const Buffer *, Buffer *);
typedef Variant *(*IoFilterResult)(const void *data);
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
IoFilter *ioFilterNew(const String *type, void *data, IoFilterProcessIn processIn, IoFilterResult result);
IoFilter *ioFilterNew(
const String *type, void *data, IoFilterDone done, IoFilterInputSame input, IoFilterProcessIn processIn,
IoFilterProcessInOut processInOut, IoFilterResult result);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void ioFilterProcessIn(IoFilter *this, const Buffer *input);
const Variant *ioFilterResult(IoFilter *this);
void ioFilterProcessInOut(IoFilter *this, const Buffer *input, Buffer *output);
IoFilter *ioFilterMove(IoFilter *this, MemContext *parentNew);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
const String *ioFilterType(IoFilter *this);
bool ioFilterDone(const IoFilter *this);
bool ioFilterInputSame(const IoFilter *this);
bool ioFilterOutput(const IoFilter *this);
const Variant *ioFilterResult(const IoFilter *this);
const String *ioFilterType(const IoFilter *this);
/***********************************************************************************************************************************
Macros for function logging

340
src/common/io/filter/group.c
View File

@ -1,8 +1,11 @@
/***********************************************************************************************************************************
Filter Group
IO Filter Group
***********************************************************************************************************************************/
#include <stdio.h>
#include "common/assert.h"
#include "common/debug.h"
#include "common/io/filter/buffer.h"
#include "common/io/filter/group.h"
#include "common/io/io.h"
#include "common/log.h"
@ -10,17 +13,45 @@ Filter Group
#include "common/type/list.h"
/***********************************************************************************************************************************
Filter group structure
Filter and buffer structure
Contains the filter object and inout/output buffers.
***********************************************************************************************************************************/
typedef struct IoFilterData
{
const Buffer *input; // Input buffer for filter
Buffer *inputLocal; // Non-null if a locally created buffer that can be cleared
IoFilter *filter; // Filter to apply
Buffer *output; // Output buffer for filter
} IoFilterData;
// Macros for logging
#define FUNCTION_DEBUG_IO_FILTER_DATA_TYPE \
IoFilterData *
#define FUNCTION_DEBUG_IO_FILTER_DATA_FORMAT(value, buffer, bufferSize) \
objToLog(value, "IoFilterData", buffer, bufferSize)
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct IoFilterGroup
{
MemContext *memContext; // Mem context
List *filterList; // List of filters to apply to read buffer
List *filterList; // List of filters to apply
unsigned int firstOutputFilter; // Index of the first output filter
KeyValue *filterResult; // Filter results (if any)
bool inputSame; // Same input required again?
bool done; // Is processing done?
#ifdef DEBUG
bool opened; // Has the filter set been opened?
bool flushing; // Is output being flushed?
bool closed; // Has the filter set been closed?
#endif
};
/***********************************************************************************************************************************
Create new filter group
New Object
***********************************************************************************************************************************/
IoFilterGroup *
ioFilterGroupNew(void)
@ -33,6 +64,8 @@ ioFilterGroupNew(void)
{
this = memNew(sizeof(IoFilterGroup));
this->memContext = memContextCurrent();
this->done = true;
this->filterList = lstNew(sizeof(IoFilterData));
}
MEM_CONTEXT_NEW_END();
@ -45,29 +78,100 @@ Add a filter
void
ioFilterGroupAdd(IoFilterGroup *this, IoFilter *filter)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_BEGIN(logLevelDebug);
FUNCTION_DEBUG_PARAM(IO_FILTER_GROUP, this);
FUNCTION_DEBUG_PARAM(IO_FILTER, filter);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(!this->opened && !this->closed);
FUNCTION_TEST_ASSERT(filter != NULL);
FUNCTION_DEBUG_END();
// Create the filter list if it has not been created
if (this->filterList == NULL)
{
MEM_CONTEXT_BEGIN(this->memContext)
{
this->filterList = lstNew(sizeof(IoFilter *));
}
MEM_CONTEXT_END();
}
// Move the filter to this object's mem context
ioFilterMove(filter, this->memContext);
// Add the filter
lstAdd(this->filterList, &filter);
IoFilterData filterData = {.filter = filter};
lstAdd(this->filterList, &filterData);
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Get a filter
***********************************************************************************************************************************/
static IoFilterData *
ioFilterGroupGet(IoFilterGroup *this, unsigned int filterIdx)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER_GROUP, this);
FUNCTION_TEST_PARAM(UINT, filterIdx);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(IO_FILTER_DATA, (IoFilterData *)lstGet(this->filterList, filterIdx));
}
/***********************************************************************************************************************************
Open filter group
Setup the filter group and allocate any required buffers.
***********************************************************************************************************************************/
void
ioFilterGroupOpen(IoFilterGroup *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_FILTER_GROUP, this);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_END();
MEM_CONTEXT_BEGIN(this->memContext)
{
// If the last filter is not an output filter then add a filter to buffer/copy data. Input filters won't copy to an output
// buffer so we need some way to get the data to the output buffer.
if (lstSize(this->filterList) == 0 || !ioFilterOutput((ioFilterGroupGet(this, lstSize(this->filterList) - 1))->filter))
ioFilterGroupAdd(this, ioBufferFilter(ioBufferNew()));
// Create filter input/output buffers. Input filters do not get an output buffer since they don't produce output.
Buffer *lastOutputBuffer = NULL;
for (unsigned int filterIdx = 0; filterIdx < lstSize(this->filterList); filterIdx++)
{
IoFilterData *filterData = ioFilterGroupGet(this, filterIdx);
// Assign the last output buffer to the input. At first there won't be an input filter because it will be passed into
// the process function as an input.
if (lastOutputBuffer != NULL)
{
filterData->input = lastOutputBuffer;
filterData->inputLocal = lastOutputBuffer;
}
// Is this an output filter?
if (ioFilterOutput(filterData->filter))
{
// If this is the first output buffer found, store the index so it can be easily found during processing
if (lastOutputBuffer == NULL)
this->firstOutputFilter = filterIdx;
// If this is not the last output filter then create a new output buffer for it. The output buffer for the last
// filter will be provided to the process function.
if (filterIdx < lstSize(this->filterList) - 1)
{
lastOutputBuffer = bufNew(ioBufferSize());
filterData->output = lastOutputBuffer;
}
}
}
}
MEM_CONTEXT_END();
// Filter group is open
#ifdef DEBUG
this->opened = true;
#endif
FUNCTION_DEBUG_RESULT_VOID();
}
@ -76,24 +180,127 @@ ioFilterGroupAdd(IoFilterGroup *this, IoFilter *filter)
Process filters
***********************************************************************************************************************************/
void
ioFilterGroupProcess(IoFilterGroup *this, const Buffer *input)
ioFilterGroupProcess(IoFilterGroup *this, const Buffer *input, Buffer *output)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_FILTER_GROUP, this);
FUNCTION_DEBUG_PARAM(BUFFER, input);
FUNCTION_DEBUG_PARAM(BUFFER, output);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_ASSERT(input != NULL);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_TEST_ASSERT(!this->flushing || input == NULL);
FUNCTION_TEST_ASSERT(output != NULL);
FUNCTION_TEST_ASSERT(bufRemains(output) > 0);
FUNCTION_DEBUG_END();
// Once input goes to NULL then flushing has started
#ifdef DEBUG
if (input == NULL)
this->flushing = true;
#endif
// Assign the input buffer up to the first output filter. After this point the input buffers were locally created during open.
if (!this->inputSame)
{
for (unsigned int filterIdx = 0; filterIdx <= this->firstOutputFilter; filterIdx++)
(ioFilterGroupGet(this, filterIdx))->input = input;
}
// Assign the output buffer
(ioFilterGroupGet(this, lstSize(this->filterList) - 1))->output = output;
//
do
{
// Start from the first filter by default
unsigned int filterIdx = 0;
// Search from the end of the list for a filter that needs the same input. This indicates that the filter was not able to
// empty the input buffer on the last call. Maybe it won't this time either -- we can but try.
if (this->inputSame)
{
this->inputSame = false;
filterIdx = lstSize(this->filterList);
do
{
filterIdx--;
if (ioFilterInputSame((ioFilterGroupGet(this, filterIdx))->filter))
{
this->inputSame = true;
break;
}
}
while (filterIdx != this->firstOutputFilter);
// If no filter is found that needs the same input that means we are done with the current input. So end the loop and
// get some more input.
if (!this->inputSame)
break;
}
// Process forward from the filter that has input to process. This may be a filter that needs the same input or it may be
// new input for the first filter.
for (; filterIdx < lstSize(this->filterList); filterIdx++)
{
IoFilterData *filterData = ioFilterGroupGet(this, filterIdx);
// If the filter produces output
if (ioFilterOutput(filterData->filter))
{
// Keep processing while the filter is not done or there is input
if (!ioFilterDone(filterData->filter) || filterData->input != NULL)
{
ioFilterProcessInOut(filterData->filter, filterData->input, filterData->output);
// If inputSame is set then the output buffer for this filter is full and it will need to be pre-processed with
// the same input once the output buffer is cleared.
if (ioFilterInputSame(filterData->filter))
this->inputSame = true;
// Else clear the buffer if it was locally allocated. If this is an input buffer that was passed in then the
// caller is reponsible for clearing it.
else if (filterData->inputLocal != NULL)
bufUsedZero(filterData->inputLocal);
}
}
// Else the filter does not produce output. No need to flush these filters because they don't buffer data.
else if (filterData->input != NULL)
ioFilterProcessIn(filterData->filter, filterData->input);
}
}
while (!bufFull(output) && this->inputSame);
// Scan the filter list to determine if inputSame is set or done is not set for any filter. We can't trust this->inputSame
// when it is true without going through the loop above again. We need to scan to set this->done anyway so set this->inputSame
// in the same loop.
this->done = true;
this->inputSame = false;
for (unsigned int filterIdx = 0; filterIdx < lstSize(this->filterList); filterIdx++)
ioFilterProcessIn(*(IoFilter **)lstGet(this->filterList, filterIdx), input);
{
IoFilterData *filterData = ioFilterGroupGet(this, filterIdx);
// When inputSame then this->done = false and we can exit the loop immediately
if (ioFilterInputSame(filterData->filter))
{
this->done = false;
this->inputSame = true;
break;
}
// Set this->done = false if any filter is not done
if (!ioFilterDone(filterData->filter))
this->done = false;
}
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Close filter group
Close filter group and gather results
***********************************************************************************************************************************/
void
ioFilterGroupClose(IoFilterGroup *this)
@ -102,13 +309,13 @@ ioFilterGroupClose(IoFilterGroup *this)
FUNCTION_DEBUG_PARAM(IO_FILTER_GROUP, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_DEBUG_END();
for (unsigned int filterIdx = 0; filterIdx < lstSize(this->filterList); filterIdx++)
{
IoFilter *filter = *(IoFilter **)lstGet(this->filterList, filterIdx);
const Variant *filterResult = ioFilterResult(filter);
IoFilterData *filterData = ioFilterGroupGet(this, filterIdx);
const Variant *filterResult = ioFilterResult(filterData->filter);
if (this->filterResult == NULL)
{
@ -121,40 +328,101 @@ ioFilterGroupClose(IoFilterGroup *this)
MEM_CONTEXT_TEMP_BEGIN()
{
kvPut(this->filterResult, varNewStr(ioFilterType(filter)), filterResult);
kvAdd(this->filterResult, varNewStr(ioFilterType(filterData->filter)), filterResult);
}
MEM_CONTEXT_TEMP_END();
}
// Filter group is open
#ifdef DEBUG
this->closed = true;
#endif
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Is the filter group done processing?
***********************************************************************************************************************************/
bool
ioFilterGroupDone(const IoFilterGroup *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER_GROUP, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->done);
}
/***********************************************************************************************************************************
Should the same input be passed again?
A buffer of input can produce multiple buffers of output, e.g. when a file containing all zeroes is being decompressed.
***********************************************************************************************************************************/
bool
ioFilterGroupInputSame(const IoFilterGroup *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER_GROUP, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->inputSame);
}
/***********************************************************************************************************************************
Get filter results
***********************************************************************************************************************************/
const Variant *
ioFilterGroupResult(const IoFilterGroup *this, const String *filterType)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER_GROUP, this);
FUNCTION_TEST_PARAM(STRING, filterType);
FUNCTION_DEBUG_BEGIN(logLevelDebug);
FUNCTION_DEBUG_PARAM(IO_FILTER_GROUP, this);
FUNCTION_DEBUG_PARAM(STRING, filterType);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && this->closed);
FUNCTION_TEST_ASSERT(filterType != NULL);
FUNCTION_TEST_END();
FUNCTION_DEBUG_END();
const Variant *result = NULL;
if (this->filterResult != NULL)
MEM_CONTEXT_TEMP_BEGIN()
{
MEM_CONTEXT_TEMP_BEGIN()
{
result = kvGet(this->filterResult, varNewStr(filterType));
}
MEM_CONTEXT_TEMP_END();
result = kvGet(this->filterResult, varNewStr(filterType));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RESULT(CONST_VARIANT, result);
FUNCTION_DEBUG_RESULT(CONST_VARIANT, result);
}
/***********************************************************************************************************************************
Convert to a zero-terminated string for logging
***********************************************************************************************************************************/
size_t
ioFilterGroupToLog(const IoFilterGroup *this, char *buffer, size_t bufferSize)
{
size_t result = 0;
MEM_CONTEXT_TEMP_BEGIN()
{
String *string = NULL;
if (this == NULL)
string = strNew("null");
else
string = strNewFmt("{inputSame: %s, done: %s}", cvtBoolToConstZ(this->inputSame), cvtBoolToConstZ(this->done));
result = (size_t)snprintf(buffer, bufferSize, "%s", strPtr(string));
}
MEM_CONTEXT_TEMP_END();
return result;
}
/***********************************************************************************************************************************

27
src/common/io/filter/group.h
View File

@ -1,11 +1,17 @@
/***********************************************************************************************************************************
Filter Group
IO Filter Group
Process data through an arbitrary group of filters in the order added by the user using ioFilterGroupAdd(). After processing
results can be gathered using ioFilterGroupResult() for any filters that produce results.
Processing is complex and asymmetric for read/write so should be done via the IoRead and IoWrite objects. General users need
only call ioFilterGroupNew(), ioFilterGroupAdd(), and ioFilterGroupResult().
***********************************************************************************************************************************/
#ifndef IO_FILTER_GROUP_H
#define IO_FILTER_GROUP_H
#ifndef COMMON_IO_FILTER_GROUP_H
#define COMMON_IO_FILTER_GROUP_H
/***********************************************************************************************************************************
Storage file read object
Object type
***********************************************************************************************************************************/
typedef struct IoFilterGroup IoFilterGroup;
@ -21,12 +27,15 @@ IoFilterGroup *ioFilterGroupNew(void);
Functions
***********************************************************************************************************************************/
void ioFilterGroupAdd(IoFilterGroup *this, IoFilter *filter);
void ioFilterGroupProcess(IoFilterGroup *this, const Buffer *input);
void ioFilterGroupOpen(IoFilterGroup *this);
void ioFilterGroupProcess(IoFilterGroup *this, const Buffer *input, Buffer *output);
void ioFilterGroupClose(IoFilterGroup *this);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool ioFilterGroupDone(const IoFilterGroup *this);
bool ioFilterGroupInputSame(const IoFilterGroup *this);
const Variant *ioFilterGroupResult(const IoFilterGroup *this, const String *filterType);
/***********************************************************************************************************************************
@ -37,9 +46,11 @@ void ioFilterGroupFree(IoFilterGroup *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
#define FUNCTION_DEBUG_IO_FILTER_GROUP_TYPE \
size_t ioFilterGroupToLog(const IoFilterGroup *this, char *buffer, size_t bufferSize);
#define FUNCTION_DEBUG_IO_FILTER_GROUP_TYPE \
IoFilterGroup *
#define FUNCTION_DEBUG_IO_FILTER_GROUP_FORMAT(value, buffer, bufferSize) \
objToLog(value, "IoFilterGroup", buffer, bufferSize)
#define FUNCTION_DEBUG_IO_FILTER_GROUP_FORMAT(value, buffer, bufferSize) \
ioFilterGroupToLog(value, buffer, bufferSize)
#endif

145
src/common/io/filter/size.c Normal file
View File

@ -0,0 +1,145 @@
/***********************************************************************************************************************************
IO Size Filter
***********************************************************************************************************************************/
#include <stdio.h>
#include "common/debug.h"
#include "common/io/filter/size.h"
#include "common/log.h"
#include "common/memContext.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define SIZE_FILTER_TYPE "size"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct IoSize
{
MemContext *memContext; // Mem context of filter
IoFilter *filter; // Filter interface
uint64_t size; // Total size of al input
};
/***********************************************************************************************************************************
New object
***********************************************************************************************************************************/
IoSize *
ioSizeNew(void)
{
FUNCTION_DEBUG_VOID(logLevelTrace);
IoSize *this = NULL;
MEM_CONTEXT_NEW_BEGIN("IoSize")
{
this = memNew(sizeof(IoSize));
this->memContext = memContextCurrent();
this->filter = ioFilterNew(
strNew(SIZE_FILTER_TYPE), this, NULL, NULL, (IoFilterProcessIn)ioSizeProcess, NULL, (IoFilterResult)ioSizeResult);
}
MEM_CONTEXT_NEW_END();
FUNCTION_DEBUG_RESULT(IO_SIZE, this);
}
/***********************************************************************************************************************************
Count bytes in the input
***********************************************************************************************************************************/
void
ioSizeProcess(IoSize *this, const Buffer *input)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_SIZE, this);
FUNCTION_DEBUG_PARAM(BUFFER, input);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_ASSERT(input != NULL);
FUNCTION_DEBUG_END();
this->size += bufUsed(input);
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Get filter interface
***********************************************************************************************************************************/
IoFilter *
ioSizeFilter(const IoSize *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_SIZE, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(IO_FILTER, this->filter);
}
/***********************************************************************************************************************************
Convert to a zero-terminated string for logging
***********************************************************************************************************************************/
size_t
ioSizeToLog(const IoSize *this, char *buffer, size_t bufferSize)
{
size_t result = 0;
MEM_CONTEXT_TEMP_BEGIN()
{
String *string = NULL;
if (this == NULL)
string = strNew("null");
else
string = strNewFmt("{size: %" PRIu64 "}", this->size);
result = (size_t)snprintf(buffer, bufferSize, "%s", strPtr(string));
}
MEM_CONTEXT_TEMP_END();
return result;
}
/***********************************************************************************************************************************
Return filter result
***********************************************************************************************************************************/
const Variant *
ioSizeResult(IoSize *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_SIZE, this);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_END();
Variant *result = NULL;
MEM_CONTEXT_BEGIN(this->memContext)
{
result = varNewUInt64(this->size);
}
MEM_CONTEXT_END();
FUNCTION_DEBUG_RESULT(VARIANT, result);
}
/***********************************************************************************************************************************
Free the filter group
***********************************************************************************************************************************/
void
ioSizeFree(IoSize *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_SIZE, this);
FUNCTION_DEBUG_END();
if (this != NULL)
memContextFree(this->memContext);
FUNCTION_DEBUG_RESULT_VOID();
}

49
src/common/io/filter/size.h Normal file
View File

@ -0,0 +1,49 @@
/***********************************************************************************************************************************
IO Size Filter
Count all bytes that pass through the filter. Useful for getting file/IO size if added first in a FilterGroup with IoRead or last
in a FilterGroup with IoWrite.
***********************************************************************************************************************************/
#ifndef COMMON_IO_FILTER_SIZE_H
#define COMMON_IO_FILTER_SIZE_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct IoSize IoSize;
#include "common/io/filter/filter.h"
#include "common/type/buffer.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
IoSize *ioSizeNew(void);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void ioSizeProcess(IoSize *this, const Buffer *input);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
IoFilter *ioSizeFilter(const IoSize *this);
const Variant *ioSizeResult(IoSize *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void ioSizeFree(IoSize *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
size_t ioSizeToLog(const IoSize *this, char *buffer, size_t bufferSize);
#define FUNCTION_DEBUG_IO_SIZE_TYPE \
IoSize *
#define FUNCTION_DEBUG_IO_SIZE_FORMAT(value, buffer, bufferSize) \
ioSizeToLog(value, buffer, bufferSize)
#endif

2
src/common/io/handle.h
View File

@ -1,5 +1,7 @@
/***********************************************************************************************************************************
Handle IO
Simple IO functions operating on handles. These will be superceded by general IoRead/IoWrite functionality.
***********************************************************************************************************************************/
#ifndef COMMON_IO_HANDLE_H
#define COMMON_IO_HANDLE_H

2
src/common/io/io.h
View File

@ -1,5 +1,7 @@
/***********************************************************************************************************************************
IO Functions
Common IO functions.
***********************************************************************************************************************************/
#ifndef COMMON_IO_IO_H
#define COMMON_IO_IO_H

123
src/common/io/read.c
View File

@ -1,5 +1,5 @@
/***********************************************************************************************************************************
IO Read
IO Read Interface
***********************************************************************************************************************************/
#include "common/debug.h"
#include "common/io/io.h"
@ -8,24 +8,30 @@ IO Read
#include "common/memContext.h"
/***********************************************************************************************************************************
IO read object
Object type
***********************************************************************************************************************************/
struct IoRead
{
MemContext *memContext; // Mem context of driver
void *driver; // Driver object
IoFilterGroup *filterGroup; // IO filters
Buffer *input; // Input buffer
IoReadOpen open; // Driver open
IoReadProcess read; // Driver read
IoReadClose close; // Driver close
IoReadEof eof; // Driver eof
size_t size; // Total bytes read
bool eofAll; // Is the read done (read and filters complete)?
#ifdef DEBUG
bool opened; // Has the io been opened?
bool closed; // Has the io been closed?
#endif
};
/***********************************************************************************************************************************
Create a new read IO
New object
Allocations will be in the memory context of the caller.
***********************************************************************************************************************************/
@ -45,6 +51,8 @@ ioReadNew(void *driver, IoReadOpen open, IoReadProcess read, IoReadClose close,
IoRead *this = memNew(sizeof(IoRead));
this->memContext = memContextCurrent();
this->input = bufNew(ioBufferSize());
this->driver = driver;
this->open = open;
this->read = read;
@ -64,13 +72,49 @@ ioReadOpen(IoRead *this)
FUNCTION_DEBUG_PARAM(IO_READ, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(!this->opened && !this->closed);
FUNCTION_DEBUG_END();
FUNCTION_DEBUG_RESULT(BOOL, this->open != NULL ? this->open(this->driver) : true);
// Open if the driver has an open function
bool result = this->open != NULL ? this->open(this->driver) : true;
// Only open the filter group if the read was opened
if (result)
{
// If no filter group exists create one to do buffering
if (this->filterGroup == NULL)
this->filterGroup = ioFilterGroupNew();
ioFilterGroupOpen(this->filterGroup);
}
#ifdef DEBUG
this->opened = result;
#endif
FUNCTION_DEBUG_RESULT(BOOL, result);
}
/***********************************************************************************************************************************
Read data from IO
Is the driver at EOF?
This is different from the overall eof because filters may still be holding buffered data.
***********************************************************************************************************************************/
static bool
ioReadEofDriver(const IoRead *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_READ, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_DEBUG_END();
FUNCTION_DEBUG_RESULT(BOOL, this->eof != NULL ? this->eof(this->driver) : false);
}
/***********************************************************************************************************************************
Read data from IO and process filters
***********************************************************************************************************************************/
size_t
ioRead(IoRead *this, Buffer *buffer)
@ -80,23 +124,41 @@ ioRead(IoRead *this, Buffer *buffer)
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(buffer != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_DEBUG_END();
// Read unless eof or the output buffer is full
size_t result = 0;
// Loop until EOF or the output buffer is full
size_t outputRemains = bufRemains(buffer);
if(!ioReadEof(this) && bufRemains(buffer) > 0)
while (!this->eofAll && bufRemains(buffer) > 0)
{
size_t resultBytes = this->read(this->driver, buffer);
result += resultBytes;
this->size += resultBytes;
// Process input buffer again to get more output
if (ioFilterGroupInputSame(this->filterGroup))
{
ioFilterGroupProcess(this->filterGroup, this->input, buffer);
}
// Else new input can be accepted
else
{
// Read if not EOF
if (!ioReadEofDriver(this))
{
bufUsedZero(this->input);
this->read(this->driver, this->input);
}
// Set input to NULL and flush (no need to actually free the buffer here as it will be freed with the mem context)
else
this->input = NULL;
// Apply filters
if (this->filterGroup != NULL)
ioFilterGroupProcess(this->filterGroup, buffer);
// Process the input buffer (or flush if NULL)
ioFilterGroupProcess(this->filterGroup, this->input, buffer);
}
// Eof when no more input and the filter group is done
this->eofAll = ioReadEofDriver(this) && ioFilterGroupDone(this->filterGroup);
}
FUNCTION_DEBUG_RESULT(SIZE, result);
FUNCTION_DEBUG_RESULT(SIZE, outputRemains - bufRemains(buffer));
}
/***********************************************************************************************************************************
@ -109,21 +171,27 @@ ioReadClose(IoRead *this)
FUNCTION_DEBUG_PARAM(IO_READ, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_DEBUG_END();
// Close the filter group and gather results
if (this->filterGroup != NULL)
ioFilterGroupClose(this->filterGroup);
ioFilterGroupClose(this->filterGroup);
// Close the driver if there is a close function
if (this->close != NULL)
this->close(this->driver);
#ifdef DEBUG
this->closed = true;
#endif
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Is IO at EOF?
All driver reads are complete and all data has been flushed from the filters (if any).
***********************************************************************************************************************************/
bool
ioReadEof(const IoRead *this)
@ -132,9 +200,10 @@ ioReadEof(const IoRead *this)
FUNCTION_DEBUG_PARAM(IO_READ, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_DEBUG_END();
FUNCTION_DEBUG_RESULT(BOOL, this->eof != NULL ? this->eof(this->driver) : false);
FUNCTION_DEBUG_RESULT(BOOL, this->eofAll);
}
/***********************************************************************************************************************************
@ -164,24 +233,10 @@ ioReadFilterGroupSet(IoRead *this, IoFilterGroup *filterGroup)
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(filterGroup != NULL);
FUNCTION_TEST_ASSERT(this->filterGroup == NULL);
FUNCTION_TEST_ASSERT(!this->opened && !this->closed);
FUNCTION_DEBUG_END();
this->filterGroup = filterGroup;
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Total bytes read
***********************************************************************************************************************************/
size_t
ioReadSize(const IoRead *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_READ, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(SIZE, this->size);
}

12
src/common/io/read.h
View File

@ -1,11 +1,16 @@
/***********************************************************************************************************************************
IO Read
IO Read Interface
Objects that read from some IO source (file, socket, etc.) are implemented using this interface. All objects are required to
implement IoReadProcess and can optionally implement IoReadOpen, IoReadClose, or IoReadEof. IoReadOpen and IoReadClose can be used
to allocate/open or deallocate/free resources. If IoReadEof is not implemented then ioReadEof() will always return false. An
example of an IoRead object is IoBufferRead.
***********************************************************************************************************************************/
#ifndef COMMON_IO_READ_H
#define COMMON_IO_READ_H
/***********************************************************************************************************************************
IO read object
Object type
***********************************************************************************************************************************/
typedef struct IoRead IoRead;
@ -37,8 +42,7 @@ Getters/Setters
***********************************************************************************************************************************/
bool ioReadEof(const IoRead *this);
const IoFilterGroup *ioReadFilterGroup(const IoRead *this);
void ioReadFilterGroupSet(IoRead *this, IoFilterGroup *FilterGroup);
size_t ioReadSize(const IoRead *this);
void ioReadFilterGroupSet(IoRead *this, IoFilterGroup *filterGroup);
/***********************************************************************************************************************************
Macros for function logging

90
src/common/io/write.c
View File

@ -1,29 +1,34 @@
/***********************************************************************************************************************************
IO Write
IO Write Interface
***********************************************************************************************************************************/
#include "common/debug.h"
#include "common/io/io.h"
#include "common/io/write.h"
#include "common/log.h"
#include "common/memContext.h"
/***********************************************************************************************************************************
IO write object
Object type
***********************************************************************************************************************************/
struct IoWrite
{
MemContext *memContext; // Mem context of driver
void *driver; // Driver object
IoFilterGroup *filterGroup; // IO filters
Buffer *output; // Output buffer
IoWriteOpen open; // Driver open
IoWriteProcess write; // Driver write
IoWriteClose close; // Driver close
size_t size; // Total bytes written
#ifdef DEBUG
bool opened; // Has the io been opened?
bool closed; // Has the io been closed?
#endif
};
/***********************************************************************************************************************************
Create a new write IO
New object
Allocations will be in the memory context of the caller.
***********************************************************************************************************************************/
@ -42,6 +47,8 @@ ioWriteNew(void *driver, IoWriteOpen open, IoWriteProcess write, IoWriteClose cl
IoWrite *this = memNew(sizeof(IoWrite));
this->memContext = memContextCurrent();
this->output = bufNew(ioBufferSize());
this->driver = driver;
this->open = open;
this->write = write;
@ -60,43 +67,61 @@ ioWriteOpen(IoWrite *this)
FUNCTION_DEBUG_PARAM(IO_WRITE, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(!this->opened && !this->closed);
FUNCTION_DEBUG_END();
if (this->open != NULL)
this->open(this->driver);
// If no filter group exists create one to do buffering
if (this->filterGroup == NULL)
this->filterGroup = ioFilterGroupNew();
ioFilterGroupOpen(this->filterGroup);
#ifdef DEBUG
this->opened = true;
#endif
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Write data to IO
Write data to IO and process filters
***********************************************************************************************************************************/
void
ioWrite(IoWrite *this, const Buffer *buffer)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(IO_WRITE, this);
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_DEBUG_END();
// Only write if there is data to write
if (buffer != NULL && bufSize(buffer) > 0)
if (buffer != NULL && bufUsed(buffer) > 0)
{
// Apply filters
if (this->filterGroup != NULL)
ioFilterGroupProcess(this->filterGroup, buffer);
do
{
ioFilterGroupProcess(this->filterGroup, buffer, this->output);
// Write data
this->write(this->driver, buffer);
this->size += bufUsed(buffer);
// Write data if the buffer is full
if (bufRemains(this->output) == 0)
{
this->write(this->driver, this->output);
bufUsedZero(this->output);
}
}
while (ioFilterGroupInputSame(this->filterGroup));
}
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Close the IO
Close the IO and write any additional data that has not been written yet
***********************************************************************************************************************************/
void
ioWriteClose(IoWrite *this)
@ -105,16 +130,34 @@ ioWriteClose(IoWrite *this)
FUNCTION_DEBUG_PARAM(IO_WRITE, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && !this->closed);
FUNCTION_DEBUG_END();
// Flush remaining data
do
{
ioFilterGroupProcess(this->filterGroup, NULL, this->output);
// Write data if the buffer is full or if this is the last buffer to be written
if (bufRemains(this->output) == 0 || (ioFilterGroupDone(this->filterGroup) && bufUsed(this->output) > 0))
{
this->write(this->driver, this->output);
bufUsedZero(this->output);
}
}
while (!ioFilterGroupDone(this->filterGroup));
// Close the filter group and gather results
if (this->filterGroup != NULL)
ioFilterGroupClose(this->filterGroup);
ioFilterGroupClose(this->filterGroup);
// Close the driver if there is a close function
if (this->close != NULL)
this->close(this->driver);
#ifdef DEBUG
this->closed = true;
#endif
FUNCTION_DEBUG_RESULT_VOID();
}
@ -130,6 +173,7 @@ ioWriteFilterGroup(const IoWrite *this)
FUNCTION_TEST_PARAM(IO_WRITE, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(this->opened && this->closed);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(IO_FILTER_GROUP, this->filterGroup);
@ -145,24 +189,10 @@ ioWriteFilterGroupSet(IoWrite *this, IoFilterGroup *filterGroup)
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(filterGroup != NULL);
FUNCTION_TEST_ASSERT(this->filterGroup == NULL);
FUNCTION_TEST_ASSERT(!this->opened && !this->closed);
FUNCTION_DEBUG_END();
this->filterGroup = filterGroup;
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Total bytes written
***********************************************************************************************************************************/
size_t
ioWriteSize(const IoWrite *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_WRITE, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(SIZE, this->size);
}

9
src/common/io/write.h
View File

@ -1,11 +1,15 @@
/***********************************************************************************************************************************
IO Write
IO Write Interface
Objects that write to some IO destination (file, socket, etc.) are implemented using this interface. All objects are required to
implement IoWriteProcess and can optionally implement IoWriteOpen or IoWriteClose. IoWriteOpen and IoWriteClose can be used to
allocate/open or deallocate/free resources. An example of an IoWrite object is IoBufferRead.
***********************************************************************************************************************************/
#ifndef COMMON_IO_WRITE_H
#define COMMON_IO_WRITE_H
/***********************************************************************************************************************************
IO write object
Object type
***********************************************************************************************************************************/
typedef struct IoWrite IoWrite;
@ -36,7 +40,6 @@ Getters/Setters
***********************************************************************************************************************************/
const IoFilterGroup *ioWriteFilterGroup(const IoWrite *this);
void ioWriteFilterGroupSet(IoWrite *this, IoFilterGroup *filterGroup);
size_t ioWriteSize(const IoWrite *this);
/***********************************************************************************************************************************
Macros for function logging

13
src/crypto/hash.c
View File

@ -1,5 +1,5 @@
/***********************************************************************************************************************************
Cryptographic Hashes
Cryptographic Hash
***********************************************************************************************************************************/
#include <string.h>
@ -18,19 +18,19 @@ Filter type constant
#define CRYPTO_HASH_FILTER_TYPE "hash"
/***********************************************************************************************************************************
Track state during block encrypt/decrypt
Object type
***********************************************************************************************************************************/
struct CryptoHash
{
MemContext *memContext; // Context to store data
const EVP_MD *hashType; // Hash type
const EVP_MD *hashType; // Hash type (sha1, md5, etc.)
EVP_MD_CTX *hashContext; // Message hash context
Buffer *hash; // Hash in binary form
IoFilter *filter; // Filter interface
};
/***********************************************************************************************************************************
New hash object
New object
***********************************************************************************************************************************/
CryptoHash *
cryptoHashNew(const String *type)
@ -70,7 +70,8 @@ cryptoHashNew(const String *type)
// Create filter interface
this->filter = ioFilterNew(
strNew(CRYPTO_HASH_FILTER_TYPE), this, (IoFilterProcessIn)cryptoHashProcess, (IoFilterResult)cryptoHashResult);
strNew(CRYPTO_HASH_FILTER_TYPE), this, NULL, NULL, (IoFilterProcessIn)cryptoHashProcess, NULL,
(IoFilterResult)cryptoHashResult);
}
MEM_CONTEXT_NEW_END();
@ -212,7 +213,7 @@ cryptoHashFilter(CryptoHash *this)
}
/***********************************************************************************************************************************
Return filter result
Get string representation of the hash as a filter result
***********************************************************************************************************************************/
const Variant *
cryptoHashResult(CryptoHash *this)

4
src/crypto/hash.h
View File

@ -1,5 +1,7 @@
/***********************************************************************************************************************************
Cryptographic Hashes
Cryptographic Hash
Generate a hash (sha1, md5, etc.) from a string, Buffer, or using an IoFilter.
***********************************************************************************************************************************/
#ifndef CRYPTO_HASH_H
#define CRYPTO_HASH_H

2
test/define.yaml
View File

@ -206,8 +206,10 @@ unit:
coverage:
common/io/bufferRead: full
common/io/bufferWrite: full
common/io/filter/buffer: full
common/io/filter/filter: full
common/io/filter/group: full
common/io/filter/size: full
common/io/handle: full
common/io/io: full
common/io/read: full

188
test/src/module/common/ioTest.c
View File

@ -11,8 +11,10 @@ Test functions for IoRead that are not covered by testing the IoBufferRead objec
static bool
testIoReadOpen(void *driver)
{
ASSERT(driver == (void *)999);
return false;
if (driver == (void *)998)
return false;
return true;
}
static size_t
@ -73,7 +75,19 @@ typedef struct IoTestFilterSize
static void
ioTestFilterSizeProcess(IoTestFilterSize *this, const Buffer *buffer)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(VOIDP, this);
FUNCTION_DEBUG_PARAM(BUFFER, buffer);
FUNCTION_DEBUG_PARAM(STRING, ioFilterType(this->filter));
FUNCTION_DEBUG_PARAM(SIZE, this->size);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(buffer != NULL);
FUNCTION_DEBUG_END();
this->size += bufUsed(buffer);
FUNCTION_DEBUG_RESULT_VOID();
}
static const Variant *
@ -93,7 +107,96 @@ ioTestFilterSizeNew(const char *type)
this->memContext = MEM_CONTEXT_NEW();
this->filter = ioFilterNew(
strNew(type), this, (IoFilterProcessIn)ioTestFilterSizeProcess, (IoFilterResult)ioTestFilterSizeResult);
strNew(type), this, NULL, NULL, (IoFilterProcessIn)ioTestFilterSizeProcess, NULL,
(IoFilterResult)ioTestFilterSizeResult);
}
MEM_CONTEXT_NEW_END();
return this;
}
/***********************************************************************************************************************************
Test filter to double input to the output. It can also flush out a variable number of bytes at the end.
***********************************************************************************************************************************/
typedef struct IoTestFilterDouble
{
MemContext *memContext;
unsigned int flushTotal;
Buffer *doubleBuffer;
IoFilter *bufferFilter;
IoFilter *filter;
} IoTestFilterDouble;
static void
ioTestFilterDoubleProcess(IoTestFilterDouble *this, const Buffer *input, Buffer *output)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(VOIDP, this);
FUNCTION_DEBUG_PARAM(BUFFER, input);
FUNCTION_DEBUG_PARAM(BUFFER, output);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_ASSERT(output != NULL && bufRemains(output) > 0);
FUNCTION_DEBUG_END();
if (input == NULL)
{
bufCat(output, bufNewC(1, "X"));
this->flushTotal--;
}
else
{
if (this->doubleBuffer == NULL)
{
this->doubleBuffer = bufNew(bufUsed(input) * 2);
unsigned char *inputPtr = bufPtr(input);
unsigned char *bufferPtr = bufPtr(this->doubleBuffer);
for (unsigned int charIdx = 0; charIdx < bufUsed(input); charIdx++)
{
bufferPtr[charIdx * 2] = inputPtr[charIdx];
bufferPtr[charIdx * 2 + 1] = inputPtr[charIdx];
}
bufUsedSet(this->doubleBuffer, bufSize(this->doubleBuffer));
}
ioFilterProcessInOut(this->bufferFilter, this->doubleBuffer, output);
if (!ioFilterInputSame(this->bufferFilter))
this->doubleBuffer = NULL;
}
FUNCTION_DEBUG_RESULT_VOID();
}
static bool
ioTestFilterDoubleDone(IoTestFilterDouble *this)
{
return this->flushTotal == 0;
}
static bool
ioTestFilterDoubleInputSame(IoTestFilterDouble *this)
{
return ioFilterInputSame(this->bufferFilter);
}
static IoTestFilterDouble *
ioTestFilterDoubleNew(const char *type, unsigned int flushTotal)
{
IoTestFilterDouble *this = NULL;
MEM_CONTEXT_NEW_BEGIN("IoTestFilterDouble")
{
this = memNew(sizeof(IoTestFilterDouble));
this->memContext = MEM_CONTEXT_NEW();
this->bufferFilter = ioBufferFilter(ioBufferNew());
this->flushTotal = flushTotal;
this->filter = ioFilterNew(
strNew(type), this, (IoFilterDone)ioTestFilterDoubleDone, (IoFilterInputSame)ioTestFilterDoubleInputSame, NULL,
(IoFilterProcessInOut)ioTestFilterDoubleProcess, NULL);
}
MEM_CONTEXT_NEW_END();
@ -117,22 +220,29 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("IoRead and IoBufferRead"))
if (testBegin("IoRead, IoBufferRead, IoBuffer, IoSize, IoFilter, and IoFilterGroup"))
{
IoRead *read = NULL;
Buffer *buffer = bufNew(2);
ioBufferSizeSet(2);
TEST_ASSIGN(
read, ioReadNew((void *)998, testIoReadOpen, testIoReadProcess, testIoReadClose, NULL), "create io read object");
TEST_RESULT_BOOL(ioReadOpen(read), false, " open io object");
TEST_ASSIGN(
read, ioReadNew((void *)999, testIoReadOpen, testIoReadProcess, testIoReadClose, NULL), "create io read object");
TEST_RESULT_BOOL(ioReadOpen(read), false, " open io object");
TEST_RESULT_SIZE(ioRead(read, buffer), 1, " read 1 byte");
TEST_RESULT_BOOL(ioReadOpen(read), true, " open io object");
TEST_RESULT_SIZE(ioRead(read, buffer), 2, " read 2 bytes");
TEST_RESULT_BOOL(ioReadEof(read), false, " no eof");
TEST_RESULT_VOID(ioReadClose(read), " close io object");
TEST_RESULT_BOOL(testIoReadCloseCalled, true, " check io object closed");
// -------------------------------------------------------------------------------------------------------------------------
IoBufferRead *bufferRead = NULL;
ioBufferSizeSet(2);
buffer = bufNew(2);
Buffer *bufferOriginal = bufNewStr(strNew("123"));
@ -146,45 +256,65 @@ testRun(void)
IoFilterGroup *filterGroup = NULL;
TEST_ASSIGN(filterGroup, ioFilterGroupNew(), " create new filter group");
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, ioTestFilterSizeNew("size")->filter), " add filter to filter group");
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, ioTestFilterSizeNew("size2")->filter), " add filter to filter group");
IoSize *sizeFilter = ioSizeNew();
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, ioSizeFilter(sizeFilter)), " add filter to filter group");
TEST_RESULT_VOID(
ioFilterGroupAdd(filterGroup, ioTestFilterDoubleNew("double", 1)->filter), " add filter to filter group");
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, ioSizeFilter(ioSizeNew())), " add filter to filter group");
IoBuffer *bufferFilter = ioBufferNew();
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, ioBufferFilter(bufferFilter)), " add filter to filter group");
TEST_RESULT_VOID(ioReadFilterGroupSet(ioBufferReadIo(bufferRead), filterGroup), " add filter group to read io");
TEST_RESULT_PTR(ioFilterMove(NULL, memContextTop()), NULL, " move NULL filter to top context");
TEST_RESULT_PTR(ioFilterGroupResult(filterGroup, strNew("size")), NULL, " check filter result is NULL");
TEST_RESULT_PTR(ioFilterGroupResult(filterGroup, strNew("size2")), NULL, " check filter result is NULL");
TEST_RESULT_BOOL(ioReadOpen(ioBufferReadIo(bufferRead)), true, " open");
TEST_RESULT_BOOL(ioReadEof(ioBufferReadIo(bufferRead)), false, " not eof");
TEST_RESULT_SIZE(ioRead(ioBufferReadIo(bufferRead), buffer), 2, " read 2 bytes");
TEST_RESULT_SIZE(ioRead(ioBufferReadIo(bufferRead), buffer), 0, " read 0 bytes (full buffer)");
TEST_RESULT_BOOL(memcmp(bufPtr(buffer), "12", 2) == 0, true, " memcmp");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "12", " check read");
TEST_RESULT_SIZE(ioReadSize(ioBufferReadIo(bufferRead)), 2, " read size is 2");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "11", " check read");
TEST_RESULT_STR(strPtr(ioFilterType(ioSizeFilter(sizeFilter))), "size", "check filter type");
TEST_RESULT_BOOL(ioReadEof(ioBufferReadIo(bufferRead)), false, " not eof");
TEST_RESULT_VOID(bufUsedZero(buffer), " zero buffer");
TEST_RESULT_SIZE(ioRead(ioBufferReadIo(bufferRead), buffer), 1, " read 1 byte");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "3", " check read");
TEST_RESULT_SIZE(ioRead(ioBufferReadIo(bufferRead), buffer), 2, " read 2 bytes");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "22", " check read");
TEST_ASSIGN(buffer, bufNew(3), "change output buffer size to 3");
TEST_RESULT_SIZE(ioRead(ioBufferReadIo(bufferRead), buffer), 3, " read 3 bytes");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "33X", " check read");
TEST_RESULT_VOID(bufUsedZero(buffer), " zero buffer");
TEST_RESULT_BOOL(ioReadEof(ioBufferReadIo(bufferRead)), true, " eof");
TEST_RESULT_BOOL(ioBufferRead(bufferRead, buffer), 0, " eof from driver");
TEST_RESULT_SIZE(ioRead(ioBufferReadIo(bufferRead), buffer), 0, " read 0 bytes");
TEST_RESULT_SIZE(ioReadSize(ioBufferReadIo(bufferRead)), 3, " read size is 3");
TEST_RESULT_VOID(ioReadClose(ioBufferReadIo(bufferRead)), " close buffer read object");
TEST_RESULT_PTR(ioReadFilterGroup(ioBufferReadIo(bufferRead)), filterGroup, " check filter group");
TEST_RESULT_UINT(varUInt64(ioFilterGroupResult(filterGroup, strNew("size"))), 3, " check filter result");
TEST_RESULT_UINT(varUInt64(ioFilterGroupResult(filterGroup, strNew("size2"))), 3, " check filter result");
TEST_RESULT_UINT(
varUInt64(varLstGet(varVarLst(ioFilterGroupResult(filterGroup, ioFilterType(ioSizeFilter(sizeFilter)))), 0)), 3,
" check filter result");
TEST_RESULT_PTR(ioFilterGroupResult(filterGroup, strNew("double")), NULL, " check filter result is NULL");
TEST_RESULT_UINT(
varUInt64(varLstGet(varVarLst(ioFilterGroupResult(filterGroup, ioFilterType(ioSizeFilter(sizeFilter)))), 1)), 7,
" check filter result");
TEST_RESULT_VOID(ioBufferReadFree(bufferRead), " free buffer read object");
TEST_RESULT_VOID(ioBufferReadFree(NULL), " free NULL buffer read object");
TEST_RESULT_VOID(ioSizeFree(sizeFilter), " free size filter object");
TEST_RESULT_VOID(ioSizeFree(NULL), " free null size filter object");
TEST_RESULT_VOID(ioBufferFree(bufferFilter), " free buffer filter object");
TEST_RESULT_VOID(ioBufferFree(NULL), " free null buffer filter object");
TEST_RESULT_VOID(ioFilterGroupFree(filterGroup), " free filter group object");
TEST_RESULT_VOID(ioFilterGroupFree(NULL), " free NULL filter group object");
}
// *****************************************************************************************************************************
if (testBegin("IoWrite and IoBufferWrite"))
if (testBegin("IoWrite, IoBufferWrite, IoBuffer, IoSize, IoFilter, and IoFilterGroup"))
{
IoWrite *write = NULL;
ioBufferSizeSet(3);
TEST_ASSIGN(
write, ioWriteNew((void *)999, testIoWriteOpen, testIoWriteProcess, testIoWriteClose), "create io write object");
@ -196,6 +326,7 @@ testRun(void)
TEST_RESULT_BOOL(testIoWriteCloseCalled, true, " check io object closed");
// -------------------------------------------------------------------------------------------------------------------------
ioBufferSizeSet(3);
IoBufferWrite *bufferWrite = NULL;
Buffer *buffer = bufNew(0);
@ -209,24 +340,29 @@ testRun(void)
IoFilterGroup *filterGroup = NULL;
TEST_ASSIGN(filterGroup, ioFilterGroupNew(), " create new filter group");
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, ioTestFilterSizeNew("size")->filter), " add filter to filter group");
IoSize *sizeFilter = ioSizeNew();
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, ioSizeFilter(sizeFilter)), " add filter to filter group");
TEST_RESULT_VOID(
ioFilterGroupAdd(filterGroup, ioTestFilterDoubleNew("double", 3)->filter), " add filter to filter group");
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, ioTestFilterSizeNew("size2")->filter), " add filter to filter group");
TEST_RESULT_VOID(ioWriteFilterGroupSet(ioBufferWriteIo(bufferWrite), filterGroup), " add filter group to write io");
TEST_RESULT_VOID(ioWriteOpen(ioBufferWriteIo(bufferWrite)), " open buffer write object");
TEST_RESULT_VOID(ioWrite(ioBufferWriteIo(bufferWrite), bufNewStr(strNew("ABC"))), " write 3 bytes");
TEST_RESULT_VOID(ioWrite(ioBufferWriteIo(bufferWrite), bufNewStr(strNew(""))), " write 0 bytes");
TEST_RESULT_VOID(ioWrite(ioBufferWriteIo(bufferWrite), NULL), " write 0 bytes");
TEST_RESULT_SIZE(ioWriteSize(ioBufferWriteIo(bufferWrite)), 3, " write size is 3");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "ABC", " check write");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "AABBCC", " check write");
TEST_RESULT_VOID(ioWrite(ioBufferWriteIo(bufferWrite), bufNewStr(strNew("1234"))), " write 4 bytes");
TEST_RESULT_SIZE(ioWriteSize(ioBufferWriteIo(bufferWrite)), 7, " write size is 7");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "ABC1234", " check write");
TEST_RESULT_VOID(ioWrite(ioBufferWriteIo(bufferWrite), bufNewStr(strNew("12345"))), " write 4 bytes");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "AABBCC112233445", " check write");
TEST_RESULT_VOID(ioWriteClose(ioBufferWriteIo(bufferWrite)), " close buffer write object");
TEST_RESULT_STR(strPtr(strNewBuf(buffer)), "AABBCC1122334455XXX", " check write after close");
TEST_RESULT_PTR(ioWriteFilterGroup(ioBufferWriteIo(bufferWrite)), filterGroup, " check filter group");
TEST_RESULT_UINT(varUInt64(ioFilterGroupResult(filterGroup, strNew("size"))), 7, " check filter result");
TEST_RESULT_UINT(
varUInt64(ioFilterGroupResult(filterGroup, ioFilterType(ioSizeFilter(sizeFilter)))), 8, " check filter result");
TEST_RESULT_UINT(varUInt64(ioFilterGroupResult(filterGroup, strNew("size2"))), 19, " check filter result");
TEST_RESULT_VOID(ioBufferWriteFree(bufferWrite), " free buffer write object");
TEST_RESULT_VOID(ioBufferWriteFree(NULL), " free NULL buffer write object");

13
test/src/module/storage/fileTest.c
View File

@ -128,7 +128,6 @@ testRun(void)
TEST_RESULT_BOOL(ioReadOpen(storageFileReadIo(file)), true, " open file");
TEST_RESULT_STR(strPtr(storageFileReadName(file)), strPtr(fileName), " check file name");
TEST_RESULT_INT(ioReadSize(storageFileReadIo(file)), 0, " check size");
TEST_RESULT_VOID(ioRead(storageFileReadIo(file), outBuffer), " load data");
bufCat(buffer, outBuffer);
@ -143,7 +142,6 @@ testRun(void)
bufCat(buffer, outBuffer);
bufUsedZero(outBuffer);
TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), false, " check file contents (not all loaded yet)");
TEST_RESULT_INT(ioReadSize(storageFileReadIo(file)), 8, " check size");
TEST_RESULT_VOID(ioRead(storageFileReadIo(file), outBuffer), " load data");
bufCat(buffer, outBuffer);
@ -156,14 +154,11 @@ testRun(void)
TEST_RESULT_INT(bufUsed(outBuffer), 0, " buffer is empty");
TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), true, " check file contents (all loaded)");
TEST_RESULT_INT(ioReadSize(storageFileReadIo(file)), 9, " check size");
TEST_RESULT_BOOL(ioReadEof(storageFileReadIo(file)), true, " eof");
TEST_RESULT_BOOL(ioReadEof(storageFileReadIo(file)), true, " still eof");
TEST_RESULT_VOID(ioReadClose(storageFileReadIo(file)), " close file");
TEST_RESULT_VOID(ioReadClose(storageFileReadIo(file)), " close again");
TEST_RESULT_INT(ioReadSize(storageFileReadIo(file)), 9, " check size");
TEST_RESULT_VOID(storageFileReadFree(file), " free file");
TEST_RESULT_VOID(storageFileReadFree(NULL), " free null file");
@ -210,6 +205,7 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
String *fileTmp = strNewFmt("%s.pgbackrest.tmp", strPtr(fileName));
ioBufferSizeSet(10);
Buffer *buffer = bufNewStr(strNew("TESTFILE\n"));
TEST_ASSIGN(file, storageNewWriteNP(storageTest, fileName), "new write file");
@ -221,17 +217,17 @@ testRun(void)
storageRemoveP(storageTest, fileTmp, .errorOnMissing = true);
TEST_ERROR_FMT(
ioWrite(storageFileWriteIo(file), buffer), FileWriteError,
storageFileWritePosix(storageFileWriteFileDriver(file), buffer), FileWriteError,
"unable to write '%s': [9] Bad file descriptor", strPtr(fileName));
TEST_ERROR_FMT(
ioWriteClose(storageFileWriteIo(file)), FileSyncError,
storageFileWritePosixClose(storageFileWriteFileDriver(file)), FileSyncError,
"unable to sync '%s': [9] Bad file descriptor", strPtr(fileName));
// Disable file sync so the close can be reached
file->fileDriver->noSyncFile = true;
TEST_ERROR_FMT(
ioWriteClose(storageFileWriteIo(file)), FileCloseError,
storageFileWritePosixClose(storageFileWriteFileDriver(file)), FileCloseError,
"unable to close '%s': [9] Bad file descriptor", strPtr(fileName));
// Set file handle to -1 so the close on free with not fail
@ -265,7 +261,6 @@ testRun(void)
TEST_RESULT_VOID(ioWrite(storageFileWriteIo(file), bufNew(0)), " write zero buffer to file");
TEST_RESULT_VOID(ioWrite(storageFileWriteIo(file), buffer), " write to file");
TEST_RESULT_VOID(ioWriteClose(storageFileWriteIo(file)), " close file");
TEST_RESULT_SIZE(ioWriteSize(storageFileWriteIo(file)), 9, " check size");
TEST_RESULT_VOID(storageFileWriteFree(file), " free file");
TEST_RESULT_VOID(storageFileWriteFree(NULL), " free null file");
TEST_RESULT_VOID(storageFileWritePosixFree(NULL), " free null posix file");

13
test/src/module/storage/storageTest.c
View File

@ -485,8 +485,9 @@ testRun(void)
TEST_RESULT_VOID(ioReadClose(storageFileReadIo(file)), " close file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(file, storageNewReadP(storageTest, fileName, .filterGroup = (IoFilterGroup *)55), "new read file with filters");
TEST_RESULT_PTR(ioReadFilterGroup(storageFileReadIo(file)), (IoFilterGroup *)55, " check filter group is set");
IoFilterGroup *filterGroup = ioFilterGroupNew();
TEST_ASSIGN(file, storageNewReadP(storageTest, fileName, .filterGroup = filterGroup), "new read file with filters");
TEST_RESULT_PTR(ioReadFilterGroup(storageFileReadIo(file)), filterGroup, " check filter group is set");
}
// *****************************************************************************************************************************
@ -519,9 +520,11 @@ testRun(void)
TEST_RESULT_INT(storageInfoNP(storageTest, fileName).mode, 0600, " check file mode");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(
file, storageNewWriteP(storageTest, fileName, .filterGroup = (IoFilterGroup *)55), "new write file with filters");
TEST_RESULT_PTR(ioWriteFilterGroup(storageFileWriteIo(file)), (IoFilterGroup *)55, " check filter group is set");
IoFilterGroup *filterGroup = ioFilterGroupNew();
TEST_ASSIGN(file, storageNewWriteP(storageTest, fileName, .filterGroup = filterGroup), "new write file with filters");
TEST_RESULT_VOID(ioWriteOpen(storageFileWriteIo(file)), " open file");
TEST_RESULT_VOID(ioWriteClose(storageFileWriteIo(file)), " close file");
TEST_RESULT_PTR(ioWriteFilterGroup(storageFileWriteIo(file)), filterGroup, " check filter group is set");
}
// *****************************************************************************************************************************