db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-10-30 23:37:45 +02:00
Code Issues Releases Activity

Add ProtocolParallel* objects for parallelizing commands.

Browse Source 
Allows commands to be easily parallelized if the jobs are broken up into discrete, non-overlapping chunks.
This commit is contained in:
David Steele
2019-02-27 21:10:52 +02:00
parent 35acfae7c2
commit 35abd4cd95
8 changed files with 911 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@@ -63,6 +63,10 @@
<p>Add separate <cmd>archive-get-async</cmd> command.</p>
</release-item>
<release-item>
<p>Add <code>ProtocolParallel*</code> objects for parallelizing commands.</p>
</release-item>
<release-item>
<p>Add <code>ProtocolCommand</code> object.</p>
</release-item>

10
src/Makefile
View File

@@ -139,10 +139,12 @@ SRCS = \
postgres/interface/v100.c \
postgres/interface/v110.c \
postgres/pageChecksum.c \
protocol/server.c \
protocol/client.c \
protocol/command.c \
protocol/helper.c \
protocol/parallel.c \
protocol/parallelJob.c \
protocol/server.c \
storage/driver/posix/storage.c \
storage/driver/posix/common.c \
storage/driver/posix/fileRead.c \
@@ -435,6 +437,12 @@ protocol/command.o: protocol/command.c common/assert.h common/debug.h common/err
protocol/helper.o: protocol/helper.c common/assert.h common/debug.h common/error.auto.h common/error.h common/exec.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h config/protocol.h crypto/crypto.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
$(CC) $(CFLAGS) -c protocol/helper.c -o protocol/helper.o
protocol/parallel.o: protocol/parallel.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/parallel.h protocol/parallelJob.h
$(CC) $(CFLAGS) -c protocol/parallel.c -o protocol/parallel.o
protocol/parallelJob.o: protocol/parallelJob.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/parallelJob.h
$(CC) $(CFLAGS) -c protocol/parallelJob.c -o protocol/parallelJob.o
protocol/server.o: protocol/server.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/command.h protocol/server.h version.h
$(CC) $(CFLAGS) -c protocol/server.c -o protocol/server.o

300
src/protocol/parallel.c Normal file
View File

@@ -0,0 +1,300 @@
/***********************************************************************************************************************************
Protocol Parallel Executor
***********************************************************************************************************************************/
#include <sys/select.h>
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/keyValue.h"
#include "common/type/list.h"
#include "protocol/command.h"
#include "protocol/parallel.h"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct ProtocolParallel
{
MemContext *memContext;
TimeMSec timeout; // Max time to wait for jobs before returning
List *clientList; // List of clients to process jobs
List *jobList; // List of jobs to be processed
ProtocolParallelJob **clientJobList; // Jobs being processing by each client
ProtocolParallelJobState state; // Overall state of job processing
};
/***********************************************************************************************************************************
Create object
***********************************************************************************************************************************/
ProtocolParallel *
protocolParallelNew(TimeMSec timeout)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(UINT64, timeout);
FUNCTION_LOG_END();
ProtocolParallel *this = NULL;
MEM_CONTEXT_NEW_BEGIN("ProtocolParallel")
{
this = memNew(sizeof(ProtocolParallel));
this->memContext = memContextCurrent();
this->timeout = timeout;
this->clientList = lstNew(sizeof(ProtocolClient *));
this->jobList = lstNew(sizeof(ProtocolParallelJob *));
this->state = protocolParallelJobStatePending;
}
MEM_CONTEXT_NEW_END();
FUNCTION_LOG_RETURN(PROTOCOL_PARALLEL, this);
}
/***********************************************************************************************************************************
Add client
***********************************************************************************************************************************/
void
protocolParallelClientAdd(ProtocolParallel *this, ProtocolClient *client)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL, this);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, client);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(client != NULL);
ASSERT(this->state == protocolParallelJobStatePending);
if (ioReadHandle(protocolClientIoRead(client)) == -1)
THROW(AssertError, "client with read handle is required");
lstAdd(this->clientList, &client);
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Add job
***********************************************************************************************************************************/
void
protocolParallelJobAdd(ProtocolParallel *this, ProtocolParallelJob *job)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL, this);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, job);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(job != NULL);
ASSERT(this->state == protocolParallelJobStatePending);
protocolParallelJobMove(job, lstMemContext(this->jobList));
lstAdd(this->jobList, &job);
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Process jobs
***********************************************************************************************************************************/
unsigned int
protocolParallelProcess(ProtocolParallel *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(this->state != protocolParallelJobStateDone);
unsigned int result = 0;
// If called for the first time, initialize processing
if (this->state == protocolParallelJobStatePending)
{
MEM_CONTEXT_BEGIN(this->memContext)
{
this->clientJobList = (ProtocolParallelJob **)memNew(sizeof(ProtocolParallelJob *) * lstSize(this->clientList));
}
MEM_CONTEXT_END();
this->state = protocolParallelJobStateRunning;
}
// Initialize the file descriptor set used for select
fd_set selectSet;
FD_ZERO(&selectSet);
int handleMax = -1;
// Find clients that are running jobs
unsigned int clientRunningTotal = 0;
for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
{
if (this->clientJobList[clientIdx] != NULL)
{
int handle = ioReadHandle(protocolClientIoRead(*(ProtocolClient **)lstGet(this->clientList, clientIdx)));
FD_SET((unsigned int)handle, &selectSet);
// Set the max handle
if (handle > handleMax) // {+uncovered - handles are often in ascending order}
handleMax = handle;
clientRunningTotal++;
}
}
// If clients are running then wait for one to finish
if (clientRunningTotal > 0)
{
// Initialize timeout struct used for select. Recreate this structure each time since Linux (at least) will modify it.
struct timeval timeoutSelect;
timeoutSelect.tv_sec = (time_t)(this->timeout / MSEC_PER_SEC);
timeoutSelect.tv_usec = (time_t)(this->timeout % MSEC_PER_SEC * 1000);
// Determine if there is data to be read
int completed = select(handleMax + 1, &selectSet, NULL, NULL, &timeoutSelect);
THROW_ON_SYS_ERROR(completed == -1, AssertError, "unable to select from parallel client(s)");
// If any jobs have completed then get the results
if (completed > 0)
{
for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
{
ProtocolParallelJob *job = this->clientJobList[clientIdx];
if (job != NULL &&
FD_ISSET(
(unsigned int)ioReadHandle(protocolClientIoRead(*(ProtocolClient **)lstGet(this->clientList, clientIdx))),
&selectSet))
{
MEM_CONTEXT_TEMP_BEGIN()
{
TRY_BEGIN()
{
protocolParallelJobResultSet(
job, protocolClientReadOutput(*(ProtocolClient **)lstGet(this->clientList, clientIdx), true));
}
CATCH_ANY()
{
protocolParallelJobErrorSet(job, errorCode(), strNew(errorMessage()));
}
TRY_END();
protocolParallelJobStateSet(job, protocolParallelJobStateDone);
this->clientJobList[clientIdx] = NULL;
}
MEM_CONTEXT_TEMP_END();
}
}
result = (unsigned int)completed;
}
}
// Find new jobs to be run
for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
{
// If nothing is running for this client
if (this->clientJobList[clientIdx] == NULL)
{
for (unsigned int jobIdx = 0; jobIdx < lstSize(this->jobList); jobIdx++)
{
ProtocolParallelJob *job = *(ProtocolParallelJob **)lstGet(this->jobList, jobIdx);
if (protocolParallelJobState(job) == protocolParallelJobStatePending)
{
protocolClientWriteCommand(
*(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job));
protocolParallelJobStateSet(job, protocolParallelJobStateRunning);
this->clientJobList[clientIdx] = job;
break;
}
}
}
}
FUNCTION_LOG_RETURN(UINT, result);
}
/***********************************************************************************************************************************
Get a completed job result
***********************************************************************************************************************************/
ProtocolParallelJob *
protocolParallelResult(ProtocolParallel *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(this->state == protocolParallelJobStateRunning);
ProtocolParallelJob *result = NULL;
// Find the next completed job
for (unsigned int jobIdx = 0; jobIdx < lstSize(this->jobList); jobIdx++)
{
ProtocolParallelJob *job = *(ProtocolParallelJob **)lstGet(this->jobList, jobIdx);
if (protocolParallelJobState(job) == protocolParallelJobStateDone)
{
result = protocolParallelJobMove(job, memContextCurrent());
lstRemove(this->jobList, jobIdx);
break;
}
}
// If all jobs have been returned then we are done
if (lstSize(this->jobList) == 0)
this->state = protocolParallelJobStateDone;
FUNCTION_LOG_RETURN(PROTOCOL_PARALLEL_JOB, result);
}
/***********************************************************************************************************************************
Process jobs
***********************************************************************************************************************************/
bool
protocolParallelDone(const ProtocolParallel *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL, this);
FUNCTION_LOG_END();
FUNCTION_LOG_RETURN(BOOL, this->state == protocolParallelJobStateDone);
}
/***********************************************************************************************************************************
Render as string for logging
***********************************************************************************************************************************/
String *
protocolParallelToLog(const ProtocolParallel *this)
{
return strNewFmt(
"{state: %s, clientTotal: %u, jobTotal: %u}", protocolParallelJobToConstZ(this->state), lstSize(this->clientList),
lstSize(this->jobList));
}
/***********************************************************************************************************************************
Free object
***********************************************************************************************************************************/
void
protocolParallelFree(ProtocolParallel *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL, this);
FUNCTION_LOG_END();
if (this != NULL)
memContextFree(this->memContext);
FUNCTION_LOG_RETURN_VOID();
}

49
src/protocol/parallel.h Normal file
View File

@@ -0,0 +1,49 @@
/***********************************************************************************************************************************
Protocol Parallel Executor
***********************************************************************************************************************************/
#ifndef PROTOCOL_PARALLEL_H
#define PROTOCOL_PARALLEL_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct ProtocolParallel ProtocolParallel;
#include "common/time.h"
#include "protocol/client.h"
#include "protocol/parallelJob.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
ProtocolParallel *protocolParallelNew(TimeMSec timeout);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void protocolParallelClientAdd(ProtocolParallel *this, ProtocolClient *client);
void protocolParallelJobAdd(ProtocolParallel *this, ProtocolParallelJob *job);
unsigned int protocolParallelProcess(ProtocolParallel *this);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool protocolParallelDone(const ProtocolParallel *this);
ProtocolParallelJob *protocolParallelResult(ProtocolParallel *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void protocolParallelFree(ProtocolParallel *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
String *protocolParallelToLog(const ProtocolParallel *this);
#define FUNCTION_LOG_PROTOCOL_PARALLEL_TYPE \
ProtocolParallel *
#define FUNCTION_LOG_PROTOCOL_PARALLEL_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, protocolParallelToLog, buffer, bufferSize)
#endif

282
src/protocol/parallelJob.c Normal file
View File

@@ -0,0 +1,282 @@
/***********************************************************************************************************************************
Protocol Parallel Job
***********************************************************************************************************************************/
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "protocol/command.h"
#include "protocol/parallelJob.h"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct ProtocolParallelJob
{
MemContext *memContext; // Job mem context
ProtocolParallelJobState state; // Current state of the job
const Variant *key; // Unique key used to identify the job
const ProtocolCommand *command; // Command to be executed
int code; // Non-zero result indicates an error
String *message; // Message if there was a error
const Variant *result; // Result if job was successful
};
/***********************************************************************************************************************************
Create object
***********************************************************************************************************************************/
ProtocolParallelJob *
protocolParallelJobNew(const Variant *key, ProtocolCommand *command)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(VARIANT, key);
FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
FUNCTION_LOG_END();
ProtocolParallelJob *this = NULL;
MEM_CONTEXT_NEW_BEGIN("ProtocolParallelJob")
{
this = memNew(sizeof(ProtocolParallelJob));
this->memContext = memContextCurrent();
this->state = protocolParallelJobStatePending;
this->key = varDup(key);
this->command = protocolCommandMove(command, memContextCurrent());
}
MEM_CONTEXT_NEW_END();
FUNCTION_LOG_RETURN(PROTOCOL_PARALLEL_JOB, this);
}
/***********************************************************************************************************************************
Move object to a new context
***********************************************************************************************************************************/
ProtocolParallelJob *
protocolParallelJobMove(ProtocolParallelJob *this, MemContext *parentNew)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_TEST_PARAM(MEM_CONTEXT, parentNew);
FUNCTION_TEST_END();
ASSERT(parentNew != NULL);
if (this != NULL)
memContextMove(this->memContext, parentNew);
FUNCTION_TEST_RETURN(this);
}
/***********************************************************************************************************************************
Get job command
***********************************************************************************************************************************/
const ProtocolCommand *
protocolParallelJobCommand(const ProtocolParallelJob *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->command);
}
/***********************************************************************************************************************************
Get/set job error
***********************************************************************************************************************************/
int
protocolParallelJobErrorCode(const ProtocolParallelJob *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->code);
}
const String *
protocolParallelJobErrorMessage(const ProtocolParallelJob *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->message);
}
void
protocolParallelJobErrorSet(ProtocolParallelJob *this, int code, const String *message)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_LOG_PARAM(INT, code);
FUNCTION_LOG_PARAM(STRING, message);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(code != 0);
ASSERT(message != NULL);
MEM_CONTEXT_BEGIN(this->memContext)
{
this->code = code;
this->message = strDup(message);
}
MEM_CONTEXT_END();
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Get job key
***********************************************************************************************************************************/
const Variant *
protocolParallelJobKey(const ProtocolParallelJob *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->key);
}
/***********************************************************************************************************************************
Get/set job result
***********************************************************************************************************************************/
const Variant *
protocolParallelJobResult(const ProtocolParallelJob *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->result);
}
void
protocolParallelJobResultSet(ProtocolParallelJob *this, const Variant *result)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_LOG_PARAM(VARIANT, result);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(result != NULL);
ASSERT(this->code == 0);
MEM_CONTEXT_BEGIN(this->memContext)
{
this->result = varDup(result);
}
MEM_CONTEXT_END();
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Get/set state
***********************************************************************************************************************************/
ProtocolParallelJobState
protocolParallelJobState(const ProtocolParallelJob *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->state);
}
void
protocolParallelJobStateSet(ProtocolParallelJob *this, ProtocolParallelJobState state)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_LOG_PARAM(ENUM, state);
FUNCTION_LOG_END();
ASSERT(this != NULL);
if (this->state == protocolParallelJobStatePending && state == protocolParallelJobStateRunning)
this->state = protocolParallelJobStateRunning;
else if (this->state == protocolParallelJobStateRunning && state == protocolParallelJobStateDone)
this->state = protocolParallelJobStateDone;
else
{
THROW_FMT(
AssertError, "invalid state transition from '%s' to '%s'", protocolParallelJobToConstZ(this->state),
protocolParallelJobToConstZ(state));
}
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Render as string for logging
***********************************************************************************************************************************/
const char *
protocolParallelJobToConstZ(ProtocolParallelJobState state)
{
const char *result = NULL;
switch (state)
{
case protocolParallelJobStatePending:
{
result = "pending";
break;
}
case protocolParallelJobStateRunning:
{
result = "running";
break;
}
case protocolParallelJobStateDone:
{
result = "done";
break;
}
}
return result;
}
String *
protocolParallelJobToLog(const ProtocolParallelJob *this)
{
return strNewFmt(
"{state: %s, key: %s, command: %s, code: %d, message: %s, result: %s}", protocolParallelJobToConstZ(this->state),
strPtr(varToLog(this->key)), strPtr(protocolCommandToLog(this->command)), this->code, strPtr(strToLog(this->message)),
strPtr(varToLog(this->result)));
}
/***********************************************************************************************************************************
Free object
***********************************************************************************************************************************/
void
protocolParallelJobFree(ProtocolParallelJob *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_LOG_END();
if (this != NULL)
memContextFree(this->memContext);
FUNCTION_LOG_RETURN_VOID();
}

64
src/protocol/parallelJob.h Normal file
View File

@@ -0,0 +1,64 @@
/***********************************************************************************************************************************
Protocol Parallel Job
***********************************************************************************************************************************/
#ifndef PROTOCOL_PARALLEL_JOB_H
#define PROTOCOL_PARALLEL_JOB_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct ProtocolParallelJob ProtocolParallelJob;
/***********************************************************************************************************************************
Job state enum
***********************************************************************************************************************************/
typedef enum
{
protocolParallelJobStatePending,
protocolParallelJobStateRunning,
protocolParallelJobStateDone,
} ProtocolParallelJobState;
#include "common/time.h"
#include "protocol/client.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
ProtocolParallelJob *protocolParallelJobNew(const Variant *key, ProtocolCommand *command);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
ProtocolParallelJob *protocolParallelJobMove(ProtocolParallelJob *this, MemContext *parentNew);
/***********************************************************************************************************************************
Getters/Setters
***********************************************************************************************************************************/
const ProtocolCommand *protocolParallelJobCommand(const ProtocolParallelJob *this);
int protocolParallelJobErrorCode(const ProtocolParallelJob *this);
const String *protocolParallelJobErrorMessage(const ProtocolParallelJob *this);
void protocolParallelJobErrorSet(ProtocolParallelJob *this, int code, const String *message);
const Variant *protocolParallelJobKey(const ProtocolParallelJob *this);
const Variant *protocolParallelJobResult(const ProtocolParallelJob *this);
void protocolParallelJobResultSet(ProtocolParallelJob *this, const Variant *result);
ProtocolParallelJobState protocolParallelJobState(const ProtocolParallelJob *this);
void protocolParallelJobStateSet(ProtocolParallelJob *this, ProtocolParallelJobState state);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void protocolParallelJobFree(ProtocolParallelJob *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
const char *protocolParallelJobToConstZ(ProtocolParallelJobState state);
String *protocolParallelJobToLog(const ProtocolParallelJob *this);
#define FUNCTION_LOG_PROTOCOL_PARALLEL_JOB_TYPE \
ProtocolParallelJob *
#define FUNCTION_LOG_PROTOCOL_PARALLEL_JOB_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, protocolParallelJobToLog, buffer, bufferSize)
#endif

4
test/define.yaml
View File

@@ -579,13 +579,15 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: protocol
total: 6
total: 7
perlReq: true
coverage:
protocol/client: full
protocol/command: full
protocol/helper: full
protocol/parallel: full
protocol/parallelJob: full
protocol/server: full
# ********************************************************************************************************************************

200
test/src/module/protocol/protocolTest.c
View File

@@ -3,6 +3,8 @@ Test Protocol
***********************************************************************************************************************************/
#include "common/io/handleRead.h"
#include "common/io/handleWrite.h"
#include "common/io/bufferRead.h"
#include "common/io/bufferWrite.h"
#include "storage/storage.h"
#include "storage/driver/posix/storage.h"
#include "version.h"
@@ -374,6 +376,204 @@ testRun(void)
HARNESS_FORK_END();
}
// *****************************************************************************************************************************
if (testBegin("ProtocolParallel and ProtocolParallelJob"))
{
ProtocolParallelJob *job = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(job, protocolParallelJobNew(varNewStr(strNew("test")), protocolCommandNew(strNew("command"))), "new job");
TEST_RESULT_PTR(protocolParallelJobMove(job, MEM_CONTEXT_OLD()), job, "move job");
TEST_RESULT_PTR(protocolParallelJobMove(NULL, MEM_CONTEXT_OLD()), NULL, "move null job");
}
MEM_CONTEXT_TEMP_END();
TEST_ERROR(
protocolParallelJobStateSet(job, protocolParallelJobStateDone), AssertError,
"invalid state transition from 'pending' to 'done'");
TEST_RESULT_VOID(protocolParallelJobStateSet(job, protocolParallelJobStateRunning), "transition to running");
TEST_ERROR(
protocolParallelJobStateSet(job, protocolParallelJobStatePending), AssertError,
"invalid state transition from 'running' to 'pending'");
// Free job
TEST_RESULT_VOID(protocolParallelJobFree(job), "free job");
TEST_RESULT_VOID(protocolParallelJobFree(NULL), "free null job");
// -------------------------------------------------------------------------------------------------------------------------
HARNESS_FORK_BEGIN()
{
// Local 1
HARNESS_FORK_CHILD_BEGIN(0, true)
{
IoRead *read = ioHandleReadIo(ioHandleReadNew(strNew("server read"), HARNESS_FORK_CHILD_READ(), 10000));
ioReadOpen(read);
IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(strNew("server write"), HARNESS_FORK_CHILD_WRITE()));
ioWriteOpen(write);
// Greeting with noop
ioWriteLine(write, strNew("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}"));
ioWriteFlush(write);
TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"cmd\":\"noop\"}", "noop");
ioWriteLine(write, strNew("{}"));
ioWriteFlush(write);
TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"cmd\":\"command1\",\"param\":[\"param1\",\"param2\"]}", "command1");
sleepMSec(4000);
ioWriteLine(write, strNew("{\"out\":1}"));
ioWriteFlush(write);
// Wait for exit
TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"cmd\":\"exit\"}", "exit command");
}
HARNESS_FORK_CHILD_END();
// Local 2
HARNESS_FORK_CHILD_BEGIN(0, true)
{
IoRead *read = ioHandleReadIo(ioHandleReadNew(strNew("server read"), HARNESS_FORK_CHILD_READ(), 10000));
ioReadOpen(read);
IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(strNew("server write"), HARNESS_FORK_CHILD_WRITE()));
ioWriteOpen(write);
// Greeting with noop
ioWriteLine(write, strNew("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}"));
ioWriteFlush(write);
TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"cmd\":\"noop\"}", "noop");
ioWriteLine(write, strNew("{}"));
ioWriteFlush(write);
TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"cmd\":\"command2\",\"param\":[\"param1\"]}", "command2");
sleepMSec(1000);
ioWriteLine(write, strNew("{\"out\":2}"));
ioWriteFlush(write);
TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"cmd\":\"command3\",\"param\":[\"param1\"]}", "command3");
ioWriteLine(write, strNew("{\"err\":39,\"out\":\"very serious error\"}"));
ioWriteFlush(write);
// Wait for exit
TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"cmd\":\"exit\"}", "exit command");
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN()
{
// -----------------------------------------------------------------------------------------------------------------
ProtocolParallel *parallel = NULL;
TEST_ASSIGN(parallel, protocolParallelNew(2000), "create parallel");
TEST_RESULT_STR(
strPtr(protocolParallelToLog(parallel)), "{state: pending, clientTotal: 0, jobTotal: 0}", "check log");
// Add client
unsigned int clientTotal = 2;
ProtocolClient *client[HARNESS_FORK_CHILD_MAX];
for (unsigned int clientIdx = 0; clientIdx < clientTotal; clientIdx++)
{
IoRead *read = ioHandleReadIo(
ioHandleReadNew(strNewFmt("client %u read", clientIdx), HARNESS_FORK_PARENT_READ_PROCESS(clientIdx), 2000));
ioReadOpen(read);
IoWrite *write = ioHandleWriteIo(
ioHandleWriteNew(strNewFmt("client %u write", clientIdx), HARNESS_FORK_PARENT_WRITE_PROCESS(clientIdx)));
ioWriteOpen(write);
TEST_ASSIGN(
client[clientIdx],
protocolClientNew(strNewFmt("test client %u", clientIdx), strNew("test"), read, write),
"create client %u", clientIdx);
TEST_RESULT_VOID(protocolParallelClientAdd(parallel, client[clientIdx]), "add client %u", clientIdx);
}
// Attempt to add client without handle io
String *protocolString = strNew(
"{\"name\":\"pgBackRest\",\"service\":\"error\",\"version\":\"" PROJECT_VERSION "\"}\n"
"{}\n");
IoRead *read = ioBufferReadIo(ioBufferReadNew(bufNewStr(protocolString)));
ioReadOpen(read);
IoWrite *write = ioBufferWriteIo(ioBufferWriteNew(bufNew(1024)));
ioWriteOpen(write);
ProtocolClient *clientError = protocolClientNew(strNew("error"), strNew("error"), read, write);
TEST_ERROR(protocolParallelClientAdd(parallel, clientError), AssertError, "client with read handle is required");
protocolClientFree(clientError);
// Add jobs
ProtocolCommand *command = protocolCommandNew(strNew("command1"));
protocolCommandParamAdd(command, varNewStr(strNew("param1")));
protocolCommandParamAdd(command, varNewStr(strNew("param2")));
TEST_RESULT_VOID(
protocolParallelJobAdd(parallel, protocolParallelJobNew(varNewStr(strNew("job1")), command)), "add job");
command = protocolCommandNew(strNew("command2"));
protocolCommandParamAdd(command, varNewStr(strNew("param1")));
TEST_RESULT_VOID(
protocolParallelJobAdd(parallel, protocolParallelJobNew(varNewStr(strNew("job2")), command)), "add job");
command = protocolCommandNew(strNew("command3"));
protocolCommandParamAdd(command, varNewStr(strNew("param1")));
TEST_RESULT_VOID(
protocolParallelJobAdd(parallel, protocolParallelJobNew(varNewStr(strNew("job3")), command)), "add job");
// Process jobs
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
// Process jobs
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
TEST_RESULT_STR(strPtr(varStr(protocolParallelJobKey(job))), "job2", "check key is job2");
TEST_RESULT_INT(varIntForce(protocolParallelJobResult(job)), 2, "check result is 2");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
// Process jobs
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
TEST_RESULT_STR(strPtr(varStr(protocolParallelJobKey(job))), "job3", "check key is job3");
TEST_RESULT_INT(protocolParallelJobErrorCode(job), 39, "check error code");
TEST_RESULT_STR(
strPtr(protocolParallelJobErrorMessage(job)), "raised from test client 1: very serious error",
"check error message");
TEST_RESULT_PTR(protocolParallelJobResult(job), NULL, "check result is null");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
// Process jobs
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
// Process jobs
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
TEST_RESULT_STR(strPtr(varStr(protocolParallelJobKey(job))), "job1", "check key is job1");
TEST_RESULT_INT(varIntForce(protocolParallelJobResult(job)), 1, "check result is 1");
TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check done");
// Free client
for (unsigned int clientIdx = 0; clientIdx < clientTotal; clientIdx++)
TEST_RESULT_VOID(protocolClientFree(client[clientIdx]), "free client %u", clientIdx);
// Free parallel
TEST_RESULT_VOID(protocolParallelFree(parallel), "free parallel");
TEST_RESULT_VOID(protocolParallelFree(NULL), "free null parallel");
}
HARNESS_FORK_PARENT_END();
}
HARNESS_FORK_END();
}
// *****************************************************************************************************************************
if (testBegin("protocolGet()"))
{