db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-09 00:45:49 +02:00
Code Issues Releases Activity

Protocol command multiplexing.

Browse Source 
Previously it was not possible to read or write two files at the same time on the same remote because the protocol was entirely taken over by the read or write command. Multiple reads are required to make restores efficient when a list of bundled files is being read but blocks need to be retrieved from a separate file or a different part of the same file.

Improve that situation with sessions that allow related commands to be run with shared state. Also break read/write into separate requests (rather than pushing all data at once) so they can be multiplexed.

The disadvantage for read/write is that they now require more back and forth to transfer a file. This is mitigated by sending asynchronous read/write requests to keep both server and client as busy as possible. Reads that can fit into a single buffer are optimized to transfer in a single command. Reads that transfer the entire file can also skip the close command since it is implicit on end-of-file.

These changes allow the protocol to be simplified to provide one response per request, which makes the data end message obsolete. Any data sent for the request is now added to the parameters so no data needs to be sent separately to the server outside the request parameters.

Also update the Db protocol to use the new sessions. Previously this code had tracked its own sessions.
This commit is contained in:
David Steele
2024-07-22 11:48:32 +07:00
committed by GitHub
parent e7f4e8d800
commit df8cbc91c3
50 changed files with 1811 additions and 1254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
doc/xml/release/2024/2.54.xml
View File

@ -1,2 +1,16 @@
<release date="XXXX-XX-XX" version="2.54dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-development-list>
<release-item>
<github-pull-request id="2108"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="david.christensen"/>
</release-item-contributor-list>
<p>Protocol command multiplexing.</p>
</release-item>
</release-development-list>
</release-core-list>
</release>

1
src/Makefile.in
View File

@ -169,7 +169,6 @@ SRCS = \
postgres/interface/crc32.c \
postgres/interface/page.c \
protocol/client.c \
protocol/command.c \
protocol/helper.c \
protocol/parallel.c \
protocol/parallelJob.c \

5
src/command/archive/get/get.c
View File

@ -878,8 +878,7 @@ archiveGetAsyncCallback(void *const data, const unsigned int clientIdx)
const ArchiveFileMap *const archiveFileMap = lstGet(jobData->archiveFileMapList, jobData->archiveFileIdx);
jobData->archiveFileIdx++;
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_GET_FILE);
PackWrite *const param = protocolCommandParam(command);
PackWrite *const param = protocolPackNew();
pckWriteStrP(param, archiveFileMap->request);
@ -897,7 +896,7 @@ archiveGetAsyncCallback(void *const data, const unsigned int clientIdx)
MEM_CONTEXT_PRIOR_BEGIN()
{
result = protocolParallelJobNew(VARSTR(archiveFileMap->request), command);
result = protocolParallelJobNew(VARSTR(archiveFileMap->request), PROTOCOL_COMMAND_ARCHIVE_GET_FILE, param);
}
MEM_CONTEXT_PRIOR_END();
}

19
src/command/archive/get/protocol.c
View File

@ -14,16 +14,16 @@ Archive Get Protocol Handler
#include "storage/write.intern.h"
/**********************************************************************************************************************************/
FN_EXTERN void
archiveGetFileProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
archiveGetFileProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
@ -50,14 +50,11 @@ archiveGetFileProtocol(PackRead *const param, ProtocolServer *const server)
strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s." STORAGE_FILE_TEMP_EXT, strZ(request)));
// Return result
PackWrite *const resultPack = protocolPackNew();
pckWriteU32P(resultPack, fileResult.actualIdx);
pckWriteStrLstP(resultPack, fileResult.warnList);
protocolServerDataPut(server, resultPack);
protocolServerDataEndPut(server);
PackWrite *const data = protocolServerResultData(result);
pckWriteU32P(data, fileResult.actualIdx);
pckWriteStrLstP(data, fileResult.warnList);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}

4
src/command/archive/get/protocol.h
View File

@ -11,7 +11,7 @@ Archive Get Protocol Handler
Functions
***********************************************************************************************************************************/
// Process protocol requests
FN_EXTERN void archiveGetFileProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN ProtocolServerResult *archiveGetFileProtocol(PackRead *param);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
#define PROTOCOL_COMMAND_ARCHIVE_GET_FILE STRID5("ag-f", 0x36ce10)
#define PROTOCOL_SERVER_HANDLER_ARCHIVE_GET_LIST \
{.command = PROTOCOL_COMMAND_ARCHIVE_GET_FILE, .handler = archiveGetFileProtocol},
{.command = PROTOCOL_COMMAND_ARCHIVE_GET_FILE, .process = archiveGetFileProtocol},
#endif

13
src/command/archive/push/protocol.c
View File

@ -13,16 +13,16 @@ Archive Push Protocol Handler
#include "storage/helper.h"
/**********************************************************************************************************************************/
FN_EXTERN void
archivePushFileProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
archivePushFileProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
@ -63,10 +63,9 @@ archivePushFileProtocol(PackRead *const param, ProtocolServer *const server)
priorErrorList);
// Return result
protocolServerDataPut(server, pckWriteStrLstP(protocolPackNew(), fileResult.warnList));
protocolServerDataEndPut(server);
pckWriteStrLstP(protocolServerResultData(result), fileResult.warnList);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}

4
src/command/archive/push/protocol.h
View File

@ -11,7 +11,7 @@ Archive Push Protocol Handler
Functions
***********************************************************************************************************************************/
// Process protocol requests
FN_EXTERN void archivePushFileProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN ProtocolServerResult *archivePushFileProtocol(PackRead *param);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
#define PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE STRID5("ap-f", 0x36e010)
#define PROTOCOL_SERVER_HANDLER_ARCHIVE_PUSH_LIST \
{.command = PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE, .handler = archivePushFileProtocol},
{.command = PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE, .process = archivePushFileProtocol},
#endif

5
src/command/archive/push/push.c
View File

@ -464,8 +464,7 @@ archivePushAsyncCallback(void *const data, const unsigned int clientIdx)
const String *const walFile = strLstGet(jobData->walFileList, jobData->walFileIdx);
jobData->walFileIdx++;
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE);
PackWrite *const param = protocolCommandParam(command);
PackWrite *const param = protocolPackNew();
pckWriteStrP(param, strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile)));
pckWriteBoolP(param, cfgOptionBool(cfgOptArchiveHeaderCheck));
@ -496,7 +495,7 @@ archivePushAsyncCallback(void *const data, const unsigned int clientIdx)
MEM_CONTEXT_PRIOR_BEGIN()
{
result = protocolParallelJobNew(VARSTR(walFile), command);
result = protocolParallelJobNew(VARSTR(walFile), PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE, param);
}
MEM_CONTEXT_PRIOR_END();
}

6
src/command/backup/backup.c
View File

@ -1951,7 +1951,6 @@ backupJobCallback(void *const data, const unsigned int clientIdx)
const int queueEnd = queueIdx;
// Create backup job
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
PackWrite *param = NULL;
uint64_t fileTotal = 0;
uint64_t fileSize = 0;
@ -1980,7 +1979,7 @@ backupJobCallback(void *const data, const unsigned int clientIdx)
// Add common parameters before first file
if (param == NULL)
{
param = protocolCommandParam(command);
param = protocolPackNew();
if (bundle && file.size <= jobData->bundleLimit)
{
@ -2068,7 +2067,8 @@ backupJobCallback(void *const data, const unsigned int clientIdx)
// Assign job to result
MEM_CONTEXT_PRIOR_BEGIN()
{
result = protocolParallelJobNew(bundle ? VARUINT64(jobData->bundleId) : VARSTR(fileName), command);
result = protocolParallelJobNew(
bundle ? VARUINT64(jobData->bundleId) : VARSTR(fileName), PROTOCOL_COMMAND_BACKUP_FILE, param);
if (bundle)
jobData->bundleId++;

41
src/command/backup/protocol.c
View File

@ -15,16 +15,16 @@ Backup Protocol Handler
#include "storage/helper.h"
/**********************************************************************************************************************************/
FN_EXTERN void
backupFileProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
backupFileProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
@ -79,37 +79,34 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
}
// Backup file
const List *const result = backupFile(
const List *const resultList = backupFile(
repoFile, bundleId, bundleRaw, blockIncrReference, repoFileCompressType, repoFileCompressLevel, cipherType, cipherPass,
pgVersionForce, pageSize, fileList);
// Return result
PackWrite *const resultPack = protocolPackNew();
PackWrite *const data = protocolServerResultData(result);
for (unsigned int resultIdx = 0; resultIdx < lstSize(result); resultIdx++)
for (unsigned int resultIdx = 0; resultIdx < lstSize(resultList); resultIdx++)
{
const BackupFileResult *const fileResult = lstGet(result, resultIdx);
const BackupFileResult *const fileResult = lstGet(resultList, resultIdx);
ASSERT(
fileResult->backupCopyResult == backupCopyResultSkip || fileResult->copySize != 0 ||
bufEq(fileResult->copyChecksum, HASH_TYPE_SHA1_ZERO_BUF));
pckWriteStrP(resultPack, fileResult->manifestFile);
pckWriteU32P(resultPack, fileResult->backupCopyResult);
pckWriteBoolP(resultPack, fileResult->repoInvalid);
pckWriteU64P(resultPack, fileResult->copySize);
pckWriteU64P(resultPack, fileResult->bundleOffset);
pckWriteU64P(resultPack, fileResult->blockIncrMapSize);
pckWriteU64P(resultPack, fileResult->repoSize);
pckWriteBinP(resultPack, fileResult->copyChecksum);
pckWriteBinP(resultPack, fileResult->repoChecksum);
pckWritePackP(resultPack, fileResult->pageChecksumResult);
pckWriteStrP(data, fileResult->manifestFile);
pckWriteU32P(data, fileResult->backupCopyResult);
pckWriteBoolP(data, fileResult->repoInvalid);
pckWriteU64P(data, fileResult->copySize);
pckWriteU64P(data, fileResult->bundleOffset);
pckWriteU64P(data, fileResult->blockIncrMapSize);
pckWriteU64P(data, fileResult->repoSize);
pckWriteBinP(data, fileResult->copyChecksum);
pckWriteBinP(data, fileResult->repoChecksum);
pckWritePackP(data, fileResult->pageChecksumResult);
}
protocolServerDataPut(server, resultPack);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}

4
src/command/backup/protocol.h
View File

@ -11,7 +11,7 @@ Backup Protocol Handler
Functions
***********************************************************************************************************************************/
// Process protocol requests
FN_EXTERN void backupFileProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN ProtocolServerResult *backupFileProtocol(PackRead *param);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
#define PROTOCOL_COMMAND_BACKUP_FILE STRID5("bp-f", 0x36e020)
#define PROTOCOL_SERVER_HANDLER_BACKUP_LIST \
{.command = PROTOCOL_COMMAND_BACKUP_FILE, .handler = backupFileProtocol},
{.command = PROTOCOL_COMMAND_BACKUP_FILE, .process = backupFileProtocol},
#endif

4
src/command/remote/remote.c
View File

@ -66,7 +66,7 @@ cmdRemote(ProtocolServer *const server)
TRY_BEGIN()
{
// Get the command. No need to check parameters since we know this is the first noop.
CHECK(FormatError, protocolServerCommandGet(server).id == PROTOCOL_COMMAND_NOOP, "noop expected");
CHECK(FormatError, protocolServerRequest(server).id == PROTOCOL_COMMAND_NOOP, "noop expected");
// Only try the lock if this is process 0, i.e. the remote started from the main process
if (cfgOptionUInt(cfgOptProcess) == 0)
@ -83,7 +83,7 @@ cmdRemote(ProtocolServer *const server)
}
// Notify the client of success
protocolServerDataEndPut(server);
protocolServerResponseP(server);
success = true;
}
CATCH_ANY()

27
src/command/restore/protocol.c
View File

@ -13,16 +13,16 @@ Restore Protocol Handler
#include "storage/helper.h"
/**********************************************************************************************************************************/
FN_EXTERN void
restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
restoreFileProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
@ -72,26 +72,23 @@ restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
}
// Restore files
const List *const result = restoreFile(
const List *const resultList = restoreFile(
repoFile, repoIdx, repoFileCompressType, copyTimeBegin, delta, deltaForce, bundleRaw, cipherPass, referenceList,
fileList);
// Return result
PackWrite *const resultPack = protocolPackNew();
PackWrite *const data = protocolServerResultData(result);
for (unsigned int resultIdx = 0; resultIdx < lstSize(result); resultIdx++)
for (unsigned int resultIdx = 0; resultIdx < lstSize(resultList); resultIdx++)
{
const RestoreFileResult *const fileResult = lstGet(result, resultIdx);
const RestoreFileResult *const fileResult = lstGet(resultList, resultIdx);
pckWriteStrP(resultPack, fileResult->manifestFile);
pckWriteU32P(resultPack, fileResult->result);
pckWriteU64P(resultPack, fileResult->blockIncrDeltaSize);
pckWriteStrP(data, fileResult->manifestFile);
pckWriteU32P(data, fileResult->result);
pckWriteU64P(data, fileResult->blockIncrDeltaSize);
}
protocolServerDataPut(server, resultPack);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}

4
src/command/restore/protocol.h
View File

@ -11,7 +11,7 @@ Restore Protocol Handler
Functions
***********************************************************************************************************************************/
// Process protocol requests
FN_EXTERN void restoreFileProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN ProtocolServerResult *restoreFileProtocol(PackRead *param);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
#define PROTOCOL_COMMAND_RESTORE_FILE STRID5("rs-f", 0x36e720)
#define PROTOCOL_SERVER_HANDLER_RESTORE_LIST \
{.command = PROTOCOL_COMMAND_RESTORE_FILE, .handler = restoreFileProtocol},
{.command = PROTOCOL_COMMAND_RESTORE_FILE, .process = restoreFileProtocol},
#endif

6
src/command/restore/restore.c
View File

@ -2309,7 +2309,6 @@ restoreJobCallback(void *const data, const unsigned int clientIdx)
RestoreJobData *const jobData = data;
// Determine where to begin scanning the queue (we'll stop when we get back here)
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
PackWrite *param = NULL;
int queueIdx = (int)(clientIdx % lstSize(jobData->queueList));
const int queueEnd = queueIdx;
@ -2334,7 +2333,7 @@ restoreJobCallback(void *const data, const unsigned int clientIdx)
// Add common parameters before first file
if (param == NULL)
{
param = protocolCommandParam(command);
param = protocolPackNew();
if (file.bundleId != 0)
{
@ -2414,7 +2413,8 @@ restoreJobCallback(void *const data, const unsigned int clientIdx)
// Assign job to result
MEM_CONTEXT_PRIOR_BEGIN()
{
result = protocolParallelJobNew(bundleId != 0 ? VARUINT64(bundleId) : VARSTR(fileName), command);
result = protocolParallelJobNew(
bundleId != 0 ? VARUINT64(bundleId) : VARSTR(fileName), PROTOCOL_COMMAND_RESTORE_FILE, param);
}
MEM_CONTEXT_PRIOR_END();

17
src/command/verify/protocol.c
View File

@ -13,16 +13,16 @@ Verify Protocol Handler
#include "storage/helper.h"
/**********************************************************************************************************************************/
FN_EXTERN void
verifyFileProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
verifyFileProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
@ -43,13 +43,12 @@ verifyFileProtocol(PackRead *const param, ProtocolServer *const server)
const uint64_t fileSize = pckReadU64P(param);
const String *const cipherPass = pckReadStrP(param);
const VerifyResult result = verifyFile(filePathName, offset, limit, compressType, fileChecksum, fileSize, cipherPass);
// Return result
protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), result));
protocolServerDataEndPut(server);
pckWriteU32P(
protocolServerResultData(result),
verifyFile(filePathName, offset, limit, compressType, fileChecksum, fileSize, cipherPass));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}

4
src/command/verify/protocol.h
View File

@ -11,7 +11,7 @@ Verify Protocol Handler
Functions
***********************************************************************************************************************************/
// Process protocol requests
FN_EXTERN void verifyFileProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN ProtocolServerResult *verifyFileProtocol(PackRead *param);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
#define PROTOCOL_COMMAND_VERIFY_FILE STRID5("vf-f", 0x36cd60)
#define PROTOCOL_SERVER_HANDLER_VERIFY_LIST \
{.command = PROTOCOL_COMMAND_VERIFY_FILE, .handler = verifyFileProtocol},
{.command = PROTOCOL_COMMAND_VERIFY_FILE, .process = verifyFileProtocol},
#endif

11
src/command/verify/verify.c
View File

@ -754,8 +754,7 @@ verifyArchive(VerifyJobData *const jobData)
encodingHex, strSubN(fileName, WAL_SEGMENT_NAME_SIZE + 1, HASH_TYPE_SHA1_SIZE_HEX));
// Set up the job
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
PackWrite *const param = protocolCommandParam(command);
PackWrite *const param = protocolPackNew();
pckWriteStrP(param, filePathName);
pckWriteBoolP(param, false);
@ -769,7 +768,7 @@ verifyArchive(VerifyJobData *const jobData)
MEM_CONTEXT_PRIOR_BEGIN()
{
result = protocolParallelJobNew(VARSTR(jobKey), command);
result = protocolParallelJobNew(VARSTR(jobKey), PROTOCOL_COMMAND_VERIFY_FILE, param);
}
MEM_CONTEXT_PRIOR_END();
@ -980,8 +979,8 @@ verifyBackup(VerifyJobData *const jobData)
if (fileBackupLabel != NULL)
{
// Set up the job
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
PackWrite *const param = protocolCommandParam(command);
PackWrite *const param = protocolPackNew();
const String *const filePathName = backupFileRepoPathP(
fileBackupLabel, .manifestName = fileData.name, .bundleId = fileData.bundleId,
.compressType = manifestData(jobData->manifest)->backupOptionCompressType,
@ -1021,7 +1020,7 @@ verifyBackup(VerifyJobData *const jobData)
MEM_CONTEXT_PRIOR_BEGIN()
{
result = protocolParallelJobNew(VARSTR(jobKey), command);
result = protocolParallelJobNew(VARSTR(jobKey), PROTOCOL_COMMAND_VERIFY_FILE, param);
}
MEM_CONTEXT_PRIOR_END();
}

19
src/config/protocol.c
View File

@ -13,16 +13,16 @@ Configuration Protocol Handler
#include "config/protocol.h"
/**********************************************************************************************************************************/
FN_EXTERN void
configOptionProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
configOptionProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
@ -36,12 +36,11 @@ configOptionProtocol(PackRead *const param, ProtocolServer *const server)
varLstAdd(optionList, varDup(cfgOptionIdxVar(option.id, cfgOptionKeyToIdx(option.id, option.keyIdx + 1))));
}
protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), jsonFromVar(varNewVarLst(optionList))));
protocolServerDataEndPut(server);
pckWriteStrP(protocolServerResultData(result), jsonFromVar(varNewVarLst(optionList)));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
/**********************************************************************************************************************************/
@ -57,13 +56,13 @@ configOptionRemote(ProtocolClient *const client, const VariantList *const paramL
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_CONFIG_OPTION);
PackWrite *const param = protocolCommandParam(command);
PackWrite *const param = protocolPackNew();
for (unsigned int paramIdx = 0; paramIdx < varLstSize(paramList); paramIdx++)
pckWriteStrP(param, varStr(varLstGet(paramList, paramIdx)));
const VariantList *const list = varVarLst(jsonToVar(pckReadStrP(protocolClientExecute(client, command, true))));
const VariantList *const list = varVarLst(
jsonToVar(pckReadStrP(protocolClientRequestP(client, PROTOCOL_COMMAND_CONFIG_OPTION, .param = param))));
MEM_CONTEXT_PRIOR_BEGIN()
{

4
src/config/protocol.h
View File

@ -12,7 +12,7 @@ Configuration Protocol Handler
Functions
***********************************************************************************************************************************/
// Process config protocol requests
FN_EXTERN void configOptionProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN ProtocolServerResult *configOptionProtocol(PackRead *param);
// Get option values from a remote process
FN_EXTERN VariantList *configOptionRemote(ProtocolClient *client, const VariantList *paramList);
@ -23,6 +23,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
#define PROTOCOL_COMMAND_CONFIG_OPTION STRID5("opt-g", 0x7dd20f0)
#define PROTOCOL_SERVER_HANDLER_OPTION_LIST \
{.command = PROTOCOL_COMMAND_CONFIG_OPTION, .handler = configOptionProtocol},
{.command = PROTOCOL_COMMAND_CONFIG_OPTION, .process = configOptionProtocol},
#endif

18
src/db/db.c
View File

@ -30,7 +30,7 @@ struct Db
DbPub pub; // Publicly accessible variables
PgClient *client; // Local PostgreSQL client
ProtocolClient *remoteClient; // Protocol client for remote db queries
unsigned int remoteIdx; // Index provided by the remote on open for subsequent calls
ProtocolClientSession *session; // Protocol session for requests
const Storage *storage; // PostgreSQL storage
const String *applicationName; // Used to identify this connection in PostgreSQL
time_t pingTimeLast; // Last time cluster was pinged
@ -50,11 +50,7 @@ dbFreeResource(THIS_VOID)
ASSERT(this != NULL);
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_CLOSE);
pckWriteU32P(protocolCommandParam(command), this->remoteIdx);
protocolClientExecute(this->remoteClient, command, false);
protocolCommandFree(command);
protocolClientSessionFree(this->session);
FUNCTION_LOG_RETURN_VOID();
}
@ -83,6 +79,7 @@ dbNew(PgClient *const client, ProtocolClient *const remoteClient, const Storage
.memContext = memContextCurrent(),
},
.remoteClient = remoteClient,
.session = remoteClient != NULL ? protocolClientSessionNewP(remoteClient, PROTOCOL_COMMAND_DB) : NULL,
.storage = storage,
.applicationName = strDup(applicationName),
};
@ -117,14 +114,12 @@ dbQuery(Db *const this, const PgClientQueryResult resultType, const String *cons
{
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_DB_QUERY);
PackWrite *const param = protocolCommandParam(command);
PackWrite *const param = protocolPackNew();
pckWriteU32P(param, this->remoteIdx);
pckWriteStrIdP(param, resultType);
pckWriteStrP(param, query);
PackRead *const read = protocolClientExecute(this->remoteClient, command, true);
PackRead *const read = protocolClientSessionRequestP(this->session, .param = param);
MEM_CONTEXT_PRIOR_BEGIN()
{
@ -240,8 +235,7 @@ dbOpen(Db *const this)
// Open the connection
if (this->remoteClient != NULL)
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_OPEN);
this->remoteIdx = pckReadU32P(protocolClientExecute(this->remoteClient, command, true));
protocolClientSessionOpenP(this->session);
// Set a callback to notify the remote when a connection is closed
memContextCallbackSet(this->pub.memContext, dbFreeResource, this);

81
src/db/protocol.c
View File

@ -14,104 +14,57 @@ Db Protocol Handler
#include "postgres/client.h"
#include "postgres/interface.h"
/***********************************************************************************************************************************
Local variables
***********************************************************************************************************************************/
static struct
{
List *pgClientList; // List of db objects
} dbProtocolLocal;
/**********************************************************************************************************************************/
FN_EXTERN void
dbOpenProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
dbOpenProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param == NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
// If the db list does not exist then create it in the top context
if (dbProtocolLocal.pgClientList == NULL)
{
MEM_CONTEXT_BEGIN(memContextTop())
{
dbProtocolLocal.pgClientList = lstNewP(sizeof(PgClient *));
}
MEM_CONTEXT_END();
}
// Add db to the list
MEM_CONTEXT_BEGIN(lstMemContext(dbProtocolLocal.pgClientList))
{
// Only a single db is passed to the remote
PgClient *const pgClient = pgClientNew(
cfgOptionStrNull(cfgOptPgSocketPath), cfgOptionUInt(cfgOptPgPort), cfgOptionStr(cfgOptPgDatabase),
cfgOptionStrNull(cfgOptPgUser), cfgOptionUInt64(cfgOptDbTimeout));
pgClientOpen(pgClient);
lstAdd(dbProtocolLocal.pgClientList, &pgClient);
}
MEM_CONTEXT_END();
// Return db index which should be included in subsequent calls
protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), lstSize(dbProtocolLocal.pgClientList) - 1));
protocolServerDataEndPut(server);
// Set session data
protocolServerResultSessionDataSet(result, pgClient);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
dbQueryProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
dbQueryProtocol(PackRead *const param, void *const pgClient)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_PARAM(PG_CLIENT, pgClient);
FUNCTION_LOG_END();
FUNCTION_AUDIT_STRUCT();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(pgClient != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
PgClient *const pgClient = *(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param));
const PgClientQueryResult resultType = (PgClientQueryResult)pckReadStrIdP(param);
const String *const query = pckReadStrP(param);
protocolServerDataPut(server, pckWritePackP(protocolPackNew(), pgClientQuery(pgClient, query, resultType)));
protocolServerDataEndPut(server);
pckWritePackP(protocolServerResultData(result), pgClientQuery(pgClient, query, resultType));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
dbCloseProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
pgClientClose(*(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param)));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}

13
src/db/protocol.h
View File

@ -11,20 +11,15 @@ Db Protocol Handler
Functions
***********************************************************************************************************************************/
// Process db protocol requests
FN_EXTERN void dbOpenProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void dbQueryProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void dbCloseProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN ProtocolServerResult *dbOpenProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *dbQueryProtocol(PackRead *param, void *sessionData);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
***********************************************************************************************************************************/
#define PROTOCOL_COMMAND_DB_OPEN STRID5("db-o", 0x7ec440)
#define PROTOCOL_COMMAND_DB_QUERY STRID5("db-q", 0x8ec440)
#define PROTOCOL_COMMAND_DB_CLOSE STRID5("db-c", 0x1ec440)
#define PROTOCOL_COMMAND_DB STRID5("db", 0x440)
#define PROTOCOL_SERVER_HANDLER_DB_LIST \
{.command = PROTOCOL_COMMAND_DB_OPEN, .handler = dbOpenProtocol}, \
{.command = PROTOCOL_COMMAND_DB_QUERY, .handler = dbQueryProtocol}, \
{.command = PROTOCOL_COMMAND_DB_CLOSE, .handler = dbCloseProtocol},
{.command = PROTOCOL_COMMAND_DB, .open = dbOpenProtocol, .processSession = dbQueryProtocol},
#endif

1
src/meson.build
View File

@ -238,7 +238,6 @@ src_pgbackrest = [
'postgres/interface/crc32.c',
'postgres/interface/page.c',
'protocol/client.c',
'protocol/command.c',
'protocol/helper.c',
'protocol/parallel.c',
'protocol/parallelJob.c',

20
src/postgres/client.c
View File

@ -354,26 +354,6 @@ pgClientQuery(PgClient *const this, const String *const query, const PgClientQue
FUNCTION_LOG_RETURN(PACK, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
pgClientClose(PgClient *const this)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PG_CLIENT, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
if (this->connection != NULL)
{
memContextCallbackClear(objMemContext(this));
PQfinish(this->connection);
this->connection = NULL;
}
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
pgClientToLog(const PgClient *const this, StringStatic *const debugLog)

3
src/postgres/client.h
View File

@ -98,9 +98,6 @@ pgClientMove(PgClient *const this, MemContext *const parentNew)
// Execute a query and return results
FN_EXTERN Pack *pgClientQuery(PgClient *this, const String *query, PgClientQueryResult resultType);
// Close connection to PostgreSQL
FN_EXTERN void pgClientClose(PgClient *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/

679
src/protocol/client.c
View File

@ -17,20 +17,14 @@ Client state enum
***********************************************************************************************************************************/
typedef enum
{
// Client is waiting for a command
// Client is waiting for a response / ready to send a request
protocolClientStateIdle = STRID5("idle", 0x2b0890),
// Command put is in progress
protocolClientStateCommandPut = STRID5("cmd-put", 0x52b0d91a30),
// Request is in progress
protocolClientStateRequest = STRID5("request", 0x5265ac4b20),
// Waiting for command data from server. Only used when dataPut is true in protocolClientCommandPut().
protocolClientStateCommandDataGet = STRID5("cmd-data-get", 0xa14fb0d024d91a30),
// Putting data to server. Only used when dataPut is true in protocolClientCommandPut().
protocolClientStateDataPut = STRID5("data-put", 0xa561b0d0240),
// Getting data from server
protocolClientStateDataGet = STRID5("data-get", 0xa14fb0d0240),
// Response is in progress
protocolClientStateResponse = STRID5("response", 0x2cdcf84cb20),
} ProtocolClientState;
/***********************************************************************************************************************************
@ -44,8 +38,246 @@ struct ProtocolClient
const String *name; // Name displayed in logging
const String *errorPrefix; // Prefix used when throwing error
TimeMSec keepAliveTime; // Last time data was put to the server
uint64_t sessionTotal; // Total sessions (used to generate session ids)
List *sessionList; // Session list
};
struct ProtocolClientSession
{
ProtocolClientSessionPub pub; // Publicly accessible variables
ProtocolClient *client; // Protocol client
StringId command; // Command
uint64_t sessionId; // Session id
bool async; // Async requests allowed?
// Stored message (read by another session and stored for later retrieval)
bool stored; // Is a message currently stored?
bool close; // Should the session be closed?
ProtocolMessageType type; // Type of last message received
PackRead *packRead; // Last message received (if any)
};
/***********************************************************************************************************************************
Check protocol state
***********************************************************************************************************************************/
static void
protocolClientStateExpectIdle(const ProtocolClient *const this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_TEST_END();
if (this->state != protocolClientStateIdle)
{
THROW_FMT(
ProtocolError, "client state is '%s' but expected '%s'", strZ(strIdToStr(this->state)),
strZ(strIdToStr(protocolClientStateIdle)));
}
FUNCTION_TEST_RETURN_VOID();
}
/***********************************************************************************************************************************
Find a client session
***********************************************************************************************************************************/
static unsigned int
protocolClientSessionFindIdx(const ProtocolClient *const this, const uint64_t sessionId)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_TEST_PARAM(UINT64, sessionId);
FUNCTION_TEST_END();
unsigned int result = 0;
for (; result < lstSize(this->sessionList); result++)
{
if ((*(ProtocolClientSession **)lstGet(this->sessionList, result))->sessionId == sessionId)
break;
}
CHECK_FMT(ProtocolError, result < lstSize(this->sessionList), "unable to find protocol client session %" PRIu64, sessionId);
FUNCTION_TEST_RETURN(UINT, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolClientRequestInternal(ProtocolClientSession *const this, const ProtocolCommandType type, PackWrite *const param)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
FUNCTION_LOG_PARAM(STRING_ID, type);
FUNCTION_LOG_PARAM(PACK_WRITE, param);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(!protocolClientSessionQueued(this));
// Expect idle state before request
protocolClientStateExpectIdle(this->client);
MEM_CONTEXT_TEMP_BEGIN()
{
// Switch state to request
this->client->state = protocolClientStateRequest;
// Write request
PackWrite *const request = pckWriteNewIo(this->client->write);
pckWriteU32P(request, protocolMessageTypeRequest, .defaultWrite = true);
pckWriteStrIdP(request, this->command);
pckWriteStrIdP(request, type);
pckWriteU64P(request, this->sessionId);
pckWriteBoolP(request, this->pub.open);
// Write request parameters
if (param != NULL)
{
pckWriteEndP(param);
pckWritePackP(request, pckWriteResult(param));
}
pckWriteEndP(request);
// Flush to send request immediately
ioWriteFlush(this->client->write);
this->pub.queued = true;
// Switch state back to idle after successful request
this->client->state = protocolClientStateIdle;
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Process errors
***********************************************************************************************************************************/
static void
protocolClientError(ProtocolClient *const this, const ProtocolMessageType type, PackRead *const error)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(ENUM, type);
FUNCTION_LOG_PARAM(PACK_READ, error);
FUNCTION_LOG_END();
ASSERT(this != NULL);
if (type == protocolMessageTypeError)
{
ASSERT(error != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
const ErrorType *const type = errorTypeFromCode(pckReadI32P(error));
const String *const message = strNewFmt("%s: %s", strZ(this->errorPrefix), strZ(pckReadStrP(error)));
const String *const stack = pckReadStrP(error);
pckReadEndP(error);
CHECK(FormatError, message != NULL && stack != NULL, "invalid error data");
errorInternalThrow(type, __FILE__, __func__, __LINE__, strZ(message), strZ(stack));
}
MEM_CONTEXT_TEMP_END();
}
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
static PackRead *
protocolClientResponseInternal(ProtocolClientSession *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
// Expect data-get state before data get
protocolClientStateExpectIdle(this->client);
PackRead *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
// Check the session for a stored response
bool found = false;
ProtocolMessageType type = protocolMessageTypeResponse;
bool close = false;
PackRead *packRead = NULL;
if (this->stored)
{
found = true;
type = this->type;
close = this->close;
packRead = pckReadMove(this->packRead, memContextCurrent());
this->stored = false;
}
// If not stored then read from protocol
while (!found)
{
// Switch state to response
this->client->state = protocolClientStateResponse;
// Read response
PackRead *const response = pckReadNewIo(this->client->pub.read);
const uint64_t sessionId = pckReadU64P(response);
type = (ProtocolMessageType)pckReadU32P(response);
close = pckReadBoolP(response);
packRead = pckReadPackReadP(response);
pckReadEndP(response);
// If this response is for another session then store it with that session. Session id 0 indicates a fatal error on the
// server that should be reported by the first session that sees it.
ASSERT(sessionId != 0 || type == protocolMessageTypeError);
if (sessionId != 0 && sessionId != this->sessionId)
{
ProtocolClientSession *const sessionOther = *(ProtocolClientSession **)lstGet(
this->client->sessionList, protocolClientSessionFindIdx(this->client, sessionId));
CHECK_FMT(
ProtocolError, !sessionOther->stored, "protocol client session %" PRIu64 " already has a stored response",
sessionOther->sessionId);
sessionOther->stored = true;
sessionOther->type = type;
sessionOther->close = close;
sessionOther->packRead = objMove(packRead, objMemContext(sessionOther));
}
else
found = true;
// Switch state back to idle after successful response
this->client->state = protocolClientStateIdle;
}
// The result is no longer queued -- either it will generate an error or be returned to the user
this->pub.queued = false;
// Close the session if requested
if (close)
this->pub.open = false;
// Check if this result is an error and if so throw it
protocolClientError(this->client, type, packRead);
CHECK(FormatError, type == protocolMessageTypeResponse, "expected response message");
// Return result the the caller
result = objMove(packRead, memContextPrior());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/***********************************************************************************************************************************
Close protocol connection
***********************************************************************************************************************************/
@ -60,13 +292,18 @@ protocolClientFreeResource(THIS_VOID)
ASSERT(this != NULL);
// Switch state to idle so the command is sent no matter the current state
this->state = protocolClientStateIdle;
// Stop client sessions from sending cancel requests
for (unsigned int sessionIdx = 0; sessionIdx < lstSize(this->sessionList); sessionIdx++)
memContextCallbackClear(objMemContext(*(ProtocolClientSession **)lstGet(this->sessionList, sessionIdx)));
// Send an exit command but don't wait to see if it succeeds
// Send an exit request but don't wait to see if it succeeds
MEM_CONTEXT_TEMP_BEGIN()
{
protocolClientCommandPut(this, protocolCommandNew(PROTOCOL_COMMAND_EXIT), false);
// Switch state to idle so the request is sent no matter the current state
this->state = protocolClientStateIdle;
// Send exit request
protocolClientRequestInternal(protocolClientSessionNewP(this, PROTOCOL_COMMAND_EXIT), protocolCommandTypeProcess, NULL);
}
MEM_CONTEXT_TEMP_END();
@ -101,6 +338,7 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
.name = strDup(name),
.errorPrefix = strNewFmt("raised from %s", strZ(name)),
.keepAliveTime = timeMSec(),
.sessionList = lstNewP(sizeof(ProtocolClientSession)),
};
// Read, parse, and check the protocol greeting
@ -153,232 +391,37 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
FUNCTION_LOG_RETURN(PROTOCOL_CLIENT, this);
}
/***********************************************************************************************************************************
Check protocol state
***********************************************************************************************************************************/
static void
protocolClientStateExpect(const ProtocolClient *const this, const ProtocolClientState expect)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_TEST_PARAM(STRING_ID, expect);
FUNCTION_TEST_END();
if (this->state != expect)
THROW_FMT(ProtocolError, "client state is '%s' but expected '%s'", strZ(strIdToStr(this->state)), strZ(strIdToStr(expect)));
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolClientDataPut(ProtocolClient *const this, PackWrite *const data)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(PACK_WRITE, data);
FUNCTION_LOG_END();
ASSERT(this != NULL);
// Expect data-put state before data put
protocolClientStateExpect(this, protocolClientStateDataPut);
MEM_CONTEXT_TEMP_BEGIN()
{
// End the pack
if (data != NULL)
pckWriteEndP(data);
// Write the data
PackWrite *dataMessage = pckWriteNewIo(this->write);
pckWriteU32P(dataMessage, protocolMessageTypeData, .defaultWrite = true);
pckWritePackP(dataMessage, pckWriteResult(data));
pckWriteEndP(dataMessage);
// Flush when there is no more data to put
if (data == NULL)
{
ioWriteFlush(this->write);
// Switch state to data-get after successful data end put
this->state = protocolClientStateDataGet;
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
// Helper to process errors
static void
protocolClientError(ProtocolClient *const this, const ProtocolMessageType type, PackRead *const error)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(ENUM, type);
FUNCTION_LOG_PARAM(PACK_READ, error);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(error != NULL);
if (type == protocolMessageTypeError)
{
MEM_CONTEXT_TEMP_BEGIN()
{
const ErrorType *const type = errorTypeFromCode(pckReadI32P(error));
const String *const message = strNewFmt("%s: %s", strZ(this->errorPrefix), strZ(pckReadStrP(error)));
const String *const stack = pckReadStrP(error);
pckReadEndP(error);
// Switch state to idle after error (server will do the same)
this->state = protocolClientStateIdle;
CHECK(FormatError, message != NULL && stack != NULL, "invalid error data");
errorInternalThrow(type, __FILE__, __func__, __LINE__, strZ(message), strZ(stack));
}
MEM_CONTEXT_TEMP_END();
}
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN PackRead *
protocolClientDataGet(ProtocolClient *const this)
protocolClientRequest(ProtocolClient *const this, const StringId command, const ProtocolClientRequestParam param)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(STRING_ID, command);
FUNCTION_LOG_PARAM(PACK_WRITE, param.param);
FUNCTION_LOG_END();
ASSERT(this != NULL);
// Expect data-get state before data get
protocolClientStateExpect(
this, this->state == protocolClientStateCommandDataGet ? protocolClientStateCommandDataGet : protocolClientStateDataGet);
PackRead *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *response = pckReadNewIo(this->pub.read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
ProtocolClientSession *const session = protocolClientSessionNewP(this, command);
protocolClientError(this, type, response);
CHECK(FormatError, type == protocolMessageTypeData, "expected data message");
protocolClientRequestInternal(session, protocolCommandTypeProcess, param.param);
MEM_CONTEXT_PRIOR_BEGIN()
{
result = pckReadPackReadP(response);
result = protocolClientResponseInternal(session);
}
MEM_CONTEXT_PRIOR_END();
pckReadEndP(response);
// Switch state to data-put after successful command data get
if (this->state == protocolClientStateCommandDataGet)
this->state = protocolClientStateDataPut;
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolClientDataEndGet(ProtocolClient *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
// Expect data-get state before data end get
protocolClientStateExpect(this, protocolClientStateDataGet);
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *response = pckReadNewIo(this->pub.read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
protocolClientError(this, type, response);
CHECK(FormatError, type == protocolMessageTypeDataEnd, "expected data end message");
pckReadEndP(response);
// Switch state to idle after successful data end get
this->state = protocolClientStateIdle;
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolClientCommandPut(ProtocolClient *const this, ProtocolCommand *const command, const bool dataPut)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
FUNCTION_LOG_PARAM(BOOL, dataPut);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(command != NULL);
// Expect idle state before command put
protocolClientStateExpect(this, protocolClientStateIdle);
// Switch state to cmd-put
this->state = protocolClientStateDataPut;
// Put command
protocolCommandPut(command, this->write);
// Switch state to data-get/data-put after successful command put
this->state = dataPut ? protocolClientStateCommandDataGet : protocolClientStateDataGet;
// Reset the keep alive time
this->keepAliveTime = timeMSec();
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN PackRead *
protocolClientExecute(ProtocolClient *const this, ProtocolCommand *const command, const bool resultRequired)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
FUNCTION_LOG_PARAM(BOOL, resultRequired);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(command != NULL);
// Put command
protocolClientCommandPut(this, command, false);
// Read result if required
PackRead *result = NULL;
if (resultRequired)
result = protocolClientDataGet(this);
// Read response
protocolClientDataEndGet(this);
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolClientNoOp(ProtocolClient *this)
@ -389,11 +432,193 @@ protocolClientNoOp(ProtocolClient *this)
ASSERT(this != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
protocolClientExecute(this, protocolCommandNew(PROTOCOL_COMMAND_NOOP), false);
protocolClientRequestP(this, PROTOCOL_COMMAND_NOOP);
FUNCTION_LOG_RETURN_VOID();
}
MEM_CONTEXT_TEMP_END();
/***********************************************************************************************************************************
Free client session
***********************************************************************************************************************************/
static void
protocolClientSessionFreeResource(THIS_VOID)
{
THIS(ProtocolClientSession);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
// If a result is queued then read it before sending cancel
if (protocolClientSessionQueued(this))
{
// Free stored result
if (this->stored)
{
pckReadFree(this->packRead);
}
// Else read and free result
else
pckReadFree(protocolClientResponseInternal(this));
}
// If open then cancel
if (this->pub.open)
{
protocolClientRequestInternal(this, protocolCommandTypeCancel, NULL);
protocolClientResponseInternal(this);
ASSERT(!this->pub.open);
}
// Remove from session list
lstRemoveIdx(this->client->sessionList, protocolClientSessionFindIdx(this->client, this->sessionId));
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN ProtocolClientSession *
protocolClientSessionNew(ProtocolClient *const client, const StringId command, const ProtocolClientSessionNewParam param)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, client);
FUNCTION_LOG_PARAM(STRING_ID, command);
FUNCTION_LOG_PARAM(BOOL, param.async);
FUNCTION_LOG_END();
ASSERT(client != NULL);
OBJ_NEW_BEGIN(ProtocolClientSession, .childQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
{
*this = (ProtocolClientSession)
{
.client = client,
.command = command,
.sessionId = ++client->sessionTotal,
.async = param.async,
};
}
OBJ_NEW_END();
// If async then the session will need to be tracked in case another session gets a result for this session
if (this->async)
{
lstAdd(this->client->sessionList, &this);
// Set a callback to cleanup the session
memContextCallbackSet(objMemContext(this), protocolClientSessionFreeResource, this);
}
FUNCTION_LOG_RETURN(PROTOCOL_CLIENT_SESSION, this);
}
/**********************************************************************************************************************************/
FN_EXTERN PackRead *
protocolClientSessionOpen(ProtocolClientSession *const this, const ProtocolClientSessionOpenParam param)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
FUNCTION_LOG_PARAM(PACK_WRITE, param.param);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(!this->pub.open);
protocolClientRequestInternal(this, protocolCommandTypeOpen, param.param);
this->pub.open = true;
// Set a callback to cleanup the session if it was not already done for async
if (!this->async)
{
lstAdd(this->client->sessionList, &this);
memContextCallbackSet(objMemContext(this), protocolClientSessionFreeResource, this);
}
FUNCTION_LOG_RETURN(PACK_READ, protocolClientResponseInternal(this));
}
/**********************************************************************************************************************************/
FN_EXTERN PackRead *
protocolClientSessionRequest(ProtocolClientSession *const this, const ProtocolClientRequestParam param)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
FUNCTION_LOG_PARAM(PACK_WRITE, param.param);
FUNCTION_LOG_END();
ASSERT(this != NULL);
protocolClientRequestInternal(this, protocolCommandTypeProcess, param.param);
FUNCTION_LOG_RETURN(PACK_READ, protocolClientResponseInternal(this));
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolClientSessionRequestAsync(ProtocolClientSession *const this, const ProtocolClientRequestParam param)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
FUNCTION_LOG_PARAM(PACK_WRITE, param.param);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(this->async);
protocolClientRequestInternal(this, protocolCommandTypeProcess, param.param);
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN PackRead *
protocolClientSessionResponse(ProtocolClientSession *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(this->async);
FUNCTION_LOG_RETURN(PACK_READ, protocolClientResponseInternal(this));
}
/**********************************************************************************************************************************/
FN_EXTERN PackRead *
protocolClientSessionClose(ProtocolClientSession *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(this->pub.open);
protocolClientRequestInternal(this, protocolCommandTypeClose, NULL);
PackRead *const result = protocolClientResponseInternal(this);
ASSERT(!this->pub.open);
memContextCallbackClear(objMemContext(this));
protocolClientSessionFreeResource(this);
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolClientSessionCancel(ProtocolClientSession *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
memContextCallbackClear(objMemContext(this));
protocolClientSessionFreeResource(this);
FUNCTION_LOG_RETURN_VOID();
}
@ -406,3 +631,17 @@ protocolClientToLog(const ProtocolClient *const this, StringStatic *const debugL
strStcResultSizeInc(debugLog, strIdToLog(this->state, strStcRemains(debugLog), strStcRemainsSize(debugLog)));
strStcCatChr(debugLog, '}');
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolClientSessionToLog(const ProtocolClientSession *const this, StringStatic *const debugLog)
{
strStcCat(debugLog, "{client: ");
protocolClientToLog(this->client, debugLog);
strStcCat(debugLog, ", command: ");
strStcResultSizeInc(debugLog, strIdToLog(this->command, strStcRemains(debugLog), strStcRemainsSize(debugLog)));
strStcFmt(
debugLog, ", sessionId: %" PRIu64 ", queued %s, stored: %s", this->sessionId,
cvtBoolToConstZ(protocolClientSessionQueued(this)), cvtBoolToConstZ(this->stored));
strStcCatChr(debugLog, '}');
}

169
src/protocol/client.h
View File

@ -1,5 +1,28 @@
/***********************************************************************************************************************************
Protocol Client
There are three ways to make a request using the client:
1. Synchronous
This is the most common way to make a request. Simply call protocolClientRequestP() and process the result.
2. Synchronous with Session
In some cases it is useful to keep session state between requests. An example of this is queries against a database where it is more
efficient to maintain the connection to the database between queries. The session is created with protocolClientSessionNewP(),
opened with protocolClientSessionOpenP(), and closed with protocolClientSessionClose(). Requests are made with
protocolClientSessionRequest().
3. Asynchronous with Session
Asynchronous requests provide better performance by allowing the client and server to process at the same time. An example of this
is reading a file where a asynchronous request for the next block of data can be sent before the current block is being processed.
The next block should be waiting when processing of the current block is complete. The same session functions are used for creation,
open, and close, except that .async = true is passed to protocolClientSessionNewP(). Asynchronous requests are made with
protocolClientSessionRequestAsync() and the responses are read with protocolClientSessionResponse(). Note that there can only be one
outstanding asynchronous request, therefore protocolClientSessionResponse() must be called before
protocolClientSessionRequestAsync() can be called again.
***********************************************************************************************************************************/
#ifndef PROTOCOL_CLIENT_H
#define PROTOCOL_CLIENT_H
@ -9,28 +32,36 @@ Message types used by the protocol
***********************************************************************************************************************************/
typedef enum
{
// Data passed between client and server in either direction. This can be used as many times as needed.
protocolMessageTypeData = 0,
// Data passed from server to client
protocolMessageTypeResponse = 0,
// Indicates no more data for the server to return to the client and ends the command
protocolMessageTypeDataEnd = 1,
// Request sent from the client to the server
protocolMessageTypeRequest = 1,
// Command sent from the client to the server
protocolMessageTypeCommand = 2,
// An error occurred on the server and the command ended abnormally. protocolMessageTypeDataEnd will not be sent to the client.
protocolMessageTypeError = 3,
// An error occurred on the server and the request ended abnormally. protocolMessageTypeResponse will not be sent to the client.
protocolMessageTypeError = 2,
} ProtocolMessageType;
/***********************************************************************************************************************************
Command types
***********************************************************************************************************************************/
typedef enum
{
protocolCommandTypeOpen = STRID5("opn", 0x3a0f0), // Open command for processing
protocolCommandTypeProcess = STRID5("prc", 0xe500), // Process command
protocolCommandTypeClose = STRID5("cls", 0x4d830), // Close command
protocolCommandTypeCancel = STRID5("cnc", 0xdc30), // Cancel command
} ProtocolCommandType;
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct ProtocolClient ProtocolClient;
typedef struct ProtocolClientSession ProtocolClientSession;
#include "common/io/read.h"
#include "common/io/write.h"
#include "common/type/object.h"
#include "protocol/command.h"
/***********************************************************************************************************************************
Constants
@ -57,12 +88,12 @@ protocolPackNew(void)
}
/***********************************************************************************************************************************
Constructors
Client Constructors
***********************************************************************************************************************************/
FN_EXTERN ProtocolClient *protocolClientNew(const String *name, const String *service, IoRead *read, IoWrite *write);
/***********************************************************************************************************************************
Getters/Setters
Client Getters/Setters
***********************************************************************************************************************************/
typedef struct ProtocolClientPub
{
@ -77,11 +108,8 @@ protocolClientIoReadFd(ProtocolClient *const this)
}
/***********************************************************************************************************************************
Functions
Client Functions
***********************************************************************************************************************************/
// Execute a command and get the result
FN_EXTERN PackRead *protocolClientExecute(ProtocolClient *this, ProtocolCommand *command, bool resultRequired);
// Move to a new parent mem context
FN_INLINE_ALWAYS ProtocolClient *
protocolClientMove(ProtocolClient *const this, MemContext *const parentNew)
@ -99,18 +127,20 @@ protocolClientNoExit(ProtocolClient *const this)
// Send noop to test connection or keep it alive
FN_EXTERN void protocolClientNoOp(ProtocolClient *this);
// Get data put by the server
FN_EXTERN PackRead *protocolClientDataGet(ProtocolClient *this);
FN_EXTERN void protocolClientDataEndGet(ProtocolClient *this);
// Simple request that does not require a session or async
typedef struct ProtocolClientRequestParam
{
VAR_PARAM_HEADER;
PackWrite *param;
} ProtocolClientRequestParam;
// Put command to the server
FN_EXTERN void protocolClientCommandPut(ProtocolClient *this, ProtocolCommand *command, const bool dataPut);
#define protocolClientRequestP(this, command, ...) \
protocolClientRequest(this, command, (ProtocolClientRequestParam){VAR_PARAM_INIT, __VA_ARGS__})
// Put data to the server
FN_EXTERN void protocolClientDataPut(ProtocolClient *this, PackWrite *data);
FN_EXTERN PackRead *protocolClientRequest(ProtocolClient *this, StringId command, ProtocolClientRequestParam param);
/***********************************************************************************************************************************
Destructor
Client Destructor
***********************************************************************************************************************************/
FN_INLINE_ALWAYS void
protocolClientFree(ProtocolClient *const this)
@ -118,6 +148,90 @@ protocolClientFree(ProtocolClient *const this)
objFree(this);
}
/***********************************************************************************************************************************
Session Constructors
***********************************************************************************************************************************/
// New session
typedef struct ProtocolClientSessionNewParam
{
VAR_PARAM_HEADER;
bool async; // Async requests allowed?
} ProtocolClientSessionNewParam;
#define protocolClientSessionNewP(client, command, ...) \
protocolClientSessionNew(client, command, (ProtocolClientSessionNewParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN ProtocolClientSession *protocolClientSessionNew(
ProtocolClient *client, StringId command, ProtocolClientSessionNewParam param);
/***********************************************************************************************************************************
Session Getters/Setters
***********************************************************************************************************************************/
typedef struct ProtocolClientSessionPub
{
bool open; // Is the session open?
bool queued; // Is a response currently queued?
} ProtocolClientSessionPub;
// Is a response currently queued?
FN_INLINE_ALWAYS bool
protocolClientSessionQueued(const ProtocolClientSession *const this)
{
return THIS_PUB(ProtocolClientSession)->queued;
}
// Is the session closed?
FN_INLINE_ALWAYS bool
protocolClientSessionClosed(const ProtocolClientSession *const this)
{
return !THIS_PUB(ProtocolClientSession)->open;
}
/***********************************************************************************************************************************
Session Functions
***********************************************************************************************************************************/
// Session open
typedef struct ProtocolClientSessionOpenParam
{
VAR_PARAM_HEADER;
PackWrite *param;
} ProtocolClientSessionOpenParam;
#define protocolClientSessionOpenP(this, ...) \
protocolClientSessionOpen(this, (ProtocolClientSessionOpenParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN PackRead *protocolClientSessionOpen(ProtocolClientSession *const this, ProtocolClientSessionOpenParam param);
// Session request
#define protocolClientSessionRequestP(this, ...) \
protocolClientSessionRequest(this, (ProtocolClientRequestParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN PackRead *protocolClientSessionRequest(ProtocolClientSession *const this, ProtocolClientRequestParam param);
// Session async request
#define protocolClientSessionRequestAsyncP(this, ...) \
protocolClientSessionRequestAsync(this, (ProtocolClientRequestParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN void protocolClientSessionRequestAsync(ProtocolClientSession *const this, ProtocolClientRequestParam param);
// Session response after a call to protocolClientSessionRequestAsyncP()
FN_EXTERN PackRead *protocolClientSessionResponse(ProtocolClientSession *const this);
// Session close
FN_EXTERN PackRead *protocolClientSessionClose(ProtocolClientSession *const this);
// Session cancel
FN_EXTERN void protocolClientSessionCancel(ProtocolClientSession *const this);
/***********************************************************************************************************************************
Session Destructor
***********************************************************************************************************************************/
FN_INLINE_ALWAYS void
protocolClientSessionFree(ProtocolClientSession *const this)
{
objFree(this);
}
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
@ -128,4 +242,11 @@ FN_EXTERN void protocolClientToLog(const ProtocolClient *this, StringStatic *deb
#define FUNCTION_LOG_PROTOCOL_CLIENT_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_OBJECT_FORMAT(value, protocolClientToLog, buffer, bufferSize)
FN_EXTERN void protocolClientSessionToLog(const ProtocolClientSession *this, StringStatic *debugLog);
#define FUNCTION_LOG_PROTOCOL_CLIENT_SESSION_TYPE \
ProtocolClientSession *
#define FUNCTION_LOG_PROTOCOL_CLIENT_SESSION_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_OBJECT_FORMAT(value, protocolClientSessionToLog, buffer, bufferSize)
#endif

107
src/protocol/command.c
View File

@ -1,107 +0,0 @@
/***********************************************************************************************************************************
Protocol Command
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/debug.h"
#include "common/log.h"
#include "common/type/keyValue.h"
#include "protocol/client.h"
#include "protocol/command.h"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct ProtocolCommand
{
StringId command;
PackWrite *pack;
};
/**********************************************************************************************************************************/
FN_EXTERN ProtocolCommand *
protocolCommandNew(const StringId command)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING_ID, command);
FUNCTION_TEST_END();
ASSERT(command != 0);
OBJ_NEW_BEGIN(ProtocolCommand, .childQty = MEM_CONTEXT_QTY_MAX)
{
*this = (ProtocolCommand)
{
.command = command,
};
}
OBJ_NEW_END();
FUNCTION_TEST_RETURN(PROTOCOL_COMMAND, this);
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolCommandPut(ProtocolCommand *const this, IoWrite *const write)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(write != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// Write the command
PackWrite *commandPack = pckWriteNewIo(write);
pckWriteU32P(commandPack, protocolMessageTypeCommand, .defaultWrite = true);
pckWriteStrIdP(commandPack, this->command);
// Write parameters
if (this->pack != NULL)
{
pckWriteEndP(this->pack);
pckWritePackP(commandPack, pckWriteResult(this->pack));
}
pckWriteEndP(commandPack);
// Flush to send command immediately
ioWriteFlush(write);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN PackWrite *
protocolCommandParam(ProtocolCommand *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
if (this->pack == NULL)
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->pack = protocolPackNew();
}
MEM_CONTEXT_OBJ_END();
}
FUNCTION_TEST_RETURN(PACK_WRITE, this->pack);
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolCommandToLog(const ProtocolCommand *const this, StringStatic *const debugLog)
{
strStcFmt(debugLog, "{name: ");
strStcResultSizeInc(debugLog, strIdToLog(this->command, strStcRemains(debugLog), strStcRemainsSize(debugLog)));
strStcCatChr(debugLog, '}');
}

55
src/protocol/command.h
View File

@ -1,55 +0,0 @@
/***********************************************************************************************************************************
Protocol Command
***********************************************************************************************************************************/
#ifndef PROTOCOL_COMMAND_H
#define PROTOCOL_COMMAND_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct ProtocolCommand ProtocolCommand;
#include "common/type/object.h"
#include "common/type/pack.h"
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN ProtocolCommand *protocolCommandNew(const StringId command);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Move to a new parent mem context
FN_INLINE_ALWAYS ProtocolCommand *
protocolCommandMove(ProtocolCommand *const this, MemContext *const parentNew)
{
return objMove(this, parentNew);
}
// Read the command output
FN_EXTERN PackWrite *protocolCommandParam(ProtocolCommand *this);
// Write protocol command
FN_EXTERN void protocolCommandPut(ProtocolCommand *this, IoWrite *write);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
FN_INLINE_ALWAYS void
protocolCommandFree(ProtocolCommand *const this)
{
objFree(this);
}
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
FN_EXTERN void protocolCommandToLog(const ProtocolCommand *this, StringStatic *debugLog);
#define FUNCTION_LOG_PROTOCOL_COMMAND_TYPE \
ProtocolCommand *
#define FUNCTION_LOG_PROTOCOL_COMMAND_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_OBJECT_FORMAT(value, protocolCommandToLog, buffer, bufferSize)
#endif

28
src/protocol/helper.c
View File

@ -413,6 +413,11 @@ protocolServer(IoServer *const tlsServer, IoSession *const socketSession)
{
TRY_BEGIN()
{
// Get parameter list from the client. This needs to happen even if we cannot authorize the client so that the
// session id gets set for the error.
const ProtocolServerRequestResult command = protocolServerRequest(result);
CHECK(FormatError, command.id == PROTOCOL_COMMAND_CONFIG, "expected config command");
// Get list of authorized stanzas for this client
CHECK(AssertError, cfgOptionTest(cfgOptTlsServerAuth), "missing auth data");
@ -423,10 +428,7 @@ protocolServer(IoServer *const tlsServer, IoSession *const socketSession)
if (clientAuthList == NULL)
THROW(AccessError, "access denied");
// Get parameter list from the client and load it
const ProtocolServerCommandGetResult command = protocolServerCommandGet(result);
CHECK(FormatError, command.id == PROTOCOL_COMMAND_CONFIG, "expected config command");
// Load parameter list from the client
StringList *const paramList = pckReadStrLstP(pckReadNew(command.param));
strLstInsert(paramList, 0, cfgExe());
cfgLoad(strLstSize(paramList), strLstPtr(paramList));
@ -443,7 +445,7 @@ protocolServer(IoServer *const tlsServer, IoSession *const socketSession)
TRY_END();
// Ack the config command
protocolServerDataEndPut(result);
protocolServerResponseP(result);
// Move result to prior context and move session into result so there is only one return value
protocolServerMove(result, memContextPrior());
@ -452,8 +454,12 @@ protocolServer(IoServer *const tlsServer, IoSession *const socketSession)
// Else the client can only detect that the server is alive
else
{
// Send a data end message and return a NULL server. Do not waste time looking at what the client wrote.
protocolServerDataEndPut(result);
// A noop command should have been received
const ProtocolServerRequestResult command = protocolServerRequest(result);
CHECK(FormatError, command.id == PROTOCOL_COMMAND_NOOP, "expected config command");
// Send a data end message and return a NULL server
protocolServerResponseP(result);
// Set result to NULL so there is no server for the caller to use. The TLS session will be freed when the temp mem
// context ends.
@ -775,10 +781,10 @@ protocolRemoteExec(
TRY_BEGIN()
{
// Pass parameters to server
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_CONFIG);
pckWriteStrLstP(protocolCommandParam(command), protocolRemoteParam(protocolStorageType, hostIdx));
protocolClientExecute(helper->client, command, false);
protocolCommandFree(command);
PackWrite *const param = protocolPackNew();
pckWriteStrLstP(param, protocolRemoteParam(protocolStorageType, hostIdx));
protocolClientRequestP(helper->client, PROTOCOL_COMMAND_CONFIG, .param = param);
}
CATCH_ANY()
{

49
src/protocol/parallel.c
View File

@ -11,13 +11,18 @@ Protocol Parallel Executor
#include "common/macro.h"
#include "common/type/keyValue.h"
#include "common/type/list.h"
#include "protocol/command.h"
#include "protocol/helper.h"
#include "protocol/parallel.h"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct ProtocolParallelJobData
{
ProtocolParallelJob *job; // Job
ProtocolClientSession *session; // Protocol session for the job
} ProtocolParallelJobData;
struct ProtocolParallel
{
TimeMSec timeout; // Max time to wait for jobs before returning
@ -27,7 +32,7 @@ struct ProtocolParallel
List *clientList; // List of clients to process jobs
List *jobList; // List of jobs to be processed
ProtocolParallelJob **clientJobList; // Jobs being processing by each client
ProtocolParallelJobData *clientJobList; // Jobs being processing by each client
ProtocolParallelJobState state; // Overall state of job processing
};
@ -103,7 +108,10 @@ protocolParallelProcess(ProtocolParallel *this)
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->clientJobList = memNewPtrArray(lstSize(this->clientList));
this->clientJobList = memNew(lstSize(this->clientList) * sizeof(ProtocolParallelJobData));
for (unsigned int jobIdx = 0; jobIdx < lstSize(this->clientList); jobIdx++)
this->clientJobList[jobIdx] = (ProtocolParallelJobData){0};
}
MEM_CONTEXT_OBJ_END();
@ -120,7 +128,7 @@ protocolParallelProcess(ProtocolParallel *this)
for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
{
if (this->clientJobList[clientIdx] != NULL)
if (this->clientJobList[clientIdx].job != NULL)
{
int fd = protocolClientIoReadFd(*(ProtocolClient **)lstGet(this->clientList, clientIdx));
FD_SET(fd, &selectSet);
@ -149,7 +157,7 @@ protocolParallelProcess(ProtocolParallel *this)
{
for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
{
ProtocolParallelJob *job = this->clientJobList[clientIdx];
ProtocolParallelJob *const job = this->clientJobList[clientIdx].job;
if (job != NULL &&
FD_ISSET(
@ -160,10 +168,8 @@ protocolParallelProcess(ProtocolParallel *this)
{
TRY_BEGIN()
{
ProtocolClient *const client = *(ProtocolClient **)lstGet(this->clientList, clientIdx);
protocolParallelJobResultSet(job, protocolClientDataGet(client));
protocolClientDataEndGet(client);
protocolParallelJobResultSet(
job, protocolClientSessionResponse(this->clientJobList[clientIdx].session));
}
CATCH_ANY()
{
@ -172,7 +178,8 @@ protocolParallelProcess(ProtocolParallel *this)
TRY_END();
protocolParallelJobStateSet(job, protocolParallelJobStateDone);
this->clientJobList[clientIdx] = NULL;
this->clientJobList[clientIdx].job = NULL;
protocolClientSessionFree(this->clientJobList[clientIdx].session);
}
MEM_CONTEXT_TEMP_END();
}
@ -186,16 +193,12 @@ protocolParallelProcess(ProtocolParallel *this)
for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
{
// If nothing is running for this client
if (this->clientJobList[clientIdx] == NULL)
if (this->clientJobList[clientIdx].job == NULL)
{
// Get a new job
ProtocolParallelJob *job = NULL;
MEM_CONTEXT_BEGIN(lstMemContext(this->jobList))
{
job = this->callbackFunction(this->callbackData, clientIdx);
}
MEM_CONTEXT_END();
// Get a new job
ProtocolParallelJob *const job = this->callbackFunction(this->callbackData, clientIdx);
// If a new job was found
if (job != NULL)
@ -204,18 +207,24 @@ protocolParallelProcess(ProtocolParallel *this)
lstAdd(this->jobList, &job);
// Put command
protocolClientCommandPut(
*(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job), false);
ProtocolClientSession *const session = protocolClientSessionNewP(
*(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job),
.async = true);
protocolClientSessionRequestAsyncP(session, .param = protocolParallelJobParam(job));
// Set client id and running state
protocolParallelJobProcessIdSet(job, clientIdx + 1);
protocolParallelJobStateSet(job, protocolParallelJobStateRunning);
this->clientJobList[clientIdx] = job;
this->clientJobList[clientIdx].job = job;
this->clientJobList[clientIdx].session = session;
}
// Else no more jobs for this client so free it
else
protocolLocalFree(clientIdx + 1);
}
MEM_CONTEXT_END();
}
}
}
MEM_CONTEXT_TEMP_END();

13
src/protocol/parallelJob.c
View File

@ -5,7 +5,6 @@ Protocol Parallel Job
#include "common/debug.h"
#include "common/log.h"
#include "protocol/command.h"
#include "protocol/parallelJob.h"
/***********************************************************************************************************************************
@ -18,11 +17,12 @@ struct ProtocolParallelJob
/**********************************************************************************************************************************/
FN_EXTERN ProtocolParallelJob *
protocolParallelJobNew(const Variant *key, ProtocolCommand *command)
protocolParallelJobNew(const Variant *key, const StringId command, PackWrite *const param)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(VARIANT, key);
FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
FUNCTION_LOG_PARAM(STRING_ID, command);
FUNCTION_LOG_PARAM(PACK_WRITE, param);
FUNCTION_LOG_END();
OBJ_NEW_BEGIN(ProtocolParallelJob, .childQty = MEM_CONTEXT_QTY_MAX)
@ -33,10 +33,10 @@ protocolParallelJobNew(const Variant *key, ProtocolCommand *command)
{
.state = protocolParallelJobStatePending,
.key = varDup(key),
.command = command,
.param = pckWriteMove(param, objMemContext(this)),
},
};
this->pub.command = protocolCommandMove(command, objMemContext(this));
}
OBJ_NEW_END();
@ -136,7 +136,8 @@ protocolParallelJobToLog(const ProtocolParallelJob *const this, StringStatic *co
varToLog(protocolParallelJobKey(this), debugLog);
strStcCat(debugLog, ", command: ");
protocolCommandToLog(protocolParallelJobCommand(this), debugLog);
strStcResultSizeInc(
debugLog, strIdToLog(protocolParallelJobCommand(this), strStcRemains(debugLog), strStcRemainsSize(debugLog)));
strStcCat(debugLog, ", result: ");
strStcResultSizeInc(

14
src/protocol/parallelJob.h
View File

@ -29,7 +29,7 @@ typedef enum
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN ProtocolParallelJob *protocolParallelJobNew(const Variant *key, ProtocolCommand *command);
FN_EXTERN ProtocolParallelJob *protocolParallelJobNew(const Variant *key, StringId command, PackWrite *param);
/***********************************************************************************************************************************
Getters/Setters
@ -37,7 +37,8 @@ Getters/Setters
typedef struct ProtocolParallelJobPub
{
const Variant *key; // Unique key used to identify the job
ProtocolCommand *command; // Command to be executed
StringId command; // Command to be executed
PackWrite *param; // Command parameters
unsigned int processId; // Process that executed this job
ProtocolParallelJobState state; // Current state of the job
int code; // Non-zero result indicates an error
@ -46,12 +47,19 @@ typedef struct ProtocolParallelJobPub
} ProtocolParallelJobPub;
// Job command
FN_INLINE_ALWAYS ProtocolCommand *
FN_INLINE_ALWAYS StringId
protocolParallelJobCommand(const ProtocolParallelJob *const this)
{
return THIS_PUB(ProtocolParallelJob)->command;
}
// Job command parameters
FN_INLINE_ALWAYS PackWrite *
protocolParallelJobParam(const ProtocolParallelJob *const this)
{
return THIS_PUB(ProtocolParallelJob)->param;
}
// Job error
FN_INLINE_ALWAYS int
protocolParallelJobErrorCode(const ProtocolParallelJob *const this)

388
src/protocol/server.c
View File

@ -24,8 +24,25 @@ struct ProtocolServer
IoRead *read; // Read interface
IoWrite *write; // Write interface
const String *name; // Name displayed in logging
List *sessionList; // List of active sessions
uint64_t sessionId; // Current session being processed
};
struct ProtocolServerResult
{
void *sessionData; // Session data
PackWrite *data; // Result data
size_t extra; // Extra bytes for initializing data
bool close; // Close session?
};
// Track server sessions
typedef struct ProtocolServerSession
{
uint64_t id; // Session id
void *data; // Data for the session
} ProtocolServerSession;
/**********************************************************************************************************************************/
FN_EXTERN ProtocolServer *
protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write)
@ -48,6 +65,7 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
.read = read,
.write = write,
.name = strDup(name),
.sessionList = lstNewP(sizeof(ProtocolServerSession)),
};
// Send the protocol greeting
@ -69,6 +87,40 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
FUNCTION_LOG_RETURN(PROTOCOL_SERVER, this);
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolServerResponse(ProtocolServer *const this, const ProtocolServerResponseParam param)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_SERVER, this);
FUNCTION_TEST_PARAM(ENUM, param.type);
FUNCTION_TEST_PARAM(BOOL, param.close);
FUNCTION_TEST_PARAM(PACK_WRITE, param.data);
FUNCTION_TEST_END();
ASSERT(this != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// End the pack
if (param.data != NULL)
pckWriteEndP(param.data);
// Write the result
PackWrite *const resultMessage = pckWriteNewIo(this->write);
pckWriteU64P(resultMessage, this->sessionId);
pckWriteU32P(resultMessage, param.type, .defaultWrite = true);
pckWriteBoolP(resultMessage, param.close);
pckWritePackP(resultMessage, pckWriteResult(param.data));
pckWriteEndP(resultMessage);
ioWriteFlush(this->write);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolServerError(ProtocolServer *this, int code, const String *message, const String *stack)
@ -87,15 +139,13 @@ protocolServerError(ProtocolServer *this, int code, const String *message, const
MEM_CONTEXT_TEMP_BEGIN()
{
// Write the error and flush to be sure it gets sent immediately
PackWrite *error = pckWriteNewIo(this->write);
pckWriteU32P(error, protocolMessageTypeError);
pckWriteI32P(error, code);
pckWriteStrP(error, message);
pckWriteStrP(error, stack);
pckWriteEndP(error);
PackWrite *const packWrite = protocolPackNew();
ioWriteFlush(this->write);
pckWriteI32P(packWrite, code);
pckWriteStrP(packWrite, message);
pckWriteStrP(packWrite, stack);
protocolServerResponseP(this, .type = protocolMessageTypeError, .data = packWrite);
}
MEM_CONTEXT_TEMP_END();
@ -103,8 +153,8 @@ protocolServerError(ProtocolServer *this, int code, const String *message, const
}
/**********************************************************************************************************************************/
FN_EXTERN ProtocolServerCommandGetResult
protocolServerCommandGet(ProtocolServer *const this)
FN_EXTERN ProtocolServerRequestResult
protocolServerRequest(ProtocolServer *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
@ -112,23 +162,28 @@ protocolServerCommandGet(ProtocolServer *const this)
FUNCTION_AUDIT_STRUCT();
ProtocolServerCommandGetResult result = {0};
ProtocolServerRequestResult result = {0};
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *const command = pckReadNewIo(this->read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(command);
PackRead *const request = pckReadNewIo(this->read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(request);
CHECK(FormatError, type == protocolMessageTypeCommand, "expected command message");
CHECK(FormatError, type == protocolMessageTypeRequest, "expected request message");
MEM_CONTEXT_PRIOR_BEGIN()
{
result.id = pckReadStrIdP(command);
result.param = pckReadPackP(command);
result.id = pckReadStrIdP(request);
result.type = (ProtocolCommandType)pckReadStrIdP(request);
this->sessionId = pckReadU64P(request);
result.sessionRequired = pckReadBoolP(request);
result.param = pckReadPackP(request);
ASSERT(this->sessionId != 0);
}
MEM_CONTEXT_PRIOR_END();
pckReadEndP(command);
pckReadEndP(request);
}
MEM_CONTEXT_TEMP_END();
@ -152,7 +207,7 @@ protocolServerProcess(
ASSERT(handlerList != NULL);
ASSERT(handlerListSize > 0);
// Loop until exit command is received
// Loop until exit request is received
bool exit = false;
do
@ -161,17 +216,17 @@ protocolServerProcess(
{
MEM_CONTEXT_TEMP_BEGIN()
{
// Get command
ProtocolServerCommandGetResult command = protocolServerCommandGet(this);
// Get request
ProtocolServerRequestResult request = protocolServerRequest(this);
// Find the handler
ProtocolServerCommandHandler handler = NULL;
const ProtocolServerHandler *handler = NULL;
for (unsigned int handlerIdx = 0; handlerIdx < handlerListSize; handlerIdx++)
{
if (command.id == handlerList[handlerIdx].command)
if (request.id == handlerList[handlerIdx].command)
{
handler = handlerList[handlerIdx].handler;
handler = &handlerList[handlerIdx];
break;
}
}
@ -179,14 +234,14 @@ protocolServerProcess(
// If handler was found then process
if (handler != NULL)
{
// Send the command to the handler
// Send the request to the handler
MEM_CONTEXT_TEMP_BEGIN()
{
// Variables to store first error message and retry messages
ErrorRetry *const errRetry = errRetryNew();
const String *errStackTrace = NULL;
// Initialize retries in case of command failure
// Initialize retries in case of request failure
bool retry = false;
unsigned int retryRemaining = retryInterval != NULL ? varLstSize(retryInterval) : 0;
TimeMSec retrySleepMs = 0;
@ -198,7 +253,166 @@ protocolServerProcess(
TRY_BEGIN()
{
handler(pckReadNew(command.param), this);
// Process request type
switch (request.type)
{
// Open a protocol session
case protocolCommandTypeOpen:
{
ASSERT(handler->open != NULL);
// Call open handler
ProtocolServerResult *const openResult = handler->open(pckReadNew(request.param));
ASSERT(openResult != NULL);
ASSERT(!openResult->close);
ProtocolServerSession session = {.id = this->sessionId};
// Send data
protocolServerResponseP(
this, .data = openResult->data, .close = openResult->sessionData == NULL);
// Create session if open returned session data
if (openResult->sessionData != NULL)
{
session.data = objMove(openResult->sessionData, objMemContext(this->sessionList));
lstAdd(this->sessionList, &session);
}
break;
}
// Process or close/cancel protocol session
default:
{
// Find session data
void *sessionData = NULL;
unsigned int sessionListIdx = 0;
if (request.sessionRequired)
{
ASSERT(handler->processSession != NULL);
ASSERT(handler->process == NULL);
for (; sessionListIdx < lstSize(this->sessionList); sessionListIdx++)
{
ProtocolServerSession *const session = lstGet(this->sessionList, sessionListIdx);
if (session->id == this->sessionId)
{
sessionData = session->data;
break;
}
}
// Error when session not found
if (sessionData == NULL && request.type != protocolCommandTypeCancel)
{
THROW_FMT(
ProtocolError, "unable to find session id %" PRIu64 " for request %s:%s",
this->sessionId, strZ(strIdToStr(request.id)),
strZ(strIdToStr(request.type)));
}
}
// Process request type
switch (request.type)
{
// Process request
case protocolCommandTypeProcess:
{
// Process session
ProtocolServerResult *processResult;
if (handler->processSession != NULL)
{
ASSERT(handler->process == NULL);
CHECK_FMT(
ProtocolError, this->sessionId != 0, "no session id for request %s:%s",
strZ(strIdToStr(request.id)), strZ(strIdToStr(request.type)));
processResult = handler->processSession(pckReadNew(request.param), sessionData);
}
// Standalone process
else
{
ASSERT(handler->process != NULL);
ASSERT(!request.sessionRequired);
processResult = handler->process(pckReadNew(request.param));
}
if (processResult == NULL)
protocolServerResponseP(this);
else
{
ASSERT(processResult->sessionData == NULL);
ASSERT(handler->processSession != NULL || !processResult->close);
protocolServerResponseP(
this, .data = processResult->data, .close = processResult->close);
// Free session when close is true. This optimization allows an explicit
// close to be skipped.
if (processResult->close)
{
objFree(sessionData);
lstRemoveIdx(this->sessionList, sessionListIdx);
}
}
break;
}
// Close protocol session
case protocolCommandTypeClose:
{
// If there is a close handler then call it
PackWrite *data = NULL;
if (handler->close != NULL)
{
ProtocolServerResult *const closeResult = handler->close(
pckReadNew(request.param), sessionData);
ASSERT(closeResult != NULL);
ASSERT(closeResult->sessionData == NULL);
ASSERT(!closeResult->close);
data = closeResult->data;
}
// Send data
protocolServerResponseP(this, .data = data, .close = true);
// Free the session
objFree(sessionData);
lstRemoveIdx(this->sessionList, sessionListIdx);
break;
}
// Cancel protocol session
default:
{
CHECK_FMT(
ProtocolError, request.type == protocolCommandTypeCancel,
"unknown request type '%s'", strZ(strIdToStr(request.type)));
// Send NULL data
protocolServerResponseP(this, .close = true);
// Free the session
if (sessionData != NULL)
{
objFree(sessionData);
lstRemoveIdx(this->sessionList, sessionListIdx);
}
break;
}
}
}
}
}
CATCH_ANY()
{
@ -228,7 +442,7 @@ protocolServerProcess(
retryRemaining--;
retry = true;
// Send keep-alive to remotes. A retry means the command is taking longer than usual so make
// Send keep-alive to remotes. A retry means the request is taking longer than usual so make
// sure the remote does not timeout.
protocolKeepAlive();
}
@ -245,22 +459,22 @@ protocolServerProcess(
}
MEM_CONTEXT_TEMP_END();
}
// Else check built-in commands
// Else check built-in requests
else
{
switch (command.id)
switch (request.id)
{
case PROTOCOL_COMMAND_EXIT:
exit = true;
break;
case PROTOCOL_COMMAND_NOOP:
protocolServerDataEndPut(this);
protocolServerResponseP(this);
break;
default:
THROW_FMT(
ProtocolError, "invalid command '%s' (0x%" PRIx64 ")", strZ(strIdToStr(command.id)), command.id);
ProtocolError, "invalid request '%s' (0x%" PRIx64 ")", strZ(strIdToStr(request.id)), request.id);
}
}
@ -273,6 +487,9 @@ protocolServerProcess(
}
CATCH_FATAL()
{
// Zero session id so a fatal error will be handled by the first client that sees it
this->sessionId = 0;
// Report error to the client
protocolServerError(this, errorCode(), STR(errorMessage()), STR(errorStackTrace()));
@ -287,85 +504,75 @@ protocolServerProcess(
}
/**********************************************************************************************************************************/
FN_EXTERN PackRead *
protocolServerDataGet(ProtocolServer *const this)
FN_EXTERN ProtocolServerResult *
protocolServerResultNew(const ProtocolServerResultNewParam param)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_END();
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(SIZE, param.extra);
FUNCTION_TEST_END();
PackRead *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
OBJ_NEW_BEGIN(ProtocolServerResult, .childQty = MEM_CONTEXT_QTY_MAX)
{
PackRead *data = pckReadNewIo(this->read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(data);
CHECK(FormatError, type == protocolMessageTypeData, "expected data message");
MEM_CONTEXT_PRIOR_BEGIN()
*this = (ProtocolServerResult)
{
result = pckReadPackReadP(data);
.extra = param.extra,
};
}
MEM_CONTEXT_PRIOR_END();
OBJ_NEW_END();
pckReadEndP(data);
FUNCTION_TEST_RETURN(PROTOCOL_SERVER_RESULT, this);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(PACK_READ, result);
/**********************************************************************************************************************************/
FN_EXTERN PackWrite *
protocolServerResultData(ProtocolServerResult *const this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_SERVER_RESULT, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(this->data == NULL);
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->data = pckWriteNewP(.size = PROTOCOL_PACK_DEFAULT_SIZE + this->extra);
}
MEM_CONTEXT_OBJ_END();
FUNCTION_TEST_RETURN(PACK_WRITE, this->data);
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolServerDataPut(ProtocolServer *const this, PackWrite *const data)
protocolServerResultSessionDataSet(ProtocolServerResult *const this, void *const sessionData)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_PARAM(PACK_WRITE, data);
FUNCTION_LOG_END();
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_SERVER_RESULT, this);
FUNCTION_TEST_PARAM_P(VOID, sessionData);
FUNCTION_TEST_END();
MEM_CONTEXT_TEMP_BEGIN()
{
// End the pack
if (data != NULL)
pckWriteEndP(data);
ASSERT(this != NULL);
ASSERT(sessionData != NULL);
// Write the result
PackWrite *resultMessage = pckWriteNewIo(this->write);
pckWriteU32P(resultMessage, protocolMessageTypeData, .defaultWrite = true);
pckWritePackP(resultMessage, pckWriteResult(data));
pckWriteEndP(resultMessage);
this->sessionData = objMove(sessionData, objMemContext(this));
// Flush on NULL result since it might be used to synchronize
if (data == NULL)
ioWriteFlush(this->write);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolServerDataEndPut(ProtocolServer *const this)
protocolServerResultCloseSet(ProtocolServerResult *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_END();
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_SERVER_RESULT, this);
FUNCTION_TEST_END();
MEM_CONTEXT_TEMP_BEGIN()
{
// Write the response and flush to be sure it gets sent immediately
PackWrite *response = pckWriteNewIo(this->write);
pckWriteU32P(response, protocolMessageTypeDataEnd, .defaultWrite = true);
pckWriteEndP(response);
}
MEM_CONTEXT_TEMP_END();
ASSERT(this != NULL);
ioWriteFlush(this->write);
this->close = true;
FUNCTION_LOG_RETURN_VOID();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
@ -374,3 +581,12 @@ protocolServerToLog(const ProtocolServer *const this, StringStatic *const debugL
{
strStcFmt(debugLog, "{name: %s}", strZ(this->name));
}
/**********************************************************************************************************************************/
FN_EXTERN void
protocolServerResultToLog(const ProtocolServerResult *const this, StringStatic *const debugLog)
{
strStcFmt(
debugLog, "{data: %s, sessionData: %s, close: %s}", cvtBoolToConstZ(this->data != NULL),
cvtBoolToConstZ(this->sessionData != NULL), cvtBoolToConstZ(this->close));
}

77
src/protocol/server.h
View File

@ -8,6 +8,7 @@ Protocol Server
Object type
***********************************************************************************************************************************/
typedef struct ProtocolServer ProtocolServer;
typedef struct ProtocolServerResult ProtocolServerResult;
#include "common/io/read.h"
#include "common/io/write.h"
@ -22,42 +23,55 @@ Protocol command handler type and structure
An array of this struct must be passed to protocolServerProcess() for the server to process commands. Each command handler should
implement a single command, as defined by the command string.
***********************************************************************************************************************************/
typedef void (*ProtocolServerCommandHandler)(PackRead *param, ProtocolServer *server);
typedef ProtocolServerResult *(*ProtocolServerCommandOpenHandler)(PackRead *param);
typedef ProtocolServerResult *(*ProtocolServerCommandProcessHandler)(PackRead *param);
typedef ProtocolServerResult *(*ProtocolServerCommandProcessSessionHandler)(PackRead *param, void *sessionData);
typedef ProtocolServerResult *(*ProtocolServerCommandCloseHandler)(PackRead *param, void *sessionData);
typedef struct ProtocolServerHandler
{
StringId command; // 5-bit StringId that identifies the protocol command
ProtocolServerCommandHandler handler; // Function that handles the protocol command
ProtocolServerCommandOpenHandler open; // Function that opens the protocol session
ProtocolServerCommandProcessHandler process; // Function that processes the protocol command
ProtocolServerCommandProcessSessionHandler processSession; // Function that processes the protocol command for a session
ProtocolServerCommandCloseHandler close; // Function that closes the protocol session
} ProtocolServerHandler;
/***********************************************************************************************************************************
Constructors
Server Constructors
***********************************************************************************************************************************/
FN_EXTERN ProtocolServer *protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write);
/***********************************************************************************************************************************
Functions
Server Functions
***********************************************************************************************************************************/
// Get command from the client. Outside ProtocolServer, this is used when the first noop command needs to be processed before
// running protocolServerProcess(), which allows an error to be returned to the client if initialization fails.
typedef struct ProtocolServerCommandGetResult
typedef struct ProtocolServerRequestResult
{
StringId id; // Command identifier
ProtocolCommandType type; // Command type
bool sessionRequired; // Session with more than one command
Pack *param; // Parameter pack
} ProtocolServerCommandGetResult;
} ProtocolServerRequestResult;
FN_EXTERN ProtocolServerCommandGetResult protocolServerCommandGet(ProtocolServer *this);
FN_EXTERN ProtocolServerRequestResult protocolServerRequest(ProtocolServer *this);
// Get data from the client
FN_EXTERN PackRead *protocolServerDataGet(ProtocolServer *this);
// Send response to client
typedef struct ProtocolServerResponseParam
{
VAR_PARAM_HEADER;
bool close; // Has the session been closed?
ProtocolMessageType type; // Message type
PackWrite *data; // Response data
} ProtocolServerResponseParam;
// Put data to the client
FN_EXTERN void protocolServerDataPut(ProtocolServer *this, PackWrite *data);
#define protocolServerResponseP(this, ...) \
protocolServerResponse(this, (ProtocolServerResponseParam){VAR_PARAM_INIT, __VA_ARGS__})
// Put data end to the client. This ends command processing and no more data should be sent.
FN_EXTERN void protocolServerDataEndPut(ProtocolServer *this);
FN_EXTERN void protocolServerResponse(ProtocolServer *const this, ProtocolServerResponseParam param);
// Return an error
// Send an error to client
FN_EXTERN void protocolServerError(ProtocolServer *this, int code, const String *message, const String *stack);
// Process requests
@ -73,7 +87,7 @@ protocolServerMove(ProtocolServer *const this, MemContext *const parentNew)
}
/***********************************************************************************************************************************
Destructor
Server Destructor
***********************************************************************************************************************************/
FN_INLINE_ALWAYS void
protocolServerFree(ProtocolServer *const this)
@ -81,6 +95,32 @@ protocolServerFree(ProtocolServer *const this)
objFree(this);
}
/***********************************************************************************************************************************
Result Constructors
***********************************************************************************************************************************/
typedef struct ProtocolServerResultNewParam
{
VAR_PARAM_HEADER;
size_t extra;
} ProtocolServerResultNewParam;
#define protocolServerResultNewP(...) \
protocolServerResultNew((ProtocolServerResultNewParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN ProtocolServerResult *protocolServerResultNew(ProtocolServerResultNewParam param);
/***********************************************************************************************************************************
Result Getters/Setters
***********************************************************************************************************************************/
// Create PackWrite object required to send data to the client
FN_EXTERN PackWrite *protocolServerResultData(ProtocolServerResult *this);
// Set session data
FN_EXTERN void protocolServerResultSessionDataSet(ProtocolServerResult *const this, void *const sessionData);
// Close session
FN_EXTERN void protocolServerResultCloseSet(ProtocolServerResult *const this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
@ -91,4 +131,11 @@ FN_EXTERN void protocolServerToLog(const ProtocolServer *this, StringStatic *deb
#define FUNCTION_LOG_PROTOCOL_SERVER_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_OBJECT_FORMAT(value, protocolServerToLog, buffer, bufferSize)
FN_EXTERN void protocolServerResultToLog(const ProtocolServerResult *this, StringStatic *debugLog);
#define FUNCTION_LOG_PROTOCOL_SERVER_RESULT_TYPE \
ProtocolServerResult *
#define FUNCTION_LOG_PROTOCOL_SERVER_RESULT_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_OBJECT_FORMAT(value, protocolServerResultToLog, buffer, bufferSize)
#endif

385
src/storage/remote/protocol.c
View File

@ -124,18 +124,18 @@ storageRemoteFilterGroup(IoFilterGroup *const filterGroup, const Pack *const fil
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageRemoteFeatureProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
storageRemoteFeatureProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
FUNCTION_AUDIT_HELPER();
ASSERT(param == NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
@ -159,16 +159,14 @@ storageRemoteFeatureProtocol(PackRead *const param, ProtocolServer *const server
}
// Return storage features
PackWrite *result = protocolPackNew();
pckWriteStrP(result, storagePathP(storage, NULL));
pckWriteU64P(result, storageInterface(storage).feature);
PackWrite *const data = protocolServerResultData(result);
protocolServerDataPut(server, result);
protocolServerDataEndPut(server);
pckWriteStrP(data, storagePathP(storage, NULL));
pckWriteU64P(data, storageInterface(storage).feature);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
/**********************************************************************************************************************************/
@ -264,18 +262,18 @@ storageRemoteInfoProtocolPut(
FUNCTION_TEST_RETURN_VOID();
}
FN_EXTERN void
storageRemoteInfoProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
storageRemoteInfoProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
// Get file info
@ -286,31 +284,26 @@ storageRemoteInfoProtocol(PackRead *const param, ProtocolServer *const server)
StorageInfo info = storageInterfaceInfoP(storageRemoteProtocolLocal.driver, file, level, .followLink = followLink);
// Write file info to protocol
PackWrite *write = protocolPackNew();
pckWriteBoolP(write, info.exists, .defaultWrite = true);
PackWrite *const data = protocolServerResultData(result);
pckWriteBoolP(data, info.exists, .defaultWrite = true);
if (info.exists)
storageRemoteInfoProtocolPut(&(StorageRemoteInfoProtocolWriteData){0}, write, &info);
protocolServerDataPut(server, write);
protocolServerDataEndPut(server);
storageRemoteInfoProtocolPut(&(StorageRemoteInfoProtocolWriteData){0}, data, &info);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageRemoteLinkCreateProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
storageRemoteLinkCreateProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
@ -320,32 +313,35 @@ storageRemoteLinkCreateProtocol(PackRead *const param, ProtocolServer *const ser
const StorageLinkType linkType = (StorageLinkType)pckReadU32P(param);
storageInterfaceLinkCreateP(storageRemoteProtocolLocal.driver, target, linkPath, .linkType = linkType);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageRemoteListProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
storageRemoteListProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
ProtocolServerResult *const result = protocolServerResultNewP(.extra = ioBufferSize());
MEM_CONTEXT_TEMP_BEGIN()
{
const String *const path = pckReadStrP(param);
const StorageInfoLevel level = (StorageInfoLevel)pckReadU32P(param);
StorageRemoteInfoProtocolWriteData writeData = {0};
StorageList *const list = storageInterfaceListP(storageRemoteProtocolLocal.driver, path, level);
PackWrite *const data = protocolServerResultData(result);
// Indicate whether or not the path was found
pckWriteBoolP(data, list != NULL, .defaultWrite = true);
// Put list
if (list != NULL)
@ -354,194 +350,235 @@ storageRemoteListProtocol(PackRead *const param, ProtocolServer *const server)
{
const StorageInfo info = storageLstGet(list, listIdx);
PackWrite *const write = protocolPackNew();
pckWriteStrP(write, info.name);
storageRemoteInfoProtocolPut(&writeData, write, &info);
protocolServerDataPut(server, write);
pckWriteFree(write);
pckWriteObjBeginP(data);
pckWriteStrP(data, info.name);
storageRemoteInfoProtocolPut(&writeData, data, &info);
pckWriteObjEndP(data);
}
}
// Indicate whether or not the path was found
PackWrite *write = protocolPackNew();
pckWriteBoolP(write, list != NULL, .defaultWrite = true);
protocolServerDataPut(server, write);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageRemoteOpenReadProtocol(PackRead *const param, ProtocolServer *const server)
static bool
storageRemoteReadInternal(StorageRead *const fileRead, PackWrite *const packWrite)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_READ, fileRead);
FUNCTION_LOG_PARAM(PACK_WRITE, packWrite);
FUNCTION_LOG_END();
ASSERT(fileRead != NULL);
ASSERT(packWrite != NULL);
FUNCTION_AUDIT_HELPER();
// Read block and send to client
Buffer *const buffer = bufNew(ioBufferSize());
ioRead(storageReadIo(fileRead), buffer);
pckWriteBoolP(packWrite, bufEmpty(buffer));
if (!bufEmpty(buffer))
pckWriteBinP(packWrite, buffer);
// On eof
bool result = true;
if (ioReadEof(storageReadIo(fileRead)))
{
// Close file (needed to get filter results)
ioReadClose(storageReadIo(fileRead));
// Write filter results
pckWritePackP(packWrite, ioFilterGroupResultAll(ioReadFilterGroup(storageReadIo(fileRead))));
// Let the server know to close the session
result = false;
}
FUNCTION_LOG_RETURN(BOOL, result);
}
FN_EXTERN ProtocolServerResult *
storageRemoteReadOpenProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
ProtocolServerResult *const result = protocolServerResultNewP(.extra = ioBufferSize());
MEM_CONTEXT_TEMP_BEGIN()
{
const String *file = pckReadStrP(param);
bool ignoreMissing = pckReadBoolP(param);
const bool ignoreMissing = pckReadBoolP(param);
const uint64_t offset = pckReadU64P(param);
const Variant *const limit = pckReadNullP(param) ? NULL : VARUINT64(pckReadU64P(param));
const Pack *const filter = pckReadPackP(param);
// Create the read object
IoRead *fileRead = storageReadIo(
storageInterfaceNewReadP(storageRemoteProtocolLocal.driver, file, ignoreMissing, .offset = offset, .limit = limit));
StorageRead *const fileRead = storageInterfaceNewReadP(
storageRemoteProtocolLocal.driver, file, ignoreMissing, .offset = offset, .limit = limit);
// Set filter group based on passed filters
storageRemoteFilterGroup(ioReadFilterGroup(fileRead), filter);
storageRemoteFilterGroup(ioReadFilterGroup(storageReadIo(fileRead)), filter);
// Check if the file exists
bool exists = ioReadOpen(fileRead);
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), exists, .defaultWrite = true));
// Determine if file exists
PackWrite *const data = protocolServerResultData(result);
const bool exists = ioReadOpen(storageReadIo(fileRead));
// Transfer the file if it exists
pckWriteBoolP(data, exists, .defaultWrite = true);
// If the file exists
if (exists)
{
Buffer *buffer = bufNew(ioBufferSize());
// Write file out to protocol layer
do
{
ioRead(fileRead, buffer);
if (!bufEmpty(buffer))
{
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *write = pckWriteNewP(.size = ioBufferSize() + PROTOCOL_PACK_DEFAULT_SIZE);
pckWriteBinP(write, buffer);
protocolServerDataPut(server, write);
// If there is more to read then set session data
if (storageRemoteReadInternal(fileRead, data))
protocolServerResultSessionDataSet(result, fileRead);
}
}
MEM_CONTEXT_TEMP_END();
bufUsedZero(buffer);
}
}
while (!ioReadEof(fileRead));
ioReadClose(fileRead);
// Write filter results
protocolServerDataPut(server, pckWritePackP(protocolPackNew(), ioFilterGroupResultAll(ioReadFilterGroup(fileRead))));
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageRemoteOpenWriteProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
storageRemoteReadProtocol(PackRead *const param, void *const fileRead)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_PARAM(STORAGE_READ, fileRead);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
FUNCTION_AUDIT_STRUCT();
ASSERT(param == NULL);
ASSERT(fileRead != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
ProtocolServerResult *const result = protocolServerResultNewP(.extra = ioBufferSize());
if (!storageRemoteReadInternal(fileRead, protocolServerResultData(result)))
protocolServerResultCloseSet(result);
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
/**********************************************************************************************************************************/
FN_EXTERN ProtocolServerResult *
storageRemoteWriteOpenProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_END();
FUNCTION_AUDIT_STRUCT();
ASSERT(param != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
// Create the write object
const String *file = pckReadStrP(param);
mode_t modeFile = pckReadModeP(param);
mode_t modePath = pckReadModeP(param);
const String *user = pckReadStrP(param);
const String *group = pckReadStrP(param);
time_t timeModified = pckReadTimeP(param);
bool createPath = pckReadBoolP(param);
bool syncFile = pckReadBoolP(param);
bool syncPath = pckReadBoolP(param);
bool atomic = pckReadBoolP(param);
const String *const file = pckReadStrP(param);
const mode_t modeFile = pckReadModeP(param);
const mode_t modePath = pckReadModeP(param);
const String *const user = pckReadStrP(param);
const String *const group = pckReadStrP(param);
const time_t timeModified = pckReadTimeP(param);
const bool createPath = pckReadBoolP(param);
const bool syncFile = pckReadBoolP(param);
const bool syncPath = pckReadBoolP(param);
const bool atomic = pckReadBoolP(param);
const Pack *const filter = pckReadPackP(param);
IoWrite *fileWrite = storageWriteIo(
storageInterfaceNewWriteP(
StorageWrite *const fileWrite = storageInterfaceNewWriteP(
storageRemoteProtocolLocal.driver, file, .modeFile = modeFile, .modePath = modePath, .user = user, .group = group,
.timeModified = timeModified, .createPath = createPath, .syncFile = syncFile, .syncPath = syncPath,
.atomic = atomic, .truncate = true));
.timeModified = timeModified, .createPath = createPath, .syncFile = syncFile, .syncPath = syncPath, .atomic = atomic,
.truncate = true);
// Set filter group based on passed filters
storageRemoteFilterGroup(ioWriteFilterGroup(fileWrite), filter);
storageRemoteFilterGroup(ioWriteFilterGroup(storageWriteIo(fileWrite)), filter);
// Open file
ioWriteOpen(fileWrite);
protocolServerDataPut(server, NULL);
ioWriteOpen(storageWriteIo(fileWrite));
// Write data
do
{
PackRead *read = protocolServerDataGet(server);
// Write is complete
if (read == NULL)
{
ioWriteClose(fileWrite);
// Push filter results
protocolServerDataPut(
server, pckWritePackP(protocolPackNew(), ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite))));
break;
}
// Else more data to write
else
{
pckReadNext(read);
// Write data
if (pckReadType(read) == pckTypeBin)
{
Buffer *const buffer = pckReadBinP(read);
ioWrite(fileWrite, buffer);
bufFree(buffer);
}
// Else write terminated unexpectedly
else
{
protocolServerDataGet(server);
ioWriteFree(fileWrite);
break;
}
}
}
while (true);
protocolServerDataEndPut(server);
// Set session data
protocolServerResultSessionDataSet(result, fileWrite);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageRemotePathCreateProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
storageRemoteWriteProtocol(PackRead *const param, void *const fileWrite)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(STORAGE_READ, fileWrite);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(fileWrite != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
Buffer *const buffer = pckReadBinP(param);
ioWrite(storageWriteIo(fileWrite), buffer);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
}
FN_EXTERN ProtocolServerResult *
storageRemoteWriteCloseProtocol(PackRead *const param, void *const fileWrite)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(STORAGE_WRITE, fileWrite);
FUNCTION_LOG_END();
FUNCTION_AUDIT_STRUCT();
ASSERT(param == NULL);
ASSERT(fileWrite != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
ioWriteClose(storageWriteIo(fileWrite));
// Send filter results
pckWritePackP(protocolServerResultData(result), ioFilterGroupResultAll(ioWriteFilterGroup(storageWriteIo(fileWrite))));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
/**********************************************************************************************************************************/
FN_EXTERN ProtocolServerResult *
storageRemotePathCreateProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
@ -552,52 +589,48 @@ storageRemotePathCreateProtocol(PackRead *const param, ProtocolServer *const ser
mode_t mode = pckReadModeP(param);
storageInterfacePathCreateP(storageRemoteProtocolLocal.driver, path, errorOnExists, noParentCreate, mode);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageRemotePathRemoveProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
storageRemotePathRemoveProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
const String *path = pckReadStrP(param);
bool recurse = pckReadBoolP(param);
const bool result = storageInterfacePathRemoveP(storageRemoteProtocolLocal.driver, path, recurse);
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), result, .defaultWrite = true));
protocolServerDataEndPut(server);
pckWriteBoolP(
protocolServerResultData(result), storageInterfacePathRemoveP(storageRemoteProtocolLocal.driver, path, recurse),
.defaultWrite = true);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageRemotePathSyncProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
storageRemotePathSyncProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
@ -605,24 +638,21 @@ storageRemotePathSyncProtocol(PackRead *const param, ProtocolServer *const serve
const String *path = pckReadStrP(param);
storageInterfacePathSyncP(storageRemoteProtocolLocal.driver, path);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageRemoteRemoveProtocol(PackRead *const param, ProtocolServer *const server)
FN_EXTERN ProtocolServerResult *
storageRemoteRemoveProtocol(PackRead *const param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
@ -631,9 +661,8 @@ storageRemoteRemoveProtocol(PackRead *const param, ProtocolServer *const server)
bool errorOnMissing = pckReadBoolP(param);
storageInterfaceRemoveP(storageRemoteProtocolLocal.driver, file, .errorOnMissing = errorOnMissing);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
}

50
src/storage/remote/protocol.h
View File

@ -11,16 +11,19 @@ Remote Storage Protocol Handler
Functions
***********************************************************************************************************************************/
// Process storage protocol requests
FN_EXTERN void storageRemoteFeatureProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void storageRemoteInfoProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void storageRemoteLinkCreateProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void storageRemoteListProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void storageRemoteOpenReadProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void storageRemoteOpenWriteProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void storageRemotePathCreateProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void storageRemotePathRemoveProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void storageRemotePathSyncProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN void storageRemoteRemoveProtocol(PackRead *param, ProtocolServer *server);
FN_EXTERN ProtocolServerResult *storageRemoteFeatureProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemoteInfoProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemoteLinkCreateProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemoteListProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemotePathCreateProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemotePathRemoveProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemotePathSyncProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemoteReadOpenProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemoteReadProtocol(PackRead *param, void *fileRead);
FN_EXTERN ProtocolServerResult *storageRemoteRemoveProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemoteWriteOpenProtocol(PackRead *param);
FN_EXTERN ProtocolServerResult *storageRemoteWriteProtocol(PackRead *param, void *fileWrite);
FN_EXTERN ProtocolServerResult *storageRemoteWriteCloseProtocol(PackRead *param, void *fileWrite);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@ -29,24 +32,25 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
#define PROTOCOL_COMMAND_STORAGE_INFO STRID5("s-i", 0x27730)
#define PROTOCOL_COMMAND_STORAGE_LINK_CREATE STRID5("s-lc", 0x1b3730)
#define PROTOCOL_COMMAND_STORAGE_LIST STRID5("s-l", 0x33730)
#define PROTOCOL_COMMAND_STORAGE_OPEN_READ STRID5("s-or", 0x93f730)
#define PROTOCOL_COMMAND_STORAGE_OPEN_WRITE STRID5("s-ow", 0xbbf730)
#define PROTOCOL_COMMAND_STORAGE_PATH_CREATE STRID5("s-pc", 0x1c3730)
#define PROTOCOL_COMMAND_STORAGE_REMOVE STRID5("s-r", 0x4b730)
#define PROTOCOL_COMMAND_STORAGE_PATH_REMOVE STRID5("s-pr", 0x943730)
#define PROTOCOL_COMMAND_STORAGE_PATH_SYNC STRID5("s-ps", 0x9c3730)
#define PROTOCOL_COMMAND_STORAGE_READ STRID5("s-rd", 0x24b730)
#define PROTOCOL_COMMAND_STORAGE_REMOVE STRID5("s-r", 0x4b730)
#define PROTOCOL_COMMAND_STORAGE_WRITE STRID5("s-wr", 0x95f730)
#define PROTOCOL_SERVER_HANDLER_STORAGE_REMOTE_LIST \
{.command = PROTOCOL_COMMAND_STORAGE_FEATURE, .handler = storageRemoteFeatureProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_INFO, .handler = storageRemoteInfoProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_LINK_CREATE, .handler = storageRemoteLinkCreateProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_LIST, .handler = storageRemoteListProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_OPEN_READ, .handler = storageRemoteOpenReadProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_OPEN_WRITE, .handler = storageRemoteOpenWriteProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_PATH_CREATE, .handler = storageRemotePathCreateProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_PATH_REMOVE, .handler = storageRemotePathRemoveProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_PATH_SYNC, .handler = storageRemotePathSyncProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_REMOVE, .handler = storageRemoteRemoveProtocol},
{.command = PROTOCOL_COMMAND_STORAGE_FEATURE, .process = storageRemoteFeatureProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_INFO, .process = storageRemoteInfoProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_LINK_CREATE, .process = storageRemoteLinkCreateProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_LIST, .process = storageRemoteListProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_PATH_CREATE, .process = storageRemotePathCreateProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_PATH_REMOVE, .process = storageRemotePathRemoveProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_PATH_SYNC, .process = storageRemotePathSyncProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_READ, .open = storageRemoteReadOpenProtocol, .processSession = storageRemoteReadProtocol},\
{.command = PROTOCOL_COMMAND_STORAGE_REMOVE, .process = storageRemoteRemoveProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_WRITE, .open = storageRemoteWriteOpenProtocol, \
.processSession = storageRemoteWriteProtocol, .close = storageRemoteWriteCloseProtocol},
/***********************************************************************************************************************************
Filters that may be passed to a remote

109
src/storage/remote/read.c
View File

@ -27,9 +27,11 @@ typedef struct StorageReadRemote
StorageRead *read; // Storage read interface
ProtocolClient *client; // Protocol client for requests
ProtocolClientSession *session; // Protocol session for requests
size_t remaining; // Bytes remaining to be read in block
Buffer *block; // Block currently being read
bool eof; // Has the file reached eof?
bool eofFound; // Eof found but a block is remaining to be read
#ifdef DEBUG
uint64_t protocolReadBytes; // How many bytes were read from the protocol layer?
@ -45,55 +47,50 @@ Macros for function logging
objNameToLog(value, "StorageReadRemote", buffer, bufferSize)
/***********************************************************************************************************************************
Clear protocol if the entire file is not read or an error occurs before the read is complete. This is required to clear the
protocol state so a subsequent command can succeed.
Read from a file
***********************************************************************************************************************************/
static void
storageReadRemoteFreeResource(THIS_VOID)
storageReadRemoteInternal(StorageReadRemote *const this, PackRead *const packRead)
{
THIS(StorageReadRemote);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_READ_REMOTE, this);
FUNCTION_LOG_PARAM(PACK_READ, packRead);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(packRead != NULL);
// Read if eof has not been reached
if (!this->eof)
{
do
{
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *const read = protocolClientDataGet(this->client);
pckReadNext(read);
FUNCTION_AUDIT_HELPER();
// If binary then discard
if (pckReadType(read) == pckTypeBin)
// If not done then read the next block
if (!pckReadBoolP(packRead))
{
pckReadBinP(read);
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->block = pckReadBinP(packRead);
this->remaining = bufUsed(this->block);
}
MEM_CONTEXT_OBJ_END();
}
// Else read is complete so discard the filter list
else
{
pckReadPackP(read);
protocolClientDataEndGet(this->client);
// If eof then get results
if (protocolClientSessionClosed(this->session))
{
ioFilterGroupResultAllSet(ioReadFilterGroup(storageReadIo(this->read)), pckReadPackP(packRead));
this->eofFound = true;
if (this->remaining == 0)
this->eof = true;
}
}
MEM_CONTEXT_TEMP_END();
}
while (!this->eof);
}
#ifdef DEBUG
this->protocolReadBytes += this->remaining;
#endif
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Read from a file
***********************************************************************************************************************************/
static size_t
storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
{
@ -120,33 +117,13 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
{
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *const read = protocolClientDataGet(this->client);
pckReadNext(read);
if (!protocolClientSessionQueued(this->session))
protocolClientSessionRequestAsyncP(this->session);
// If binary then read the next block
if (pckReadType(read) == pckTypeBin)
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->block = pckReadBinP(read);
this->remaining = bufUsed(this->block);
}
MEM_CONTEXT_OBJ_END();
}
// Else read is complete and get the filter list
else
{
bufFree(this->block);
storageReadRemoteInternal(this, protocolClientSessionResponse(this->session));
ioFilterGroupResultAllSet(ioReadFilterGroup(storageReadIo(this->read)), pckReadPackP(read));
this->eof = true;
protocolClientDataEndGet(this->client);
}
#ifdef DEBUG
this->protocolReadBytes += this->remaining;
#endif
if (!this->eofFound)
protocolClientSessionRequestAsyncP(this->session);
}
MEM_CONTEXT_TEMP_END();
}
@ -167,6 +144,9 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
{
bufFree(this->block);
this->block = NULL;
if (this->eofFound)
this->eof = true;
}
}
}
@ -219,8 +199,7 @@ storageReadRemoteOpen(THIS_VOID)
compressFilterP(compressTypeGz, (int)this->interface.compressLevel, .raw = true));
}
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ);
PackWrite *const param = protocolCommandParam(command);
PackWrite *const param = protocolPackNew();
pckWriteStrP(param, this->interface.name);
pckWriteBoolP(param, this->interface.ignoreMissing);
@ -233,10 +212,9 @@ storageReadRemoteOpen(THIS_VOID)
pckWritePackP(param, ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read))));
protocolClientCommandPut(this->client, command, false);
// If the file exists
result = pckReadBoolP(protocolClientDataGet(this->client));
PackRead *const packRead = protocolClientSessionOpenP(this->session, .param = param);
result = pckReadBoolP(packRead);
if (result)
{
@ -247,12 +225,9 @@ storageReadRemoteOpen(THIS_VOID)
if (this->interface.compressible)
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilterP(compressTypeGz, .raw = true));
// Set free callback to ensure the protocol is cleared on a short read
memContextCallbackSet(objMemContext(this), storageReadRemoteFreeResource, this);
// Read the first block or eof
storageReadRemoteInternal(this, packRead);
}
// Else nothing to do
else
protocolClientDataEndGet(this->client);
}
MEM_CONTEXT_TEMP_END();
@ -274,8 +249,7 @@ storageReadRemoteClose(THIS_VOID)
ASSERT(this != NULL);
memContextCallbackClear(objMemContext(this));
storageReadRemoteFreeResource(this);
protocolClientSessionCancel(this->session);
FUNCTION_LOG_RETURN_VOID();
}
@ -307,6 +281,7 @@ storageReadRemoteNew(
{
.storage = storage,
.client = client,
.session = protocolClientSessionNewP(client, PROTOCOL_COMMAND_STORAGE_READ, .async = true),
.interface = (StorageReadInterface)
{

65
src/storage/remote/storage.c
View File

@ -136,18 +136,14 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, Storage
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO);
PackWrite *const commandParam = protocolCommandParam(command);
PackWrite *const commandParam = protocolPackNew();
pckWriteStrP(commandParam, file);
pckWriteU32P(commandParam, level);
pckWriteBoolP(commandParam, param.followLink);
// Put command
protocolClientCommandPut(this->client, command, false);
// Read info from protocol
PackRead *read = protocolClientDataGet(this->client);
PackRead *const read = protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_INFO, .param = commandParam);
result.exists = pckReadBoolP(read);
@ -162,8 +158,6 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, Storage
}
MEM_CONTEXT_PRIOR_END();
}
protocolClientDataEndGet(this->client);
}
MEM_CONTEXT_TEMP_END();
@ -190,14 +184,13 @@ storageRemoteLinkCreate(
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_LINK_CREATE);
PackWrite *const commandParam = protocolCommandParam(command);
PackWrite *const commandParam = protocolPackNew();
pckWriteStrP(commandParam, target);
pckWriteStrP(commandParam, linkPath);
pckWriteU32P(commandParam, param.linkType);
protocolClientExecute(this->client, command, false);
protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_LINK_CREATE, .param = commandParam);
}
MEM_CONTEXT_TEMP_END();
@ -224,46 +217,40 @@ storageRemoteList(THIS_VOID, const String *const path, const StorageInfoLevel le
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_LIST);
PackWrite *const commandParam = protocolCommandParam(command);
PackWrite *const commandParam = protocolPackNew();
pckWriteStrP(commandParam, path);
pckWriteU32P(commandParam, level);
// Put command
protocolClientCommandPut(this->client, command, false);
// Read list
StorageRemoteInfoData parseData = {.memContext = memContextCurrent()};
PackRead *const read = protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_LIST, .param = commandParam);
if (pckReadBoolP(read))
{
result = storageLstNew(level);
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
PackRead *read = protocolClientDataGet(this->client);
pckReadNext(read);
while (pckReadType(read) == pckTypeStr)
while (pckReadNext(read))
{
pckReadObjBeginP(read);
StorageInfo info = {.exists = true, .level = level, .name = pckReadStrP(read)};
storageRemoteInfoGet(&parseData, read, &info);
storageLstAdd(result, &info);
pckReadObjEndP(read);
// Reset the memory context occasionally so we don't use too much memory or slow down processing
MEM_CONTEXT_TEMP_RESET(1000);
read = protocolClientDataGet(this->client);
pckReadNext(read);
}
if (!pckReadBoolP(read))
result = NULL;
}
MEM_CONTEXT_TEMP_END();
protocolClientDataEndGet(this->client);
storageLstMove(result, memContextPrior());
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STORAGE_LIST, result);
@ -349,15 +336,14 @@ storageRemotePathCreate(
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_CREATE);
PackWrite *const commandParam = protocolCommandParam(command);
PackWrite *const commandParam = protocolPackNew();
pckWriteStrP(commandParam, path);
pckWriteBoolP(commandParam, errorOnExists);
pckWriteBoolP(commandParam, noParentCreate);
pckWriteModeP(commandParam, mode);
protocolClientExecute(this->client, command, false);
protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_PATH_CREATE, .param = commandParam);
}
MEM_CONTEXT_TEMP_END();
@ -384,13 +370,12 @@ storageRemotePathRemove(THIS_VOID, const String *path, bool recurse, StorageInte
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE);
PackWrite *const commandParam = protocolCommandParam(command);
PackWrite *const commandParam = protocolPackNew();
pckWriteStrP(commandParam, path);
pckWriteBoolP(commandParam, recurse);
result = pckReadBoolP(protocolClientExecute(this->client, command, true));
result = pckReadBoolP(protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE, .param = commandParam));
}
MEM_CONTEXT_TEMP_END();
@ -414,10 +399,11 @@ storageRemotePathSync(THIS_VOID, const String *path, StorageInterfacePathSyncPar
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_SYNC);
pckWriteStrP(protocolCommandParam(command), path);
PackWrite *const commandParam = protocolPackNew();
protocolClientExecute(this->client, command, false);
pckWriteStrP(commandParam, path);
protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_PATH_SYNC, .param = commandParam);
}
MEM_CONTEXT_TEMP_END();
@ -441,13 +427,12 @@ storageRemoteRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam p
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_REMOVE);
PackWrite *const commandParam = protocolCommandParam(command);
PackWrite *const commandParam = protocolPackNew();
pckWriteStrP(commandParam, file);
pckWriteBoolP(commandParam, param.errorOnMissing);
protocolClientExecute(this->client, command, false);
protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_REMOVE, .param = commandParam);
}
MEM_CONTEXT_TEMP_END();
@ -501,7 +486,7 @@ storageRemoteNew(
MEM_CONTEXT_TEMP_BEGIN()
{
// Execute command and get result
PackRead *result = protocolClientExecute(this->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE), true);
PackRead *const result = protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_FEATURE);
// Get path in parent context
MEM_CONTEXT_PRIOR_BEGIN()

54
src/storage/remote/write.c
View File

@ -22,6 +22,7 @@ typedef struct StorageWriteRemote
StorageRemote *storage; // Storage that created this object
StorageWrite *write; // Storage write interface
ProtocolClient *client; // Protocol client to make requests with
ProtocolClientSession *session; // Protocol session for requests
#ifdef DEBUG
uint64_t protocolWriteBytes; // How many bytes were written to the protocol layer?
@ -36,30 +37,6 @@ Macros for function logging
#define FUNCTION_LOG_STORAGE_WRITE_REMOTE_FORMAT(value, buffer, bufferSize) \
objNameToLog(value, "StorageWriteRemote", buffer, bufferSize)
/***********************************************************************************************************************************
Close file on the remote
***********************************************************************************************************************************/
static void
storageWriteRemoteFreeResource(THIS_VOID)
{
THIS(StorageWriteRemote);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_WRITE_REMOTE, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
PackWrite *const write = protocolPackNew();
protocolClientDataPut(this->client, pckWriteBoolP(write, false));
pckWriteFree(write);
protocolClientDataPut(this->client, NULL);
protocolClientDataEndGet(this->client);
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Open the file
***********************************************************************************************************************************/
@ -80,8 +57,7 @@ storageWriteRemoteOpen(THIS_VOID)
if (this->interface.compressible)
ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilterP(compressTypeGz));
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE);
PackWrite *const param = protocolCommandParam(command);
PackWrite *const param = protocolPackNew();
pckWriteStrP(param, this->interface.name);
pckWriteModeP(param, this->interface.modeFile);
@ -95,8 +71,7 @@ storageWriteRemoteOpen(THIS_VOID)
pckWriteBoolP(param, this->interface.atomic);
pckWritePackP(param, ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write))));
protocolClientCommandPut(this->client, command, true);
protocolClientDataGet(this->client);
protocolClientSessionOpenP(this->session, .param = param);
// Clear filters since they will be run on the remote side
ioFilterGroupClear(ioWriteFilterGroup(storageWriteIo(this->write)));
@ -108,9 +83,6 @@ storageWriteRemoteOpen(THIS_VOID)
ioWriteFilterGroup(storageWriteIo(this->write)),
compressFilterP(compressTypeGz, (int)this->interface.compressLevel));
}
// Set free callback to ensure remote file is freed
memContextCallbackSet(objMemContext(this), storageWriteRemoteFreeResource, this);
}
MEM_CONTEXT_TEMP_END();
@ -121,7 +93,7 @@ storageWriteRemoteOpen(THIS_VOID)
Write to the file
***********************************************************************************************************************************/
static void
storageWriteRemote(THIS_VOID, const Buffer *buffer)
storageWriteRemote(THIS_VOID, const Buffer *const buffer)
{
THIS(StorageWriteRemote);
@ -135,8 +107,13 @@ storageWriteRemote(THIS_VOID, const Buffer *buffer)
MEM_CONTEXT_TEMP_BEGIN()
{
protocolClientDataPut(
this->client, pckWriteBinP(pckWriteNewP(.size = ioBufferSize() + PROTOCOL_PACK_DEFAULT_SIZE), buffer));
if (protocolClientSessionQueued(this->session))
protocolClientSessionResponse(this->session);
PackWrite *const param = pckWriteNewP(.size = PROTOCOL_PACK_DEFAULT_SIZE + bufUsed(buffer));
pckWriteBinP(param, buffer);
protocolClientSessionRequestAsyncP(this->session, .param = param);
}
MEM_CONTEXT_TEMP_END();
@ -166,15 +143,15 @@ storageWriteRemoteClose(THIS_VOID)
{
MEM_CONTEXT_TEMP_BEGIN()
{
protocolClientDataPut(this->client, NULL);
if (protocolClientSessionQueued(this->session))
protocolClientSessionResponse(this->session);
ioFilterGroupResultAllSet(
ioWriteFilterGroup(storageWriteIo(this->write)), pckReadPackP(protocolClientDataGet(this->client)));
protocolClientDataEndGet(this->client);
ioWriteFilterGroup(storageWriteIo(this->write)), pckReadPackP(protocolClientSessionClose(this->session)));
}
MEM_CONTEXT_TEMP_END();
this->client = NULL;
memContextCallbackClear(objMemContext(this));
}
FUNCTION_LOG_RETURN_VOID();
@ -215,6 +192,7 @@ storageWriteRemoteNew(
{
.storage = storage,
.client = client,
.session = protocolClientSessionNewP(client, PROTOCOL_COMMAND_STORAGE_WRITE, .async = true),
.interface = (StorageWriteInterface)
{

8
test/code-count/file-type.yaml
View File

@ -2175,14 +2175,6 @@ src/protocol/client.h:
class: core
type: c/h
src/protocol/command.c:
class: core
type: c
src/protocol/command.h:
class: core
type: c/h
src/protocol/helper.c:
class: core
type: c

3
test/define.yaml
View File

@ -453,7 +453,6 @@ unit:
- postgres/interface/crc32
- postgres/interface/page
- protocol/client
- protocol/command
- protocol/helper
- protocol/server
- storage/helper
@ -481,7 +480,6 @@ unit:
coverage:
- protocol/client
- protocol/command
- protocol/helper
- protocol/parallel
- protocol/parallelJob
@ -562,6 +560,7 @@ unit:
- storage/storage
include:
- protocol/client
- storage/read
- storage/write

8
test/src/common/harnessDebug.h
View File

@ -52,6 +52,14 @@ C Debug Harness
} \
while (0)
#define FUNCTION_HARNESS_RETURN_STRUCT(...) \
do \
{ \
STACK_TRACE_POP(false); \
return __VA_ARGS__; \
} \
while (0)
#define FUNCTION_HARNESS_RETURN_VOID() \
STACK_TRACE_POP(false);

2
test/src/common/harnessProtocol.c
View File

@ -235,7 +235,7 @@ protocolRemoteExec(
protocolServerProcess(server, NULL, hrnProtocolStatic.remoteHandlerList, hrnProtocolStatic.remoteHandlerListSize);
// Put an end message here to sync with the client to ensure that coverage data is written before exiting
protocolServerDataEndPut(server);
protocolServerResponseP(server);
// Exit when done
exit(0);

4
test/src/module/command/backupTest.c
View File

@ -1823,7 +1823,7 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("report job error");
ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("key"), protocolCommandNew(strIdFromZ("x")));
ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("key"), strIdFromZ("x"), NULL);
protocolParallelJobErrorSet(job, errorTypeCode(&AssertError), STRDEF("error message"));
unsigned int currentPercentComplete = 0;
@ -1837,7 +1837,7 @@ testRun(void)
TEST_TITLE("report host/100% progress on noop result");
// Create job that skips file
job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ("x")));
job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), strIdFromZ("x"), NULL);
PackWrite *const resultPack = protocolPackNew();
pckWriteStrP(resultPack, STRDEF("pg_data/test"));

2
test/src/module/db/dbTest.c
View File

@ -151,7 +151,6 @@ testRun(void)
TRY_BEGIN()
{
TEST_RESULT_VOID(dbOpen(db), "open db");
TEST_RESULT_UINT(db->remoteIdx, 0, "check remote idx");
TEST_RESULT_VOID(dbFree(db), "free db");
db = NULL;
}
@ -170,7 +169,6 @@ testRun(void)
TRY_BEGIN()
{
TEST_RESULT_VOID(dbOpen(db), "open db");
TEST_RESULT_UINT(db->remoteIdx, 1, "check idx");
TEST_RESULT_STR_Z(dbWalSwitch(db), "000000030000000200000003", "wal switch");
TEST_RESULT_UINT(dbDbTimeout(db), 777000, "check timeout");
TEST_RESULT_VOID(memContextCallbackClear(db->pub.memContext), "clear context so close is not called");

3
test/src/module/postgres/clientTest.c
View File

@ -497,8 +497,7 @@ testRun(void)
{.function = HRN_PQ_FINISH},
{.function = HRN_PQ_GETRESULT, .resultNull = true});
#endif
TEST_RESULT_VOID(pgClientClose(client), "close client");
TEST_RESULT_VOID(pgClientClose(client), "close client again");
TEST_RESULT_VOID(pgClientFree(client), "free client");
}
FUNCTION_HARNESS_RETURN_VOID();

332
test/src/module/protocol/protocolTest.c
View File

@ -21,113 +21,150 @@ Test protocol server command handlers
***********************************************************************************************************************************/
#define TEST_PROTOCOL_COMMAND_ASSERT STRID5("assert", 0x2922ce610)
__attribute__((__noreturn__)) static void
testCommandAssertProtocol(PackRead *const param, ProtocolServer *const server)
__attribute__((__noreturn__)) static ProtocolServerResult *
testCommandAssertProtocol(PackRead *const param)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(server != NULL);
hrnErrorThrowP();
// No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
// No FUNCTION_HARNESS_RETURN() because the function does not return
}
#define TEST_PROTOCOL_COMMAND_ERROR STRID5("error", 0x127ca450)
static unsigned int testCommandErrorProtocolTotal = 0;
__attribute__((__noreturn__)) static void
testCommandErrorProtocol(PackRead *const param, ProtocolServer *const server)
__attribute__((__noreturn__)) static ProtocolServerResult *
testCommandErrorProtocol(PackRead *const param)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(server != NULL);
testCommandErrorProtocolTotal++;
hrnErrorThrowP(.errorType = &FormatError, .message = testCommandErrorProtocolTotal <= 2 ? NULL : "ERR_MESSAGE_RETRY");
// No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
// No FUNCTION_HARNESS_RETURN() because the function does not return
}
#define TEST_PROTOCOL_COMMAND_SIMPLE STRID5("c-simple", 0x2b20d4cf630)
static void
testCommandRequestSimpleProtocol(PackRead *const param, ProtocolServer *const server)
static ProtocolServerResult *
testCommandRequestSimpleProtocol(PackRead *const param)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = param != NULL ? protocolServerResultNewP() : NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), STRDEF("output")));
protocolServerDataEndPut(server);
if (param != NULL)
pckWriteStrP(protocolServerResultData(result), strNewFmt("output%u", pckReadU32P(param)));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID();
FUNCTION_HARNESS_RETURN(PROTOCOL_SERVER_RESULT, result);
}
#define TEST_PROTOCOL_COMMAND_COMPLEX STRID5("c-complex", 0x182b20d78f630)
#define TEST_PROTOCOL_COMMAND_COMPLEX_CLOSE STRID5("c-complex-c", 0xf782b20d78f630)
static void
testCommandRequestComplexProtocol(PackRead *const param, ProtocolServer *const server)
static bool testCommandRequestComplexOpenReturn = false;
static ProtocolServerResult *
testCommandRequestComplexOpenProtocol(PackRead *const param)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_RESULT_UINT(pckReadU32P(param), 87, "param check");
TEST_RESULT_STR_Z(pckReadStrP(param), "data", "param check");
TEST_RESULT_VOID(protocolServerDataPut(server, NULL), "sync");
pckWriteBoolP(protocolServerResultData(result), testCommandRequestComplexOpenReturn);
TEST_RESULT_BOOL(pckReadBoolP(protocolServerDataGet(server)), true, "data get");
TEST_RESULT_UINT(pckReadModeP(protocolServerDataGet(server)), 0644, "data get");
TEST_RESULT_PTR(protocolServerDataGet(server), NULL, "data end get");
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true)), "data put");
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteI32P(protocolPackNew(), -1)), "data put");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
if (testCommandRequestComplexOpenReturn)
protocolServerResultSessionDataSet(result, strNewZ("DATA"));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID();
testCommandRequestComplexOpenReturn = true;
FUNCTION_HARNESS_RETURN(PROTOCOL_SERVER_RESULT, result);
}
static bool testCommandRequestComplexReturn = false;
static ProtocolServerResult *
testCommandRequestComplexProtocol(PackRead *const param, void *const data)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(STRING, data);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(data != NULL);
ASSERT(strEqZ(data, "DATA"));
ProtocolServerResult *const result = protocolServerResultNewP();
pckWriteBoolP(protocolServerResultData(result), testCommandRequestComplexReturn);
if (!testCommandRequestComplexReturn)
protocolServerResultCloseSet(result);
testCommandRequestComplexReturn = true;
FUNCTION_HARNESS_RETURN_STRUCT(result);
}
static ProtocolServerResult *
testCommandRequestComplexCloseProtocol(PackRead *const param, void *const data)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(STRING, data);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(data != NULL);
ASSERT(strEqZ(data, "DATA"));
ProtocolServerResult *const result = protocolServerResultNewP();
pckWriteBoolP(protocolServerResultData(result), true);
FUNCTION_HARNESS_RETURN(PROTOCOL_SERVER_RESULT, result);
}
#define TEST_PROTOCOL_COMMAND_RETRY STRID5("retry", 0x19950b20)
static unsigned int testCommandRetryTotal = 1;
static void
testCommandRetryProtocol(PackRead *const param, ProtocolServer *const server)
static ProtocolServerResult *
testCommandRetryProtocol(PackRead *const param)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(server != NULL);
ProtocolServerResult *const result = protocolServerResultNewP();
MEM_CONTEXT_TEMP_BEGIN()
{
@ -137,20 +174,22 @@ testCommandRetryProtocol(PackRead *const param, ProtocolServer *const server)
THROW(FormatError, "error-until-0");
}
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true));
protocolServerDataEndPut(server);
pckWriteBoolP(protocolServerResultData(result), true);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID();
FUNCTION_HARNESS_RETURN(PROTOCOL_SERVER_RESULT, result);
}
#define TEST_PROTOCOL_SERVER_HANDLER_LIST \
{.command = TEST_PROTOCOL_COMMAND_ASSERT, .handler = testCommandAssertProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_ERROR, .handler = testCommandErrorProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_SIMPLE, .handler = testCommandRequestSimpleProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_COMPLEX, .handler = testCommandRequestComplexProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_RETRY, .handler = testCommandRetryProtocol},
{.command = TEST_PROTOCOL_COMMAND_ASSERT, .process = testCommandAssertProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_ERROR, .process = testCommandErrorProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_SIMPLE, .process = testCommandRequestSimpleProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_COMPLEX, .open = testCommandRequestComplexOpenProtocol, \
.processSession = testCommandRequestComplexProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_COMPLEX_CLOSE, .open = testCommandRequestComplexOpenProtocol, \
.processSession = testCommandRequestComplexProtocol, .close = testCommandRequestComplexCloseProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_RETRY, .process = testCommandRetryProtocol},
/***********************************************************************************************************************************
Test ParallelJobCallback
@ -290,6 +329,7 @@ testRun(void)
.name = strNewZ("test"),
.state = protocolClientStateIdle,
.write = write,
.sessionList = lstNewP(sizeof(ProtocolClientSession)),
};
memContextCallbackSet(memContextCurrent(), protocolClientFreeResource, protocolHelperClient.client);
@ -527,7 +567,7 @@ testRun(void)
TEST_ERROR(
protocolServerProcess(server, NULL, commandHandler, LENGTH_OF(commandHandler)), ProtocolError,
"invalid command 'BOGUS' (0x38eacd271)");
"invalid request 'BOGUS' (0x38eacd271)");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("server restart and assert");
@ -615,24 +655,24 @@ testRun(void)
TEST_TITLE("invalid command");
TEST_ERROR(
protocolClientExecute(client, protocolCommandNew(strIdFromZ("BOGUS")), false), ProtocolError,
"raised from test client: invalid command 'BOGUS' (0x38eacd271)");
protocolClientRequestP(client, strIdFromZ("BOGUS")), ProtocolError,
"raised from test client: invalid request 'BOGUS' (0x38eacd271)");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command throws assert");
TRY_BEGIN()
{
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ASSERT), false);
protocolClientRequestP(client, TEST_PROTOCOL_COMMAND_ASSERT);
THROW(TestError, "error was expected");
}
CATCH_FATAL()
{
TEST_RESULT_Z(errorMessage(), "raised from test client: ERR_MESSAGE", "check message");
TEST_RESULT_PTR(errorType(), &AssertError, "check type");
TEST_RESULT_Z(errorFileName(), TEST_PGB_PATH "/src/protocol/client.c", "check file");
TEST_RESULT_Z(errorFunctionName(), "protocolClientError", "check function");
TEST_RESULT_BOOL(errorFileLine() > 0, true, "check file line > 0");
TEST_RESULT_Z(errorMessage(), "raised from test client: ERR_MESSAGE", "check message");
TEST_RESULT_Z(errorStackTrace(), "ERR_STACK_TRACE", "check stack trace");
}
TRY_END();
@ -645,36 +685,141 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("simple command");
PackWrite *commandParam = protocolPackNew();
pckWriteU32P(commandParam, 99);
TEST_RESULT_STR_Z(
pckReadStrP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_SIMPLE), true)), "output",
pckReadStrP(protocolClientRequestP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .param = commandParam)), "output99",
"execute");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("complex command");
TEST_TITLE("simple command with out of order results");
// Put the command to the server
ProtocolCommand *command = NULL;
TEST_ASSIGN(command, protocolCommandNew(TEST_PROTOCOL_COMMAND_COMPLEX), "command");
TEST_RESULT_VOID(pckWriteU32P(protocolCommandParam(command), 87), "param");
TEST_RESULT_VOID(pckWriteStrP(protocolCommandParam(command), STRDEF("data")), "param");
TEST_RESULT_VOID(protocolClientCommandPut(client, command, true), "command put");
ProtocolClientSession *session1 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .async = true);
PackWrite *param1 = protocolPackNew();
pckWriteU32P(param1, 1);
protocolClientSessionRequestAsyncP(session1, .param = param1);
ProtocolClientSession *session2 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .async = true);
protocolClientSessionRequestAsyncP(session2);
ProtocolClientSession *session3 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE);
PackWrite *param3 = protocolPackNew();
pckWriteU32P(param3, 3);
TEST_RESULT_STR_Z(pckReadStrP(protocolClientSessionRequestP(session3, .param = param3)), "output3", "output 3");
ProtocolClientSession *session4 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .async = true);
PackWrite *param4 = protocolPackNew();
pckWriteU32P(param4, 4);
protocolClientSessionRequestAsyncP(session4, .param = param4);
ProtocolClientSession *session5 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .async = true);
PackWrite *param5 = protocolPackNew();
pckWriteU32P(param5, 5);
protocolClientSessionRequestAsyncP(session5, .param = param5);
TEST_RESULT_VOID(protocolClientSessionCancel(session2), "cancel 2");
TEST_RESULT_STR_Z(pckReadStrP(protocolClientSessionResponse(session1)), "output1", "output 1");
TEST_RESULT_VOID(protocolClientSessionCancel(session4), "cancel 4");
TEST_RESULT_STR_Z(pckReadStrP(protocolClientSessionResponse(session5)), "output5", "output 5");
TEST_RESULT_VOID(protocolClientSessionFree(session1), "free 1");
TEST_RESULT_VOID(protocolClientSessionFree(session5), "free 5");
TEST_RESULT_UINT(lstSize(client->sessionList), 0, "session list is empty");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("invalid session");
TEST_ERROR(protocolClientSessionFindIdx(client, 999), ProtocolError, "unable to find protocol client session 999");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("open returns false");
PackWrite *commandOpenParam = protocolPackNew();
pckWriteU32P(commandOpenParam, 87);
pckWriteStrP(commandOpenParam, STRDEF("data"));
TEST_RESULT_BOOL(
pckReadBoolP(
protocolClientSessionOpenP(
protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX), .param = commandOpenParam)),
false, "open request");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("process returns false (no close needed)");
ProtocolClientSession *session = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX);
TEST_RESULT_BOOL(
pckReadBoolP(protocolClientSessionOpenP(session, .param = commandOpenParam)), true, "open succeed");
uint64_t sessionIdOld = session->sessionId;
session->sessionId = 9999;
TEST_ERROR(
protocolClientStateExpect(client, protocolClientStateIdle), ProtocolError,
"client state is 'cmd-data-get' but expected 'idle'");
protocolClientSessionRequestP(session), ProtocolError,
"raised from test client: unable to find session id 9999 for request c-complex:prc");
// Read null data to indicate that the server has started the command and is read to receive data
TEST_RESULT_PTR(protocolClientDataGet(client), NULL, "command started and ready for data");
session->sessionId = sessionIdOld;
// Write data to the server
TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteBoolP(protocolPackNew(), true)), "data put");
TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteModeP(protocolPackNew(), 0644)), "data put");
TEST_RESULT_VOID(protocolClientDataPut(client, NULL), "data end put");
TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionRequestP(session)), false, "no more to process");
// Get data from the server
TEST_RESULT_BOOL(pckReadBoolP(protocolClientDataGet(client)), true, "data get");
TEST_RESULT_INT(pckReadI32P(protocolClientDataGet(client)), -1, "data get");
TEST_RESULT_VOID(protocolClientDataEndGet(client), "data end get");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error if state not idle");
client->state = protocolClientStateResponse;
session->pub.open = true;
TEST_ERROR(
protocolClientSessionRequestP(session), ProtocolError,
"client state is 'response' but expected 'idle'");
client->state = protocolClientStateIdle;
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("process returns true");
session = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX);
TEST_RESULT_BOOL(
pckReadBoolP(protocolClientSessionOpenP(session, .param = commandOpenParam)), true, "open succeed");
TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionRequestP(session)), true, "more to process");
TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionRequestP(session)), true, "more to process");
TEST_RESULT_PTR(protocolClientSessionClose(session), NULL, "close request");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("close handler");
session = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX_CLOSE, .async = true);
TEST_RESULT_BOOL(
pckReadBoolP(protocolClientSessionOpenP(session, .param = commandOpenParam)), true, "open succeed");
TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionRequestP(session)), true, "more to process");
TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionClose(session)), true, "close request");
session->pub.open = true;
TEST_RESULT_VOID(protocolClientRequestInternal(session, protocolCommandTypeClose, NULL), "close after close");
TEST_ERROR_FMT(
protocolClientResponseInternal(session), ProtocolError,
"raised from test client: unable to find session id %" PRIu64 " for request c-complex-c:cls",
session->sessionId);
TEST_RESULT_VOID(protocolClientRequestInternal(session, protocolCommandTypeCancel, NULL), "cancel after close");
TEST_RESULT_VOID(protocolClientResponseInternal(session), "cancel request succeeds");
session->pub.open = false;
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("cancel handler");
session = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX);
TEST_RESULT_BOOL(
pckReadBoolP(protocolClientSessionOpenP(session, .param = commandOpenParam)), true, "open succeed");
TEST_RESULT_VOID(protocolClientSessionFree(session), "free request");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("free client");
@ -692,15 +837,13 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command with retry");
TEST_RESULT_BOOL(
pckReadBoolP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_RETRY), true)), true,
"execute");
TEST_RESULT_BOOL(pckReadBoolP(protocolClientRequestP(client, TEST_PROTOCOL_COMMAND_RETRY)), true, "execute");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command throws assert with retry messages");
TEST_ERROR(
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ERROR), false), FormatError,
protocolClientRequestP(client, TEST_PROTOCOL_COMMAND_ERROR), FormatError,
"raised from test client: ERR_MESSAGE\n"
"[RETRY DETAIL OMITTED]");
@ -843,7 +986,7 @@ testRun(void)
TEST_ASSIGN(server, protocolServer(tlsServer, socketSession), "server start");
TEST_RESULT_PTR_NE(server, NULL, "server is not null");
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "server exit");
TEST_RESULT_UINT(protocolServerRequest(server).id, PROTOCOL_COMMAND_EXIT, "server exit");
// Repo server access denied (archive-get) invalid stanza
// -----------------------------------------------------------------------------------------------------------------
@ -885,7 +1028,7 @@ testRun(void)
TEST_ASSIGN(server, protocolServer(tlsServer, socketSession), "server start");
TEST_RESULT_PTR_NE(server, NULL, "server is not null");
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "server exit");
TEST_RESULT_UINT(protocolServerRequest(server).id, PROTOCOL_COMMAND_EXIT, "server exit");
}
HRN_FORK_PARENT_END();
}
@ -906,7 +1049,7 @@ testRun(void)
{
TEST_ASSIGN(
job,
protocolParallelJobNew(VARSTRDEF("test"), protocolCommandNew(strIdFromZ("c"))), "new job");
protocolParallelJobNew(VARSTRDEF("test"), strIdFromZ("c"), NULL), "new job");
TEST_RESULT_PTR(protocolParallelJobMove(job, memContextPrior()), job, "move job");
TEST_RESULT_PTR(protocolParallelJobMove(NULL, memContextPrior()), NULL, "move null job");
}
@ -938,16 +1081,15 @@ testRun(void)
"local server 1");
// Command with output
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ("c-one"), "c-one command get");
TEST_RESULT_UINT(protocolServerRequest(server).id, strIdFromZ("c-one"), "c-one command get");
// Wait for notify from parent
HRN_FORK_CHILD_NOTIFY_GET();
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 1)), "data end put");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
TEST_RESULT_VOID(protocolServerResponseP(server, .data = pckWriteU32P(protocolPackNew(), 1)), "data end put");
// Wait for exit
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "noop command get");
TEST_RESULT_UINT(protocolServerRequest(server).id, PROTOCOL_COMMAND_EXIT, "noop command get");
}
HRN_FORK_CHILD_END();
@ -961,20 +1103,19 @@ testRun(void)
"local server 2");
// Command with output
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ("c2"), "c2 command get");
TEST_RESULT_UINT(protocolServerRequest(server).id, strIdFromZ("c2"), "c2 command get");
// Wait for notify from parent
HRN_FORK_CHILD_NOTIFY_GET();
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 2)), "data end put");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
TEST_RESULT_VOID(protocolServerResponseP(server, .data = pckWriteU32P(protocolPackNew(), 2)), "data end put");
// Command with error
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ("c-three"), "c-three command get");
TEST_RESULT_UINT(protocolServerRequest(server).id, strIdFromZ("c-three"), "c-three command get");
TEST_RESULT_VOID(protocolServerError(server, 39, STRDEF("very serious error"), STRDEF("stack")), "error put");
// Wait for exit
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "wait for exit");
TEST_RESULT_UINT(protocolServerRequest(server).id, PROTOCOL_COMMAND_EXIT, "wait for exit");
}
HRN_FORK_CHILD_END();
@ -1018,23 +1159,26 @@ testRun(void)
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("add jobs");
ProtocolCommand *command = protocolCommandNew(strIdFromZ("c-one"));
pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
pckWriteStrP(protocolCommandParam(command), STRDEF("param2"));
StringId command = strIdFromZ("c-one");
PackWrite *param = protocolPackNew();
pckWriteStrP(param, STRDEF("param1"));
pckWriteStrP(param, STRDEF("param2"));
ProtocolParallelJob *job = protocolParallelJobNew(varNewStr(STRDEF("job1")), command);
ProtocolParallelJob *job = protocolParallelJobNew(varNewStr(STRDEF("job1")), command, param);
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
command = protocolCommandNew(strIdFromZ("c2"));
pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
command = strIdFromZ("c2");
param = protocolPackNew();
pckWriteStrP(param, STRDEF("param1"));
job = protocolParallelJobNew(varNewStr(STRDEF("job2")), command);
job = protocolParallelJobNew(varNewStr(STRDEF("job2")), command, param);
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
command = protocolCommandNew(strIdFromZ("c-three"));
pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
command = strIdFromZ("c-three");
param = protocolPackNew();
pckWriteStrP(param, STRDEF("param1"));
job = protocolParallelJobNew(varNewStr(STRDEF("job3")), command);
job = protocolParallelJobNew(varNewStr(STRDEF("job3")), command, param);
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
// -----------------------------------------------------------------------------------------------------------------

57
test/src/module/storage/remoteTest.c
View File

@ -325,16 +325,30 @@ testRun(void)
ioBufferSizeSet(bufferOld);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("read partial file then free");
TEST_TITLE("read partial file then free (interleaved with normal read)");
buffer = bufNew(6);
Buffer *buffer2 = bufNew(7);
StorageRead *fileRead2 = NULL;
TEST_ASSIGN(fileRead, storageNewReadP(storageRepo, STRDEF("test.txt")), "get file");
TEST_ASSIGN(fileRead2, storageNewReadP(storageRepo, STRDEF("test.txt"), .limit = VARUINT64(11)), "get file");
TEST_RESULT_BOOL(ioReadOpen(storageReadIo(fileRead)), true, "open read");
TEST_RESULT_BOOL(ioReadOpen(storageReadIo(fileRead2)), true, "open read file 2");
TEST_RESULT_UINT(ioRead(storageReadIo(fileRead), buffer), 6, "partial read");
TEST_RESULT_STR_Z(strNewBuf(buffer), "BABABA", "check contents");
TEST_RESULT_BOOL(ioReadEof(storageReadIo(fileRead)), false, "no eof");
TEST_RESULT_UINT(ioRead(storageReadIo(fileRead2), buffer2), 7, "partial read file 2");
TEST_RESULT_STR_Z(strNewBuf(buffer2), "BABABAB", "check contents");
bufUsedZero(buffer2);
TEST_RESULT_UINT(ioRead(storageReadIo(fileRead2), buffer2), 4, "partial read file 2");
TEST_RESULT_STR_Z(strNewBuf(buffer2), "ABAB", "check contents");
TEST_RESULT_VOID(storageReadFree(fileRead), "free");
TEST_RESULT_VOID(ioReadClose(storageReadIo(fileRead2)), "close");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("read file with compression");
@ -452,17 +466,35 @@ testRun(void)
((StorageRemote *)storageDriver(storageRepoWrite))->compressLevel = 3;
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write file, free before close, make sure the .tmp file remains");
TEST_TITLE("write file, free before close, make sure the .tmp file remains (interleaved with normal write)");
StorageWrite *write3 = NULL;
TEST_ASSIGN(write, storageNewWriteP(storageRepoWrite, STRDEF("test2.txt")), "new write file");
TEST_ASSIGN(write3, storageNewWriteP(storageRepoWrite, STRDEF("test3.txt")), "new write file");
TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(write)), "open file");
TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(write3)), "open file 3");
TEST_RESULT_VOID(ioWrite(storageWriteIo(write), contentBuf), "write bytes");
TEST_RESULT_VOID(ioWrite(storageWriteIo(write3), contentBuf), "write bytes to file 3");
TEST_RESULT_VOID(storageWriteFree(write), "free file");
TEST_RESULT_VOID(ioWriteClose(storageWriteIo(write3)), "close file 3");
TEST_RESULT_UINT(
storageInfoP(storageTest, STRDEF("repo128/test2.txt.pgbackrest.tmp")).size, 16384, "file exists and is partial");
TEST_RESULT_BOOL(bufEq(storageGetP(storageNewReadP(storageRepo, STRDEF("test3.txt"))), contentBuf), true, "check file 3");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write free and close before write");
TEST_ASSIGN(write, storageNewWriteP(storageRepoWrite, STRDEF("test2.txt")), "new write file");
TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(write)), "open file");
TEST_RESULT_VOID(storageWriteFree(write), "free file");
TEST_ASSIGN(write, storageNewWriteP(storageRepoWrite, STRDEF("test2.txt")), "new write file");
TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(write)), "open file");
TEST_RESULT_VOID(ioWriteClose(storageWriteIo(write)), "close file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write the file again with protocol compression");
@ -711,10 +743,23 @@ testRun(void)
// sure coverage data has been written by the remote. We also need to make sure that the mem context callback is cleared so that
// protocolClientFreeResource() will not be called and send another exit. protocolFree() is still required to free the client
// objects.
memContextCallbackClear(objMemContext(protocolRemoteGet(protocolStorageTypeRepo, 0)));
protocolClientExecute(protocolRemoteGet(protocolStorageTypeRepo, 0), protocolCommandNew(PROTOCOL_COMMAND_EXIT), false);
memContextCallbackClear(objMemContext(protocolRemoteGet(protocolStorageTypePg, 1)));
protocolClientExecute(protocolRemoteGet(protocolStorageTypePg, 1), protocolCommandNew(PROTOCOL_COMMAND_EXIT), false);
ProtocolClient *client = protocolRemoteGet(protocolStorageTypeRepo, 0);
memContextCallbackClear(objMemContext(client));
for (unsigned int sessionIdx = 0; sessionIdx < lstSize(client->sessionList); sessionIdx++)
memContextCallbackClear(objMemContext(*(ProtocolClientSession **)lstGet(client->sessionList, sessionIdx)));
protocolClientRequestP(client, PROTOCOL_COMMAND_EXIT);
client = protocolRemoteGet(protocolStorageTypePg, 1);
for (unsigned int sessionIdx = 0; sessionIdx < lstSize(client->sessionList); sessionIdx++)
memContextCallbackClear(objMemContext(*(ProtocolClientSession **)lstGet(client->sessionList, sessionIdx)));
memContextCallbackClear(objMemContext(client));
protocolClientRequestP(client, PROTOCOL_COMMAND_EXIT);
protocolFree();
FUNCTION_HARNESS_RETURN_VOID();