1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-15 01:04:37 +02:00

Binary protocol.

Switch from JSON-based to binary protocol for communicating with local and remote process. The pack type is used to implement the binary protocol.

There are a number advantages:

* The pack type is more compact than JSON and are more efficient to render/parse.
* Packs are more strictly typed than JSON.
* Each protocol message is written entirely within ProtocolServer/ProtocolClient so is less likely to get interrupted by an error and leave the protocol in a bad state.
* There is no limit on message size. Previously this was limited by buffer size without a custom implementation, as was done for read/writing files.

Some cruft from the Perl days was removed, specifically allowing NULL messages and stack traces. This is no longer possible in C.

There is room for improvement here, in particular locking down the allowed sequence of protocol messages and building a state machine to enforce it. This will be useful for resetting the protocol when it gets in a bad state.
This commit is contained in:
David Steele
2021-06-24 13:31:16 -04:00
committed by GitHub
parent bffb43ea3f
commit 6a1c0337dd
41 changed files with 1397 additions and 1227 deletions

View File

@ -15,6 +15,17 @@
<release date="XXXX-XX-XX" version="2.35dev" title="UNDER DEVELOPMENT"> <release date="XXXX-XX-XX" version="2.35dev" title="UNDER DEVELOPMENT">
<release-core-list> <release-core-list>
<release-development-list> <release-development-list>
<release-item>
<github-pull-request id="1424"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>Binary protocol.</p>
</release-item>
<release-item> <release-item>
<github-pull-request id="1421"/> <github-pull-request id="1421"/>

View File

@ -851,19 +851,21 @@ static ProtocolParallelJob *archiveGetAsyncCallback(void *data, unsigned int cli
const ArchiveFileMap *archiveFileMap = lstGet(jobData->archiveFileMapList, jobData->archiveFileIdx); const ArchiveFileMap *archiveFileMap = lstGet(jobData->archiveFileMapList, jobData->archiveFileIdx);
jobData->archiveFileIdx++; jobData->archiveFileIdx++;
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_GET_FILE); ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_GET_FILE);
protocolCommandParamAdd(command, VARSTR(archiveFileMap->request)); PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, archiveFileMap->request);
// Add actual files to get // Add actual files to get
for (unsigned int actualIdx = 0; actualIdx < lstSize(archiveFileMap->actualList); actualIdx++) for (unsigned int actualIdx = 0; actualIdx < lstSize(archiveFileMap->actualList); actualIdx++)
{ {
const ArchiveGetFile *actual = lstGet(archiveFileMap->actualList, actualIdx); const ArchiveGetFile *const actual = lstGet(archiveFileMap->actualList, actualIdx);
protocolCommandParamAdd(command, VARSTR(actual->file)); pckWriteStrP(param, actual->file);
protocolCommandParamAdd(command, VARUINT(actual->repoIdx)); pckWriteU32P(param, actual->repoIdx);
protocolCommandParamAdd(command, VARSTR(actual->archiveId)); pckWriteStrP(param, actual->archiveId);
protocolCommandParamAdd(command, VARUINT64(actual->cipherType)); pckWriteU64P(param, actual->cipherType);
protocolCommandParamAdd(command, VARSTR(actual->cipherPassArchive)); pckWriteStrP(param, actual->cipherPassArchive);
} }
FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(archiveFileMap->request), command)); FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(archiveFileMap->request), command));
@ -941,12 +943,12 @@ cmdArchiveGetAsync(void)
if (protocolParallelJobErrorCode(job) == 0) if (protocolParallelJobErrorCode(job) == 0)
{ {
// Get the actual file retrieved // Get the actual file retrieved
const VariantList *fileResult = varVarLst(protocolParallelJobResult(job)); PackRead *const fileResult = protocolParallelJobResult(job);
ArchiveGetFile *file = lstGet(fileMap->actualList, varUIntForce(varLstGet(fileResult, 0))); ArchiveGetFile *file = lstGet(fileMap->actualList, pckReadU32P(fileResult));
ASSERT(file != NULL); ASSERT(file != NULL);
// Output file warnings // Output file warnings
StringList *fileWarnList = strLstNewVarLst(varVarLst(varLstGet(fileResult, 1))); StringList *fileWarnList = pckReadStrLstP(fileResult);
for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++) for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++)
LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx))); LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx)));

View File

@ -15,54 +15,47 @@ Archive Get Protocol Handler
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
archiveGetFileProtocol(const VariantList *paramList, ProtocolServer *server) archiveGetFileProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
const String *request = varStr(varLstGet(paramList, 0)); // Get request
const String *const request = pckReadStrP(param);
const unsigned int paramFixed = 1; // Fixed params before the actual list
const unsigned int paramActual = 5; // Parameters in each index of the actual list
// Check that the correct number of list parameters were passed
CHECK((varLstSize(paramList) - paramFixed) % paramActual == 0);
// Build the actual list // Build the actual list
List *actualList = lstNewP(sizeof(ArchiveGetFile)); List *actualList = lstNewP(sizeof(ArchiveGetFile));
unsigned int actualListSize = (varLstSize(paramList) - paramFixed) / paramActual;
for (unsigned int actualIdx = 0; actualIdx < actualListSize; actualIdx++) while (!pckReadNullP(param))
{ {
lstAdd( ArchiveGetFile actual = {.file = pckReadStrP(param)};
actualList, actual.repoIdx = pckReadU32P(param);
&(ArchiveGetFile) actual.archiveId = pckReadStrP(param);
{ actual.cipherType = pckReadU64P(param);
.file = varStr(varLstGet(paramList, paramFixed + (actualIdx * paramActual))), actual.cipherPassArchive = pckReadStrP(param);
.repoIdx = varUIntForce(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 1)),
.archiveId = varStr(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 2)), lstAdd(actualList, &actual);
.cipherType = varUInt64(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 3)),
.cipherPassArchive = varStr(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 4)),
});
} }
// Return result // Get file
ArchiveGetFileResult fileResult = archiveGetFile( ArchiveGetFileResult fileResult = archiveGetFile(
storageSpoolWrite(), request, actualList, storageSpoolWrite(), request, actualList,
strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s." STORAGE_FILE_TEMP_EXT, strZ(request))); strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s." STORAGE_FILE_TEMP_EXT, strZ(request)));
VariantList *result = varLstNew(); // Return result
varLstAdd(result, varNewUInt(fileResult.actualIdx)); PackWrite *const resultPack = protocolPackNew();
varLstAdd(result, varNewVarLst(varLstNewStrLst(fileResult.warnList))); pckWriteU32P(resultPack, fileResult.actualIdx);
pckWriteStrLstP(resultPack, fileResult.warnList);
protocolServerResponse(server, varNewVarLst(result)); protocolServerDataPut(server, resultPack);
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();

View File

@ -4,16 +4,14 @@ Archive Get Protocol Handler
#ifndef COMMAND_ARCHIVE_GET_PROTOCOL_H #ifndef COMMAND_ARCHIVE_GET_PROTOCOL_H
#define COMMAND_ARCHIVE_GET_PROTOCOL_H #define COMMAND_ARCHIVE_GET_PROTOCOL_H
#include "common/type/string.h" #include "common/type/pack.h"
#include "common/type/stringId.h"
#include "common/type/variantList.h"
#include "protocol/server.h" #include "protocol/server.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Process protocol requests // Process protocol requests
void archiveGetFileProtocol(const VariantList *paramList, ProtocolServer *server); void archiveGetFileProtocol(PackRead *param, ProtocolServer *server);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess() Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

View File

@ -14,50 +14,55 @@ Archive Push Protocol Handler
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server) archivePushFileProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Build the repo data list // Read parameters
const String *const walSource = pckReadStrP(param);
const bool headerCheck = pckReadBoolP(param);
const unsigned int pgVersion = pckReadU32P(param);
const uint64_t pgSystemId = pckReadU64P(param);
const String *const archiveFile = pckReadStrP(param);
const CompressType compressType = pckReadU32P(param);
const int compressLevel = pckReadI32P(param);
const StringList *const priorErrorList = pckReadStrLstP(param);
// Read repo data
List *repoList = lstNewP(sizeof(ArchivePushFileRepoData)); List *repoList = lstNewP(sizeof(ArchivePushFileRepoData));
unsigned int repoListSize = varUIntForce(varLstGet(paramList, 8));
unsigned int paramIdx = 9;
for (unsigned int repoListIdx = 0; repoListIdx < repoListSize; repoListIdx++) pckReadArrayBeginP(param);
while (!pckReadNullP(param))
{ {
lstAdd( pckReadObjBeginP(param);
repoList,
&(ArchivePushFileRepoData)
{
.repoIdx = varUIntForce(varLstGet(paramList, paramIdx)),
.archiveId = varStr(varLstGet(paramList, paramIdx + 1)),
.cipherType = varUInt64(varLstGet(paramList, paramIdx + 2)),
.cipherPass = varStr(varLstGet(paramList, paramIdx + 3)),
});
paramIdx += 4; ArchivePushFileRepoData repo = {.repoIdx = pckReadU32P(param)};
repo.archiveId = pckReadStrP(param);
repo.cipherType = pckReadU64P(param);
repo.cipherPass = pckReadStrP(param);
pckReadObjEndP(param);
lstAdd(repoList, &repo);
} }
// Push the file pckReadArrayEndP(param);
ArchivePushFileResult fileResult = archivePushFile(
varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUIntForce(varLstGet(paramList, 2)), // Push file
varUInt64(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)), const ArchivePushFileResult fileResult = archivePushFile(
(CompressType)varUIntForce(varLstGet(paramList, 5)), varIntForce(varLstGet(paramList, 6)), repoList, walSource, headerCheck, pgVersion, pgSystemId, archiveFile, compressType, compressLevel, repoList, priorErrorList);
strLstNewVarLst(varVarLst(varLstGet(paramList, 7))));
// Return result // Return result
VariantList *result = varLstNew(); protocolServerDataPut(server, pckWriteStrLstP(protocolPackNew(), fileResult.warnList));
varLstAdd(result, varNewVarLst(varLstNewStrLst(fileResult.warnList))); protocolServerDataEndPut(server);
protocolServerResponse(server, varNewVarLst(result));
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();

View File

@ -4,15 +4,14 @@ Archive Push Protocol Handler
#ifndef COMMAND_ARCHIVE_PUSH_PROTOCOL_H #ifndef COMMAND_ARCHIVE_PUSH_PROTOCOL_H
#define COMMAND_ARCHIVE_PUSH_PROTOCOL_H #define COMMAND_ARCHIVE_PUSH_PROTOCOL_H
#include "common/type/string.h" #include "common/type/pack.h"
#include "common/type/variantList.h"
#include "protocol/server.h" #include "protocol/server.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Process protocol requests // Process protocol requests
void archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server); void archivePushFileProtocol(PackRead *param, ProtocolServer *server);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess() Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

View File

@ -460,28 +460,35 @@ archivePushAsyncCallback(void *data, unsigned int clientIdx)
const String *walFile = strLstGet(jobData->walFileList, jobData->walFileIdx); const String *walFile = strLstGet(jobData->walFileList, jobData->walFileIdx);
jobData->walFileIdx++; jobData->walFileIdx++;
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE); ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE);
protocolCommandParamAdd(command, VARSTR(strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile)))); PackWrite *const param = protocolCommandParam(command);
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptArchiveHeaderCheck)));
protocolCommandParamAdd(command, VARUINT(jobData->archiveInfo.pgVersion)); pckWriteStrP(param, strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile)));
protocolCommandParamAdd(command, VARUINT64(jobData->archiveInfo.pgSystemId)); pckWriteBoolP(param, cfgOptionBool(cfgOptArchiveHeaderCheck));
protocolCommandParamAdd(command, VARSTR(walFile)); pckWriteU32P(param, jobData->archiveInfo.pgVersion);
protocolCommandParamAdd(command, VARUINT(jobData->compressType)); pckWriteU64P(param, jobData->archiveInfo.pgSystemId);
protocolCommandParamAdd(command, VARINT(jobData->compressLevel)); pckWriteStrP(param, walFile);
protocolCommandParamAdd(command, varNewVarLst(varLstNewStrLst(jobData->archiveInfo.errorList))); pckWriteU32P(param, jobData->compressType);
protocolCommandParamAdd(command, VARUINT(lstSize(jobData->archiveInfo.repoList))); pckWriteI32P(param, jobData->compressLevel);
pckWriteStrLstP(param, jobData->archiveInfo.errorList);
// Add data for each repo to push to // Add data for each repo to push to
pckWriteArrayBeginP(param);
for (unsigned int repoListIdx = 0; repoListIdx < lstSize(jobData->archiveInfo.repoList); repoListIdx++) for (unsigned int repoListIdx = 0; repoListIdx < lstSize(jobData->archiveInfo.repoList); repoListIdx++)
{ {
ArchivePushFileRepoData *data = lstGet(jobData->archiveInfo.repoList, repoListIdx); ArchivePushFileRepoData *data = lstGet(jobData->archiveInfo.repoList, repoListIdx);
protocolCommandParamAdd(command, VARUINT(data->repoIdx)); pckWriteObjBeginP(param);
protocolCommandParamAdd(command, VARSTR(data->archiveId)); pckWriteU32P(param, data->repoIdx);
protocolCommandParamAdd(command, VARUINT64(data->cipherType)); pckWriteStrP(param, data->archiveId);
protocolCommandParamAdd(command, VARSTR(data->cipherPass)); pckWriteU64P(param, data->cipherType);
pckWriteStrP(param, data->cipherPass);
pckWriteObjEndP(param);
} }
pckWriteArrayEndP(param);
FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(walFile), command)); FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(walFile), command));
} }
@ -572,11 +579,8 @@ cmdArchivePushAsync(void)
// The job was successful // The job was successful
if (protocolParallelJobErrorCode(job) == 0) if (protocolParallelJobErrorCode(job) == 0)
{ {
// Get job result
const VariantList *fileResult = varVarLst(protocolParallelJobResult(job));
// Output file warnings // Output file warnings
StringList *fileWarnList = strLstNewVarLst(varVarLst(varLstGet(fileResult, 0))); StringList *fileWarnList = pckReadStrLstP(protocolParallelJobResult(job));
for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++) for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++)
LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx))); LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx)));

View File

@ -23,6 +23,7 @@ Backup Command
#include "common/log.h" #include "common/log.h"
#include "common/time.h" #include "common/time.h"
#include "common/type/convert.h" #include "common/type/convert.h"
#include "common/type/json.h"
#include "config/config.h" #include "config/config.h"
#include "db/helper.h" #include "db/helper.h"
#include "info/infoArchive.h" #include "info/infoArchive.h"
@ -1043,12 +1044,12 @@ backupJobResult(
const ManifestFile *const file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job))); const ManifestFile *const file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
const unsigned int processId = protocolParallelJobProcessId(job); const unsigned int processId = protocolParallelJobProcessId(job);
const VariantList *const jobResult = varVarLst(protocolParallelJobResult(job)); PackRead *const jobResult = protocolParallelJobResult(job);
const BackupCopyResult copyResult = (BackupCopyResult)varUIntForce(varLstGet(jobResult, 0)); const BackupCopyResult copyResult = (BackupCopyResult)pckReadU32P(jobResult);
const uint64_t copySize = varUInt64(varLstGet(jobResult, 1)); const uint64_t copySize = pckReadU64P(jobResult);
const uint64_t repoSize = varUInt64(varLstGet(jobResult, 2)); const uint64_t repoSize = pckReadU64P(jobResult);
const String *const copyChecksum = varStr(varLstGet(jobResult, 3)); const String *const copyChecksum = pckReadStrP(jobResult);
const KeyValue *const checksumPageResult = varKv(varLstGet(jobResult, 4)); const KeyValue *const checksumPageResult = varKv(jsonToVar(pckReadStrP(jobResult, .defaultValue = NULL_STR)));
// Increment backup copy progress // Increment backup copy progress
sizeCopied += copySize; sizeCopied += copySize;
@ -1439,23 +1440,23 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
// Create backup job // Create backup job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
PackWrite *const param = protocolCommandParam(command);
protocolCommandParamAdd(command, VARSTR(manifestPathPg(file->name))); pckWriteStrP(param, manifestPathPg(file->name));
protocolCommandParamAdd( pckWriteBoolP(param, !strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
command, VARBOOL(!strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)))); pckWriteU64P(param, file->size);
protocolCommandParamAdd(command, VARUINT64(file->size)); pckWriteBoolP(param, !file->primary);
protocolCommandParamAdd(command, VARBOOL(!file->primary)); pckWriteStrP(param, file->checksumSha1[0] != 0 ? STR(file->checksumSha1) : NULL);
protocolCommandParamAdd(command, file->checksumSha1[0] != 0 ? VARSTRZ(file->checksumSha1) : NULL); pckWriteBoolP(param, file->checksumPage);
protocolCommandParamAdd(command, VARBOOL(file->checksumPage)); pckWriteU64P(param, jobData->lsnStart);
protocolCommandParamAdd(command, VARUINT64(jobData->lsnStart)); pckWriteStrP(param, file->name);
protocolCommandParamAdd(command, VARSTR(file->name)); pckWriteBoolP(param, file->reference != NULL);
protocolCommandParamAdd(command, VARBOOL(file->reference != NULL)); pckWriteU32P(param, jobData->compressType);
protocolCommandParamAdd(command, VARUINT(jobData->compressType)); pckWriteI32P(param, jobData->compressLevel);
protocolCommandParamAdd(command, VARINT(jobData->compressLevel)); pckWriteStrP(param, jobData->backupLabel);
protocolCommandParamAdd(command, VARSTR(jobData->backupLabel)); pckWriteBoolP(param, jobData->delta);
protocolCommandParamAdd(command, VARBOOL(jobData->delta)); pckWriteU64P(param, jobData->cipherSubPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc);
protocolCommandParamAdd(command, VARUINT64(jobData->cipherType)); pckWriteStrP(param, jobData->cipherSubPass);
protocolCommandParamAdd(command, VARSTR(jobData->cipherSubPass));
// Remove job from the queue // Remove job from the queue
lstRemoveIdx(queue, 0); lstRemoveIdx(queue, 0);

View File

@ -9,41 +9,56 @@ Backup Protocol Handler
#include "common/io/io.h" #include "common/io/io.h"
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/type/json.h"
#include "config/config.h" #include "config/config.h"
#include "storage/helper.h" #include "storage/helper.h"
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
backupFileProtocol(const VariantList *paramList, ProtocolServer *server) backupFileProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Backup the file // Backup file
BackupFileResult result = backupFile( const String *const pgFile = pckReadStrP(param);
varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)), const bool pgFileIgnoreMissing = pckReadBoolP(param);
varBool(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)), varBool(varLstGet(paramList, 5)), const uint64_t pgFileSize = pckReadU64P(param);
varUInt64(varLstGet(paramList, 6)), varStr(varLstGet(paramList, 7)), varBool(varLstGet(paramList, 8)), const bool pgFileCopyExactSize = pckReadBoolP(param);
(CompressType)varUIntForce(varLstGet(paramList, 9)), varIntForce(varLstGet(paramList, 10)), const String *const pgFileChecksum = pckReadStrP(param);
varStr(varLstGet(paramList, 11)), varBool(varLstGet(paramList, 12)), varUInt64(varLstGet(paramList, 13)), const bool pgFileChecksumPage = pckReadBoolP(param);
varStr(varLstGet(paramList, 14))); const uint64_t pgFileChecksumPageLsnLimit = pckReadU64P(param);
const String *const repoFile = pckReadStrP(param);
const bool repoFileHasReference = pckReadBoolP(param);
const CompressType repoFileCompressType = (CompressType)pckReadU32P(param);
const int repoFileCompressLevel = pckReadI32P(param);
const String *const backupLabel = pckReadStrP(param);
const bool delta = pckReadBoolP(param);
const CipherType cipherType = (CipherType)pckReadU64P(param);
const String *const cipherPass = pckReadStrP(param);
// Return backup result const BackupFileResult result = backupFile(
VariantList *resultList = varLstNew(); pgFile, pgFileIgnoreMissing, pgFileSize, pgFileCopyExactSize, pgFileChecksum, pgFileChecksumPage,
varLstAdd(resultList, varNewUInt(result.backupCopyResult)); pgFileChecksumPageLsnLimit, repoFile, repoFileHasReference, repoFileCompressType, repoFileCompressLevel,
varLstAdd(resultList, varNewUInt64(result.copySize)); backupLabel, delta, cipherType, cipherPass);
varLstAdd(resultList, varNewUInt64(result.repoSize));
varLstAdd(resultList, varNewStr(result.copyChecksum));
varLstAdd(resultList, result.pageChecksumResult != NULL ? varNewKv(result.pageChecksumResult) : NULL);
protocolServerResponse(server, varNewVarLst(resultList)); // Return result
PackWrite *const resultPack = protocolPackNew();
pckWriteU32P(resultPack, result.backupCopyResult);
pckWriteU64P(resultPack, result.copySize);
pckWriteU64P(resultPack, result.repoSize);
pckWriteStrP(resultPack, result.copyChecksum);
pckWriteStrP(resultPack, result.pageChecksumResult != NULL ? jsonFromKv(result.pageChecksumResult) : NULL);
protocolServerDataPut(server, resultPack);
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();

View File

@ -4,15 +4,14 @@ Backup Protocol Handler
#ifndef COMMAND_BACKUP_PROTOCOL_H #ifndef COMMAND_BACKUP_PROTOCOL_H
#define COMMAND_BACKUP_PROTOCOL_H #define COMMAND_BACKUP_PROTOCOL_H
#include "common/type/string.h" #include "common/type/pack.h"
#include "common/type/variantList.h"
#include "protocol/server.h" #include "protocol/server.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Process protocol requests // Process protocol requests
void backupFileProtocol(const VariantList *paramList, ProtocolServer *server); void backupFileProtocol(PackRead *param, ProtocolServer *server);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess() Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

View File

@ -50,8 +50,8 @@ cmdRemote(int fdRead, int fdWrite)
TRY_BEGIN() TRY_BEGIN()
{ {
// Read the command. No need to parse it since we know this is the first noop. // Get the command. No need to check parameters since we know this is the first noop.
ioReadLine(read); CHECK(protocolServerCommandGet(server).id == PROTOCOL_COMMAND_NOOP);
// Only try the lock if this is process 0, i.e. the remote started from the main process // Only try the lock if this is process 0, i.e. the remote started from the main process
if (cfgOptionUInt(cfgOptProcess) == 0) if (cfgOptionUInt(cfgOptProcess) == 0)
@ -70,7 +70,7 @@ cmdRemote(int fdRead, int fdWrite)
} }
// Notify the client of success // Notify the client of success
protocolServerResponse(server, NULL); protocolServerDataEndPut(server);
success = true; success = true;
} }
CATCH_ANY() CATCH_ANY()

View File

@ -14,30 +14,43 @@ Restore Protocol Handler
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
restoreFileProtocol(const VariantList *paramList, ProtocolServer *server) restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
protocolServerResponse( // Restore file
server, const String *const repoFile = pckReadStrP(param);
VARBOOL( const unsigned int repoIdx = pckReadU32P(param);
restoreFile( const String *const repoFileReference = pckReadStrP(param);
varStr(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)), varStr(varLstGet(paramList, 2)), const CompressType repoFileCompressType = (CompressType)pckReadU32P(param);
(CompressType)varUIntForce(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)), const String *const pgFile = pckReadStrP(param);
varStr(varLstGet(paramList, 5)), varBoolForce(varLstGet(paramList, 6)), varUInt64(varLstGet(paramList, 7)), const String *const pgFileChecksum = pckReadStrP(param);
(time_t)varInt64Force(varLstGet(paramList, 8)), const bool pgFileZero = pckReadBoolP(param);
(mode_t)cvtZToUIntBase(strZ(varStr(varLstGet(paramList, 9))), 8), const uint64_t pgFileSize = pckReadU64P(param);
varStr(varLstGet(paramList, 10)), varStr(varLstGet(paramList, 11)), const time_t pgFileModified = pckReadTimeP(param);
(time_t)varInt64Force(varLstGet(paramList, 12)), varBoolForce(varLstGet(paramList, 13)), const mode_t pgFileMode = pckReadModeP(param);
varBoolForce(varLstGet(paramList, 14)), varStr(varLstGet(paramList, 15))))); const String *const pgFileUser = pckReadStrP(param);
const String *const pgFileGroup = pckReadStrP(param);
const time_t copyTimeBegin = pckReadTimeP(param);
const bool delta = pckReadBoolP(param);
const bool deltaForce = pckReadBoolP(param);
const String *const cipherPass = pckReadStrP(param);
const bool result = restoreFile(
repoFile, repoIdx, repoFileReference, repoFileCompressType, pgFile, pgFileChecksum, pgFileZero, pgFileSize,
pgFileModified, pgFileMode, pgFileUser, pgFileGroup, copyTimeBegin, delta, deltaForce, cipherPass);
// Return result
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), result));
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();

View File

@ -4,15 +4,14 @@ Restore Protocol Handler
#ifndef COMMAND_RESTORE_PROTOCOL_H #ifndef COMMAND_RESTORE_PROTOCOL_H
#define COMMAND_RESTORE_PROTOCOL_H #define COMMAND_RESTORE_PROTOCOL_H
#include "common/type/string.h" #include "common/type/pack.h"
#include "common/type/variantList.h"
#include "protocol/server.h" #include "protocol/server.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Process protocol requests // Process protocol requests
void restoreFileProtocol(const VariantList *paramList, ProtocolServer *server); void restoreFileProtocol(PackRead *param, ProtocolServer *server);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess() Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

View File

@ -2015,7 +2015,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
{ {
const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job))); const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
bool zeroed = restoreFileZeroed(file->name, zeroExp); bool zeroed = restoreFileZeroed(file->name, zeroExp);
bool copy = varBool(protocolParallelJobResult(job)); bool copy = pckReadBoolP(protocolParallelJobResult(job));
String *log = strNewZ("restore"); String *log = strNewZ("restore");
@ -2141,24 +2141,24 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
// Create restore job // Create restore job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
protocolCommandParamAdd(command, VARSTR(file->name)); PackWrite *const param = protocolCommandParam(command);
protocolCommandParamAdd(command, VARUINT(jobData->repoIdx));
protocolCommandParamAdd( pckWriteStrP(param, file->name);
command, file->reference != NULL ? pckWriteU32P(param, jobData->repoIdx);
VARSTR(file->reference) : VARSTR(manifestData(jobData->manifest)->backupLabel)); pckWriteStrP(param, file->reference != NULL ? file->reference : manifestData(jobData->manifest)->backupLabel);
protocolCommandParamAdd(command, VARUINT(manifestData(jobData->manifest)->backupOptionCompressType)); pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
protocolCommandParamAdd(command, VARSTR(restoreFilePgPath(jobData->manifest, file->name))); pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file->name));
protocolCommandParamAdd(command, VARSTRZ(file->checksumSha1)); pckWriteStrP(param, STR(file->checksumSha1));
protocolCommandParamAdd(command, VARBOOL(restoreFileZeroed(file->name, jobData->zeroExp))); pckWriteBoolP(param, restoreFileZeroed(file->name, jobData->zeroExp));
protocolCommandParamAdd(command, VARUINT64(file->size)); pckWriteU64P(param, file->size);
protocolCommandParamAdd(command, VARUINT64((uint64_t)file->timestamp)); pckWriteTimeP(param, file->timestamp);
protocolCommandParamAdd(command, VARSTR(strNewFmt("%04o", file->mode))); pckWriteModeP(param, file->mode);
protocolCommandParamAdd(command, VARSTR(file->user)); pckWriteStrP(param, file->user);
protocolCommandParamAdd(command, VARSTR(file->group)); pckWriteStrP(param, file->group);
protocolCommandParamAdd(command, VARUINT64((uint64_t)manifestData(jobData->manifest)->backupTimestampCopyStart)); pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptDelta))); pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce))); pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
protocolCommandParamAdd(command, VARSTR(jobData->cipherSubPass)); pckWriteStrP(param, jobData->cipherSubPass);
// Remove job from the queue // Remove job from the queue
lstRemoveIdx(queue, 0); lstRemoveIdx(queue, 0);

View File

@ -14,25 +14,29 @@ Verify Protocol Handler
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
verifyFileProtocol(const VariantList *paramList, ProtocolServer *server) verifyFileProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
VerifyResult result = verifyFile( // Verify file
varStr(varLstGet(paramList, 0)), // Full filename const String *const filePathName = pckReadStrP(param);
varStr(varLstGet(paramList, 1)), // Checksum const String *const fileChecksum = pckReadStrP(param);
varUInt64(varLstGet(paramList, 2)), // File size const uint64_t fileSize = pckReadU64P(param);
varStr(varLstGet(paramList, 3))); // Cipher pass const String *const cipherPass = pckReadStrP(param);
protocolServerResponse(server, VARUINT(result)); const VerifyResult result = verifyFile(filePathName, fileChecksum, fileSize, cipherPass);
// Return result
protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), result));
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();

View File

@ -4,15 +4,14 @@ Verify Protocol Handler
#ifndef COMMAND_VERIFY_PROTOCOL_H #ifndef COMMAND_VERIFY_PROTOCOL_H
#define COMMAND_VERIFY_PROTOCOL_H #define COMMAND_VERIFY_PROTOCOL_H
#include "common/type/string.h" #include "common/type/pack.h"
#include "common/type/variantList.h"
#include "protocol/server.h" #include "protocol/server.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Process protocol requests // Process protocol requests
void verifyFileProtocol(const VariantList *paramList, ProtocolServer *server); void verifyFileProtocol(PackRead *param, ProtocolServer *server);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess() Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

View File

@ -755,10 +755,12 @@ verifyArchive(void *data)
// Set up the job // Set up the job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
protocolCommandParamAdd(command, VARSTR(filePathName)); PackWrite *const param = protocolCommandParam(command);
protocolCommandParamAdd(command, VARSTR(checksum));
protocolCommandParamAdd(command, VARUINT64(archiveResult->pgWalInfo.size)); pckWriteStrP(param, filePathName);
protocolCommandParamAdd(command, VARSTR(jobData->walCipherPass)); pckWriteStrP(param, checksum);
pckWriteU64P(param, archiveResult->pgWalInfo.size);
pckWriteStrP(param, jobData->walCipherPass);
// Assign job to result, prepending the archiveId to the key for consistency with backup processing // Assign job to result, prepending the archiveId to the key for consistency with backup processing
result = protocolParallelJobNew( result = protocolParallelJobNew(
@ -978,12 +980,13 @@ verifyBackup(void *data)
{ {
// Set up the job // Set up the job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
protocolCommandParamAdd(command, VARSTR(filePathName)); PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, filePathName);
// If the checksum is not present in the manifest, it will be calculated by manifest load // If the checksum is not present in the manifest, it will be calculated by manifest load
protocolCommandParamAdd(command, VARSTRZ(fileData->checksumSha1)); pckWriteStrP(param, STR(fileData->checksumSha1));
protocolCommandParamAdd(command, VARUINT64(fileData->size)); pckWriteU64P(param, fileData->size);
protocolCommandParamAdd(command, VARSTR(jobData->backupCipherPass)); pckWriteStrP(param, jobData->backupCipherPass);
// Assign job to result (prepend backup label being processed to the key since some files are in a prior backup) // Assign job to result (prepend backup label being processed to the key since some files are in a prior backup)
result = protocolParallelJobNew( result = protocolParallelJobNew(
@ -1535,7 +1538,7 @@ verifyProcess(unsigned int *errorTotal)
// The job was successful // The job was successful
if (protocolParallelJobErrorCode(job) == 0) if (protocolParallelJobErrorCode(job) == 0)
{ {
const VerifyResult verifyResult = (VerifyResult)varUIntForce(protocolParallelJobResult(job)); const VerifyResult verifyResult = (VerifyResult)pckReadU32P(protocolParallelJobResult(job));
// Update the result set for the type of file being processed // Update the result set for the type of file being processed
if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR)) if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR))

View File

@ -17,13 +17,18 @@ Fields in a pack are identified by IDs. A field ID is stored as a delta from the
that reading from the middle is generally not practical. The size of the gap between field IDs is important -- a gap of 1 never that reading from the middle is generally not practical. The size of the gap between field IDs is important -- a gap of 1 never
incurs extra cost, but depending on the field type larger gaps may require additional bytes to store the field ID delta. incurs extra cost, but depending on the field type larger gaps may require additional bytes to store the field ID delta.
The standard default is the C default for that type (e.g. bool = false, int = 0) but can be changed with the .defaultValue
parameter. For example, pckWriteBoolP(write, false, .defaultWrite = true) will write a 0 (i.e. false) with a field ID into the pack,
but pckWriteBoolP(write, false) will not write to the pack, it will simply skip the ID. Note that
pckWriteStrP(packWrite, NULL, .defaultWrite = true) is not valid since there is no way to explicitly write a NULL.
NULLs are not stored in a pack and are therefore not typed. A NULL is essentially just a gap in the field IDs. Fields that are NULLs are not stored in a pack and are therefore not typed. A NULL is essentially just a gap in the field IDs. Fields that are
frequently NULL are best stored at the end of an object. When using .defaultWrite = false in write functions a NULL will be written frequently NULL are best stored at the end of an object. When using read functions the default will always be returned
(by making a gap in the IDs) if the value matches the default. When using read functions the default will always be returned when the field is NULL (i.e. missing). There are times when NULL must be explicitly passed, for example:
when the field is NULL (i.e. missing). The standard default is the C default for that type (e.g. bool = false, int = 0) but can be pckWriteStrP(resultPack, result.pageChecksumResult != NULL ? jsonFromKv(result.pageChecksumResult) : NULL);
changed with the .defaultValue parameter. For example, pckWriteBoolP(write, false, .defaultWrite = true) will write a 0 with an ID In this case, NULL is declared since jsonFromKv() does not accept a NULL parameter and, following the rules for NULLs the field ID
into the pack, but pckWriteBoolP(write, false) will not write to the pack, it will simply skip the ID. Note that is skipped when result.pageChecksumResult == NULL. Upon reading, we can declare a NULL_STR when a NULL (field ID gap) is
pckWriteStrP(packWrite, NULL, .defaultWrite = true) is not valid since there is no way to explcitly write a NULL. encountered, e.g. jsonToVar(pckReadStrP(jobResult, .defaultValue = NULL_STR)).
A pack is an object by default. Objects can store fields, objects, or arrays. Objects and arrays will be referred to collectively as A pack is an object by default. Objects can store fields, objects, or arrays. Objects and arrays will be referred to collectively as
containers. Fields contain data to be stored, e.g. integers, strings, etc. containers. Fields contain data to be stored, e.g. integers, strings, etc.
@ -38,8 +43,8 @@ pckWriteStringP(write, STRDEF("sample"));
pckWriteEndP(); pckWriteEndP();
A string representation of this pack is `1:uint64:77,2:bool:false,4:str:sample`. The boolean was stored even though it was the A string representation of this pack is `1:uint64:77,2:bool:false,4:str:sample`. The boolean was stored even though it was the
default because a write was explcitly requested. The int32 field was not stored because the value matched the expicitly set default. default because a write was explicitly requested. The int32 field was not stored because the value matched the explicitly set
Note that there is a gap in the ID stream, which represents the NULL/default value. default. Note that there is a gap in the ID stream, which represents the NULL/default value.
This pack can be read with: This pack can be read with:
@ -50,8 +55,8 @@ pckReadI32P(read, .defaultValue = -1);
pckReadStringP(read); pckReadStringP(read);
pckReadEndP(); pckReadEndP();
Note that defaults are not stored in the pack so any defaults that were applied when writing (by setting .defaulWrite and Note that defaults are not stored in the pack so any defaults that were applied when writing (by setting .defaultValue) must be
optionally .defaultValue) must be applied again when reading (by optionally setting .defaultValue). applied again when reading by setting .defaultValue if the default value is not a standard C default.
If we don't care about the NULL/default, another way to read is: If we don't care about the NULL/default, another way to read is:

View File

@ -7,35 +7,37 @@ Configuration Protocol Handler
#include "common/io/io.h" #include "common/io/io.h"
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/type/json.h"
#include "config/config.intern.h" #include "config/config.intern.h"
#include "config/parse.h" #include "config/parse.h"
#include "config/protocol.h" #include "config/protocol.h"
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
configOptionProtocol(const VariantList *paramList, ProtocolServer *server) configOptionProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
VariantList *optionList = varLstNew(); VariantList *optionList = varLstNew();
for (unsigned int optionIdx = 0; optionIdx < varLstSize(paramList); optionIdx++) while (pckReadNext(param))
{ {
CfgParseOptionResult option = cfgParseOptionP(varStr(varLstGet(paramList, optionIdx))); CfgParseOptionResult option = cfgParseOptionP(pckReadStrP(param));
CHECK(option.found); CHECK(option.found);
varLstAdd(optionList, varDup(cfgOptionIdx(option.id, cfgOptionKeyToIdx(option.id, option.keyIdx + 1)))); varLstAdd(optionList, varDup(cfgOptionIdx(option.id, cfgOptionKeyToIdx(option.id, option.keyIdx + 1))));
} }
protocolServerResponse(server, varNewVarLst(optionList)); protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), jsonFromVar(varNewVarLst(optionList))));
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -56,13 +58,14 @@ configOptionRemote(ProtocolClient *client, const VariantList *paramList)
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_CONFIG_OPTION); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_CONFIG_OPTION);
PackWrite *const param = protocolCommandParam(command);
for (unsigned int paramIdx = 0; paramIdx < varLstSize(paramList); paramIdx++) for (unsigned int paramIdx = 0; paramIdx < varLstSize(paramList); paramIdx++)
protocolCommandParamAdd(command, varLstGet(paramList, paramIdx)); pckWriteStrP(param, varStr(varLstGet(paramList, paramIdx)));
MEM_CONTEXT_PRIOR_BEGIN() MEM_CONTEXT_PRIOR_BEGIN()
{ {
result = varVarLst(protocolClientExecute(client, command, true)); result = varVarLst(jsonToVar(pckReadStrP(protocolClientExecute(client, command, true))));
} }
MEM_CONTEXT_PRIOR_END(); MEM_CONTEXT_PRIOR_END();
} }

View File

@ -4,8 +4,7 @@ Configuration Protocol Handler
#ifndef CONFIG_PROTOCOL_H #ifndef CONFIG_PROTOCOL_H
#define CONFIG_PROTOCOL_H #define CONFIG_PROTOCOL_H
#include "common/type/string.h" #include "common/type/pack.h"
#include "common/type/variantList.h"
#include "protocol/client.h" #include "protocol/client.h"
#include "protocol/server.h" #include "protocol/server.h"
@ -13,7 +12,7 @@ Configuration Protocol Handler
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Process config protocol requests // Process config protocol requests
void configOptionProtocol(const VariantList *paramList, ProtocolServer *server); void configOptionProtocol(PackRead *param, ProtocolServer *server);
// Get option values from a remote process // Get option values from a remote process
VariantList *configOptionRemote(ProtocolClient *client, const VariantList *paramList); VariantList *configOptionRemote(ProtocolClient *client, const VariantList *paramList);

View File

@ -6,6 +6,7 @@ Database Client
#include "common/debug.h" #include "common/debug.h"
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/type/json.h"
#include "common/wait.h" #include "common/wait.h"
#include "db/db.h" #include "db/db.h"
#include "db/protocol.h" #include "db/protocol.h"
@ -46,7 +47,7 @@ dbFreeResource(THIS_VOID)
ASSERT(this != NULL); ASSERT(this != NULL);
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_CLOSE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_CLOSE);
protocolCommandParamAdd(command, VARUINT(this->remoteIdx)); pckWriteU32P(protocolCommandParam(command), this->remoteIdx);
protocolClientExecute(this->remoteClient, command, false); protocolClientExecute(this->remoteClient, command, false);
@ -109,10 +110,12 @@ dbQuery(Db *this, const String *query)
if (this->remoteClient != NULL) if (this->remoteClient != NULL)
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_QUERY); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_QUERY);
protocolCommandParamAdd(command, VARUINT(this->remoteIdx)); PackWrite *const param = protocolCommandParam(command);
protocolCommandParamAdd(command, VARSTR(query));
result = varVarLst(protocolClientExecute(this->remoteClient, command, true)); pckWriteU32P(param, this->remoteIdx);
pckWriteStrP(param, query);
result = varVarLst(jsonToVar(pckReadStrP(protocolClientExecute(this->remoteClient, command, true))));
} }
// Else locally // Else locally
else else
@ -199,7 +202,7 @@ dbOpen(Db *this)
if (this->remoteClient != NULL) if (this->remoteClient != NULL)
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_OPEN); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_OPEN);
this->remoteIdx = varUIntForce(protocolClientExecute(this->remoteClient, command, true)); this->remoteIdx = pckReadU32P(protocolClientExecute(this->remoteClient, command, true));
// Set a callback to notify the remote when a connection is closed // Set a callback to notify the remote when a connection is closed
memContextCallbackSet(this->pub.memContext, dbFreeResource, this); memContextCallbackSet(this->pub.memContext, dbFreeResource, this);

View File

@ -7,6 +7,7 @@ Db Protocol Handler
#include "common/io/io.h" #include "common/io/io.h"
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/list.h" #include "common/type/list.h"
#include "config/config.h" #include "config/config.h"
#include "db/protocol.h" #include "db/protocol.h"
@ -23,14 +24,14 @@ static struct
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
dbOpenProtocol(const VariantList *paramList, ProtocolServer *server) dbOpenProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList == NULL); ASSERT(param == NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
@ -46,8 +47,6 @@ dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
} }
// Add db to the list // Add db to the list
unsigned int dbIdx = lstSize(dbProtocolLocal.pgClientList);
MEM_CONTEXT_BEGIN(lstMemContext(dbProtocolLocal.pgClientList)) MEM_CONTEXT_BEGIN(lstMemContext(dbProtocolLocal.pgClientList))
{ {
// Only a single db is passed to the remote // Only a single db is passed to the remote
@ -61,7 +60,8 @@ dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
MEM_CONTEXT_END(); MEM_CONTEXT_END();
// Return db index which should be included in subsequent calls // Return db index which should be included in subsequent calls
protocolServerResponse(server, VARUINT(dbIdx)); protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), lstSize(dbProtocolLocal.pgClientList) - 1));
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -70,24 +70,23 @@ dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
dbQueryProtocol(const VariantList *paramList, ProtocolServer *server) dbQueryProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
protocolServerResponse( PgClient *const pgClient = *(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param));
server, const String *const query = pckReadStrP(param);
varNewVarLst(
pgClientQuery( protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), jsonFromVar(varNewVarLst(pgClientQuery(pgClient, query)))));
*(PgClient **)lstGet(dbProtocolLocal.pgClientList, varUIntForce(varLstGet(paramList, 0))), protocolServerDataEndPut(server);
varStr(varLstGet(paramList, 1)))));
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -96,20 +95,20 @@ dbQueryProtocol(const VariantList *paramList, ProtocolServer *server)
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
dbCloseProtocol(const VariantList *paramList, ProtocolServer *server) dbCloseProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
pgClientClose(*(PgClient **)lstGet(dbProtocolLocal.pgClientList, varUIntForce(varLstGet(paramList, 0)))); pgClientClose(*(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param)));
protocolServerResponse(server, NULL); protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();

View File

@ -4,18 +4,16 @@ Db Protocol Handler
#ifndef DB_PROTOCOL_H #ifndef DB_PROTOCOL_H
#define DB_PROTOCOL_H #define DB_PROTOCOL_H
#include "common/type/string.h" #include "common/type/pack.h"
#include "common/type/variantList.h"
#include "protocol/client.h"
#include "protocol/server.h" #include "protocol/server.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Process db protocol requests // Process db protocol requests
void dbOpenProtocol(const VariantList *paramList, ProtocolServer *server); void dbOpenProtocol(PackRead *param, ProtocolServer *server);
void dbQueryProtocol(const VariantList *paramList, ProtocolServer *server); void dbQueryProtocol(PackRead *param, ProtocolServer *server);
void dbCloseProtocol(const VariantList *paramList, ProtocolServer *server); void dbCloseProtocol(PackRead *param, ProtocolServer *server);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess() Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

View File

@ -10,6 +10,7 @@ Protocol Client
#include "common/type/json.h" #include "common/type/json.h"
#include "common/type/keyValue.h" #include "common/type/keyValue.h"
#include "protocol/client.h" #include "protocol/client.h"
#include "protocol/server.h"
#include "version.h" #include "version.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
@ -19,20 +20,16 @@ STRING_EXTERN(PROTOCOL_GREETING_NAME_STR, PROTOCOL_GRE
STRING_EXTERN(PROTOCOL_GREETING_SERVICE_STR, PROTOCOL_GREETING_SERVICE); STRING_EXTERN(PROTOCOL_GREETING_SERVICE_STR, PROTOCOL_GREETING_SERVICE);
STRING_EXTERN(PROTOCOL_GREETING_VERSION_STR, PROTOCOL_GREETING_VERSION); STRING_EXTERN(PROTOCOL_GREETING_VERSION_STR, PROTOCOL_GREETING_VERSION);
STRING_EXTERN(PROTOCOL_ERROR_STR, PROTOCOL_ERROR);
STRING_EXTERN(PROTOCOL_ERROR_STACK_STR, PROTOCOL_ERROR_STACK);
STRING_EXTERN(PROTOCOL_OUTPUT_STR, PROTOCOL_OUTPUT);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Object type Object type
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
struct ProtocolClient struct ProtocolClient
{ {
ProtocolClientPub pub; // Publicly accessible variables ProtocolClientPub pub; // Publicly accessible variables
const String *name; IoWrite *write; // Write interface
const String *errorPrefix; const String *name; // Name displayed in logging
TimeMSec keepAliveTime; const String *errorPrefix; // Prefix used when throwing error
TimeMSec keepAliveTime; // Last time data was put to the server
}; };
/*********************************************************************************************************************************** /***********************************************************************************************************************************
@ -52,7 +49,7 @@ protocolClientFreeResource(THIS_VOID)
// Send an exit command but don't wait to see if it succeeds // Send an exit command but don't wait to see if it succeeds
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
protocolClientWriteCommand(this, protocolCommandNew(PROTOCOL_COMMAND_EXIT)); protocolClientCommandPut(this, protocolCommandNew(PROTOCOL_COMMAND_EXIT));
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -86,8 +83,8 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
{ {
.memContext = memContextCurrent(), .memContext = memContextCurrent(),
.read = read, .read = read,
.write = write,
}, },
.write = write,
.name = strDup(name), .name = strDup(name),
.errorPrefix = strNewFmt("raised from %s", strZ(name)), .errorPrefix = strNewFmt("raised from %s", strZ(name)),
.keepAliveTime = timeMSec(), .keepAliveTime = timeMSec(),
@ -96,7 +93,7 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
// Read, parse, and check the protocol greeting // Read, parse, and check the protocol greeting
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
String *greeting = ioReadLine(protocolClientIoRead(this)); String *greeting = ioReadLine(this->pub.read);
KeyValue *greetingKv = jsonToKv(greeting); KeyValue *greetingKv = jsonToKv(greeting);
const String *expected[] = const String *expected[] =
@ -143,93 +140,127 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
// Helper to process errors void
static void protocolClientDataPut(ProtocolClient *const this, PackWrite *const data)
protocolClientProcessError(ProtocolClient *this, KeyValue *errorKv)
{ {
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this); FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(KEY_VALUE, errorKv); FUNCTION_LOG_PARAM(PACK_WRITE, data);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(errorKv != NULL);
// End the pack
if (data != NULL)
pckWriteEndP(data);
// Write the data
PackWrite *dataMessage = pckWriteNew(this->write);
pckWriteU32P(dataMessage, protocolMessageTypeData, .defaultWrite = true);
pckWritePackP(dataMessage, data);
pckWriteEndP(dataMessage);
// Flush when there is no more data to put
if (data == NULL)
ioWriteFlush(this->write);
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
// Helper to process errors
static void
protocolClientError(ProtocolClient *const this, const ProtocolMessageType type, PackRead *const error)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(ENUM, type);
FUNCTION_LOG_PARAM(PACK_READ, error);
FUNCTION_LOG_END();
if (type == protocolMessageTypeError)
{
const ErrorType *type = errorTypeFromCode(pckReadI32P(error));
String *const message = strNewFmt("%s: %s", strZ(this->errorPrefix), strZ(pckReadStrP(error)));
const String *const stack = pckReadStrP(error);
pckReadEndP(error);
CHECK(message != NULL);
// Add stack trace if the error is an assertion or debug-level logging is enabled
if (type == &AssertError || logAny(logLevelDebug))
{
CHECK(stack != NULL);
strCat(message, LF_STR);
strCat(message, stack);
}
THROWP(type, strZ(message));
}
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
PackRead *
protocolClientDataGet(ProtocolClient *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_END();
PackRead *result = NULL;
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Process error if any PackRead *response = pckReadNew(this->pub.read);
const Variant *error = kvGet(errorKv, VARSTR(PROTOCOL_ERROR_STR)); ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
if (error != NULL) protocolClientError(this, type, response);
CHECK(type == protocolMessageTypeData);
MEM_CONTEXT_PRIOR_BEGIN()
{ {
const ErrorType *type = errorTypeFromCode(varIntForce(error)); result = pckReadPackP(response);
const String *message = varStr(kvGet(errorKv, VARSTR(PROTOCOL_OUTPUT_STR)));
// Required part of the message
String *throwMessage = strNewFmt(
"%s: %s", strZ(this->errorPrefix), message == NULL ? "no details available" : strZ(message));
// Add stack trace if the error is an assertion or debug-level logging is enabled
if (type == &AssertError || logAny(logLevelDebug))
{
const String *stack = varStr(kvGet(errorKv, VARSTR(PROTOCOL_ERROR_STACK_STR)));
strCat(throwMessage, LF_STR);
strCat(throwMessage, stack == NULL ? STRDEF("no stack trace available") : stack);
}
THROWP(type, strZ(throwMessage));
} }
MEM_CONTEXT_PRIOR_END();
pckReadEndP(response);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/**********************************************************************************************************************************/
void
protocolClientDataEndGet(ProtocolClient *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_END();
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *response = pckReadNew(this->pub.read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
protocolClientError(this, type, response);
CHECK(type == protocolMessageTypeDataEnd);
pckReadEndP(response);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }
const Variant *
protocolClientReadOutput(ProtocolClient *this, bool outputRequired)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(BOOL, outputRequired);
FUNCTION_LOG_END();
ASSERT(this != NULL);
const Variant *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
// Read the response
String *response = ioReadLine(protocolClientIoRead(this));
KeyValue *responseKv = varKv(jsonToVar(response));
// Process error if any
protocolClientProcessError(this, responseKv);
// Get output
result = kvGet(responseKv, VARSTR(PROTOCOL_OUTPUT_STR));
if (outputRequired)
{
// Just move the entire response kv since the output is the largest part if it
kvMove(responseKv, memContextPrior());
}
// Else if no output is required then there should not be any
else if (result != NULL)
THROW(AssertError, "no output required by command");
// Reset the keep alive time
this->keepAliveTime = timeMSec();
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_CONST(VARIANT, result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command) protocolClientCommandPut(ProtocolClient *const this, ProtocolCommand *const command)
{ {
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this); FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
@ -239,9 +270,8 @@ protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command)
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(command != NULL); ASSERT(command != NULL);
// Write out the command // Put command
ioWriteStrLine(protocolClientIoWrite(this), protocolCommandJson(command)); protocolCommandPut(command, this->write);
ioWriteFlush(protocolClientIoWrite(this));
// Reset the keep alive time // Reset the keep alive time
this->keepAliveTime = timeMSec(); this->keepAliveTime = timeMSec();
@ -250,21 +280,31 @@ protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command)
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
const Variant * PackRead *
protocolClientExecute(ProtocolClient *this, const ProtocolCommand *command, bool outputRequired) protocolClientExecute(ProtocolClient *const this, ProtocolCommand *const command, const bool resultRequired)
{ {
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this); FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command); FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
FUNCTION_LOG_PARAM(BOOL, outputRequired); FUNCTION_LOG_PARAM(BOOL, resultRequired);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(command != NULL); ASSERT(command != NULL);
protocolClientWriteCommand(this, command); // Put command
protocolClientCommandPut(this, command);
FUNCTION_LOG_RETURN_CONST(VARIANT, protocolClientReadOutput(this, outputRequired)); // Read result if required
PackRead *result = NULL;
if (resultRequired)
result = protocolClientDataGet(this);
// Read response
protocolClientDataEndGet(this);
FUNCTION_LOG_RETURN(PACK_READ, result);
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
@ -286,50 +326,6 @@ protocolClientNoOp(ProtocolClient *this)
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }
/**********************************************************************************************************************************/
String *
protocolClientReadLine(ProtocolClient *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
String *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
result = ioReadLine(protocolClientIoRead(this));
if (strSize(result) == 0)
{
THROW(FormatError, "unexpected empty line");
}
else if (strZ(result)[0] == '{')
{
KeyValue *responseKv = varKv(jsonToVar(result));
// Process expected error
protocolClientProcessError(this, responseKv);
// If not an error then there is probably a protocol bug
THROW(FormatError, "expected error but got output");
}
else if (strZ(result)[0] != '.')
THROW_FMT(FormatError, "invalid prefix in '%s'", strZ(result));
MEM_CONTEXT_PRIOR_BEGIN()
{
result = strSub(result, 1);
}
MEM_CONTEXT_PRIOR_END();
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING, result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
String * String *
protocolClientToLog(const ProtocolClient *this) protocolClientToLog(const ProtocolClient *this)

View File

@ -4,6 +4,24 @@ Protocol Client
#ifndef PROTOCOL_CLIENT_H #ifndef PROTOCOL_CLIENT_H
#define PROTOCOL_CLIENT_H #define PROTOCOL_CLIENT_H
/***********************************************************************************************************************************
Message types used by the protocol
***********************************************************************************************************************************/
typedef enum
{
// Data passed between client and server in either direction. This can be used as many times as needed.
protocolMessageTypeData = 0,
// Indicates no more data for the server to return to the client and ends the command
protocolMessageTypeDataEnd = 1,
// Command sent from the client to the server
protocolMessageTypeCommand = 2,
// An error occurred on the server and the command ended abnormally. protocolMessageTypeDataEnd will not be sent to the client.
protocolMessageTypeError = 3,
} ProtocolMessageType;
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Object type Object type
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
@ -27,13 +45,18 @@ Constants
#define PROTOCOL_COMMAND_EXIT STRID5("exit", 0xa27050) #define PROTOCOL_COMMAND_EXIT STRID5("exit", 0xa27050)
#define PROTOCOL_COMMAND_NOOP STRID5("noop", 0x83dee0) #define PROTOCOL_COMMAND_NOOP STRID5("noop", 0x83dee0)
#define PROTOCOL_ERROR "err" /***********************************************************************************************************************************
STRING_DECLARE(PROTOCOL_ERROR_STR); This size should be safe for most pack data without wasting a lot of space. If binary data is being transferred then this size can
#define PROTOCOL_ERROR_STACK "errStack" be added to the expected binary size to account for overhead.
STRING_DECLARE(PROTOCOL_ERROR_STACK_STR); ***********************************************************************************************************************************/
#define PROTOCOL_PACK_DEFAULT_SIZE 1024
#define PROTOCOL_OUTPUT "out" // Pack large enough for standard data. Note that the buffer will automatically resize when required.
STRING_DECLARE(PROTOCOL_OUTPUT_STR); __attribute__((always_inline)) static inline PackWrite *
protocolPackNew(void)
{
return pckWriteNewBuf(bufNew(PROTOCOL_PACK_DEFAULT_SIZE));
}
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Constructors Constructors
@ -47,28 +70,20 @@ typedef struct ProtocolClientPub
{ {
MemContext *memContext; // Mem context MemContext *memContext; // Mem context
IoRead *read; // Read interface IoRead *read; // Read interface
IoWrite *write; // Write interface
} ProtocolClientPub; } ProtocolClientPub;
// Read interface // Read file descriptor
__attribute__((always_inline)) static inline IoRead * __attribute__((always_inline)) static inline int
protocolClientIoRead(ProtocolClient *const this) protocolClientIoReadFd(ProtocolClient *const this)
{ {
return THIS_PUB(ProtocolClient)->read; return ioReadFd(THIS_PUB(ProtocolClient)->read);
}
// Write interface
__attribute__((always_inline)) static inline IoWrite *
protocolClientIoWrite(ProtocolClient *const this)
{
return THIS_PUB(ProtocolClient)->write;
} }
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Execute a protocol command and get the output // Execute a command and get the result
const Variant *protocolClientExecute(ProtocolClient *this, const ProtocolCommand *command, bool outputRequired); PackRead *protocolClientExecute(ProtocolClient *this, ProtocolCommand *command, bool resultRequired);
// Move to a new parent mem context // Move to a new parent mem context
__attribute__((always_inline)) static inline ProtocolClient * __attribute__((always_inline)) static inline ProtocolClient *
@ -80,14 +95,15 @@ protocolClientMove(ProtocolClient *const this, MemContext *const parentNew)
// Send noop to test connection or keep it alive // Send noop to test connection or keep it alive
void protocolClientNoOp(ProtocolClient *this); void protocolClientNoOp(ProtocolClient *this);
// Read a line // Get data put by the server
String *protocolClientReadLine(ProtocolClient *this); PackRead *protocolClientDataGet(ProtocolClient *this);
void protocolClientDataEndGet(ProtocolClient *this);
// Read the command output // Put command to the server
const Variant *protocolClientReadOutput(ProtocolClient *this, bool outputRequired); void protocolClientCommandPut(ProtocolClient *this, ProtocolCommand *command);
// Write the protocol command // Put data to the server
void protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command); void protocolClientDataPut(ProtocolClient *this, PackWrite *data);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Destructor Destructor

View File

@ -6,15 +6,9 @@ Protocol Command
#include "common/debug.h" #include "common/debug.h"
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/keyValue.h" #include "common/type/keyValue.h"
#include "protocol/command.h" #include "protocol/command.h"
#include "protocol/client.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
STRING_EXTERN(PROTOCOL_KEY_COMMAND_STR, PROTOCOL_KEY_COMMAND);
STRING_EXTERN(PROTOCOL_KEY_PARAMETER_STR, PROTOCOL_KEY_PARAMETER);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Object type Object type
@ -23,7 +17,7 @@ struct ProtocolCommand
{ {
MemContext *memContext; MemContext *memContext;
StringId command; StringId command;
Variant *parameterList; PackWrite *pack;
}; };
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
@ -54,33 +48,39 @@ protocolCommandNew(const StringId command)
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
ProtocolCommand * void
protocolCommandParamAdd(ProtocolCommand *this, const Variant *param) protocolCommandPut(ProtocolCommand *const this, IoWrite *const write)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this); FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
FUNCTION_TEST_PARAM(VARIANT, param);
FUNCTION_TEST_END(); FUNCTION_TEST_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(write != NULL);
MEM_CONTEXT_BEGIN(this->memContext) // Write the command and flush to be sure the command gets sent immediately
PackWrite *commandPack = pckWriteNew(write);
pckWriteU32P(commandPack, protocolMessageTypeCommand, .defaultWrite = true);
pckWriteStrIdP(commandPack, this->command);
// Only write params if there were any
if (this->pack != NULL)
{ {
// Create parameter list if not already created pckWriteEndP(this->pack);
if (this->parameterList == NULL) pckWritePackP(commandPack, this->pack);
this->parameterList = varNewVarLst(varLstNew());
// Add parameter to the list
varLstAdd(varVarLst(this->parameterList), varDup(param));
} }
MEM_CONTEXT_END();
FUNCTION_TEST_RETURN(this); pckWriteEndP(commandPack);
// Flush to send command immediately
ioWriteFlush(write);
FUNCTION_TEST_RETURN_VOID();
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
String * PackWrite *
protocolCommandJson(const ProtocolCommand *this) protocolCommandParam(ProtocolCommand *this)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this); FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
@ -88,27 +88,16 @@ protocolCommandJson(const ProtocolCommand *this)
ASSERT(this != NULL); ASSERT(this != NULL);
String *result = NULL; if (this->pack == NULL)
MEM_CONTEXT_TEMP_BEGIN()
{ {
char commandStrId[STRID_MAX + 1]; MEM_CONTEXT_BEGIN(this->memContext)
strIdToZ(this->command, commandStrId);
KeyValue *command = kvPut(kvNew(), VARSTR(PROTOCOL_KEY_COMMAND_STR), VARSTRZ(commandStrId));
if (this->parameterList != NULL)
kvPut(command, VARSTR(PROTOCOL_KEY_PARAMETER_STR), this->parameterList);
MEM_CONTEXT_PRIOR_BEGIN()
{ {
result = jsonFromKv(command); this->pack = protocolPackNew();
} }
MEM_CONTEXT_PRIOR_END(); MEM_CONTEXT_END();
} }
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN(result); FUNCTION_TEST_RETURN(this->pack);
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/

View File

@ -10,28 +10,13 @@ Object type
typedef struct ProtocolCommand ProtocolCommand; typedef struct ProtocolCommand ProtocolCommand;
#include "common/type/object.h" #include "common/type/object.h"
#include "common/type/stringId.h" #include "common/type/pack.h"
#include "common/type/variant.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define PROTOCOL_KEY_COMMAND "cmd"
STRING_DECLARE(PROTOCOL_KEY_COMMAND_STR);
#define PROTOCOL_KEY_PARAMETER "param"
STRING_DECLARE(PROTOCOL_KEY_PARAMETER_STR);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Constructors Constructors
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
ProtocolCommand *protocolCommandNew(const StringId command); ProtocolCommand *protocolCommandNew(const StringId command);
/***********************************************************************************************************************************
Getters/Setters
***********************************************************************************************************************************/
// Command JSON
String *protocolCommandJson(const ProtocolCommand *this);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
@ -44,7 +29,10 @@ protocolCommandMove(ProtocolCommand *const this, MemContext *const parentNew)
} }
// Read the command output // Read the command output
ProtocolCommand *protocolCommandParamAdd(ProtocolCommand *this, const Variant *param); PackWrite *protocolCommandParam(ProtocolCommand *this);
// Write protocol command
void protocolCommandPut(ProtocolCommand *this, IoWrite *write);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Destructor Destructor

View File

@ -10,7 +10,6 @@ Protocol Parallel Executor
#include "common/log.h" #include "common/log.h"
#include "common/macro.h" #include "common/macro.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/keyValue.h" #include "common/type/keyValue.h"
#include "common/type/list.h" #include "common/type/list.h"
#include "protocol/command.h" #include "protocol/command.h"
@ -83,7 +82,7 @@ protocolParallelClientAdd(ProtocolParallel *this, ProtocolClient *client)
ASSERT(client != NULL); ASSERT(client != NULL);
ASSERT(this->state == protocolParallelJobStatePending); ASSERT(this->state == protocolParallelJobStatePending);
if (ioReadFd(protocolClientIoRead(client)) == -1) if (protocolClientIoReadFd(client) == -1)
THROW(AssertError, "client with read fd is required"); THROW(AssertError, "client with read fd is required");
lstAdd(this->clientList, &client); lstAdd(this->clientList, &client);
@ -128,7 +127,7 @@ protocolParallelProcess(ProtocolParallel *this)
{ {
if (this->clientJobList[clientIdx] != NULL) if (this->clientJobList[clientIdx] != NULL)
{ {
int fd = ioReadFd(protocolClientIoRead(*(ProtocolClient **)lstGet(this->clientList, clientIdx))); int fd = protocolClientIoReadFd(*(ProtocolClient **)lstGet(this->clientList, clientIdx));
FD_SET((unsigned int)fd, &selectSet); FD_SET((unsigned int)fd, &selectSet);
// Find the max file descriptor needed for select() // Find the max file descriptor needed for select()
@ -159,15 +158,17 @@ protocolParallelProcess(ProtocolParallel *this)
if (job != NULL && if (job != NULL &&
FD_ISSET( FD_ISSET(
(unsigned int)ioReadFd(protocolClientIoRead(*(ProtocolClient **)lstGet(this->clientList, clientIdx))), (unsigned int)protocolClientIoReadFd(*(ProtocolClient **)lstGet(this->clientList, clientIdx)),
&selectSet)) &selectSet))
{ {
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
TRY_BEGIN() TRY_BEGIN()
{ {
protocolParallelJobResultSet( ProtocolClient *const client = *(ProtocolClient **)lstGet(this->clientList, clientIdx);
job, protocolClientReadOutput(*(ProtocolClient **)lstGet(this->clientList, clientIdx), true));
protocolParallelJobResultSet(job, protocolClientDataGet(client));
protocolClientDataEndGet(client);
} }
CATCH_ANY() CATCH_ANY()
{ {
@ -207,9 +208,8 @@ protocolParallelProcess(ProtocolParallel *this)
// Add to the job list // Add to the job list
lstAdd(this->jobList, &job); lstAdd(this->jobList, &job);
// Send the job to the client // Put command
protocolClientWriteCommand( protocolClientCommandPut(*(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job));
*(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job));
// Set client id and running state // Set client id and running state
protocolParallelJobProcessIdSet(job, clientIdx + 1); protocolParallelJobProcessIdSet(job, clientIdx + 1);

View File

@ -92,21 +92,17 @@ protocolParallelJobProcessIdSet(ProtocolParallelJob *this, unsigned int processI
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
protocolParallelJobResultSet(ProtocolParallelJob *this, const Variant *result) protocolParallelJobResultSet(ProtocolParallelJob *const this, PackRead *const result)
{ {
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, this); FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_LOG_PARAM(VARIANT, result); FUNCTION_LOG_PARAM(PACK_READ, result);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(protocolParallelJobErrorCode(this) == 0); ASSERT(protocolParallelJobErrorCode(this) == 0);
MEM_CONTEXT_BEGIN(this->pub.memContext) this->pub.result = pckReadMove(result, this->pub.memContext);
{
this->pub.result = varDup(result);
}
MEM_CONTEXT_END();
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }
@ -143,5 +139,6 @@ protocolParallelJobToLog(const ProtocolParallelJob *this)
"{state: %s, key: %s, command: %s, code: %d, message: %s, result: %s}", "{state: %s, key: %s, command: %s, code: %d, message: %s, result: %s}",
strZ(strIdToStr(protocolParallelJobState(this))), strZ(varToLog(protocolParallelJobKey(this))), strZ(strIdToStr(protocolParallelJobState(this))), strZ(varToLog(protocolParallelJobKey(this))),
strZ(protocolCommandToLog(protocolParallelJobCommand(this))), protocolParallelJobErrorCode(this), strZ(protocolCommandToLog(protocolParallelJobCommand(this))), protocolParallelJobErrorCode(this),
strZ(strToLog(protocolParallelJobErrorMessage(this))), strZ(varToLog(protocolParallelJobResult(this)))); strZ(strToLog(protocolParallelJobErrorMessage(this))),
protocolParallelJobResult(this) == NULL ? NULL_Z : strZ(pckReadToLog(protocolParallelJobResult(this))));
} }

View File

@ -23,6 +23,7 @@ typedef enum
#include "common/time.h" #include "common/time.h"
#include "common/type/object.h" #include "common/type/object.h"
#include "common/type/pack.h"
#include "protocol/client.h" #include "protocol/client.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
@ -37,16 +38,16 @@ typedef struct ProtocolParallelJobPub
{ {
MemContext *memContext; // Mem context MemContext *memContext; // Mem context
const Variant *key; // Unique key used to identify the job const Variant *key; // Unique key used to identify the job
const ProtocolCommand *command; // Command to be executed ProtocolCommand *command; // Command to be executed
unsigned int processId; // Process that executed this job unsigned int processId; // Process that executed this job
ProtocolParallelJobState state; // Current state of the job ProtocolParallelJobState state; // Current state of the job
int code; // Non-zero result indicates an error int code; // Non-zero result indicates an error
String *message; // Message if there was a error String *message; // Message if there was a error
const Variant *result; // Result if job was successful PackRead *result; // Result if job was successful
} ProtocolParallelJobPub; } ProtocolParallelJobPub;
// Job command // Job command
__attribute__((always_inline)) static inline const ProtocolCommand * __attribute__((always_inline)) static inline ProtocolCommand *
protocolParallelJobCommand(const ProtocolParallelJob *const this) protocolParallelJobCommand(const ProtocolParallelJob *const this)
{ {
return THIS_PUB(ProtocolParallelJob)->command; return THIS_PUB(ProtocolParallelJob)->command;
@ -84,13 +85,13 @@ protocolParallelJobProcessId(const ProtocolParallelJob *const this)
void protocolParallelJobProcessIdSet(ProtocolParallelJob *this, unsigned int processId); void protocolParallelJobProcessIdSet(ProtocolParallelJob *this, unsigned int processId);
// Job result // Job result
__attribute__((always_inline)) static inline const Variant * __attribute__((always_inline)) static inline PackRead *
protocolParallelJobResult(const ProtocolParallelJob *const this) protocolParallelJobResult(const ProtocolParallelJob *const this)
{ {
return THIS_PUB(ProtocolParallelJob)->result; return THIS_PUB(ProtocolParallelJob)->result;
} }
void protocolParallelJobResultSet(ProtocolParallelJob *this, const Variant *result); void protocolParallelJobResultSet(ProtocolParallelJob *const this, PackRead *const result);
// Job state // Job state
__attribute__((always_inline)) static inline ProtocolParallelJobState __attribute__((always_inline)) static inline ProtocolParallelJobState

View File

@ -12,7 +12,6 @@ Protocol Server
#include "common/type/json.h" #include "common/type/json.h"
#include "common/type/keyValue.h" #include "common/type/keyValue.h"
#include "common/type/list.h" #include "common/type/list.h"
#include "protocol/client.h"
#include "protocol/helper.h" #include "protocol/helper.h"
#include "protocol/server.h" #include "protocol/server.h"
#include "version.h" #include "version.h"
@ -22,8 +21,10 @@ Object type
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
struct ProtocolServer struct ProtocolServer
{ {
ProtocolServerPub pub; // Publicly accessible variables MemContext *memContext; // Mem context
const String *name; IoRead *read; // Read interface
IoWrite *write; // Write interface
const String *name; // Name displayed in logging
}; };
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
@ -49,12 +50,9 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
*this = (ProtocolServer) *this = (ProtocolServer)
{ {
.pub = .memContext = memContextCurrent(),
{ .read = read,
.memContext = memContextCurrent(), .write = write,
.read = read,
.write = write,
},
.name = strDup(name), .name = strDup(name),
}; };
@ -66,8 +64,8 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_SERVICE_STR), VARSTR(service)); kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_SERVICE_STR), VARSTR(service));
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_VERSION_STR), VARSTRZ(PROJECT_VERSION)); kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_VERSION_STR), VARSTRZ(PROJECT_VERSION));
ioWriteStrLine(protocolServerIoWrite(this), jsonFromKv(greetingKv)); ioWriteStrLine(this->write, jsonFromKv(greetingKv));
ioWriteFlush(protocolServerIoWrite(this)); ioWriteFlush(this->write);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
} }
@ -92,17 +90,50 @@ protocolServerError(ProtocolServer *this, int code, const String *message, const
ASSERT(message != NULL); ASSERT(message != NULL);
ASSERT(stack != NULL); ASSERT(stack != NULL);
KeyValue *error = kvNew(); // Write the error and flush to be sure it gets sent immediately
kvPut(error, VARSTR(PROTOCOL_ERROR_STR), VARINT(code)); PackWrite *error = pckWriteNew(this->write);
kvPut(error, VARSTR(PROTOCOL_OUTPUT_STR), VARSTR(message)); pckWriteU32P(error, protocolMessageTypeError);
kvPut(error, VARSTR(PROTOCOL_ERROR_STACK_STR), VARSTR(stack)); pckWriteI32P(error, code);
pckWriteStrP(error, message);
pckWriteStrP(error, stack);
pckWriteEndP(error);
ioWriteStrLine(protocolServerIoWrite(this), jsonFromKv(error)); ioWriteFlush(this->write);
ioWriteFlush(protocolServerIoWrite(this));
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }
/**********************************************************************************************************************************/
ProtocolServerCommandGetResult
protocolServerCommandGet(ProtocolServer *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_END();
ProtocolServerCommandGetResult result = {0};
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *const command = pckReadNew(this->read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(command);
CHECK(type == protocolMessageTypeCommand);
MEM_CONTEXT_PRIOR_BEGIN()
{
result.id = pckReadStrIdP(command);
result.param = pckReadPackBufP(command);
}
MEM_CONTEXT_PRIOR_END();
pckReadEndP(command);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_STRUCT(result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
protocolServerProcess( protocolServerProcess(
@ -129,17 +160,15 @@ protocolServerProcess(
{ {
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Read command // Get command
KeyValue *commandKv = jsonToKv(ioReadLine(protocolServerIoRead(this))); ProtocolServerCommandGetResult command = protocolServerCommandGet(this);
const StringId command = strIdFromStr(stringIdBit5, varStr(kvGet(commandKv, VARSTR(PROTOCOL_KEY_COMMAND_STR))));
VariantList *paramList = varVarLst(kvGet(commandKv, VARSTR(PROTOCOL_KEY_PARAMETER_STR)));
// Find the handler // Find the handler
ProtocolServerCommandHandler handler = NULL; ProtocolServerCommandHandler handler = NULL;
for (unsigned int handlerIdx = 0; handlerIdx < handlerListSize; handlerIdx++) for (unsigned int handlerIdx = 0; handlerIdx < handlerListSize; handlerIdx++)
{ {
if (command == handlerList[handlerIdx].command) if (command.id == handlerList[handlerIdx].command)
{ {
handler = handlerList[handlerIdx].handler; handler = handlerList[handlerIdx].handler;
break; break;
@ -151,7 +180,7 @@ protocolServerProcess(
{ {
// Send the command to the handler. Run the handler in the server's memory context in case any persistent data // Send the command to the handler. Run the handler in the server's memory context in case any persistent data
// needs to be stored by the handler. // needs to be stored by the handler.
MEM_CONTEXT_BEGIN(this->pub.memContext) MEM_CONTEXT_BEGIN(this->memContext)
{ {
// Initialize retries in case of command failure // Initialize retries in case of command failure
bool retry = false; bool retry = false;
@ -164,7 +193,7 @@ protocolServerProcess(
TRY_BEGIN() TRY_BEGIN()
{ {
handler(paramList, this); handler(pckReadNewBuf(command.param), this);
} }
CATCH_ANY() CATCH_ANY()
{ {
@ -180,9 +209,8 @@ protocolServerProcess(
"retry %s after %" PRIu64 "ms: %s", errorTypeName(errorType()), retrySleepMs, "retry %s after %" PRIu64 "ms: %s", errorTypeName(errorType()), retrySleepMs,
errorMessage()); errorMessage());
// Sleep if there is an interval // Sleep for interval
if (retrySleepMs > 0) sleepMSec(retrySleepMs);
sleepMSec(retrySleepMs);
// Decrement retries remaining and retry // Decrement retries remaining and retry
retryRemaining--; retryRemaining--;
@ -204,18 +232,19 @@ protocolServerProcess(
// Else check built-in commands // Else check built-in commands
else else
{ {
switch (command) switch (command.id)
{ {
case PROTOCOL_COMMAND_EXIT: case PROTOCOL_COMMAND_EXIT:
exit = true; exit = true;
break; break;
case PROTOCOL_COMMAND_NOOP: case PROTOCOL_COMMAND_NOOP:
protocolServerResponse(this, NULL); protocolServerDataEndPut(this);
break; break;
default: default:
THROW_FMT(ProtocolError, "invalid command '%s' (0x%" PRIx64 ")", strZ(strIdToStr(command)), command); THROW_FMT(
ProtocolError, "invalid command '%s' (0x%" PRIx64 ")", strZ(strIdToStr(command.id)), command.id);
} }
} }
@ -239,45 +268,75 @@ protocolServerProcess(
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void PackRead *
protocolServerResponse(ProtocolServer *this, const Variant *output) protocolServerDataGet(ProtocolServer *const this)
{ {
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_PARAM(VARIANT, output);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
KeyValue *result = kvNew(); PackRead *result = NULL;
if (output != NULL) MEM_CONTEXT_TEMP_BEGIN()
kvPut(result, VARSTR(PROTOCOL_OUTPUT_STR), output); {
PackRead *data = pckReadNew(this->read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(data);
ioWriteStrLine(protocolServerIoWrite(this), jsonFromKv(result)); CHECK(type == protocolMessageTypeData);
ioWriteFlush(protocolServerIoWrite(this));
MEM_CONTEXT_PRIOR_BEGIN()
{
result = pckReadPackP(data);
}
MEM_CONTEXT_PRIOR_END();
pckReadEndP(data);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/**********************************************************************************************************************************/
void
protocolServerDataPut(ProtocolServer *const this, PackWrite *const data)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_PARAM(PACK_WRITE, data);
FUNCTION_LOG_END();
// End the pack
if (data != NULL)
pckWriteEndP(data);
// Write the result
PackWrite *resultMessage = pckWriteNew(this->write);
pckWriteU32P(resultMessage, protocolMessageTypeData, .defaultWrite = true);
pckWritePackP(resultMessage, data);
pckWriteEndP(resultMessage);
// Flush on NULL result since it might be used to synchronize
if (data == NULL)
ioWriteFlush(this->write);
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
protocolServerWriteLine(ProtocolServer *this, const String *line) protocolServerDataEndPut(ProtocolServer *const this)
{ {
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_PARAM(STRING, line);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); // Write the response and flush to be sure it gets sent immediately
PackWrite *response = pckWriteNew(this->write);
pckWriteU32P(response, protocolMessageTypeDataEnd, .defaultWrite = true);
pckWriteEndP(response);
// Dot indicates the start of an lf-terminated line ioWriteFlush(this->write);
ioWrite(protocolServerIoWrite(this), DOT_BUF);
// Write the line if it exists
if (line != NULL)
ioWriteStr(protocolServerIoWrite(this), line);
// Terminate with a linefeed
ioWrite(protocolServerIoWrite(this), LF_BUF);
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }

View File

@ -12,7 +12,9 @@ typedef struct ProtocolServer ProtocolServer;
#include "common/io/read.h" #include "common/io/read.h"
#include "common/io/write.h" #include "common/io/write.h"
#include "common/type/object.h" #include "common/type/object.h"
#include "common/type/pack.h"
#include "common/type/stringId.h" #include "common/type/stringId.h"
#include "protocol/client.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Protocol command handler type and structure Protocol command handler type and structure
@ -20,7 +22,7 @@ Protocol command handler type and structure
An array of this struct must be passed to protocolServerProcess() for the server to process commands. Each command handler should An array of this struct must be passed to protocolServerProcess() for the server to process commands. Each command handler should
implement a single command, as defined by the command string. implement a single command, as defined by the command string.
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
typedef void (*ProtocolServerCommandHandler)(const VariantList *paramList, ProtocolServer *server); typedef void (*ProtocolServerCommandHandler)(PackRead *param, ProtocolServer *server);
typedef struct ProtocolServerHandler typedef struct ProtocolServerHandler
{ {
@ -35,44 +37,36 @@ Constructors
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
ProtocolServer *protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write); ProtocolServer *protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write);
/***********************************************************************************************************************************
Getters/Setters
***********************************************************************************************************************************/
typedef struct ProtocolServerPub
{
MemContext *memContext; // Mem context
IoRead *read; // Read interface
IoWrite *write; // Write interface
} ProtocolServerPub;
// Read interface
__attribute__((always_inline)) static inline IoRead *
protocolServerIoRead(ProtocolServer *const this)
{
return THIS_PUB(ProtocolServer)->read;
}
// Write interface
__attribute__((always_inline)) static inline IoWrite *
protocolServerIoWrite(ProtocolServer *const this)
{
return THIS_PUB(ProtocolServer)->write;
}
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Get command from the client. Outside ProtocolServer, this is used when the first noop command needs to be processed before
// running protocolServerProcess(), which allows an error to be returned to the client if initialization fails.
typedef struct ProtocolServerCommandGetResult
{
StringId id; // Command identifier
Buffer *param; // Parameter pack
} ProtocolServerCommandGetResult;
ProtocolServerCommandGetResult protocolServerCommandGet(ProtocolServer *this);
// Get data from the client
PackRead *protocolServerDataGet(ProtocolServer *this);
// Put data to the client
void protocolServerDataPut(ProtocolServer *this, PackWrite *data);
// Put data end to the client. This ends command processing and no more data should be sent.
void protocolServerDataEndPut(ProtocolServer *this);
// Return an error // Return an error
void protocolServerError(ProtocolServer *this, int code, const String *message, const String *stack); void protocolServerError(ProtocolServer *this, int code, const String *message, const String *stack);
// Process requests // Process requests
void protocolServerProcess( void protocolServerProcess(
ProtocolServer *this, const VariantList *retryInterval, const ProtocolServerHandler *const handlerList, ProtocolServer *this, const VariantList *retryInterval, const ProtocolServerHandler *handlerList,
const unsigned int handlerListSize); const unsigned int handlerListSize);
// Respond to request with output if provided
void protocolServerResponse(ProtocolServer *this, const Variant *output);
// Move to a new parent mem context // Move to a new parent mem context
__attribute__((always_inline)) static inline ProtocolServer * __attribute__((always_inline)) static inline ProtocolServer *
protocolServerMove(ProtocolServer *const this, MemContext *const parentNew) protocolServerMove(ProtocolServer *const this, MemContext *const parentNew)
@ -80,9 +74,6 @@ protocolServerMove(ProtocolServer *const this, MemContext *const parentNew)
return objMove(this, parentNew); return objMove(this, parentNew);
} }
// Write a line
void protocolServerWriteLine(ProtocolServer *this, const String *line);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Destructor Destructor
***********************************************************************************************************************************/ ***********************************************************************************************************************************/

View File

@ -14,6 +14,7 @@ Remote Storage Protocol Handler
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/regExp.h" #include "common/regExp.h"
#include "common/type/pack.h"
#include "common/type/json.h" #include "common/type/json.h"
#include "config/config.h" #include "config/config.h"
#include "protocol/helper.h" #include "protocol/helper.h"
@ -21,11 +22,6 @@ Remote Storage Protocol Handler
#include "storage/helper.h" #include "storage/helper.h"
#include "storage/storage.intern.h" #include "storage/storage.intern.h"
/***********************************************************************************************************************************
Regular expressions
***********************************************************************************************************************************/
STRING_STATIC(BLOCK_REG_EXP_STR, PROTOCOL_BLOCK_HEADER "-1|[0-9]+");
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Local variables Local variables
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
@ -76,64 +72,16 @@ storageRemoteFilterGroup(IoFilterGroup *filterGroup, const Variant *filterList)
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
} }
/***********************************************************************************************************************************
Callback to write info list into the protocol
***********************************************************************************************************************************/
// Helper to write storage info into the protocol. This function is not called unless the info exists so no need to write exists or
// check for level == storageInfoLevelExists.
static void
storageRemoteInfoWrite(ProtocolServer *server, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_SERVER, server);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
protocolServerWriteLine(server, jsonFromUInt(info->type));
protocolServerWriteLine(server, jsonFromInt64(info->timeModified));
if (info->type == storageTypeFile)
protocolServerWriteLine(server, jsonFromUInt64(info->size));
if (info->level >= storageInfoLevelDetail)
{
protocolServerWriteLine(server, jsonFromUInt(info->userId));
protocolServerWriteLine(server, jsonFromStr(info->user));
protocolServerWriteLine(server, jsonFromUInt(info->groupId));
protocolServerWriteLine(server, jsonFromStr(info->group));
protocolServerWriteLine(server, jsonFromUInt(info->mode));
if (info->type == storageTypeLink)
protocolServerWriteLine(server, jsonFromStr(info->linkDestination));
}
FUNCTION_TEST_RETURN_VOID();
}
static void
storageRemoteProtocolInfoListCallback(void *server, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
protocolServerWriteLine(server, jsonFromStr(info->name));
storageRemoteInfoWrite(server, info);
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *server) storageRemoteFeatureProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList == NULL); ASSERT(param == NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
@ -158,10 +106,12 @@ storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *serve
} }
// Return storage features // Return storage features
protocolServerWriteLine(server, jsonFromStr(storagePathP(storage, NULL))); PackWrite *result = protocolPackNew();
protocolServerWriteLine(server, jsonFromUInt64(storageInterface(storage).feature)); pckWriteStrP(result, storagePathP(storage, NULL));
pckWriteU64P(result, storageInterface(storage).feature);
protocolServerResponse(server, NULL); protocolServerDataPut(server, result);
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -169,58 +119,203 @@ storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *serve
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void typedef struct StorageRemoteInfoProcotolWriteData
storageRemoteInfoProtocol(const VariantList *paramList, ProtocolServer *server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); MemContext *memContext; // Mem context used to store values from last call
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); time_t timeModifiedLast; // timeModified from last call
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); mode_t modeLast; // mode from last call
FUNCTION_LOG_END(); uid_t userIdLast; // userId from last call
gid_t groupIdLast; // groupId from last call
String *user; // user from last call
String *group; // group from last call
} StorageRemoteInfoProtocolWriteData;
ASSERT(paramList != NULL); // Helper to write storage info into the protocol. This function is not called unless the info exists so no need to write exists or
ASSERT(server != NULL); // check for level == storageInfoLevelExists.
ASSERT(storageRemoteProtocolLocal.driver != NULL); //
// Fields that do not change from one call to the next are omitted to save bandwidth.
static void
storageRemoteInfoProtocolPut(
StorageRemoteInfoProtocolWriteData *const data, PackWrite *const write, const StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(PACK_WRITE, write);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
MEM_CONTEXT_TEMP_BEGIN() ASSERT(data != NULL);
ASSERT(write != NULL);
ASSERT(info != NULL);
// Write type and time
pckWriteU32P(write, info->type);
pckWriteTimeP(write, info->timeModified - data->timeModifiedLast);
// Write size for files
if (info->type == storageTypeFile)
pckWriteU64P(write, info->size);
// Write fields needed for detail level
if (info->level >= storageInfoLevelDetail)
{ {
StorageInfo info = storageInterfaceInfoP( // Write mode
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), pckWriteModeP(write, info->mode, .defaultValue = data->modeLast);
(StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)), .followLink = varBool(varLstGet(paramList, 2)));
protocolServerResponse(server, VARBOOL(info.exists)); // Write user id/name
pckWriteU32P(write, info->userId, .defaultValue = data->userIdLast);
if (info.exists) if (info->user == NULL) // {vm_covered}
pckWriteBoolP(write, true); // {vm_covered}
else
{ {
storageRemoteInfoWrite(server, &info); pckWriteBoolP(write, false);
protocolServerResponse(server, NULL); pckWriteStrP(write, info->user, .defaultValue = data->user);
}
// Write group id/name
pckWriteU32P(write, info->groupId, .defaultValue = data->groupIdLast);
if (info->group == NULL) // {vm_covered}
pckWriteBoolP(write, true); // {vm_covered}
else
{
pckWriteBoolP(write, false);
pckWriteStrP(write, info->group, .defaultValue = data->group);
}
// Write link destination
if (info->type == storageTypeLink)
pckWriteStrP(write, info->linkDestination);
}
// Store defaults to use for the next call. If memContext is NULL this function is only being called one time so there is no
// point in storing defaults.
if (data->memContext != NULL)
{
data->timeModifiedLast = info->timeModified;
data->modeLast = info->mode;
data->userIdLast = info->userId;
data->groupIdLast = info->groupId;
if (!strEq(info->user, data->user) && info->user != NULL) // {vm_covered}
{
strFree(data->user);
MEM_CONTEXT_BEGIN(data->memContext)
{
data->user = strDup(info->user);
}
MEM_CONTEXT_END();
}
if (!strEq(info->group, data->group) && info->group != NULL) // {vm_covered}
{
strFree(data->group);
MEM_CONTEXT_BEGIN(data->memContext)
{
data->group = strDup(info->group);
}
MEM_CONTEXT_END();
} }
} }
FUNCTION_TEST_RETURN_VOID();
}
void
storageRemoteInfoProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// Get file info
const String *file = pckReadStrP(param);
StorageInfoLevel level = (StorageInfoLevel)pckReadU32P(param);
bool followLink = pckReadBoolP(param);
StorageInfo info = storageInterfaceInfoP(storageRemoteProtocolLocal.driver, file, level, .followLink = followLink);
// Write file info to protocol
PackWrite *write = protocolPackNew();
pckWriteBoolP(write, info.exists, .defaultWrite = true);
if (info.exists)
storageRemoteInfoProtocolPut(&(StorageRemoteInfoProtocolWriteData){0}, write, &info);
protocolServerDataPut(server, write);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
typedef struct StorageRemoteProtocolInfoListCallbackData
{
ProtocolServer *const server;
StorageRemoteInfoProtocolWriteData writeData;
} StorageRemoteProtocolInfoListCallbackData;
static void
storageRemoteProtocolInfoListCallback(void *dataVoid, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM_P(VOID, dataVoid);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(dataVoid != NULL);
ASSERT(info != NULL);
StorageRemoteProtocolInfoListCallbackData *data = dataVoid;
PackWrite *write = protocolPackNew();
pckWriteStrP(write, info->name);
storageRemoteInfoProtocolPut(&data->writeData, write, info);
protocolServerDataPut(data->server, write);
FUNCTION_TEST_RETURN_VOID();
}
void void
storageRemoteInfoListProtocol(const VariantList *paramList, ProtocolServer *server) storageRemoteInfoListProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL); ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
bool result = storageInterfaceInfoListP( const String *const path = pckReadStrP(param);
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), const StorageInfoLevel level = (StorageInfoLevel)pckReadU32P(param);
(StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)), storageRemoteProtocolInfoListCallback, server);
protocolServerWriteLine(server, NULL); StorageRemoteProtocolInfoListCallbackData data = {.server = server, .writeData = {.memContext = memContextCurrent()}};
protocolServerResponse(server, VARBOOL(result));
const bool result = storageInterfaceInfoListP(
storageRemoteProtocolLocal.driver, path, level, storageRemoteProtocolInfoListCallback, &data);
// Indicate whether or not the path was found
PackWrite *write = protocolPackNew();
pckWriteBoolP(write, result, .defaultWrite = true);
protocolServerDataPut(server, write);
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -229,31 +324,34 @@ storageRemoteInfoListProtocol(const VariantList *paramList, ProtocolServer *serv
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *server) storageRemoteOpenReadProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL); ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
const String *file = pckReadStrP(param);
bool ignoreMissing = pckReadBoolP(param);
const Variant *limit = jsonToVar(pckReadStrP(param));
const Variant *filter = jsonToVar(pckReadStrP(param));
// Create the read object // Create the read object
IoRead *fileRead = storageReadIo( IoRead *fileRead = storageReadIo(
storageInterfaceNewReadP( storageInterfaceNewReadP(storageRemoteProtocolLocal.driver, file, ignoreMissing, .limit = limit));
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)),
.limit = varLstGet(paramList, 2)));
// Set filter group based on passed filters // Set filter group based on passed filters
storageRemoteFilterGroup(ioReadFilterGroup(fileRead), varLstGet(paramList, 3)); storageRemoteFilterGroup(ioReadFilterGroup(fileRead), filter);
// Check if the file exists // Check if the file exists
bool exists = ioReadOpen(fileRead); bool exists = ioReadOpen(fileRead);
protocolServerResponse(server, VARBOOL(exists)); protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), exists, .defaultWrite = true));
// Transfer the file if it exists // Transfer the file if it exists
if (exists) if (exists)
@ -267,9 +365,9 @@ storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *serv
if (!bufEmpty(buffer)) if (!bufEmpty(buffer))
{ {
ioWriteStrLine(protocolServerIoWrite(server), strNewFmt(PROTOCOL_BLOCK_HEADER "%zu", bufUsed(buffer))); PackWrite *write = protocolPackNew();
ioWrite(protocolServerIoWrite(server), buffer); pckWriteBinP(write, buffer);
ioWriteFlush(protocolServerIoWrite(server)); protocolServerDataPut(server, write);
bufUsedZero(buffer); bufUsedZero(buffer);
} }
@ -278,13 +376,12 @@ storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *serv
ioReadClose(fileRead); ioReadClose(fileRead);
// Write a zero block to show file is complete // Write filter results
ioWriteLine(protocolServerIoWrite(server), BUFSTRDEF(PROTOCOL_BLOCK_HEADER "0")); protocolServerDataPut(
ioWriteFlush(protocolServerIoWrite(server)); server, pckWriteStrP(protocolPackNew(), jsonFromVar(ioFilterGroupResultAll(ioReadFilterGroup(fileRead)))));
// Push filter results
protocolServerResponse(server, ioFilterGroupResultAll(ioReadFilterGroup(fileRead)));
} }
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -293,79 +390,85 @@ storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *serv
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
storageRemoteOpenWriteProtocol(const VariantList *paramList, ProtocolServer *server) storageRemoteOpenWriteProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL); ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Create the write object // Create the write object
const String *file = pckReadStrP(param);
mode_t modeFile = pckReadModeP(param);
mode_t modePath = pckReadModeP(param);
const String *user = pckReadStrP(param);
const String *group = pckReadStrP(param);
time_t timeModified = pckReadTimeP(param);
bool createPath = pckReadBoolP(param);
bool syncFile = pckReadBoolP(param);
bool syncPath = pckReadBoolP(param);
bool atomic = pckReadBoolP(param);
const Variant *filter = jsonToVar(pckReadStrP(param));
IoWrite *fileWrite = storageWriteIo( IoWrite *fileWrite = storageWriteIo(
storageInterfaceNewWriteP( storageInterfaceNewWriteP(
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), storageRemoteProtocolLocal.driver, file, .modeFile = modeFile, .modePath = modePath, .user = user, .group = group,
.modeFile = (mode_t)varUIntForce(varLstGet(paramList, 1)), .timeModified = timeModified, .createPath = createPath, .syncFile = syncFile, .syncPath = syncPath,
.modePath = (mode_t)varUIntForce(varLstGet(paramList, 2)), .user = varStr(varLstGet(paramList, 3)), .atomic = atomic));
.group = varStr(varLstGet(paramList, 4)), .timeModified = (time_t)varUInt64Force(varLstGet(paramList, 5)),
.createPath = varBool(varLstGet(paramList, 6)), .syncFile = varBool(varLstGet(paramList, 7)),
.syncPath = varBool(varLstGet(paramList, 8)), .atomic = varBool(varLstGet(paramList, 9))));
// Set filter group based on passed filters // Set filter group based on passed filters
storageRemoteFilterGroup(ioWriteFilterGroup(fileWrite), varLstGet(paramList, 10)); storageRemoteFilterGroup(ioWriteFilterGroup(fileWrite), filter);
// Open file // Open file
ioWriteOpen(fileWrite); ioWriteOpen(fileWrite);
protocolServerResponse(server, NULL); protocolServerDataPut(server, NULL);
// Write data // Write data
Buffer *buffer = bufNew(ioBufferSize());
ssize_t remaining;
do do
{ {
// How much data is remaining to write? PackRead *read = protocolServerDataGet(server);
remaining = storageRemoteProtocolBlockSize(ioReadLine(protocolServerIoRead(server)));
// Write data // Write is complete
if (remaining > 0) if (read == NULL)
{
size_t bytesToCopy = (size_t)remaining;
do
{
if (bytesToCopy < bufSize(buffer))
bufLimitSet(buffer, bytesToCopy);
bytesToCopy -= ioRead(protocolServerIoRead(server), buffer);
ioWrite(fileWrite, buffer);
bufUsedZero(buffer);
bufLimitClear(buffer);
}
while (bytesToCopy > 0);
}
// Close when all data has been written
else if (remaining == 0)
{ {
ioWriteClose(fileWrite); ioWriteClose(fileWrite);
// Push filter results // Push filter results
protocolServerResponse(server, ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite))); protocolServerDataPut(
server, pckWriteStrP(protocolPackNew(), jsonFromVar(ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite)))));
break;
} }
// Write was aborted so free the file // Else more data to write
else else
{ {
ioWriteFree(fileWrite); pckReadNext(read);
protocolServerResponse(server, NULL);
// Write data
if (pckReadType(read) == pckTypeBin)
{
Buffer *const buffer = pckReadBinP(read);
ioWrite(fileWrite, buffer);
bufFree(buffer);
}
// Else write terminated unexpectedly
else
{
protocolServerDataGet(server);
ioWriteFree(fileWrite);
break;
}
} }
} }
while (remaining > 0); while (true);
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -374,24 +477,26 @@ storageRemoteOpenWriteProtocol(const VariantList *paramList, ProtocolServer *ser
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
storageRemotePathCreateProtocol(const VariantList *paramList, ProtocolServer *server) storageRemotePathCreateProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL); ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
storageInterfacePathCreateP( const String *path = pckReadStrP(param);
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), bool errorOnExists = pckReadBoolP(param);
varBool(varLstGet(paramList, 2)), (mode_t)varUIntForce(varLstGet(paramList, 3))); bool noParentCreate = pckReadBoolP(param);
mode_t mode = pckReadModeP(param);
protocolServerResponse(server, NULL); storageInterfacePathCreateP(storageRemoteProtocolLocal.driver, path, errorOnExists, noParentCreate, mode);
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -400,24 +505,26 @@ storageRemotePathCreateProtocol(const VariantList *paramList, ProtocolServer *se
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
storageRemotePathRemoveProtocol(const VariantList *paramList, ProtocolServer *server) storageRemotePathRemoveProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL); ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
protocolServerResponse( const String *path = pckReadStrP(param);
server, bool recurse = pckReadBoolP(param);
VARBOOL(
storageInterfacePathRemoveP( const bool result = storageInterfacePathRemoveP(storageRemoteProtocolLocal.driver, path, recurse);
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)))));
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), result, .defaultWrite = true));
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -426,21 +533,23 @@ storageRemotePathRemoveProtocol(const VariantList *paramList, ProtocolServer *se
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
storageRemotePathSyncProtocol(const VariantList *paramList, ProtocolServer *server) storageRemotePathSyncProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL); ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
storageInterfacePathSyncP(storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0))); const String *path = pckReadStrP(param);
protocolServerResponse(server, NULL);
storageInterfacePathSyncP(storageRemoteProtocolLocal.driver, path);
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -449,54 +558,26 @@ storageRemotePathSyncProtocol(const VariantList *paramList, ProtocolServer *serv
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
void void
storageRemoteRemoveProtocol(const VariantList *paramList, ProtocolServer *server) storageRemoteRemoveProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList); FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server); FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(paramList != NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL); ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
storageInterfaceRemoveP( const String *file = pckReadStrP(param);
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), .errorOnMissing = varBool(varLstGet(paramList, 1))); bool errorOnMissing = pckReadBoolP(param);
protocolServerResponse(server, NULL);
storageInterfaceRemoveP(storageRemoteProtocolLocal.driver, file, .errorOnMissing = errorOnMissing);
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }
/**********************************************************************************************************************************/
ssize_t
storageRemoteProtocolBlockSize(const String *message)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STRING, message);
FUNCTION_LOG_END();
ASSERT(message != NULL);
// Regular expression to check block messages
static RegExp *blockRegExp = NULL;
// Create block regular expression if it has not been created yet
if (blockRegExp == NULL)
{
MEM_CONTEXT_BEGIN(memContextTop())
{
blockRegExp = regExpNew(BLOCK_REG_EXP_STR);
}
MEM_CONTEXT_END();
}
// Validate the header block size message
if (!regExpMatch(blockRegExp, message))
THROW_FMT(ProtocolError, "'%s' is not a valid block size message", strZ(message));
FUNCTION_LOG_RETURN(SSIZE, (ssize_t)cvtZToInt(strZ(message) + sizeof(PROTOCOL_BLOCK_HEADER) - 1));
}

View File

@ -4,31 +4,22 @@ Remote Storage Protocol Handler
#ifndef STORAGE_REMOTE_PROTOCOL_H #ifndef STORAGE_REMOTE_PROTOCOL_H
#define STORAGE_REMOTE_PROTOCOL_H #define STORAGE_REMOTE_PROTOCOL_H
#include "common/type/string.h" #include "common/type/pack.h"
#include "common/type/variantList.h"
#include "protocol/server.h" #include "protocol/server.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define PROTOCOL_BLOCK_HEADER "BRBLOCK"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Get size of the next transfer block
ssize_t storageRemoteProtocolBlockSize(const String *message);
// Process storage protocol requests // Process storage protocol requests
void storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *server); void storageRemoteFeatureProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteInfoProtocol(const VariantList *paramList, ProtocolServer *server); void storageRemoteInfoProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteInfoListProtocol(const VariantList *paramList, ProtocolServer *server); void storageRemoteInfoListProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *server); void storageRemoteOpenReadProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteOpenWriteProtocol(const VariantList *paramList, ProtocolServer *server); void storageRemoteOpenWriteProtocol(PackRead *param, ProtocolServer *server);
void storageRemotePathCreateProtocol(const VariantList *paramList, ProtocolServer *server); void storageRemotePathCreateProtocol(PackRead *param, ProtocolServer *server);
void storageRemotePathRemoveProtocol(const VariantList *paramList, ProtocolServer *server); void storageRemotePathRemoveProtocol(PackRead *param, ProtocolServer *server);
void storageRemotePathSyncProtocol(const VariantList *paramList, ProtocolServer *server); void storageRemotePathSyncProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteRemoveProtocol(const VariantList *paramList, ProtocolServer *server); void storageRemoteRemoveProtocol(PackRead *param, ProtocolServer *server);
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess() Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

View File

@ -12,6 +12,7 @@ Remote Storage Read
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/type/convert.h" #include "common/type/convert.h"
#include "common/type/json.h"
#include "common/type/object.h" #include "common/type/object.h"
#include "storage/remote/protocol.h" #include "storage/remote/protocol.h"
#include "storage/remote/read.h" #include "storage/remote/read.h"
@ -29,6 +30,7 @@ typedef struct StorageReadRemote
ProtocolClient *client; // Protocol client for requests ProtocolClient *client; // Protocol client for requests
size_t remaining; // Bytes remaining to be read in block size_t remaining; // Bytes remaining to be read in block
Buffer *block; // Block currently being read
bool eof; // Has the file reached eof? bool eof; // Has the file reached eof?
#ifdef DEBUG #ifdef DEBUG
@ -70,19 +72,30 @@ storageReadRemoteOpen(THIS_VOID)
} }
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ);
protocolCommandParamAdd(command, VARSTR(this->interface.name)); PackWrite *const param = protocolCommandParam(command);
protocolCommandParamAdd(command, VARBOOL(this->interface.ignoreMissing));
protocolCommandParamAdd(command, this->interface.limit);
protocolCommandParamAdd(command, ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read))));
result = varBool(protocolClientExecute(this->client, command, true)); pckWriteStrP(param, this->interface.name);
pckWriteBoolP(param, this->interface.ignoreMissing);
pckWriteStrP(param, jsonFromVar(this->interface.limit));
pckWriteStrP(param, jsonFromVar(ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read)))));
// Clear filters since they will be run on the remote side protocolClientCommandPut(this->client, command);
ioFilterGroupClear(ioReadFilterGroup(storageReadIo(this->read)));
// If the file is compressible add decompression filter locally // If the file exists
if (this->interface.compressible) result = pckReadBoolP(protocolClientDataGet(this->client));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilter(compressTypeGz));
if (result)
{
// Clear filters since they will be run on the remote side
ioFilterGroupClear(ioReadFilterGroup(storageReadIo(this->read)));
// If the file is compressible add decompression filter locally
if (this->interface.compressible)
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilter(compressTypeGz));
}
// Else nothing to do
else
protocolClientDataEndGet(this->client);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -118,13 +131,28 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
{ {
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
this->remaining = (size_t)storageRemoteProtocolBlockSize(ioReadLine(protocolClientIoRead(this->client))); PackRead *const read = protocolClientDataGet(this->client);
pckReadNext(read);
if (this->remaining == 0) // If binary then read the next block
if (pckReadType(read) == pckTypeBin)
{ {
ioFilterGroupResultAllSet( MEM_CONTEXT_BEGIN(this->memContext)
ioReadFilterGroup(storageReadIo(this->read)), protocolClientReadOutput(this->client, true)); {
this->block = pckReadBinP(read);
this->remaining = bufUsed(this->block);
}
MEM_CONTEXT_END();
}
// Else read is complete and get the filter list
else
{
bufFree(this->block);
ioFilterGroupResultAllSet(ioReadFilterGroup(storageReadIo(this->read)), jsonToVar(pckReadStrP(read)));
this->eof = true; this->eof = true;
protocolClientDataEndGet(this->client);
} }
#ifdef DEBUG #ifdef DEBUG
@ -140,14 +168,19 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
// If the buffer can contain all remaining bytes // If the buffer can contain all remaining bytes
if (bufRemains(buffer) >= this->remaining) if (bufRemains(buffer) >= this->remaining)
{ {
bufLimitSet(buffer, bufUsed(buffer) + this->remaining); bufCatSub(buffer, this->block, bufUsed(this->block) - this->remaining, this->remaining);
ioRead(protocolClientIoRead(this->client), buffer);
bufLimitClear(buffer);
this->remaining = 0; this->remaining = 0;
bufFree(this->block);
this->block = NULL;
} }
// Else read what we can // Else read what we can
else else
this->remaining -= ioRead(protocolClientIoRead(this->client), buffer); {
size_t remains = bufRemains(buffer);
bufCatSub(buffer, this->block, bufUsed(this->block) - this->remaining, remains);
this->remaining -= remains;
}
} }
} }
while (!this->eof && !bufFull(buffer)); while (!this->eof && !bufFull(buffer));

View File

@ -8,6 +8,7 @@ Remote Storage
#include "common/memContext.h" #include "common/memContext.h"
#include "common/type/json.h" #include "common/type/json.h"
#include "common/type/object.h" #include "common/type/object.h"
#include "common/type/pack.h"
#include "storage/remote/protocol.h" #include "storage/remote/protocol.h"
#include "storage/remote/read.h" #include "storage/remote/read.h"
#include "storage/remote/storage.intern.h" #include "storage/remote/storage.intern.h"
@ -25,31 +26,81 @@ struct StorageRemote
}; };
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
// Helper to parse storage info from the protocol output typedef struct StorageRemoteInfoData
{
time_t timeModifiedLast; // timeModified from last call
mode_t modeLast; // mode from last call
uid_t userIdLast; // userId from last call
gid_t groupIdLast; // groupId from last call
String *user; // user from last call
String *group; // group from last call
} StorageRemoteInfoData;
// Helper to get storage info from the protocol output
static void static void
storageRemoteInfoParse(ProtocolClient *client, StorageInfo *info) storageRemoteInfoGet(StorageRemoteInfoData *const data, PackRead *const read, StorageInfo *const info)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_CLIENT, client); FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(PACK_READ, read);
FUNCTION_TEST_PARAM(STORAGE_INFO, info); FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END(); FUNCTION_TEST_END();
info->type = jsonToUInt(protocolClientReadLine(client)); ASSERT(data != NULL);
info->timeModified = (time_t)jsonToUInt64(protocolClientReadLine(client)); ASSERT(read != NULL);
ASSERT(info != NULL);
// Read type and time modified
info->type = pckReadU32P(read);
info->timeModified = pckReadTimeP(read) + data->timeModifiedLast;
// Read size for files
if (info->type == storageTypeFile) if (info->type == storageTypeFile)
info->size = jsonToUInt64(protocolClientReadLine(client)); info->size = pckReadU64P(read);
// Read fields needed for detail level
if (info->level >= storageInfoLevelDetail) if (info->level >= storageInfoLevelDetail)
{ {
info->userId = jsonToUInt(protocolClientReadLine(client)); // Read mode
info->user = jsonToStr(protocolClientReadLine(client)); info->mode = pckReadModeP(read, .defaultValue = data->modeLast);
info->groupId = jsonToUInt(protocolClientReadLine(client));
info->group = jsonToStr(protocolClientReadLine(client));
info->mode = (mode_t)jsonToUInt(protocolClientReadLine(client));
// Read user id/name
info->userId = pckReadU32P(read, .defaultValue = data->userIdLast);
if (pckReadBoolP(read)) // {vm_covered}
info->user = NULL; // {vm_covered}
else
info->user = pckReadStrP(read, .defaultValue = data->user);
// Read group id/name
info->groupId = pckReadU32P(read, .defaultValue = data->groupIdLast);
if (pckReadBoolP(read)) // {vm_covered}
info->group = NULL; // {vm_covered}
else
info->group = pckReadStrP(read, .defaultValue = data->group);
// Read link destination
if (info->type == storageTypeLink) if (info->type == storageTypeLink)
info->linkDestination = jsonToStr(protocolClientReadLine(client)); info->linkDestination = pckReadStrP(read);
}
// Store defaults to use for the next call
data->timeModifiedLast = info->timeModified;
data->modeLast = info->mode;
data->userIdLast = info->userId;
data->groupIdLast = info->groupId;
if (!strEq(info->user, data->user) && info->user != NULL) // {vm_covered}
{
strFree(data->user);
data->user = strDup(info->user);
}
if (!strEq(info->group, data->group) && info->group != NULL) // {vm_covered}
{
strFree(data->group);
data->group = strDup(info->group);
} }
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
@ -74,25 +125,31 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, Storage
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO);
protocolCommandParamAdd(command, VARSTR(file)); PackWrite *const commandParam = protocolCommandParam(command);
protocolCommandParamAdd(command, VARUINT(level));
protocolCommandParamAdd(command, VARBOOL(param.followLink));
result.exists = varBool(protocolClientExecute(this->client, command, true)); pckWriteStrP(commandParam, file);
pckWriteU32P(commandParam, level);
pckWriteBoolP(commandParam, param.followLink);
// Put command
protocolClientCommandPut(this->client, command);
// Read info from protocol
PackRead *read = protocolClientDataGet(this->client);
result.exists = pckReadBoolP(read);
if (result.exists) if (result.exists)
{ {
// Read info from protocol into prior context // Read info from protocol into prior context
MEM_CONTEXT_PRIOR_BEGIN() MEM_CONTEXT_PRIOR_BEGIN()
{ {
result.name = strDup(result.name); storageRemoteInfoGet(&(StorageRemoteInfoData){0}, read, &result);
storageRemoteInfoParse(this->client, &result);
} }
MEM_CONTEXT_PRIOR_END(); MEM_CONTEXT_PRIOR_END();
// Acknowledge command completed
protocolClientReadOutput(this->client, false);
} }
protocolClientDataEndGet(this->client);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -125,36 +182,41 @@ storageRemoteInfoList(
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_LIST); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_LIST);
protocolCommandParamAdd(command, VARSTR(path)); PackWrite *const commandParam = protocolCommandParam(command);
protocolCommandParamAdd(command, VARUINT(level));
// Send command pckWriteStrP(commandParam, path);
protocolClientWriteCommand(this->client, command); pckWriteU32P(commandParam, level);
// Put command
protocolClientCommandPut(this->client, command);
// Read list
StorageRemoteInfoData parseData = {0};
// Read list. The list ends when there is a blank line -- this is safe even for file systems that allow blank filenames
// since the filename is json-encoded so will always include quotes.
MEM_CONTEXT_TEMP_RESET_BEGIN() MEM_CONTEXT_TEMP_RESET_BEGIN()
{ {
const String *name = protocolClientReadLine(this->client); PackRead *read = protocolClientDataGet(this->client);
pckReadNext(read);
while (strSize(name) != 0) while (pckReadType(read) == pckTypeStr)
{ {
StorageInfo info = {.exists = true, .level = level, .name = jsonToStr(name)}; StorageInfo info = {.exists = true, .level = level, .name = pckReadStrP(read)};
storageRemoteInfoParse(this->client, &info); storageRemoteInfoGet(&parseData, read, &info);
callback(callbackData, &info); callback(callbackData, &info);
// Reset the memory context occasionally so we don't use too much memory or slow down processing // Reset the memory context occasionally so we don't use too much memory or slow down processing
MEM_CONTEXT_TEMP_RESET(1000); MEM_CONTEXT_TEMP_RESET(1000);
// Read the next item read = protocolClientDataGet(this->client);
name = protocolClientReadLine(this->client); pckReadNext(read);
} }
result = pckReadBoolP(read);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
// Acknowledge command completed protocolClientDataEndGet(this->client);
result = varBool(protocolClientReadOutput(this->client, true));
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -240,10 +302,12 @@ storageRemotePathCreate(
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_CREATE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_CREATE);
protocolCommandParamAdd(command, VARSTR(path)); PackWrite *const commandParam = protocolCommandParam(command);
protocolCommandParamAdd(command, VARBOOL(errorOnExists));
protocolCommandParamAdd(command, VARBOOL(noParentCreate)); pckWriteStrP(commandParam, path);
protocolCommandParamAdd(command, VARUINT(mode)); pckWriteBoolP(commandParam, errorOnExists);
pckWriteBoolP(commandParam, noParentCreate);
pckWriteModeP(commandParam, mode);
protocolClientExecute(this->client, command, false); protocolClientExecute(this->client, command, false);
} }
@ -273,10 +337,12 @@ storageRemotePathRemove(THIS_VOID, const String *path, bool recurse, StorageInte
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE);
protocolCommandParamAdd(command, VARSTR(path)); PackWrite *const commandParam = protocolCommandParam(command);
protocolCommandParamAdd(command, VARBOOL(recurse));
result = varBool(protocolClientExecute(this->client, command, true)); pckWriteStrP(commandParam, path);
pckWriteBoolP(commandParam, recurse);
result = pckReadBoolP(protocolClientExecute(this->client, command, true));
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -301,7 +367,7 @@ storageRemotePathSync(THIS_VOID, const String *path, StorageInterfacePathSyncPar
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_SYNC); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_SYNC);
protocolCommandParamAdd(command, VARSTR(path)); pckWriteStrP(protocolCommandParam(command), path);
protocolClientExecute(this->client, command, false); protocolClientExecute(this->client, command, false);
} }
@ -328,8 +394,10 @@ storageRemoteRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam p
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_REMOVE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_REMOVE);
protocolCommandParamAdd(command, VARSTR(file)); PackWrite *const commandParam = protocolCommandParam(command);
protocolCommandParamAdd(command, VARBOOL(param.errorOnMissing));
pckWriteStrP(commandParam, file);
pckWriteBoolP(commandParam, param.errorOnMissing);
protocolClientExecute(this->client, command, false); protocolClientExecute(this->client, command, false);
} }
@ -388,22 +456,17 @@ storageRemoteNew(
// Get storage features from the remote // Get storage features from the remote
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Send command // Execute command and get result
protocolClientWriteCommand(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE)); PackRead *result = protocolClientExecute(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE), true);
// Read values // Get path in parent context
path = jsonToStr(protocolClientReadLine(driver->client));
driver->interface.feature = jsonToUInt64(protocolClientReadLine(driver->client));
// Acknowledge command completed
protocolClientReadOutput(driver->client, false);
// Dup path into parent context
MEM_CONTEXT_PRIOR_BEGIN() MEM_CONTEXT_PRIOR_BEGIN()
{ {
path = strDup(path); path = pckReadStrP(result);
} }
MEM_CONTEXT_PRIOR_END(); MEM_CONTEXT_PRIOR_END();
driver->interface.feature = pckReadU64P(result);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();

View File

@ -8,6 +8,7 @@ Remote Storage File write
#include "common/io/write.h" #include "common/io/write.h"
#include "common/log.h" #include "common/log.h"
#include "common/memContext.h" #include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/object.h" #include "common/type/object.h"
#include "storage/remote/protocol.h" #include "storage/remote/protocol.h"
#include "storage/remote/write.h" #include "storage/remote/write.h"
@ -51,9 +52,9 @@ storageWriteRemoteFreeResource(THIS_VOID)
ASSERT(this != NULL); ASSERT(this != NULL);
ioWriteLine(protocolClientIoWrite(this->client), BUFSTRDEF(PROTOCOL_BLOCK_HEADER "-1")); protocolClientDataPut(this->client, pckWriteBoolP(protocolPackNew(), false));
ioWriteFlush(protocolClientIoWrite(this->client)); protocolClientDataPut(this->client, NULL);
protocolClientReadOutput(this->client, false); protocolClientDataEndGet(this->client);
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }
@ -79,19 +80,22 @@ storageWriteRemoteOpen(THIS_VOID)
ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilter(compressTypeGz)); ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilter(compressTypeGz));
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE);
protocolCommandParamAdd(command, VARSTR(this->interface.name)); PackWrite *const param = protocolCommandParam(command);
protocolCommandParamAdd(command, VARUINT(this->interface.modeFile));
protocolCommandParamAdd(command, VARUINT(this->interface.modePath));
protocolCommandParamAdd(command, VARSTR(this->interface.user));
protocolCommandParamAdd(command, VARSTR(this->interface.group));
protocolCommandParamAdd(command, VARINT64(this->interface.timeModified));
protocolCommandParamAdd(command, VARBOOL(this->interface.createPath));
protocolCommandParamAdd(command, VARBOOL(this->interface.syncFile));
protocolCommandParamAdd(command, VARBOOL(this->interface.syncPath));
protocolCommandParamAdd(command, VARBOOL(this->interface.atomic));
protocolCommandParamAdd(command, ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write))));
protocolClientExecute(this->client, command, false); pckWriteStrP(param, this->interface.name);
pckWriteModeP(param, this->interface.modeFile);
pckWriteModeP(param, this->interface.modePath);
pckWriteStrP(param, this->interface.user);
pckWriteStrP(param, this->interface.group);
pckWriteTimeP(param, this->interface.timeModified);
pckWriteBoolP(param, this->interface.createPath);
pckWriteBoolP(param, this->interface.syncFile);
pckWriteBoolP(param, this->interface.syncPath);
pckWriteBoolP(param, this->interface.atomic);
pckWriteStrP(param, jsonFromVar(ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write)))));
protocolClientCommandPut(this->client, command);
protocolClientDataGet(this->client);
// Clear filters since they will be run on the remote side // Clear filters since they will be run on the remote side
ioFilterGroupClear(ioWriteFilterGroup(storageWriteIo(this->write))); ioFilterGroupClear(ioWriteFilterGroup(storageWriteIo(this->write)));
@ -128,9 +132,7 @@ storageWriteRemote(THIS_VOID, const Buffer *buffer)
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(buffer != NULL); ASSERT(buffer != NULL);
ioWriteStrLine(protocolClientIoWrite(this->client), strNewFmt(PROTOCOL_BLOCK_HEADER "%zu", bufUsed(buffer))); protocolClientDataPut(this->client, pckWriteBinP(protocolPackNew(), buffer));
ioWrite(protocolClientIoWrite(this->client), buffer);
ioWriteFlush(protocolClientIoWrite(this->client));
#ifdef DEBUG #ifdef DEBUG
this->protocolWriteBytes += bufUsed(buffer); this->protocolWriteBytes += bufUsed(buffer);
@ -156,11 +158,12 @@ storageWriteRemoteClose(THIS_VOID)
// Close if the file has not already been closed // Close if the file has not already been closed
if (this->client != NULL) if (this->client != NULL)
{ {
ioWriteLine(protocolClientIoWrite(this->client), BUFSTRDEF(PROTOCOL_BLOCK_HEADER "0")); protocolClientDataPut(this->client, NULL);
ioWriteFlush(protocolClientIoWrite(this->client)); ioFilterGroupResultAllSet(
ioFilterGroupResultAllSet(ioWriteFilterGroup(storageWriteIo(this->write)), protocolClientReadOutput(this->client, true)); ioWriteFilterGroup(storageWriteIo(this->write)), jsonToVar(pckReadStrP(protocolClientDataGet(this->client))));
this->client = NULL; protocolClientDataEndGet(this->client);
this->client = NULL;
memContextCallbackClear(this->memContext); memContextCallbackClear(this->memContext);
} }

View File

@ -454,7 +454,7 @@ unit:
test: test:
# ---------------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------------
- name: protocol - name: protocol
total: 9 total: 7
harness: harness:
name: protocol name: protocol
shim: shim:

View File

@ -1322,14 +1322,15 @@ testRun(void)
// Create job that skips file // Create job that skips file
job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ(stringIdBit5, "x"))); job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ(stringIdBit5, "x")));
VariantList *result = varLstNew(); PackWrite *const resultPack = protocolPackNew();
varLstAdd(result, varNewUInt64(backupCopyResultNoOp)); pckWriteU32P(resultPack, backupCopyResultNoOp);
varLstAdd(result, varNewUInt64(0)); pckWriteU64P(resultPack, 0);
varLstAdd(result, varNewUInt64(0)); pckWriteU64P(resultPack, 0);
varLstAdd(result, NULL); pckWriteStrP(resultPack, NULL);
varLstAdd(result, NULL); pckWriteStrP(resultPack, NULL);
pckWriteEndP(resultPack);
protocolParallelJobResultSet(job, varNewVarLst(result)); protocolParallelJobResultSet(job, pckReadNewBuf(pckWriteBuf(resultPack)));
// Create manifest with file // Create manifest with file
Manifest *manifest = manifestNewInternal(); Manifest *manifest = manifestNewInternal();

View File

@ -11,102 +11,143 @@ Test Protocol
#include "version.h" #include "version.h"
#include "common/harnessConfig.h" #include "common/harnessConfig.h"
#include "common/harnessError.h"
#include "common/harnessFork.h" #include "common/harnessFork.h"
#include "common/harnessPack.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Test protocol request handler Test protocol server command handlers
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
static unsigned int testServerProtocolErrorTotal = 0; #define TEST_PROTOCOL_COMMAND_ASSERT STRID5("assert", 0x2922ce610)
static void __attribute__((__noreturn__)) static void
testServerAssertProtocol(const VariantList *paramList, ProtocolServer *server) testCommandAssertProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_HARNESS_BEGIN(); FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList); FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server); FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END(); FUNCTION_HARNESS_END();
ASSERT(paramList == NULL); ASSERT(param == NULL);
ASSERT(server != NULL);
hrnErrorThrowP();
// No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
}
#define TEST_PROTOCOL_COMMAND_ERROR STRID5("error", 0x127ca450)
__attribute__((__noreturn__)) static void
testCommandErrorProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(server != NULL);
hrnErrorThrowP(.errorType = &FormatError);
// No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
}
#define TEST_PROTOCOL_COMMAND_SIMPLE STRID5("c-simple", 0x2b20d4cf630)
static void
testCommandRequestSimpleProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
THROW(AssertError, "test assert"); protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), STRDEF("output")));
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID(); FUNCTION_HARNESS_RETURN_VOID();
} }
#define TEST_PROTOCOL_COMMAND_COMPLEX STRID5("c-complex", 0x182b20d78f630)
static void static void
testServerRequestSimpleProtocol(const VariantList *paramList, ProtocolServer *server) testCommandRequestComplexProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_HARNESS_BEGIN(); FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList); FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server); FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END(); FUNCTION_HARNESS_END();
ASSERT(paramList == NULL); ASSERT(param != NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
protocolServerResponse(server, varNewBool(true)); TEST_RESULT_UINT(pckReadU32P(param), 87, "param check");
TEST_RESULT_STR_Z(pckReadStrP(param), "data", "param check");
TEST_RESULT_VOID(protocolServerDataPut(server, NULL), "sync");
TEST_RESULT_BOOL(pckReadBoolP(protocolServerDataGet(server)), true, "data get");
TEST_RESULT_UINT(pckReadModeP(protocolServerDataGet(server)), 0644, "data get");
TEST_RESULT_PTR(protocolServerDataGet(server), NULL, "data end get");
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true)), "data put");
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteI32P(protocolPackNew(), -1)), "data put");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID(); FUNCTION_HARNESS_RETURN_VOID();
} }
#define TEST_PROTOCOL_COMMAND_RETRY STRID5("retry", 0x19950b20)
static unsigned int testCommandRetryTotal = 1;
static void static void
testServerRequestComplexProtocol(const VariantList *paramList, ProtocolServer *server) testCommandRetryProtocol(PackRead *const param, ProtocolServer *const server)
{ {
FUNCTION_HARNESS_BEGIN(); FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList); FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server); FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END(); FUNCTION_HARNESS_END();
ASSERT(paramList == NULL); ASSERT(param == NULL);
ASSERT(server != NULL); ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
protocolServerResponse(server, varNewBool(false)); if (testCommandRetryTotal > 0)
protocolServerWriteLine(server, STRDEF("LINEOFTEXT"));
protocolServerWriteLine(server, NULL);
ioWriteFlush(protocolServerIoWrite(server));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID();
}
static void
testServerErrorUntil0Protocol(const VariantList *paramList, ProtocolServer *server)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(paramList == NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
if (testServerProtocolErrorTotal > 0)
{ {
testServerProtocolErrorTotal--; testCommandRetryTotal--;
THROW(FormatError, "error-until-0"); THROW(FormatError, "error-until-0");
} }
protocolServerResponse(server, varNewBool(true)); protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true));
protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID(); FUNCTION_HARNESS_RETURN_VOID();
} }
#define TEST_PROTOCOL_SERVER_HANDLER_LIST \
{.command = TEST_PROTOCOL_COMMAND_ASSERT, .handler = testCommandAssertProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_ERROR, .handler = testCommandErrorProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_SIMPLE, .handler = testCommandRequestSimpleProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_COMPLEX, .handler = testCommandRequestComplexProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_RETRY, .handler = testCommandRetryProtocol},
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Test ParallelJobCallback Test ParallelJobCallback
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
@ -390,35 +431,7 @@ testRun(void)
} }
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("ProtocolCommand")) if (testBegin("ProtocolClient, ProtocolCommand, and ProtocolServer"))
{
ProtocolCommand *command = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(command, protocolCommandNew(strIdFromZ(stringIdBit5, "cmd-one")), "create command");
TEST_RESULT_PTR(protocolCommandParamAdd(command, VARSTRDEF("param1")), command, "add param");
TEST_RESULT_PTR(protocolCommandParamAdd(command, VARSTRDEF("param2")), command, "add param");
TEST_RESULT_PTR(protocolCommandMove(command, memContextPrior()), command, "move protocol command");
TEST_RESULT_PTR(protocolCommandMove(NULL, memContextPrior()), NULL, "move null protocol command");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_STR_Z(protocolCommandToLog(command), "{command: cmd-one}", "check log");
TEST_RESULT_STR_Z(protocolCommandJson(command), "{\"cmd\":\"cmd-one\",\"param\":[\"param1\",\"param2\"]}", "check json");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(command, protocolCommandNew(strIdFromZ(stringIdBit5, "cmd2")), "create command");
TEST_RESULT_STR_Z(protocolCommandToLog(command), "{command: cmd2}", "check log");
TEST_RESULT_STR_Z(protocolCommandJson(command), "{\"cmd\":\"cmd2\"}", "check json");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(protocolCommandFree(command), "free command");
}
// *****************************************************************************************************************************
if (testBegin("ProtocolClient"))
{ {
HARNESS_FORK_BEGIN() HARNESS_FORK_BEGIN()
{ {
@ -430,6 +443,7 @@ testRun(void)
ioWriteOpen(write); ioWriteOpen(write);
// Various bogus greetings // Various bogus greetings
// -----------------------------------------------------------------------------------------------------------------
ioWriteStrLine(write, STRDEF("bogus greeting")); ioWriteStrLine(write, STRDEF("bogus greeting"));
ioWriteFlush(write); ioWriteFlush(write);
ioWriteStrLine(write, STRDEF("{\"name\":999}")); ioWriteStrLine(write, STRDEF("{\"name\":999}"));
@ -443,56 +457,38 @@ testRun(void)
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"bogus\"}")); ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"bogus\"}"));
ioWriteFlush(write); ioWriteFlush(write);
// Correct greeting with noop // -----------------------------------------------------------------------------------------------------------------
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}")); TEST_TITLE("server");
ioWriteFlush(write);
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop"); ProtocolServer *server = NULL;
ioWriteStrLine(write, STRDEF("{}"));
ioWriteFlush(write);
// Throw errors MEM_CONTEXT_TEMP_BEGIN()
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop with error text"); {
ioWriteStrLine(write, STRDEF("{\"err\":25,\"out\":\"sample error message\",\"errStack\":\"stack data\"}")); TEST_ASSIGN(
ioWriteFlush(write); server,
protocolServerMove(
protocolServerNew(STRDEF("test server"), STRDEF("test"), read, write), memContextPrior()),
"new server");
TEST_RESULT_VOID(protocolServerMove(NULL, memContextPrior()), "move null server");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop with no error text"); const ProtocolServerHandler commandHandler[] = {TEST_PROTOCOL_SERVER_HANDLER_LIST};
ioWriteStrLine(write, STRDEF("{\"err\":255}"));
ioWriteFlush(write);
// No output expected // This cannot run in a TEST* macro because tests are run by the command handlers
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop with parameters returned"); protocolServerProcess(server, NULL, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler));
ioWriteStrLine(write, STRDEF("{\"out\":[\"bogus\"]}"));
ioWriteFlush(write);
// Send output // -----------------------------------------------------------------------------------------------------------------
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"test\"}", "test command"); TEST_TITLE("server with retries");
ioWriteStrLine(write, STRDEF(".OUTPUT"));
ioWriteStrLine(write, STRDEF("{\"out\":[\"value1\",\"value2\"]}"));
ioWriteFlush(write);
// invalid line VariantList *retryList = varLstNew();
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"invalid-line\"}", "invalid line command"); varLstAdd(retryList, varNewUInt64(0));
ioWrite(write, LF_BUF);
ioWriteFlush(write);
// error instead of output TEST_ASSIGN(
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"err-i-o\"}", "error instead of output command"); server, protocolServerNew(STRDEF("test server"), STRDEF("test"), read, write), "new server");
ioWriteStrLine(write, STRDEF("{\"err\":255}"));
ioWriteFlush(write);
// unexpected output // This cannot run in a TEST* macro because tests are run by the command handlers
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"unexp-output\"}", "unexpected output"); protocolServerProcess(server, retryList, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler));
ioWriteStrLine(write, STRDEF("{}"));
ioWriteFlush(write);
// invalid prefix
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"i-pr\"}", "invalid prefix");
ioWriteStrLine(write, STRDEF("~line"));
ioWriteFlush(write);
// Wait for exit
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"exit\"}", "exit command");
} }
HARNESS_FORK_CHILD_END(); HARNESS_FORK_CHILD_END();
@ -503,7 +499,9 @@ testRun(void)
IoWrite *write = ioFdWriteNew(STRDEF("client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000); IoWrite *write = ioFdWriteNew(STRDEF("client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000);
ioWriteOpen(write); ioWriteOpen(write);
// Various bogus greetings // -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("bogus greetings");
TEST_ERROR( TEST_ERROR(
protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), JsonFormatError, protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), JsonFormatError,
"expected '{' at 'bogus greeting'"); "expected '{' at 'bogus greeting'");
@ -526,7 +524,9 @@ testRun(void)
"expected value '" PROJECT_VERSION "' for greeting key 'version' but got 'bogus'\n" "expected value '" PROJECT_VERSION "' for greeting key 'version' but got 'bogus'\n"
"HINT: is the same version of " PROJECT_NAME " installed on the local and remote host?"); "HINT: is the same version of " PROJECT_NAME " installed on the local and remote host?");
// Correct greeting // -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("new client with successful handshake");
ProtocolClient *client = NULL; ProtocolClient *client = NULL;
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
@ -535,193 +535,96 @@ testRun(void)
client, client,
protocolClientMove( protocolClientMove(
protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), memContextPrior()), protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), memContextPrior()),
"create client"); "new client");
TEST_RESULT_VOID(protocolClientMove(NULL, memContextPrior()), "move null client"); TEST_RESULT_VOID(protocolClientMove(NULL, memContextPrior()), "move null client");
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
TEST_RESULT_PTR(protocolClientIoRead(client), client->pub.read, "get read io"); TEST_RESULT_INT(protocolClientIoReadFd(client), ioReadFd(client->pub.read), "get read fd");
TEST_RESULT_PTR(protocolClientIoWrite(client), client->pub.write, "get write io");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("invalid command");
// Throw errors
TEST_ERROR( TEST_ERROR(
protocolClientNoOp(client), AssertError, protocolClientExecute(client, protocolCommandNew(strIdFromZ(stringIdBit6, "BOGUS")), false), ProtocolError,
"raised from test client: sample error message\nstack data"); "raised from test client: invalid command 'BOGUS' (0x38eacd271)");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command throws assert");
TEST_ERROR(
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ASSERT), false), AssertError,
"raised from test client: ERR_MESSAGE\nERR_STACK_TRACE");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command throws error");
TEST_ERROR(
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ERROR), false), FormatError,
"raised from test client: ERR_MESSAGE");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command throws error in debug log level");
harnessLogLevelSet(logLevelDebug); harnessLogLevelSet(logLevelDebug);
TEST_ERROR( TEST_ERROR(
protocolClientNoOp(client), UnknownError, protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ERROR), false), FormatError,
"raised from test client: no details available\nno stack trace available"); "raised from test client: ERR_MESSAGE\nERR_STACK_TRACE");
harnessLogLevelReset(); harnessLogLevelReset();
// No output expected // -----------------------------------------------------------------------------------------------------------------
TEST_ERROR(protocolClientNoOp(client), AssertError, "no output required by command"); TEST_TITLE("simple command");
// Get command output
const VariantList *output = NULL;
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "test"))),
"execute command with output");
TEST_RESULT_STR_Z(protocolClientReadLine(client), "OUTPUT", "check output");
TEST_ASSIGN(output, varVarLst(protocolClientReadOutput(client, true)), "execute command with output");
TEST_RESULT_UINT(varLstSize(output), 2, "check output size");
TEST_RESULT_STR_Z(varStr(varLstGet(output, 0)), "value1", "check value1");
TEST_RESULT_STR_Z(varStr(varLstGet(output, 1)), "value2", "check value2");
// Invalid line
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "invalid-line"))),
"execute command that returns invalid line");
TEST_ERROR(protocolClientReadLine(client), FormatError, "unexpected empty line");
// Error instead of output
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "err-i-o"))),
"execute command that returns error instead of output");
TEST_ERROR(protocolClientReadLine(client), UnknownError, "raised from test client: no details available");
// Unexpected output
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "unexp-output"))),
"execute command that returns unexpected output");
TEST_ERROR(protocolClientReadLine(client), FormatError, "expected error but got output");
// Invalid prefix
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "i-pr"))),
"execute command that returns an invalid prefix");
TEST_ERROR(protocolClientReadLine(client), FormatError, "invalid prefix in '~line'");
// Free client
TEST_RESULT_VOID(protocolClientFree(client), "free client");
}
HARNESS_FORK_PARENT_END();
}
HARNESS_FORK_END();
}
// *****************************************************************************************************************************
if (testBegin("ProtocolServer"))
{
HARNESS_FORK_BEGIN()
{
HARNESS_FORK_CHILD_BEGIN(0, true)
{
IoRead *read = ioFdReadNew(STRDEF("client read"), HARNESS_FORK_CHILD_READ(), 2000);
ioReadOpen(read);
IoWrite *write = ioFdWriteNew(STRDEF("client write"), HARNESS_FORK_CHILD_WRITE(), 2000);
ioWriteOpen(write);
// Check greeting
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
ioReadLine(read), "{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}", pckReadStrP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_SIMPLE), true)), "output",
"check greeting"); "execute");
// Noop
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"noop\"}")), "write noop");
TEST_RESULT_VOID(ioWriteFlush(write), "flush noop");
TEST_RESULT_STR_Z(ioReadLine(read), "{}", "noop result");
// Invalid command
KeyValue *result = NULL;
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"bogus\"}")), "write bogus");
TEST_RESULT_VOID(ioWriteFlush(write), "flush bogus");
TEST_ASSIGN(result, varKv(jsonToVar(ioReadLine(read))), "parse error result");
TEST_RESULT_INT(varIntForce(kvGet(result, VARSTRDEF("err"))), 39, " check code");
TEST_RESULT_STR_Z(
varStr(kvGet(result, VARSTRDEF("out"))), "invalid command 'bogus' (0x13a9de20)", " check message");
TEST_RESULT_BOOL(kvGet(result, VARSTRDEF("errStack")) != NULL, true, " check stack exists");
// Simple request
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"r-s\"}")), "write simple request");
TEST_RESULT_VOID(ioWriteFlush(write), "flush simple request");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"out\":true}", "simple request result");
// Throw an assert error which will include a stack trace
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"assert\"}")), "write assert");
TEST_RESULT_VOID(ioWriteFlush(write), "flush assert error");
TEST_ASSIGN(result, varKv(jsonToVar(ioReadLine(read))), "parse error result");
TEST_RESULT_INT(varIntForce(kvGet(result, VARSTRDEF("err"))), 25, " check code");
TEST_RESULT_STR_Z(varStr(kvGet(result, VARSTRDEF("out"))), "test assert", " check message");
TEST_RESULT_BOOL(kvGet(result, VARSTRDEF("errStack")) != NULL, true, " check stack exists");
// Complex request -- after process loop has been restarted
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"r-c\"}")), "write complex request");
TEST_RESULT_VOID(ioWriteFlush(write), "flush complex request");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"out\":false}", "complex request result");
TEST_RESULT_STR_Z(ioReadLine(read), ".LINEOFTEXT", "complex request result");
TEST_RESULT_STR_Z(ioReadLine(read), ".", "complex request result");
// Exit
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"exit\"}")), "write exit");
TEST_RESULT_VOID(ioWriteFlush(write), "flush exit");
// Retry errors until success
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"ezero\"}")), "write error-until-0");
TEST_RESULT_VOID(ioWriteFlush(write), "flush error-until-0");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"out\":true}", "error-until-0 result");
// Exit
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"exit\"}")), "write exit");
TEST_RESULT_VOID(ioWriteFlush(write), "flush exit");
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN()
{
IoRead *read = ioFdReadNew(STRDEF("server read"), HARNESS_FORK_PARENT_READ_PROCESS(0), 2000);
ioReadOpen(read);
IoWrite *write = ioFdWriteNew(STRDEF("server write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000);
ioWriteOpen(write);
// Send greeting
ProtocolServer *server = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(
server,
protocolServerMove(
protocolServerNew(STRDEF("test server"), STRDEF("test"), read, write), memContextPrior()),
"create server");
TEST_RESULT_VOID(protocolServerMove(NULL, memContextPrior()), "move null server");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_PTR(protocolServerIoRead(server), server->pub.read, "get read io");
TEST_RESULT_PTR(protocolServerIoWrite(server), server->pub.write, "get write io");
ProtocolServerHandler commandHandler[] =
{
{.command = strIdFromZ(stringIdBit5, "assert"), .handler = testServerAssertProtocol},
{.command = strIdFromZ(stringIdBit5, "r-s"), .handler = testServerRequestSimpleProtocol},
{.command = strIdFromZ(stringIdBit5, "r-c"), .handler = testServerRequestComplexProtocol},
{.command = strIdFromZ(stringIdBit5, "ezero"), .handler = testServerErrorUntil0Protocol},
};
TEST_RESULT_VOID(
protocolServerProcess(server, NULL, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler)),
"run process loop");
// ----------------------------------------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("run process loop with retries"); TEST_TITLE("complex command");
VariantList *retryInterval = varLstNew(); // Put the command to the server
varLstAdd(retryInterval, varNewUInt64(0)); ProtocolCommand *command = NULL;
varLstAdd(retryInterval, varNewUInt64(50)); TEST_ASSIGN(command, protocolCommandNew(TEST_PROTOCOL_COMMAND_COMPLEX), "command");
TEST_RESULT_VOID(pckWriteU32P(protocolCommandParam(command), 87), "param");
TEST_RESULT_VOID(pckWriteStrP(protocolCommandParam(command), STRDEF("data")), "param");
TEST_RESULT_VOID(protocolClientCommandPut(client, command), "command put");
testServerProtocolErrorTotal = 2; // Read null data to indicate that the server has started the command and is read to receive data
TEST_RESULT_PTR(protocolClientDataGet(client), NULL, "command started and ready for data");
TEST_RESULT_VOID( // Write data to the server
protocolServerProcess(server, retryInterval, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler)), TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteBoolP(protocolPackNew(), true)), "data put");
"run process loop"); TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteModeP(protocolPackNew(), 0644)), "data put");
TEST_RESULT_VOID(protocolClientDataPut(client, NULL), "data end put");
// Get data from the server
TEST_RESULT_BOOL(pckReadBoolP(protocolClientDataGet(client)), true, "data get");
TEST_RESULT_INT(pckReadI32P(protocolClientDataGet(client)), -1, "data get");
TEST_RESULT_VOID(protocolClientDataEndGet(client), "data end get");
// ----------------------------------------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("free server"); TEST_TITLE("free client");
TEST_RESULT_VOID(protocolServerFree(server), "free"); TEST_RESULT_VOID(protocolClientFree(client), "free");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("new client with server retries");
TEST_ASSIGN(client, protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), "new client");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command with retry");
TEST_RESULT_BOOL(
pckReadBoolP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_RETRY), true)), true,
"execute");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("free client");
TEST_RESULT_VOID(protocolClientFree(client), "free");
} }
HARNESS_FORK_PARENT_END(); HARNESS_FORK_PARENT_END();
} }
@ -731,6 +634,8 @@ testRun(void)
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("ProtocolParallel and ProtocolParallelJob")) if (testBegin("ProtocolParallel and ProtocolParallelJob"))
{ {
TEST_TITLE("job state transitions");
ProtocolParallelJob *job = NULL; ProtocolParallelJob *job = NULL;
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
@ -760,63 +665,63 @@ testRun(void)
// Local 1 // Local 1
HARNESS_FORK_CHILD_BEGIN(0, true) HARNESS_FORK_CHILD_BEGIN(0, true)
{ {
IoRead *read = ioFdReadNew(STRDEF("server read"), HARNESS_FORK_CHILD_READ(), 10000); IoRead *read = ioFdReadNew(STRDEF("local server 1 read"), HARNESS_FORK_CHILD_READ(), 10000);
ioReadOpen(read); ioReadOpen(read);
IoWrite *write = ioFdWriteNew(STRDEF("server write"), HARNESS_FORK_CHILD_WRITE(), 2000); IoWrite *write = ioFdWriteNew(STRDEF("local server 1 write"), HARNESS_FORK_CHILD_WRITE(), 2000);
ioWriteOpen(write); ioWriteOpen(write);
// Greeting with noop ProtocolServer *server = NULL;
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}")); TEST_ASSIGN(server, protocolServerNew(STRDEF("local server 1"), STRDEF("test"), read, write), "local server 1");
ioWriteFlush(write);
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop"); TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_NOOP, "noop command get");
ioWriteStrLine(write, STRDEF("{}")); TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
ioWriteFlush(write);
// Command with output
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c-one"), "c-one command get");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"c-one\",\"param\":[\"param1\",\"param2\"]}", "command1");
sleepMSec(4000); sleepMSec(4000);
ioWriteStrLine(write, STRDEF("{\"out\":1}"));
ioWriteFlush(write); TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 1)), "data end put");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
// Wait for exit // Wait for exit
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"exit\"}", "exit command"); TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "noop command get");
} }
HARNESS_FORK_CHILD_END(); HARNESS_FORK_CHILD_END();
// Local 2 // Local 2
HARNESS_FORK_CHILD_BEGIN(0, true) HARNESS_FORK_CHILD_BEGIN(0, true)
{ {
IoRead *read = ioFdReadNew(STRDEF("server read"), HARNESS_FORK_CHILD_READ(), 10000); IoRead *read = ioFdReadNew(STRDEF("local server 2 read"), HARNESS_FORK_CHILD_READ(), 10000);
ioReadOpen(read); ioReadOpen(read);
IoWrite *write = ioFdWriteNew(STRDEF("server write"), HARNESS_FORK_CHILD_WRITE(), 2000); IoWrite *write = ioFdWriteNew(STRDEF("local server 2 write"), HARNESS_FORK_CHILD_WRITE(), 2000);
ioWriteOpen(write); ioWriteOpen(write);
// Greeting with noop ProtocolServer *server = NULL;
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}")); TEST_ASSIGN(server, protocolServerNew(STRDEF("local server 2"), STRDEF("test"), read, write), "local server 2");
ioWriteFlush(write);
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop"); TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_NOOP, "noop command get");
ioWriteStrLine(write, STRDEF("{}")); TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
ioWriteFlush(write);
// Command with output
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c2"), "c2 command get");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"c2\",\"param\":[\"param1\"]}", "command2");
sleepMSec(1000); sleepMSec(1000);
ioWriteStrLine(write, STRDEF("{\"out\":2}"));
ioWriteFlush(write);
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"c-three\",\"param\":[\"param1\"]}", "command3"); TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 2)), "data end put");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
ioWriteStrLine(write, STRDEF("{\"err\":39,\"out\":\"very serious error\"}")); // Command with error
ioWriteFlush(write); TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c-three"), "c-three command get");
TEST_RESULT_VOID(protocolServerError(server, 39, STRDEF("very serious error"), STRDEF("stack")), "error put");
// Wait for exit // Wait for exit
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"exit\"}", "exit command"); CHECK(protocolServerCommandGet(server).id == PROTOCOL_COMMAND_EXIT);
} }
HARNESS_FORK_CHILD_END(); HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN() HARNESS_FORK_PARENT_BEGIN()
{ {
// -----------------------------------------------------------------------------------------------------------------
TestParallelJobCallback data = {.jobList = lstNewP(sizeof(ProtocolParallelJob *))}; TestParallelJobCallback data = {.jobList = lstNewP(sizeof(ProtocolParallelJob *))};
ProtocolParallel *parallel = NULL; ProtocolParallel *parallel = NULL;
TEST_ASSIGN(parallel, protocolParallelNew(2000, testParallelJobCallback, &data), "create parallel"); TEST_ASSIGN(parallel, protocolParallelNew(2000, testParallelJobCallback, &data), "create parallel");
@ -829,57 +734,59 @@ testRun(void)
for (unsigned int clientIdx = 0; clientIdx < clientTotal; clientIdx++) for (unsigned int clientIdx = 0; clientIdx < clientTotal; clientIdx++)
{ {
IoRead *read = ioFdReadNew( IoRead *read = ioFdReadNew(
strNewFmt("client %u read", clientIdx), HARNESS_FORK_PARENT_READ_PROCESS(clientIdx), 2000); strNewFmt("local client %u read", clientIdx), HARNESS_FORK_PARENT_READ_PROCESS(clientIdx), 2000);
ioReadOpen(read); ioReadOpen(read);
IoWrite *write = ioFdWriteNew( IoWrite *write = ioFdWriteNew(
strNewFmt("client %u write", clientIdx), HARNESS_FORK_PARENT_WRITE_PROCESS(clientIdx), 2000); strNewFmt("local client %u write", clientIdx), HARNESS_FORK_PARENT_WRITE_PROCESS(clientIdx), 2000);
ioWriteOpen(write); ioWriteOpen(write);
TEST_ASSIGN( TEST_ASSIGN(
client[clientIdx], client[clientIdx],
protocolClientNew(strNewFmt("test client %u", clientIdx), STRDEF("test"), read, write), protocolClientNew(strNewFmt("local client %u", clientIdx), STRDEF("test"), read, write),
strZ(strNewFmt("create client %u", clientIdx))); strZ(strNewFmt("local client %u new", clientIdx)));
TEST_RESULT_VOID( TEST_RESULT_VOID(
protocolParallelClientAdd(parallel, client[clientIdx]), strZ(strNewFmt("add client %u", clientIdx))); protocolParallelClientAdd(parallel, client[clientIdx]), strZ(strNewFmt("local client %u add", clientIdx)));
} }
// Attempt to add client without an fd // -----------------------------------------------------------------------------------------------------------------
const String *protocolString = STRDEF( TEST_TITLE("error on add without an fd");
"{\"name\":\"pgBackRest\",\"service\":\"error\",\"version\":\"" PROJECT_VERSION "\"}\n"
"{}\n");
IoRead *read = ioBufferReadNew(BUFSTR(protocolString)); // Fake a client without a read fd
ioReadOpen(read); ProtocolClient clientError = {.pub = {.read = ioBufferReadNew(bufNew(0))}, .name = STRDEF("test")};
IoWrite *write = ioBufferWriteNew(bufNew(1024));
ioWriteOpen(write);
ProtocolClient *clientError = protocolClientNew(STRDEF("error"), STRDEF("error"), read, write); TEST_ERROR(protocolParallelClientAdd(parallel, &clientError), AssertError, "client with read fd is required");
TEST_ERROR(protocolParallelClientAdd(parallel, clientError), AssertError, "client with read fd is required");
protocolClientFree(clientError); // -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("add jobs");
// Add jobs
ProtocolCommand *command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-one")); ProtocolCommand *command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-one"));
protocolCommandParamAdd(command, VARSTRDEF("param1")); pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
protocolCommandParamAdd(command, VARSTRDEF("param2")); pckWriteStrP(protocolCommandParam(command), STRDEF("param2"));
ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("job1"), command);
ProtocolParallelJob *job = protocolParallelJobNew(varNewStr(STRDEF("job1")), command);
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job"); TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
command = protocolCommandNew(strIdFromZ(stringIdBit5, "c2")); command = protocolCommandNew(strIdFromZ(stringIdBit5, "c2"));
protocolCommandParamAdd(command, VARSTRDEF("param1")); pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
job = protocolParallelJobNew(VARSTRDEF("job2"), command);
job = protocolParallelJobNew(varNewStr(STRDEF("job2")), command);
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job"); TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-three")); command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-three"));
protocolCommandParamAdd(command, VARSTRDEF("param1")); pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
job = protocolParallelJobNew(VARSTRDEF("job3"), command);
job = protocolParallelJobNew(varNewStr(STRDEF("job3")), command);
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job"); TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
// Process jobs // -----------------------------------------------------------------------------------------------------------------
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs"); TEST_TITLE("process jobs with no result");
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result"); TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
// Process jobs // -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("result for job 2");
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs"); TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result"); TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
@ -887,35 +794,40 @@ testRun(void)
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
protocolParallelJobProcessId(job) >= 1 && protocolParallelJobProcessId(job) <= 2, true, protocolParallelJobProcessId(job) >= 1 && protocolParallelJobProcessId(job) <= 2, true,
"check process id is valid"); "check process id is valid");
TEST_RESULT_INT(varIntForce(protocolParallelJobResult(job)), 2, "check result is 2"); TEST_RESULT_UINT(pckReadU32P(protocolParallelJobResult(job)), 2, "check result is 2");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results"); TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
// Process jobs // -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error for job 3");
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs"); TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result"); TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job3", "check key is job3"); TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job3", "check key is job3");
TEST_RESULT_INT(protocolParallelJobErrorCode(job), 39, "check error code"); TEST_RESULT_INT(protocolParallelJobErrorCode(job), 39, "check error code");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
protocolParallelJobErrorMessage(job), "raised from test client 1: very serious error", protocolParallelJobErrorMessage(job), "raised from local client 1: very serious error",
"check error message"); "check error message");
TEST_RESULT_PTR(protocolParallelJobResult(job), NULL, "check result is null"); TEST_RESULT_PTR(protocolParallelJobResult(job), NULL, "check result is null");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results"); TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
// Process jobs // -----------------------------------------------------------------------------------------------------------------
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs"); TEST_TITLE("process jobs with no result");
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result"); TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
TEST_RESULT_BOOL(protocolParallelDone(parallel), false, "check not done"); TEST_RESULT_BOOL(protocolParallelDone(parallel), false, "check not done");
// Process jobs // -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("result for job 1");
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs"); TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result"); TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job1", "check key is job1"); TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job1", "check key is job1");
TEST_RESULT_INT(varIntForce(protocolParallelJobResult(job)), 1, "check result is 1"); TEST_RESULT_UINT(pckReadU32P(protocolParallelJobResult(job)), 1, "check result is 1");
TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check done"); TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check done");
TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check still done"); TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check still done");

View File

@ -279,9 +279,6 @@ testRun(void)
((StorageReadRemote *)fileRead->driver)->protocolReadBytes < bufSize(contentBuf), true, ((StorageReadRemote *)fileRead->driver)->protocolReadBytes < bufSize(contentBuf), true,
" check compressed read size"); " check compressed read size");
TEST_ERROR(
storageRemoteProtocolBlockSize(STRDEF("bogus")), ProtocolError, "'bogus' is not a valid block size message");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("file missing"); TEST_TITLE("file missing");