db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-15 01:04:37 +02:00
Code Issues Releases Activity

Binary protocol.

Browse Source 
Switch from JSON-based to binary protocol for communicating with local and remote process. The pack type is used to implement the binary protocol.

There are a number advantages:

* The pack type is more compact than JSON and are more efficient to render/parse.
* Packs are more strictly typed than JSON.
* Each protocol message is written entirely within ProtocolServer/ProtocolClient so is less likely to get interrupted by an error and leave the protocol in a bad state.
* There is no limit on message size. Previously this was limited by buffer size without a custom implementation, as was done for read/writing files.

Some cruft from the Perl days was removed, specifically allowing NULL messages and stack traces. This is no longer possible in C.

There is room for improvement here, in particular locking down the allowed sequence of protocol messages and building a state machine to enforce it. This will be useful for resetting the protocol when it gets in a bad state.
This commit is contained in:
David Steele
2021-06-24 13:31:16 -04:00
committed by GitHub
parent bffb43ea3f
commit 6a1c0337dd
41 changed files with 1397 additions and 1227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
doc/xml/release.xml
View File

@ -15,6 +15,17 @@
<release date="XXXX-XX-XX" version="2.35dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-development-list>
<release-item>
<github-pull-request id="1424"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>Binary protocol.</p>
</release-item>
<release-item>
<github-pull-request id="1421"/>

24
src/command/archive/get/get.c
View File

@ -851,19 +851,21 @@ static ProtocolParallelJob *archiveGetAsyncCallback(void *data, unsigned int cli
const ArchiveFileMap *archiveFileMap = lstGet(jobData->archiveFileMapList, jobData->archiveFileIdx);
jobData->archiveFileIdx++;
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_GET_FILE);
protocolCommandParamAdd(command, VARSTR(archiveFileMap->request));
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_GET_FILE);
PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, archiveFileMap->request);
// Add actual files to get
for (unsigned int actualIdx = 0; actualIdx < lstSize(archiveFileMap->actualList); actualIdx++)
{
const ArchiveGetFile *actual = lstGet(archiveFileMap->actualList, actualIdx);
const ArchiveGetFile *const actual = lstGet(archiveFileMap->actualList, actualIdx);
protocolCommandParamAdd(command, VARSTR(actual->file));
protocolCommandParamAdd(command, VARUINT(actual->repoIdx));
protocolCommandParamAdd(command, VARSTR(actual->archiveId));
protocolCommandParamAdd(command, VARUINT64(actual->cipherType));
protocolCommandParamAdd(command, VARSTR(actual->cipherPassArchive));
pckWriteStrP(param, actual->file);
pckWriteU32P(param, actual->repoIdx);
pckWriteStrP(param, actual->archiveId);
pckWriteU64P(param, actual->cipherType);
pckWriteStrP(param, actual->cipherPassArchive);
}
FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(archiveFileMap->request), command));
@ -941,12 +943,12 @@ cmdArchiveGetAsync(void)
if (protocolParallelJobErrorCode(job) == 0)
{
// Get the actual file retrieved
const VariantList *fileResult = varVarLst(protocolParallelJobResult(job));
ArchiveGetFile *file = lstGet(fileMap->actualList, varUIntForce(varLstGet(fileResult, 0)));
PackRead *const fileResult = protocolParallelJobResult(job);
ArchiveGetFile *file = lstGet(fileMap->actualList, pckReadU32P(fileResult));
ASSERT(file != NULL);
// Output file warnings
StringList *fileWarnList = strLstNewVarLst(varVarLst(varLstGet(fileResult, 1)));
StringList *fileWarnList = pckReadStrLstP(fileResult);
for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++)
LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx)));

47
src/command/archive/get/protocol.c
View File

@ -15,54 +15,47 @@ Archive Get Protocol Handler
/**********************************************************************************************************************************/
void
archiveGetFileProtocol(const VariantList *paramList, ProtocolServer *server)
archiveGetFileProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
const String *request = varStr(varLstGet(paramList, 0));
const unsigned int paramFixed = 1; // Fixed params before the actual list
const unsigned int paramActual = 5; // Parameters in each index of the actual list
// Check that the correct number of list parameters were passed
CHECK((varLstSize(paramList) - paramFixed) % paramActual == 0);
// Get request
const String *const request = pckReadStrP(param);
// Build the actual list
List *actualList = lstNewP(sizeof(ArchiveGetFile));
unsigned int actualListSize = (varLstSize(paramList) - paramFixed) / paramActual;
for (unsigned int actualIdx = 0; actualIdx < actualListSize; actualIdx++)
while (!pckReadNullP(param))
{
lstAdd(
actualList,
&(ArchiveGetFile)
{
.file = varStr(varLstGet(paramList, paramFixed + (actualIdx * paramActual))),
.repoIdx = varUIntForce(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 1)),
.archiveId = varStr(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 2)),
.cipherType = varUInt64(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 3)),
.cipherPassArchive = varStr(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 4)),
});
ArchiveGetFile actual = {.file = pckReadStrP(param)};
actual.repoIdx = pckReadU32P(param);
actual.archiveId = pckReadStrP(param);
actual.cipherType = pckReadU64P(param);
actual.cipherPassArchive = pckReadStrP(param);
lstAdd(actualList, &actual);
}
// Return result
// Get file
ArchiveGetFileResult fileResult = archiveGetFile(
storageSpoolWrite(), request, actualList,
strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s." STORAGE_FILE_TEMP_EXT, strZ(request)));
VariantList *result = varLstNew();
varLstAdd(result, varNewUInt(fileResult.actualIdx));
varLstAdd(result, varNewVarLst(varLstNewStrLst(fileResult.warnList)));
// Return result
PackWrite *const resultPack = protocolPackNew();
pckWriteU32P(resultPack, fileResult.actualIdx);
pckWriteStrLstP(resultPack, fileResult.warnList);
protocolServerResponse(server, varNewVarLst(result));
protocolServerDataPut(server, resultPack);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();

6
src/command/archive/get/protocol.h
View File

@ -4,16 +4,14 @@ Archive Get Protocol Handler
#ifndef COMMAND_ARCHIVE_GET_PROTOCOL_H
#define COMMAND_ARCHIVE_GET_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/stringId.h"
#include "common/type/variantList.h"
#include "common/type/pack.h"
#include "protocol/server.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Process protocol requests
void archiveGetFileProtocol(const VariantList *paramList, ProtocolServer *server);
void archiveGetFileProtocol(PackRead *param, ProtocolServer *server);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

61
src/command/archive/push/protocol.c
View File

@ -14,50 +14,55 @@ Archive Push Protocol Handler
/**********************************************************************************************************************************/
void
archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server)
archivePushFileProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// Build the repo data list
// Read parameters
const String *const walSource = pckReadStrP(param);
const bool headerCheck = pckReadBoolP(param);
const unsigned int pgVersion = pckReadU32P(param);
const uint64_t pgSystemId = pckReadU64P(param);
const String *const archiveFile = pckReadStrP(param);
const CompressType compressType = pckReadU32P(param);
const int compressLevel = pckReadI32P(param);
const StringList *const priorErrorList = pckReadStrLstP(param);
// Read repo data
List *repoList = lstNewP(sizeof(ArchivePushFileRepoData));
unsigned int repoListSize = varUIntForce(varLstGet(paramList, 8));
unsigned int paramIdx = 9;
for (unsigned int repoListIdx = 0; repoListIdx < repoListSize; repoListIdx++)
{
lstAdd(
repoList,
&(ArchivePushFileRepoData)
{
.repoIdx = varUIntForce(varLstGet(paramList, paramIdx)),
.archiveId = varStr(varLstGet(paramList, paramIdx + 1)),
.cipherType = varUInt64(varLstGet(paramList, paramIdx + 2)),
.cipherPass = varStr(varLstGet(paramList, paramIdx + 3)),
});
pckReadArrayBeginP(param);
paramIdx += 4;
while (!pckReadNullP(param))
{
pckReadObjBeginP(param);
ArchivePushFileRepoData repo = {.repoIdx = pckReadU32P(param)};
repo.archiveId = pckReadStrP(param);
repo.cipherType = pckReadU64P(param);
repo.cipherPass = pckReadStrP(param);
pckReadObjEndP(param);
lstAdd(repoList, &repo);
}
// Push the file
ArchivePushFileResult fileResult = archivePushFile(
varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUIntForce(varLstGet(paramList, 2)),
varUInt64(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)),
(CompressType)varUIntForce(varLstGet(paramList, 5)), varIntForce(varLstGet(paramList, 6)), repoList,
strLstNewVarLst(varVarLst(varLstGet(paramList, 7))));
pckReadArrayEndP(param);
// Push file
const ArchivePushFileResult fileResult = archivePushFile(
walSource, headerCheck, pgVersion, pgSystemId, archiveFile, compressType, compressLevel, repoList, priorErrorList);
// Return result
VariantList *result = varLstNew();
varLstAdd(result, varNewVarLst(varLstNewStrLst(fileResult.warnList)));
protocolServerResponse(server, varNewVarLst(result));
protocolServerDataPut(server, pckWriteStrLstP(protocolPackNew(), fileResult.warnList));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();

5
src/command/archive/push/protocol.h
View File

@ -4,15 +4,14 @@ Archive Push Protocol Handler
#ifndef COMMAND_ARCHIVE_PUSH_PROTOCOL_H
#define COMMAND_ARCHIVE_PUSH_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/variantList.h"
#include "common/type/pack.h"
#include "protocol/server.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Process protocol requests
void archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server);
void archivePushFileProtocol(PackRead *param, ProtocolServer *server);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

40
src/command/archive/push/push.c
View File

@ -460,28 +460,35 @@ archivePushAsyncCallback(void *data, unsigned int clientIdx)
const String *walFile = strLstGet(jobData->walFileList, jobData->walFileIdx);
jobData->walFileIdx++;
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE);
protocolCommandParamAdd(command, VARSTR(strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile))));
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptArchiveHeaderCheck)));
protocolCommandParamAdd(command, VARUINT(jobData->archiveInfo.pgVersion));
protocolCommandParamAdd(command, VARUINT64(jobData->archiveInfo.pgSystemId));
protocolCommandParamAdd(command, VARSTR(walFile));
protocolCommandParamAdd(command, VARUINT(jobData->compressType));
protocolCommandParamAdd(command, VARINT(jobData->compressLevel));
protocolCommandParamAdd(command, varNewVarLst(varLstNewStrLst(jobData->archiveInfo.errorList)));
protocolCommandParamAdd(command, VARUINT(lstSize(jobData->archiveInfo.repoList)));
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE);
PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile)));
pckWriteBoolP(param, cfgOptionBool(cfgOptArchiveHeaderCheck));
pckWriteU32P(param, jobData->archiveInfo.pgVersion);
pckWriteU64P(param, jobData->archiveInfo.pgSystemId);
pckWriteStrP(param, walFile);
pckWriteU32P(param, jobData->compressType);
pckWriteI32P(param, jobData->compressLevel);
pckWriteStrLstP(param, jobData->archiveInfo.errorList);
// Add data for each repo to push to
pckWriteArrayBeginP(param);
for (unsigned int repoListIdx = 0; repoListIdx < lstSize(jobData->archiveInfo.repoList); repoListIdx++)
{
ArchivePushFileRepoData *data = lstGet(jobData->archiveInfo.repoList, repoListIdx);
protocolCommandParamAdd(command, VARUINT(data->repoIdx));
protocolCommandParamAdd(command, VARSTR(data->archiveId));
protocolCommandParamAdd(command, VARUINT64(data->cipherType));
protocolCommandParamAdd(command, VARSTR(data->cipherPass));
pckWriteObjBeginP(param);
pckWriteU32P(param, data->repoIdx);
pckWriteStrP(param, data->archiveId);
pckWriteU64P(param, data->cipherType);
pckWriteStrP(param, data->cipherPass);
pckWriteObjEndP(param);
}
pckWriteArrayEndP(param);
FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(walFile), command));
}
@ -572,11 +579,8 @@ cmdArchivePushAsync(void)
// The job was successful
if (protocolParallelJobErrorCode(job) == 0)
{
// Get job result
const VariantList *fileResult = varVarLst(protocolParallelJobResult(job));
// Output file warnings
StringList *fileWarnList = strLstNewVarLst(varVarLst(varLstGet(fileResult, 0)));
StringList *fileWarnList = pckReadStrLstP(protocolParallelJobResult(job));
for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++)
LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx)));

45
src/command/backup/backup.c
View File

@ -23,6 +23,7 @@ Backup Command
#include "common/log.h"
#include "common/time.h"
#include "common/type/convert.h"
#include "common/type/json.h"
#include "config/config.h"
#include "db/helper.h"
#include "info/infoArchive.h"
@ -1043,12 +1044,12 @@ backupJobResult(
const ManifestFile *const file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
const unsigned int processId = protocolParallelJobProcessId(job);
const VariantList *const jobResult = varVarLst(protocolParallelJobResult(job));
const BackupCopyResult copyResult = (BackupCopyResult)varUIntForce(varLstGet(jobResult, 0));
const uint64_t copySize = varUInt64(varLstGet(jobResult, 1));
const uint64_t repoSize = varUInt64(varLstGet(jobResult, 2));
const String *const copyChecksum = varStr(varLstGet(jobResult, 3));
const KeyValue *const checksumPageResult = varKv(varLstGet(jobResult, 4));
PackRead *const jobResult = protocolParallelJobResult(job);
const BackupCopyResult copyResult = (BackupCopyResult)pckReadU32P(jobResult);
const uint64_t copySize = pckReadU64P(jobResult);
const uint64_t repoSize = pckReadU64P(jobResult);
const String *const copyChecksum = pckReadStrP(jobResult);
const KeyValue *const checksumPageResult = varKv(jsonToVar(pckReadStrP(jobResult, .defaultValue = NULL_STR)));
// Increment backup copy progress
sizeCopied += copySize;
@ -1439,23 +1440,23 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
// Create backup job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
PackWrite *const param = protocolCommandParam(command);
protocolCommandParamAdd(command, VARSTR(manifestPathPg(file->name)));
protocolCommandParamAdd(
command, VARBOOL(!strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))));
protocolCommandParamAdd(command, VARUINT64(file->size));
protocolCommandParamAdd(command, VARBOOL(!file->primary));
protocolCommandParamAdd(command, file->checksumSha1[0] != 0 ? VARSTRZ(file->checksumSha1) : NULL);
protocolCommandParamAdd(command, VARBOOL(file->checksumPage));
protocolCommandParamAdd(command, VARUINT64(jobData->lsnStart));
protocolCommandParamAdd(command, VARSTR(file->name));
protocolCommandParamAdd(command, VARBOOL(file->reference != NULL));
protocolCommandParamAdd(command, VARUINT(jobData->compressType));
protocolCommandParamAdd(command, VARINT(jobData->compressLevel));
protocolCommandParamAdd(command, VARSTR(jobData->backupLabel));
protocolCommandParamAdd(command, VARBOOL(jobData->delta));
protocolCommandParamAdd(command, VARUINT64(jobData->cipherType));
protocolCommandParamAdd(command, VARSTR(jobData->cipherSubPass));
pckWriteStrP(param, manifestPathPg(file->name));
pckWriteBoolP(param, !strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
pckWriteU64P(param, file->size);
pckWriteBoolP(param, !file->primary);
pckWriteStrP(param, file->checksumSha1[0] != 0 ? STR(file->checksumSha1) : NULL);
pckWriteBoolP(param, file->checksumPage);
pckWriteU64P(param, jobData->lsnStart);
pckWriteStrP(param, file->name);
pckWriteBoolP(param, file->reference != NULL);
pckWriteU32P(param, jobData->compressType);
pckWriteI32P(param, jobData->compressLevel);
pckWriteStrP(param, jobData->backupLabel);
pckWriteBoolP(param, jobData->delta);
pckWriteU64P(param, jobData->cipherSubPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc);
pckWriteStrP(param, jobData->cipherSubPass);
// Remove job from the queue
lstRemoveIdx(queue, 0);

53
src/command/backup/protocol.c
View File

@ -9,41 +9,56 @@ Backup Protocol Handler
#include "common/io/io.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "config/config.h"
#include "storage/helper.h"
/**********************************************************************************************************************************/
void
backupFileProtocol(const VariantList *paramList, ProtocolServer *server)
backupFileProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// Backup the file
BackupFileResult result = backupFile(
varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)),
varBool(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)), varBool(varLstGet(paramList, 5)),
varUInt64(varLstGet(paramList, 6)), varStr(varLstGet(paramList, 7)), varBool(varLstGet(paramList, 8)),
(CompressType)varUIntForce(varLstGet(paramList, 9)), varIntForce(varLstGet(paramList, 10)),
varStr(varLstGet(paramList, 11)), varBool(varLstGet(paramList, 12)), varUInt64(varLstGet(paramList, 13)),
varStr(varLstGet(paramList, 14)));
// Backup file
const String *const pgFile = pckReadStrP(param);
const bool pgFileIgnoreMissing = pckReadBoolP(param);
const uint64_t pgFileSize = pckReadU64P(param);
const bool pgFileCopyExactSize = pckReadBoolP(param);
const String *const pgFileChecksum = pckReadStrP(param);
const bool pgFileChecksumPage = pckReadBoolP(param);
const uint64_t pgFileChecksumPageLsnLimit = pckReadU64P(param);
const String *const repoFile = pckReadStrP(param);
const bool repoFileHasReference = pckReadBoolP(param);
const CompressType repoFileCompressType = (CompressType)pckReadU32P(param);
const int repoFileCompressLevel = pckReadI32P(param);
const String *const backupLabel = pckReadStrP(param);
const bool delta = pckReadBoolP(param);
const CipherType cipherType = (CipherType)pckReadU64P(param);
const String *const cipherPass = pckReadStrP(param);
// Return backup result
VariantList *resultList = varLstNew();
varLstAdd(resultList, varNewUInt(result.backupCopyResult));
varLstAdd(resultList, varNewUInt64(result.copySize));
varLstAdd(resultList, varNewUInt64(result.repoSize));
varLstAdd(resultList, varNewStr(result.copyChecksum));
varLstAdd(resultList, result.pageChecksumResult != NULL ? varNewKv(result.pageChecksumResult) : NULL);
const BackupFileResult result = backupFile(
pgFile, pgFileIgnoreMissing, pgFileSize, pgFileCopyExactSize, pgFileChecksum, pgFileChecksumPage,
pgFileChecksumPageLsnLimit, repoFile, repoFileHasReference, repoFileCompressType, repoFileCompressLevel,
backupLabel, delta, cipherType, cipherPass);
protocolServerResponse(server, varNewVarLst(resultList));
// Return result
PackWrite *const resultPack = protocolPackNew();
pckWriteU32P(resultPack, result.backupCopyResult);
pckWriteU64P(resultPack, result.copySize);
pckWriteU64P(resultPack, result.repoSize);
pckWriteStrP(resultPack, result.copyChecksum);
pckWriteStrP(resultPack, result.pageChecksumResult != NULL ? jsonFromKv(result.pageChecksumResult) : NULL);
protocolServerDataPut(server, resultPack);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();

5
src/command/backup/protocol.h
View File

@ -4,15 +4,14 @@ Backup Protocol Handler
#ifndef COMMAND_BACKUP_PROTOCOL_H
#define COMMAND_BACKUP_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/variantList.h"
#include "common/type/pack.h"
#include "protocol/server.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Process protocol requests
void backupFileProtocol(const VariantList *paramList, ProtocolServer *server);
void backupFileProtocol(PackRead *param, ProtocolServer *server);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

6
src/command/remote/remote.c
View File

@ -50,8 +50,8 @@ cmdRemote(int fdRead, int fdWrite)
TRY_BEGIN()
{
// Read the command. No need to parse it since we know this is the first noop.
ioReadLine(read);
// Get the command. No need to check parameters since we know this is the first noop.
CHECK(protocolServerCommandGet(server).id == PROTOCOL_COMMAND_NOOP);
// Only try the lock if this is process 0, i.e. the remote started from the main process
if (cfgOptionUInt(cfgOptProcess) == 0)
@ -70,7 +70,7 @@ cmdRemote(int fdRead, int fdWrite)
}
// Notify the client of success
protocolServerResponse(server, NULL);
protocolServerDataEndPut(server);
success = true;
}
CATCH_ANY()

43
src/command/restore/protocol.c
View File

@ -14,30 +14,43 @@ Restore Protocol Handler
/**********************************************************************************************************************************/
void
restoreFileProtocol(const VariantList *paramList, ProtocolServer *server)
restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
protocolServerResponse(
server,
VARBOOL(
restoreFile(
varStr(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)), varStr(varLstGet(paramList, 2)),
(CompressType)varUIntForce(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)),
varStr(varLstGet(paramList, 5)), varBoolForce(varLstGet(paramList, 6)), varUInt64(varLstGet(paramList, 7)),
(time_t)varInt64Force(varLstGet(paramList, 8)),
(mode_t)cvtZToUIntBase(strZ(varStr(varLstGet(paramList, 9))), 8),
varStr(varLstGet(paramList, 10)), varStr(varLstGet(paramList, 11)),
(time_t)varInt64Force(varLstGet(paramList, 12)), varBoolForce(varLstGet(paramList, 13)),
varBoolForce(varLstGet(paramList, 14)), varStr(varLstGet(paramList, 15)))));
// Restore file
const String *const repoFile = pckReadStrP(param);
const unsigned int repoIdx = pckReadU32P(param);
const String *const repoFileReference = pckReadStrP(param);
const CompressType repoFileCompressType = (CompressType)pckReadU32P(param);
const String *const pgFile = pckReadStrP(param);
const String *const pgFileChecksum = pckReadStrP(param);
const bool pgFileZero = pckReadBoolP(param);
const uint64_t pgFileSize = pckReadU64P(param);
const time_t pgFileModified = pckReadTimeP(param);
const mode_t pgFileMode = pckReadModeP(param);
const String *const pgFileUser = pckReadStrP(param);
const String *const pgFileGroup = pckReadStrP(param);
const time_t copyTimeBegin = pckReadTimeP(param);
const bool delta = pckReadBoolP(param);
const bool deltaForce = pckReadBoolP(param);
const String *const cipherPass = pckReadStrP(param);
const bool result = restoreFile(
repoFile, repoIdx, repoFileReference, repoFileCompressType, pgFile, pgFileChecksum, pgFileZero, pgFileSize,
pgFileModified, pgFileMode, pgFileUser, pgFileGroup, copyTimeBegin, delta, deltaForce, cipherPass);
// Return result
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), result));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();

5
src/command/restore/protocol.h
View File

@ -4,15 +4,14 @@ Restore Protocol Handler
#ifndef COMMAND_RESTORE_PROTOCOL_H
#define COMMAND_RESTORE_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/variantList.h"
#include "common/type/pack.h"
#include "protocol/server.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Process protocol requests
void restoreFileProtocol(const VariantList *paramList, ProtocolServer *server);
void restoreFileProtocol(PackRead *param, ProtocolServer *server);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

38
src/command/restore/restore.c
View File

@ -2015,7 +2015,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
{
const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
bool zeroed = restoreFileZeroed(file->name, zeroExp);
bool copy = varBool(protocolParallelJobResult(job));
bool copy = pckReadBoolP(protocolParallelJobResult(job));
String *log = strNewZ("restore");
@ -2141,24 +2141,24 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
// Create restore job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
protocolCommandParamAdd(command, VARSTR(file->name));
protocolCommandParamAdd(command, VARUINT(jobData->repoIdx));
protocolCommandParamAdd(
command, file->reference != NULL ?
VARSTR(file->reference) : VARSTR(manifestData(jobData->manifest)->backupLabel));
protocolCommandParamAdd(command, VARUINT(manifestData(jobData->manifest)->backupOptionCompressType));
protocolCommandParamAdd(command, VARSTR(restoreFilePgPath(jobData->manifest, file->name)));
protocolCommandParamAdd(command, VARSTRZ(file->checksumSha1));
protocolCommandParamAdd(command, VARBOOL(restoreFileZeroed(file->name, jobData->zeroExp)));
protocolCommandParamAdd(command, VARUINT64(file->size));
protocolCommandParamAdd(command, VARUINT64((uint64_t)file->timestamp));
protocolCommandParamAdd(command, VARSTR(strNewFmt("%04o", file->mode)));
protocolCommandParamAdd(command, VARSTR(file->user));
protocolCommandParamAdd(command, VARSTR(file->group));
protocolCommandParamAdd(command, VARUINT64((uint64_t)manifestData(jobData->manifest)->backupTimestampCopyStart));
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptDelta)));
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce)));
protocolCommandParamAdd(command, VARSTR(jobData->cipherSubPass));
PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, file->name);
pckWriteU32P(param, jobData->repoIdx);
pckWriteStrP(param, file->reference != NULL ? file->reference : manifestData(jobData->manifest)->backupLabel);
pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file->name));
pckWriteStrP(param, STR(file->checksumSha1));
pckWriteBoolP(param, restoreFileZeroed(file->name, jobData->zeroExp));
pckWriteU64P(param, file->size);
pckWriteTimeP(param, file->timestamp);
pckWriteModeP(param, file->mode);
pckWriteStrP(param, file->user);
pckWriteStrP(param, file->group);
pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
pckWriteStrP(param, jobData->cipherSubPass);
// Remove job from the queue
lstRemoveIdx(queue, 0);

22
src/command/verify/protocol.c
View File

@ -14,25 +14,29 @@ Verify Protocol Handler
/**********************************************************************************************************************************/
void
verifyFileProtocol(const VariantList *paramList, ProtocolServer *server)
verifyFileProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
VerifyResult result = verifyFile(
varStr(varLstGet(paramList, 0)), // Full filename
varStr(varLstGet(paramList, 1)), // Checksum
varUInt64(varLstGet(paramList, 2)), // File size
varStr(varLstGet(paramList, 3))); // Cipher pass
// Verify file
const String *const filePathName = pckReadStrP(param);
const String *const fileChecksum = pckReadStrP(param);
const uint64_t fileSize = pckReadU64P(param);
const String *const cipherPass = pckReadStrP(param);
protocolServerResponse(server, VARUINT(result));
const VerifyResult result = verifyFile(filePathName, fileChecksum, fileSize, cipherPass);
// Return result
protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), result));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();

5
src/command/verify/protocol.h
View File

@ -4,15 +4,14 @@ Verify Protocol Handler
#ifndef COMMAND_VERIFY_PROTOCOL_H
#define COMMAND_VERIFY_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/variantList.h"
#include "common/type/pack.h"
#include "protocol/server.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Process protocol requests
void verifyFileProtocol(const VariantList *paramList, ProtocolServer *server);
void verifyFileProtocol(PackRead *param, ProtocolServer *server);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

21
src/command/verify/verify.c
View File

@ -755,10 +755,12 @@ verifyArchive(void *data)
// Set up the job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
protocolCommandParamAdd(command, VARSTR(filePathName));
protocolCommandParamAdd(command, VARSTR(checksum));
protocolCommandParamAdd(command, VARUINT64(archiveResult->pgWalInfo.size));
protocolCommandParamAdd(command, VARSTR(jobData->walCipherPass));
PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, filePathName);
pckWriteStrP(param, checksum);
pckWriteU64P(param, archiveResult->pgWalInfo.size);
pckWriteStrP(param, jobData->walCipherPass);
// Assign job to result, prepending the archiveId to the key for consistency with backup processing
result = protocolParallelJobNew(
@ -978,12 +980,13 @@ verifyBackup(void *data)
{
// Set up the job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
protocolCommandParamAdd(command, VARSTR(filePathName));
PackWrite *const param = protocolCommandParam(command);
pckWriteStrP(param, filePathName);
// If the checksum is not present in the manifest, it will be calculated by manifest load
protocolCommandParamAdd(command, VARSTRZ(fileData->checksumSha1));
protocolCommandParamAdd(command, VARUINT64(fileData->size));
protocolCommandParamAdd(command, VARSTR(jobData->backupCipherPass));
pckWriteStrP(param, STR(fileData->checksumSha1));
pckWriteU64P(param, fileData->size);
pckWriteStrP(param, jobData->backupCipherPass);
// Assign job to result (prepend backup label being processed to the key since some files are in a prior backup)
result = protocolParallelJobNew(
@ -1535,7 +1538,7 @@ verifyProcess(unsigned int *errorTotal)
// The job was successful
if (protocolParallelJobErrorCode(job) == 0)
{
const VerifyResult verifyResult = (VerifyResult)varUIntForce(protocolParallelJobResult(job));
const VerifyResult verifyResult = (VerifyResult)pckReadU32P(protocolParallelJobResult(job));
// Update the result set for the type of file being processed
if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR))

25
src/common/type/pack.h
View File

@ -17,13 +17,18 @@ Fields in a pack are identified by IDs. A field ID is stored as a delta from the
that reading from the middle is generally not practical. The size of the gap between field IDs is important -- a gap of 1 never
incurs extra cost, but depending on the field type larger gaps may require additional bytes to store the field ID delta.
The standard default is the C default for that type (e.g. bool = false, int = 0) but can be changed with the .defaultValue
parameter. For example, pckWriteBoolP(write, false, .defaultWrite = true) will write a 0 (i.e. false) with a field ID into the pack,
but pckWriteBoolP(write, false) will not write to the pack, it will simply skip the ID. Note that
pckWriteStrP(packWrite, NULL, .defaultWrite = true) is not valid since there is no way to explicitly write a NULL.
NULLs are not stored in a pack and are therefore not typed. A NULL is essentially just a gap in the field IDs. Fields that are
frequently NULL are best stored at the end of an object. When using .defaultWrite = false in write functions a NULL will be written
(by making a gap in the IDs) if the value matches the default. When using read functions the default will always be returned
when the field is NULL (i.e. missing). The standard default is the C default for that type (e.g. bool = false, int = 0) but can be
changed with the .defaultValue parameter. For example, pckWriteBoolP(write, false, .defaultWrite = true) will write a 0 with an ID
into the pack, but pckWriteBoolP(write, false) will not write to the pack, it will simply skip the ID. Note that
pckWriteStrP(packWrite, NULL, .defaultWrite = true) is not valid since there is no way to explcitly write a NULL.
frequently NULL are best stored at the end of an object. When using read functions the default will always be returned
when the field is NULL (i.e. missing). There are times when NULL must be explicitly passed, for example:
pckWriteStrP(resultPack, result.pageChecksumResult != NULL ? jsonFromKv(result.pageChecksumResult) : NULL);
In this case, NULL is declared since jsonFromKv() does not accept a NULL parameter and, following the rules for NULLs the field ID
is skipped when result.pageChecksumResult == NULL. Upon reading, we can declare a NULL_STR when a NULL (field ID gap) is
encountered, e.g. jsonToVar(pckReadStrP(jobResult, .defaultValue = NULL_STR)).
A pack is an object by default. Objects can store fields, objects, or arrays. Objects and arrays will be referred to collectively as
containers. Fields contain data to be stored, e.g. integers, strings, etc.
@ -38,8 +43,8 @@ pckWriteStringP(write, STRDEF("sample"));
pckWriteEndP();
A string representation of this pack is `1:uint64:77,2:bool:false,4:str:sample`. The boolean was stored even though it was the
default because a write was explcitly requested. The int32 field was not stored because the value matched the expicitly set default.
Note that there is a gap in the ID stream, which represents the NULL/default value.
default because a write was explicitly requested. The int32 field was not stored because the value matched the explicitly set
default. Note that there is a gap in the ID stream, which represents the NULL/default value.
This pack can be read with:
@ -50,8 +55,8 @@ pckReadI32P(read, .defaultValue = -1);
pckReadStringP(read);
pckReadEndP();
Note that defaults are not stored in the pack so any defaults that were applied when writing (by setting .defaulWrite and
optionally .defaultValue) must be applied again when reading (by optionally setting .defaultValue).
Note that defaults are not stored in the pack so any defaults that were applied when writing (by setting .defaultValue) must be
applied again when reading by setting .defaultValue if the default value is not a standard C default.
If we don't care about the NULL/default, another way to read is:

19
src/config/protocol.c
View File

@ -7,35 +7,37 @@ Configuration Protocol Handler
#include "common/io/io.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "config/config.intern.h"
#include "config/parse.h"
#include "config/protocol.h"
/**********************************************************************************************************************************/
void
configOptionProtocol(const VariantList *paramList, ProtocolServer *server)
configOptionProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
VariantList *optionList = varLstNew();
for (unsigned int optionIdx = 0; optionIdx < varLstSize(paramList); optionIdx++)
while (pckReadNext(param))
{
CfgParseOptionResult option = cfgParseOptionP(varStr(varLstGet(paramList, optionIdx)));
CfgParseOptionResult option = cfgParseOptionP(pckReadStrP(param));
CHECK(option.found);
varLstAdd(optionList, varDup(cfgOptionIdx(option.id, cfgOptionKeyToIdx(option.id, option.keyIdx + 1))));
}
protocolServerResponse(server, varNewVarLst(optionList));
protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), jsonFromVar(varNewVarLst(optionList))));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -56,13 +58,14 @@ configOptionRemote(ProtocolClient *client, const VariantList *paramList)
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_CONFIG_OPTION);
PackWrite *const param = protocolCommandParam(command);
for (unsigned int paramIdx = 0; paramIdx < varLstSize(paramList); paramIdx++)
protocolCommandParamAdd(command, varLstGet(paramList, paramIdx));
pckWriteStrP(param, varStr(varLstGet(paramList, paramIdx)));
MEM_CONTEXT_PRIOR_BEGIN()
{
result = varVarLst(protocolClientExecute(client, command, true));
result = varVarLst(jsonToVar(pckReadStrP(protocolClientExecute(client, command, true))));
}
MEM_CONTEXT_PRIOR_END();
}

5
src/config/protocol.h
View File

@ -4,8 +4,7 @@ Configuration Protocol Handler
#ifndef CONFIG_PROTOCOL_H
#define CONFIG_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/variantList.h"
#include "common/type/pack.h"
#include "protocol/client.h"
#include "protocol/server.h"
@ -13,7 +12,7 @@ Configuration Protocol Handler
Functions
***********************************************************************************************************************************/
// Process config protocol requests
void configOptionProtocol(const VariantList *paramList, ProtocolServer *server);
void configOptionProtocol(PackRead *param, ProtocolServer *server);
// Get option values from a remote process
VariantList *configOptionRemote(ProtocolClient *client, const VariantList *paramList);

13
src/db/db.c
View File

@ -6,6 +6,7 @@ Database Client
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "common/wait.h"
#include "db/db.h"
#include "db/protocol.h"
@ -46,7 +47,7 @@ dbFreeResource(THIS_VOID)
ASSERT(this != NULL);
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_CLOSE);
protocolCommandParamAdd(command, VARUINT(this->remoteIdx));
pckWriteU32P(protocolCommandParam(command), this->remoteIdx);
protocolClientExecute(this->remoteClient, command, false);
@ -109,10 +110,12 @@ dbQuery(Db *this, const String *query)
if (this->remoteClient != NULL)
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_QUERY);
protocolCommandParamAdd(command, VARUINT(this->remoteIdx));
protocolCommandParamAdd(command, VARSTR(query));
PackWrite *const param = protocolCommandParam(command);
result = varVarLst(protocolClientExecute(this->remoteClient, command, true));
pckWriteU32P(param, this->remoteIdx);
pckWriteStrP(param, query);
result = varVarLst(jsonToVar(pckReadStrP(protocolClientExecute(this->remoteClient, command, true))));
}
// Else locally
else
@ -199,7 +202,7 @@ dbOpen(Db *this)
if (this->remoteClient != NULL)
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_OPEN);
this->remoteIdx = varUIntForce(protocolClientExecute(this->remoteClient, command, true));
this->remoteIdx = pckReadU32P(protocolClientExecute(this->remoteClient, command, true));
// Set a callback to notify the remote when a connection is closed
memContextCallbackSet(this->pub.memContext, dbFreeResource, this);

39
src/db/protocol.c
View File

@ -7,6 +7,7 @@ Db Protocol Handler
#include "common/io/io.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/list.h"
#include "config/config.h"
#include "db/protocol.h"
@ -23,14 +24,14 @@ static struct
/**********************************************************************************************************************************/
void
dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
dbOpenProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList == NULL);
ASSERT(param == NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
@ -46,8 +47,6 @@ dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
}
// Add db to the list
unsigned int dbIdx = lstSize(dbProtocolLocal.pgClientList);
MEM_CONTEXT_BEGIN(lstMemContext(dbProtocolLocal.pgClientList))
{
// Only a single db is passed to the remote
@ -61,7 +60,8 @@ dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
MEM_CONTEXT_END();
// Return db index which should be included in subsequent calls
protocolServerResponse(server, VARUINT(dbIdx));
protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), lstSize(dbProtocolLocal.pgClientList) - 1));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -70,24 +70,23 @@ dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
/**********************************************************************************************************************************/
void
dbQueryProtocol(const VariantList *paramList, ProtocolServer *server)
dbQueryProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
protocolServerResponse(
server,
varNewVarLst(
pgClientQuery(
*(PgClient **)lstGet(dbProtocolLocal.pgClientList, varUIntForce(varLstGet(paramList, 0))),
varStr(varLstGet(paramList, 1)))));
PgClient *const pgClient = *(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param));
const String *const query = pckReadStrP(param);
protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), jsonFromVar(varNewVarLst(pgClientQuery(pgClient, query)))));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -96,20 +95,20 @@ dbQueryProtocol(const VariantList *paramList, ProtocolServer *server)
/**********************************************************************************************************************************/
void
dbCloseProtocol(const VariantList *paramList, ProtocolServer *server)
dbCloseProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
pgClientClose(*(PgClient **)lstGet(dbProtocolLocal.pgClientList, varUIntForce(varLstGet(paramList, 0))));
protocolServerResponse(server, NULL);
pgClientClose(*(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param)));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();

10
src/db/protocol.h
View File

@ -4,18 +4,16 @@ Db Protocol Handler
#ifndef DB_PROTOCOL_H
#define DB_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/variantList.h"
#include "protocol/client.h"
#include "common/type/pack.h"
#include "protocol/server.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Process db protocol requests
void dbOpenProtocol(const VariantList *paramList, ProtocolServer *server);
void dbQueryProtocol(const VariantList *paramList, ProtocolServer *server);
void dbCloseProtocol(const VariantList *paramList, ProtocolServer *server);
void dbOpenProtocol(PackRead *param, ProtocolServer *server);
void dbQueryProtocol(PackRead *param, ProtocolServer *server);
void dbCloseProtocol(PackRead *param, ProtocolServer *server);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

208
src/protocol/client.c
View File

@ -10,6 +10,7 @@ Protocol Client
#include "common/type/json.h"
#include "common/type/keyValue.h"
#include "protocol/client.h"
#include "protocol/server.h"
#include "version.h"
/***********************************************************************************************************************************
@ -19,20 +20,16 @@ STRING_EXTERN(PROTOCOL_GREETING_NAME_STR, PROTOCOL_GRE
STRING_EXTERN(PROTOCOL_GREETING_SERVICE_STR, PROTOCOL_GREETING_SERVICE);
STRING_EXTERN(PROTOCOL_GREETING_VERSION_STR, PROTOCOL_GREETING_VERSION);
STRING_EXTERN(PROTOCOL_ERROR_STR, PROTOCOL_ERROR);
STRING_EXTERN(PROTOCOL_ERROR_STACK_STR, PROTOCOL_ERROR_STACK);
STRING_EXTERN(PROTOCOL_OUTPUT_STR, PROTOCOL_OUTPUT);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct ProtocolClient
{
ProtocolClientPub pub; // Publicly accessible variables
const String *name;
const String *errorPrefix;
TimeMSec keepAliveTime;
IoWrite *write; // Write interface
const String *name; // Name displayed in logging
const String *errorPrefix; // Prefix used when throwing error
TimeMSec keepAliveTime; // Last time data was put to the server
};
/***********************************************************************************************************************************
@ -52,7 +49,7 @@ protocolClientFreeResource(THIS_VOID)
// Send an exit command but don't wait to see if it succeeds
MEM_CONTEXT_TEMP_BEGIN()
{
protocolClientWriteCommand(this, protocolCommandNew(PROTOCOL_COMMAND_EXIT));
protocolClientCommandPut(this, protocolCommandNew(PROTOCOL_COMMAND_EXIT));
}
MEM_CONTEXT_TEMP_END();
@ -86,8 +83,8 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
{
.memContext = memContextCurrent(),
.read = read,
.write = write,
},
.write = write,
.name = strDup(name),
.errorPrefix = strNewFmt("raised from %s", strZ(name)),
.keepAliveTime = timeMSec(),
@ -96,7 +93,7 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
// Read, parse, and check the protocol greeting
MEM_CONTEXT_TEMP_BEGIN()
{
String *greeting = ioReadLine(protocolClientIoRead(this));
String *greeting = ioReadLine(this->pub.read);
KeyValue *greetingKv = jsonToKv(greeting);
const String *expected[] =
@ -143,93 +140,127 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
}
/**********************************************************************************************************************************/
// Helper to process errors
static void
protocolClientProcessError(ProtocolClient *this, KeyValue *errorKv)
void
protocolClientDataPut(ProtocolClient *const this, PackWrite *const data)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(KEY_VALUE, errorKv);
FUNCTION_LOG_PARAM(PACK_WRITE, data);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(errorKv != NULL);
MEM_CONTEXT_TEMP_BEGIN()
// End the pack
if (data != NULL)
pckWriteEndP(data);
// Write the data
PackWrite *dataMessage = pckWriteNew(this->write);
pckWriteU32P(dataMessage, protocolMessageTypeData, .defaultWrite = true);
pckWritePackP(dataMessage, data);
pckWriteEndP(dataMessage);
// Flush when there is no more data to put
if (data == NULL)
ioWriteFlush(this->write);
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
// Helper to process errors
static void
protocolClientError(ProtocolClient *const this, const ProtocolMessageType type, PackRead *const error)
{
// Process error if any
const Variant *error = kvGet(errorKv, VARSTR(PROTOCOL_ERROR_STR));
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(ENUM, type);
FUNCTION_LOG_PARAM(PACK_READ, error);
FUNCTION_LOG_END();
if (error != NULL)
if (type == protocolMessageTypeError)
{
const ErrorType *type = errorTypeFromCode(varIntForce(error));
const String *message = varStr(kvGet(errorKv, VARSTR(PROTOCOL_OUTPUT_STR)));
const ErrorType *type = errorTypeFromCode(pckReadI32P(error));
String *const message = strNewFmt("%s: %s", strZ(this->errorPrefix), strZ(pckReadStrP(error)));
const String *const stack = pckReadStrP(error);
pckReadEndP(error);
// Required part of the message
String *throwMessage = strNewFmt(
"%s: %s", strZ(this->errorPrefix), message == NULL ? "no details available" : strZ(message));
CHECK(message != NULL);
// Add stack trace if the error is an assertion or debug-level logging is enabled
if (type == &AssertError || logAny(logLevelDebug))
{
const String *stack = varStr(kvGet(errorKv, VARSTR(PROTOCOL_ERROR_STACK_STR)));
CHECK(stack != NULL);
strCat(throwMessage, LF_STR);
strCat(throwMessage, stack == NULL ? STRDEF("no stack trace available") : stack);
strCat(message, LF_STR);
strCat(message, stack);
}
THROWP(type, strZ(throwMessage));
THROWP(type, strZ(message));
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
const Variant *
protocolClientReadOutput(ProtocolClient *this, bool outputRequired)
/**********************************************************************************************************************************/
PackRead *
protocolClientDataGet(ProtocolClient *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(BOOL, outputRequired);
FUNCTION_LOG_END();
ASSERT(this != NULL);
const Variant *result = NULL;
PackRead *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
// Read the response
String *response = ioReadLine(protocolClientIoRead(this));
KeyValue *responseKv = varKv(jsonToVar(response));
PackRead *response = pckReadNew(this->pub.read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
// Process error if any
protocolClientProcessError(this, responseKv);
protocolClientError(this, type, response);
// Get output
result = kvGet(responseKv, VARSTR(PROTOCOL_OUTPUT_STR));
CHECK(type == protocolMessageTypeData);
if (outputRequired)
MEM_CONTEXT_PRIOR_BEGIN()
{
// Just move the entire response kv since the output is the largest part if it
kvMove(responseKv, memContextPrior());
result = pckReadPackP(response);
}
// Else if no output is required then there should not be any
else if (result != NULL)
THROW(AssertError, "no output required by command");
MEM_CONTEXT_PRIOR_END();
// Reset the keep alive time
this->keepAliveTime = timeMSec();
pckReadEndP(response);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_CONST(VARIANT, result);
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/**********************************************************************************************************************************/
void
protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command)
protocolClientDataEndGet(ProtocolClient *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_END();
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *response = pckReadNew(this->pub.read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
protocolClientError(this, type, response);
CHECK(type == protocolMessageTypeDataEnd);
pckReadEndP(response);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
void
protocolClientCommandPut(ProtocolClient *const this, ProtocolCommand *const command)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
@ -239,9 +270,8 @@ protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command)
ASSERT(this != NULL);
ASSERT(command != NULL);
// Write out the command
ioWriteStrLine(protocolClientIoWrite(this), protocolCommandJson(command));
ioWriteFlush(protocolClientIoWrite(this));
// Put command
protocolCommandPut(command, this->write);
// Reset the keep alive time
this->keepAliveTime = timeMSec();
@ -250,21 +280,31 @@ protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command)
}
/**********************************************************************************************************************************/
const Variant *
protocolClientExecute(ProtocolClient *this, const ProtocolCommand *command, bool outputRequired)
PackRead *
protocolClientExecute(ProtocolClient *const this, ProtocolCommand *const command, const bool resultRequired)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
FUNCTION_LOG_PARAM(BOOL, outputRequired);
FUNCTION_LOG_PARAM(BOOL, resultRequired);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(command != NULL);
protocolClientWriteCommand(this, command);
// Put command
protocolClientCommandPut(this, command);
FUNCTION_LOG_RETURN_CONST(VARIANT, protocolClientReadOutput(this, outputRequired));
// Read result if required
PackRead *result = NULL;
if (resultRequired)
result = protocolClientDataGet(this);
// Read response
protocolClientDataEndGet(this);
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/**********************************************************************************************************************************/
@ -286,50 +326,6 @@ protocolClientNoOp(ProtocolClient *this)
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
String *
protocolClientReadLine(ProtocolClient *this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
FUNCTION_LOG_END();
ASSERT(this != NULL);
String *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
result = ioReadLine(protocolClientIoRead(this));
if (strSize(result) == 0)
{
THROW(FormatError, "unexpected empty line");
}
else if (strZ(result)[0] == '{')
{
KeyValue *responseKv = varKv(jsonToVar(result));
// Process expected error
protocolClientProcessError(this, responseKv);
// If not an error then there is probably a protocol bug
THROW(FormatError, "expected error but got output");
}
else if (strZ(result)[0] != '.')
THROW_FMT(FormatError, "invalid prefix in '%s'", strZ(result));
MEM_CONTEXT_PRIOR_BEGIN()
{
result = strSub(result, 1);
}
MEM_CONTEXT_PRIOR_END();
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING, result);
}
/**********************************************************************************************************************************/
String *
protocolClientToLog(const ProtocolClient *this)

68
src/protocol/client.h
View File

@ -4,6 +4,24 @@ Protocol Client
#ifndef PROTOCOL_CLIENT_H
#define PROTOCOL_CLIENT_H
/***********************************************************************************************************************************
Message types used by the protocol
***********************************************************************************************************************************/
typedef enum
{
// Data passed between client and server in either direction. This can be used as many times as needed.
protocolMessageTypeData = 0,
// Indicates no more data for the server to return to the client and ends the command
protocolMessageTypeDataEnd = 1,
// Command sent from the client to the server
protocolMessageTypeCommand = 2,
// An error occurred on the server and the command ended abnormally. protocolMessageTypeDataEnd will not be sent to the client.
protocolMessageTypeError = 3,
} ProtocolMessageType;
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -27,13 +45,18 @@ Constants
#define PROTOCOL_COMMAND_EXIT STRID5("exit", 0xa27050)
#define PROTOCOL_COMMAND_NOOP STRID5("noop", 0x83dee0)
#define PROTOCOL_ERROR "err"
STRING_DECLARE(PROTOCOL_ERROR_STR);
#define PROTOCOL_ERROR_STACK "errStack"
STRING_DECLARE(PROTOCOL_ERROR_STACK_STR);
/***********************************************************************************************************************************
This size should be safe for most pack data without wasting a lot of space. If binary data is being transferred then this size can
be added to the expected binary size to account for overhead.
***********************************************************************************************************************************/
#define PROTOCOL_PACK_DEFAULT_SIZE 1024
#define PROTOCOL_OUTPUT "out"
STRING_DECLARE(PROTOCOL_OUTPUT_STR);
// Pack large enough for standard data. Note that the buffer will automatically resize when required.
__attribute__((always_inline)) static inline PackWrite *
protocolPackNew(void)
{
return pckWriteNewBuf(bufNew(PROTOCOL_PACK_DEFAULT_SIZE));
}
/***********************************************************************************************************************************
Constructors
@ -47,28 +70,20 @@ typedef struct ProtocolClientPub
{
MemContext *memContext; // Mem context
IoRead *read; // Read interface
IoWrite *write; // Write interface
} ProtocolClientPub;
// Read interface
__attribute__((always_inline)) static inline IoRead *
protocolClientIoRead(ProtocolClient *const this)
// Read file descriptor
__attribute__((always_inline)) static inline int
protocolClientIoReadFd(ProtocolClient *const this)
{
return THIS_PUB(ProtocolClient)->read;
}
// Write interface
__attribute__((always_inline)) static inline IoWrite *
protocolClientIoWrite(ProtocolClient *const this)
{
return THIS_PUB(ProtocolClient)->write;
return ioReadFd(THIS_PUB(ProtocolClient)->read);
}
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Execute a protocol command and get the output
const Variant *protocolClientExecute(ProtocolClient *this, const ProtocolCommand *command, bool outputRequired);
// Execute a command and get the result
PackRead *protocolClientExecute(ProtocolClient *this, ProtocolCommand *command, bool resultRequired);
// Move to a new parent mem context
__attribute__((always_inline)) static inline ProtocolClient *
@ -80,14 +95,15 @@ protocolClientMove(ProtocolClient *const this, MemContext *const parentNew)
// Send noop to test connection or keep it alive
void protocolClientNoOp(ProtocolClient *this);
// Read a line
String *protocolClientReadLine(ProtocolClient *this);
// Get data put by the server
PackRead *protocolClientDataGet(ProtocolClient *this);
void protocolClientDataEndGet(ProtocolClient *this);
// Read the command output
const Variant *protocolClientReadOutput(ProtocolClient *this, bool outputRequired);
// Put command to the server
void protocolClientCommandPut(ProtocolClient *this, ProtocolCommand *command);
// Write the protocol command
void protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command);
// Put data to the server
void protocolClientDataPut(ProtocolClient *this, PackWrite *data);
/***********************************************************************************************************************************
Destructor

65
src/protocol/command.c
View File

@ -6,15 +6,9 @@ Protocol Command
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/keyValue.h"
#include "protocol/command.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
STRING_EXTERN(PROTOCOL_KEY_COMMAND_STR, PROTOCOL_KEY_COMMAND);
STRING_EXTERN(PROTOCOL_KEY_PARAMETER_STR, PROTOCOL_KEY_PARAMETER);
#include "protocol/client.h"
/***********************************************************************************************************************************
Object type
@ -23,7 +17,7 @@ struct ProtocolCommand
{
MemContext *memContext;
StringId command;
Variant *parameterList;
PackWrite *pack;
};
/**********************************************************************************************************************************/
@ -54,33 +48,39 @@ protocolCommandNew(const StringId command)
}
/**********************************************************************************************************************************/
ProtocolCommand *
protocolCommandParamAdd(ProtocolCommand *this, const Variant *param)
void
protocolCommandPut(ProtocolCommand *const this, IoWrite *const write)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
FUNCTION_TEST_PARAM(VARIANT, param);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(write != NULL);
MEM_CONTEXT_BEGIN(this->memContext)
// Write the command and flush to be sure the command gets sent immediately
PackWrite *commandPack = pckWriteNew(write);
pckWriteU32P(commandPack, protocolMessageTypeCommand, .defaultWrite = true);
pckWriteStrIdP(commandPack, this->command);
// Only write params if there were any
if (this->pack != NULL)
{
// Create parameter list if not already created
if (this->parameterList == NULL)
this->parameterList = varNewVarLst(varLstNew());
// Add parameter to the list
varLstAdd(varVarLst(this->parameterList), varDup(param));
pckWriteEndP(this->pack);
pckWritePackP(commandPack, this->pack);
}
MEM_CONTEXT_END();
FUNCTION_TEST_RETURN(this);
pckWriteEndP(commandPack);
// Flush to send command immediately
ioWriteFlush(write);
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
String *
protocolCommandJson(const ProtocolCommand *this)
PackWrite *
protocolCommandParam(ProtocolCommand *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
@ -88,27 +88,16 @@ protocolCommandJson(const ProtocolCommand *this)
ASSERT(this != NULL);
String *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
if (this->pack == NULL)
{
char commandStrId[STRID_MAX + 1];
strIdToZ(this->command, commandStrId);
KeyValue *command = kvPut(kvNew(), VARSTR(PROTOCOL_KEY_COMMAND_STR), VARSTRZ(commandStrId));
if (this->parameterList != NULL)
kvPut(command, VARSTR(PROTOCOL_KEY_PARAMETER_STR), this->parameterList);
MEM_CONTEXT_PRIOR_BEGIN()
MEM_CONTEXT_BEGIN(this->memContext)
{
result = jsonFromKv(command);
this->pack = protocolPackNew();
}
MEM_CONTEXT_PRIOR_END();
MEM_CONTEXT_END();
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN(result);
FUNCTION_TEST_RETURN(this->pack);
}
/**********************************************************************************************************************************/

22
src/protocol/command.h
View File

@ -10,28 +10,13 @@ Object type
typedef struct ProtocolCommand ProtocolCommand;
#include "common/type/object.h"
#include "common/type/stringId.h"
#include "common/type/variant.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define PROTOCOL_KEY_COMMAND "cmd"
STRING_DECLARE(PROTOCOL_KEY_COMMAND_STR);
#define PROTOCOL_KEY_PARAMETER "param"
STRING_DECLARE(PROTOCOL_KEY_PARAMETER_STR);
#include "common/type/pack.h"
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
ProtocolCommand *protocolCommandNew(const StringId command);
/***********************************************************************************************************************************
Getters/Setters
***********************************************************************************************************************************/
// Command JSON
String *protocolCommandJson(const ProtocolCommand *this);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
@ -44,7 +29,10 @@ protocolCommandMove(ProtocolCommand *const this, MemContext *const parentNew)
}
// Read the command output
ProtocolCommand *protocolCommandParamAdd(ProtocolCommand *this, const Variant *param);
PackWrite *protocolCommandParam(ProtocolCommand *this);
// Write protocol command
void protocolCommandPut(ProtocolCommand *this, IoWrite *write);
/***********************************************************************************************************************************
Destructor

18
src/protocol/parallel.c
View File

@ -10,7 +10,6 @@ Protocol Parallel Executor
#include "common/log.h"
#include "common/macro.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/keyValue.h"
#include "common/type/list.h"
#include "protocol/command.h"
@ -83,7 +82,7 @@ protocolParallelClientAdd(ProtocolParallel *this, ProtocolClient *client)
ASSERT(client != NULL);
ASSERT(this->state == protocolParallelJobStatePending);
if (ioReadFd(protocolClientIoRead(client)) == -1)
if (protocolClientIoReadFd(client) == -1)
THROW(AssertError, "client with read fd is required");
lstAdd(this->clientList, &client);
@ -128,7 +127,7 @@ protocolParallelProcess(ProtocolParallel *this)
{
if (this->clientJobList[clientIdx] != NULL)
{
int fd = ioReadFd(protocolClientIoRead(*(ProtocolClient **)lstGet(this->clientList, clientIdx)));
int fd = protocolClientIoReadFd(*(ProtocolClient **)lstGet(this->clientList, clientIdx));
FD_SET((unsigned int)fd, &selectSet);
// Find the max file descriptor needed for select()
@ -159,15 +158,17 @@ protocolParallelProcess(ProtocolParallel *this)
if (job != NULL &&
FD_ISSET(
(unsigned int)ioReadFd(protocolClientIoRead(*(ProtocolClient **)lstGet(this->clientList, clientIdx))),
(unsigned int)protocolClientIoReadFd(*(ProtocolClient **)lstGet(this->clientList, clientIdx)),
&selectSet))
{
MEM_CONTEXT_TEMP_BEGIN()
{
TRY_BEGIN()
{
protocolParallelJobResultSet(
job, protocolClientReadOutput(*(ProtocolClient **)lstGet(this->clientList, clientIdx), true));
ProtocolClient *const client = *(ProtocolClient **)lstGet(this->clientList, clientIdx);
protocolParallelJobResultSet(job, protocolClientDataGet(client));
protocolClientDataEndGet(client);
}
CATCH_ANY()
{
@ -207,9 +208,8 @@ protocolParallelProcess(ProtocolParallel *this)
// Add to the job list
lstAdd(this->jobList, &job);
// Send the job to the client
protocolClientWriteCommand(
*(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job));
// Put command
protocolClientCommandPut(*(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job));
// Set client id and running state
protocolParallelJobProcessIdSet(job, clientIdx + 1);

13
src/protocol/parallelJob.c
View File

@ -92,21 +92,17 @@ protocolParallelJobProcessIdSet(ProtocolParallelJob *this, unsigned int processI
/**********************************************************************************************************************************/
void
protocolParallelJobResultSet(ProtocolParallelJob *this, const Variant *result)
protocolParallelJobResultSet(ProtocolParallelJob *const this, PackRead *const result)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, this);
FUNCTION_LOG_PARAM(VARIANT, result);
FUNCTION_LOG_PARAM(PACK_READ, result);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(protocolParallelJobErrorCode(this) == 0);
MEM_CONTEXT_BEGIN(this->pub.memContext)
{
this->pub.result = varDup(result);
}
MEM_CONTEXT_END();
this->pub.result = pckReadMove(result, this->pub.memContext);
FUNCTION_LOG_RETURN_VOID();
}
@ -143,5 +139,6 @@ protocolParallelJobToLog(const ProtocolParallelJob *this)
"{state: %s, key: %s, command: %s, code: %d, message: %s, result: %s}",
strZ(strIdToStr(protocolParallelJobState(this))), strZ(varToLog(protocolParallelJobKey(this))),
strZ(protocolCommandToLog(protocolParallelJobCommand(this))), protocolParallelJobErrorCode(this),
strZ(strToLog(protocolParallelJobErrorMessage(this))), strZ(varToLog(protocolParallelJobResult(this))));
strZ(strToLog(protocolParallelJobErrorMessage(this))),
protocolParallelJobResult(this) == NULL ? NULL_Z : strZ(pckReadToLog(protocolParallelJobResult(this))));
}

11
src/protocol/parallelJob.h
View File

@ -23,6 +23,7 @@ typedef enum
#include "common/time.h"
#include "common/type/object.h"
#include "common/type/pack.h"
#include "protocol/client.h"
/***********************************************************************************************************************************
@ -37,16 +38,16 @@ typedef struct ProtocolParallelJobPub
{
MemContext *memContext; // Mem context
const Variant *key; // Unique key used to identify the job
const ProtocolCommand *command; // Command to be executed
ProtocolCommand *command; // Command to be executed
unsigned int processId; // Process that executed this job
ProtocolParallelJobState state; // Current state of the job
int code; // Non-zero result indicates an error
String *message; // Message if there was a error
const Variant *result; // Result if job was successful
PackRead *result; // Result if job was successful
} ProtocolParallelJobPub;
// Job command
__attribute__((always_inline)) static inline const ProtocolCommand *
__attribute__((always_inline)) static inline ProtocolCommand *
protocolParallelJobCommand(const ProtocolParallelJob *const this)
{
return THIS_PUB(ProtocolParallelJob)->command;
@ -84,13 +85,13 @@ protocolParallelJobProcessId(const ProtocolParallelJob *const this)
void protocolParallelJobProcessIdSet(ProtocolParallelJob *this, unsigned int processId);
// Job result
__attribute__((always_inline)) static inline const Variant *
__attribute__((always_inline)) static inline PackRead *
protocolParallelJobResult(const ProtocolParallelJob *const this)
{
return THIS_PUB(ProtocolParallelJob)->result;
}
void protocolParallelJobResultSet(ProtocolParallelJob *this, const Variant *result);
void protocolParallelJobResultSet(ProtocolParallelJob *const this, PackRead *const result);
// Job state
__attribute__((always_inline)) static inline ProtocolParallelJobState

151
src/protocol/server.c
View File

@ -12,7 +12,6 @@ Protocol Server
#include "common/type/json.h"
#include "common/type/keyValue.h"
#include "common/type/list.h"
#include "protocol/client.h"
#include "protocol/helper.h"
#include "protocol/server.h"
#include "version.h"
@ -22,8 +21,10 @@ Object type
***********************************************************************************************************************************/
struct ProtocolServer
{
ProtocolServerPub pub; // Publicly accessible variables
const String *name;
MemContext *memContext; // Mem context
IoRead *read; // Read interface
IoWrite *write; // Write interface
const String *name; // Name displayed in logging
};
/**********************************************************************************************************************************/
@ -48,13 +49,10 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
this = memNew(sizeof(ProtocolServer));
*this = (ProtocolServer)
{
.pub =
{
.memContext = memContextCurrent(),
.read = read,
.write = write,
},
.name = strDup(name),
};
@ -66,8 +64,8 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_SERVICE_STR), VARSTR(service));
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_VERSION_STR), VARSTRZ(PROJECT_VERSION));
ioWriteStrLine(protocolServerIoWrite(this), jsonFromKv(greetingKv));
ioWriteFlush(protocolServerIoWrite(this));
ioWriteStrLine(this->write, jsonFromKv(greetingKv));
ioWriteFlush(this->write);
}
MEM_CONTEXT_TEMP_END();
}
@ -92,17 +90,50 @@ protocolServerError(ProtocolServer *this, int code, const String *message, const
ASSERT(message != NULL);
ASSERT(stack != NULL);
KeyValue *error = kvNew();
kvPut(error, VARSTR(PROTOCOL_ERROR_STR), VARINT(code));
kvPut(error, VARSTR(PROTOCOL_OUTPUT_STR), VARSTR(message));
kvPut(error, VARSTR(PROTOCOL_ERROR_STACK_STR), VARSTR(stack));
// Write the error and flush to be sure it gets sent immediately
PackWrite *error = pckWriteNew(this->write);
pckWriteU32P(error, protocolMessageTypeError);
pckWriteI32P(error, code);
pckWriteStrP(error, message);
pckWriteStrP(error, stack);
pckWriteEndP(error);
ioWriteStrLine(protocolServerIoWrite(this), jsonFromKv(error));
ioWriteFlush(protocolServerIoWrite(this));
ioWriteFlush(this->write);
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
ProtocolServerCommandGetResult
protocolServerCommandGet(ProtocolServer *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_END();
ProtocolServerCommandGetResult result = {0};
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *const command = pckReadNew(this->read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(command);
CHECK(type == protocolMessageTypeCommand);
MEM_CONTEXT_PRIOR_BEGIN()
{
result.id = pckReadStrIdP(command);
result.param = pckReadPackBufP(command);
}
MEM_CONTEXT_PRIOR_END();
pckReadEndP(command);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_STRUCT(result);
}
/**********************************************************************************************************************************/
void
protocolServerProcess(
@ -129,17 +160,15 @@ protocolServerProcess(
{
MEM_CONTEXT_TEMP_BEGIN()
{
// Read command
KeyValue *commandKv = jsonToKv(ioReadLine(protocolServerIoRead(this)));
const StringId command = strIdFromStr(stringIdBit5, varStr(kvGet(commandKv, VARSTR(PROTOCOL_KEY_COMMAND_STR))));
VariantList *paramList = varVarLst(kvGet(commandKv, VARSTR(PROTOCOL_KEY_PARAMETER_STR)));
// Get command
ProtocolServerCommandGetResult command = protocolServerCommandGet(this);
// Find the handler
ProtocolServerCommandHandler handler = NULL;
for (unsigned int handlerIdx = 0; handlerIdx < handlerListSize; handlerIdx++)
{
if (command == handlerList[handlerIdx].command)
if (command.id == handlerList[handlerIdx].command)
{
handler = handlerList[handlerIdx].handler;
break;
@ -151,7 +180,7 @@ protocolServerProcess(
{
// Send the command to the handler. Run the handler in the server's memory context in case any persistent data
// needs to be stored by the handler.
MEM_CONTEXT_BEGIN(this->pub.memContext)
MEM_CONTEXT_BEGIN(this->memContext)
{
// Initialize retries in case of command failure
bool retry = false;
@ -164,7 +193,7 @@ protocolServerProcess(
TRY_BEGIN()
{
handler(paramList, this);
handler(pckReadNewBuf(command.param), this);
}
CATCH_ANY()
{
@ -180,8 +209,7 @@ protocolServerProcess(
"retry %s after %" PRIu64 "ms: %s", errorTypeName(errorType()), retrySleepMs,
errorMessage());
// Sleep if there is an interval
if (retrySleepMs > 0)
// Sleep for interval
sleepMSec(retrySleepMs);
// Decrement retries remaining and retry
@ -204,18 +232,19 @@ protocolServerProcess(
// Else check built-in commands
else
{
switch (command)
switch (command.id)
{
case PROTOCOL_COMMAND_EXIT:
exit = true;
break;
case PROTOCOL_COMMAND_NOOP:
protocolServerResponse(this, NULL);
protocolServerDataEndPut(this);
break;
default:
THROW_FMT(ProtocolError, "invalid command '%s' (0x%" PRIx64 ")", strZ(strIdToStr(command)), command);
THROW_FMT(
ProtocolError, "invalid command '%s' (0x%" PRIx64 ")", strZ(strIdToStr(command.id)), command.id);
}
}
@ -239,45 +268,75 @@ protocolServerProcess(
}
/**********************************************************************************************************************************/
void
protocolServerResponse(ProtocolServer *this, const Variant *output)
PackRead *
protocolServerDataGet(ProtocolServer *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_PARAM(VARIANT, output);
FUNCTION_LOG_END();
KeyValue *result = kvNew();
PackRead *result = NULL;
if (output != NULL)
kvPut(result, VARSTR(PROTOCOL_OUTPUT_STR), output);
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *data = pckReadNew(this->read);
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(data);
ioWriteStrLine(protocolServerIoWrite(this), jsonFromKv(result));
ioWriteFlush(protocolServerIoWrite(this));
CHECK(type == protocolMessageTypeData);
MEM_CONTEXT_PRIOR_BEGIN()
{
result = pckReadPackP(data);
}
MEM_CONTEXT_PRIOR_END();
pckReadEndP(data);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/**********************************************************************************************************************************/
void
protocolServerDataPut(ProtocolServer *const this, PackWrite *const data)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_PARAM(PACK_WRITE, data);
FUNCTION_LOG_END();
// End the pack
if (data != NULL)
pckWriteEndP(data);
// Write the result
PackWrite *resultMessage = pckWriteNew(this->write);
pckWriteU32P(resultMessage, protocolMessageTypeData, .defaultWrite = true);
pckWritePackP(resultMessage, data);
pckWriteEndP(resultMessage);
// Flush on NULL result since it might be used to synchronize
if (data == NULL)
ioWriteFlush(this->write);
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
void
protocolServerWriteLine(ProtocolServer *this, const String *line)
protocolServerDataEndPut(ProtocolServer *const this)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
FUNCTION_LOG_PARAM(STRING, line);
FUNCTION_LOG_END();
ASSERT(this != NULL);
// Write the response and flush to be sure it gets sent immediately
PackWrite *response = pckWriteNew(this->write);
pckWriteU32P(response, protocolMessageTypeDataEnd, .defaultWrite = true);
pckWriteEndP(response);
// Dot indicates the start of an lf-terminated line
ioWrite(protocolServerIoWrite(this), DOT_BUF);
// Write the line if it exists
if (line != NULL)
ioWriteStr(protocolServerIoWrite(this), line);
// Terminate with a linefeed
ioWrite(protocolServerIoWrite(this), LF_BUF);
ioWriteFlush(this->write);
FUNCTION_LOG_RETURN_VOID();
}

55
src/protocol/server.h
View File

@ -12,7 +12,9 @@ typedef struct ProtocolServer ProtocolServer;
#include "common/io/read.h"
#include "common/io/write.h"
#include "common/type/object.h"
#include "common/type/pack.h"
#include "common/type/stringId.h"
#include "protocol/client.h"
/***********************************************************************************************************************************
Protocol command handler type and structure
@ -20,7 +22,7 @@ Protocol command handler type and structure
An array of this struct must be passed to protocolServerProcess() for the server to process commands. Each command handler should
implement a single command, as defined by the command string.
***********************************************************************************************************************************/
typedef void (*ProtocolServerCommandHandler)(const VariantList *paramList, ProtocolServer *server);
typedef void (*ProtocolServerCommandHandler)(PackRead *param, ProtocolServer *server);
typedef struct ProtocolServerHandler
{
@ -35,44 +37,36 @@ Constructors
***********************************************************************************************************************************/
ProtocolServer *protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write);
/***********************************************************************************************************************************
Getters/Setters
***********************************************************************************************************************************/
typedef struct ProtocolServerPub
{
MemContext *memContext; // Mem context
IoRead *read; // Read interface
IoWrite *write; // Write interface
} ProtocolServerPub;
// Read interface
__attribute__((always_inline)) static inline IoRead *
protocolServerIoRead(ProtocolServer *const this)
{
return THIS_PUB(ProtocolServer)->read;
}
// Write interface
__attribute__((always_inline)) static inline IoWrite *
protocolServerIoWrite(ProtocolServer *const this)
{
return THIS_PUB(ProtocolServer)->write;
}
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Get command from the client. Outside ProtocolServer, this is used when the first noop command needs to be processed before
// running protocolServerProcess(), which allows an error to be returned to the client if initialization fails.
typedef struct ProtocolServerCommandGetResult
{
StringId id; // Command identifier
Buffer *param; // Parameter pack
} ProtocolServerCommandGetResult;
ProtocolServerCommandGetResult protocolServerCommandGet(ProtocolServer *this);
// Get data from the client
PackRead *protocolServerDataGet(ProtocolServer *this);
// Put data to the client
void protocolServerDataPut(ProtocolServer *this, PackWrite *data);
// Put data end to the client. This ends command processing and no more data should be sent.
void protocolServerDataEndPut(ProtocolServer *this);
// Return an error
void protocolServerError(ProtocolServer *this, int code, const String *message, const String *stack);
// Process requests
void protocolServerProcess(
ProtocolServer *this, const VariantList *retryInterval, const ProtocolServerHandler *const handlerList,
ProtocolServer *this, const VariantList *retryInterval, const ProtocolServerHandler *handlerList,
const unsigned int handlerListSize);
// Respond to request with output if provided
void protocolServerResponse(ProtocolServer *this, const Variant *output);
// Move to a new parent mem context
__attribute__((always_inline)) static inline ProtocolServer *
protocolServerMove(ProtocolServer *const this, MemContext *const parentNew)
@ -80,9 +74,6 @@ protocolServerMove(ProtocolServer *const this, MemContext *const parentNew)
return objMove(this, parentNew);
}
// Write a line
void protocolServerWriteLine(ProtocolServer *this, const String *line);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/

497
src/storage/remote/protocol.c
View File

@ -14,6 +14,7 @@ Remote Storage Protocol Handler
#include "common/log.h"
#include "common/memContext.h"
#include "common/regExp.h"
#include "common/type/pack.h"
#include "common/type/json.h"
#include "config/config.h"
#include "protocol/helper.h"
@ -21,11 +22,6 @@ Remote Storage Protocol Handler
#include "storage/helper.h"
#include "storage/storage.intern.h"
/***********************************************************************************************************************************
Regular expressions
***********************************************************************************************************************************/
STRING_STATIC(BLOCK_REG_EXP_STR, PROTOCOL_BLOCK_HEADER "-1|[0-9]+");
/***********************************************************************************************************************************
Local variables
***********************************************************************************************************************************/
@ -76,64 +72,16 @@ storageRemoteFilterGroup(IoFilterGroup *filterGroup, const Variant *filterList)
FUNCTION_TEST_RETURN_VOID();
}
/***********************************************************************************************************************************
Callback to write info list into the protocol
***********************************************************************************************************************************/
// Helper to write storage info into the protocol. This function is not called unless the info exists so no need to write exists or
// check for level == storageInfoLevelExists.
static void
storageRemoteInfoWrite(ProtocolServer *server, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_SERVER, server);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
protocolServerWriteLine(server, jsonFromUInt(info->type));
protocolServerWriteLine(server, jsonFromInt64(info->timeModified));
if (info->type == storageTypeFile)
protocolServerWriteLine(server, jsonFromUInt64(info->size));
if (info->level >= storageInfoLevelDetail)
{
protocolServerWriteLine(server, jsonFromUInt(info->userId));
protocolServerWriteLine(server, jsonFromStr(info->user));
protocolServerWriteLine(server, jsonFromUInt(info->groupId));
protocolServerWriteLine(server, jsonFromStr(info->group));
protocolServerWriteLine(server, jsonFromUInt(info->mode));
if (info->type == storageTypeLink)
protocolServerWriteLine(server, jsonFromStr(info->linkDestination));
}
FUNCTION_TEST_RETURN_VOID();
}
static void
storageRemoteProtocolInfoListCallback(void *server, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
protocolServerWriteLine(server, jsonFromStr(info->name));
storageRemoteInfoWrite(server, info);
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
void
storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *server)
storageRemoteFeatureProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList == NULL);
ASSERT(param == NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
@ -158,10 +106,12 @@ storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *serve
}
// Return storage features
protocolServerWriteLine(server, jsonFromStr(storagePathP(storage, NULL)));
protocolServerWriteLine(server, jsonFromUInt64(storageInterface(storage).feature));
PackWrite *result = protocolPackNew();
pckWriteStrP(result, storagePathP(storage, NULL));
pckWriteU64P(result, storageInterface(storage).feature);
protocolServerResponse(server, NULL);
protocolServerDataPut(server, result);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -169,31 +119,203 @@ storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *serve
}
/**********************************************************************************************************************************/
typedef struct StorageRemoteInfoProcotolWriteData
{
MemContext *memContext; // Mem context used to store values from last call
time_t timeModifiedLast; // timeModified from last call
mode_t modeLast; // mode from last call
uid_t userIdLast; // userId from last call
gid_t groupIdLast; // groupId from last call
String *user; // user from last call
String *group; // group from last call
} StorageRemoteInfoProtocolWriteData;
// Helper to write storage info into the protocol. This function is not called unless the info exists so no need to write exists or
// check for level == storageInfoLevelExists.
//
// Fields that do not change from one call to the next are omitted to save bandwidth.
static void
storageRemoteInfoProtocolPut(
StorageRemoteInfoProtocolWriteData *const data, PackWrite *const write, const StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(PACK_WRITE, write);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(data != NULL);
ASSERT(write != NULL);
ASSERT(info != NULL);
// Write type and time
pckWriteU32P(write, info->type);
pckWriteTimeP(write, info->timeModified - data->timeModifiedLast);
// Write size for files
if (info->type == storageTypeFile)
pckWriteU64P(write, info->size);
// Write fields needed for detail level
if (info->level >= storageInfoLevelDetail)
{
// Write mode
pckWriteModeP(write, info->mode, .defaultValue = data->modeLast);
// Write user id/name
pckWriteU32P(write, info->userId, .defaultValue = data->userIdLast);
if (info->user == NULL) // {vm_covered}
pckWriteBoolP(write, true); // {vm_covered}
else
{
pckWriteBoolP(write, false);
pckWriteStrP(write, info->user, .defaultValue = data->user);
}
// Write group id/name
pckWriteU32P(write, info->groupId, .defaultValue = data->groupIdLast);
if (info->group == NULL) // {vm_covered}
pckWriteBoolP(write, true); // {vm_covered}
else
{
pckWriteBoolP(write, false);
pckWriteStrP(write, info->group, .defaultValue = data->group);
}
// Write link destination
if (info->type == storageTypeLink)
pckWriteStrP(write, info->linkDestination);
}
// Store defaults to use for the next call. If memContext is NULL this function is only being called one time so there is no
// point in storing defaults.
if (data->memContext != NULL)
{
data->timeModifiedLast = info->timeModified;
data->modeLast = info->mode;
data->userIdLast = info->userId;
data->groupIdLast = info->groupId;
if (!strEq(info->user, data->user) && info->user != NULL) // {vm_covered}
{
strFree(data->user);
MEM_CONTEXT_BEGIN(data->memContext)
{
data->user = strDup(info->user);
}
MEM_CONTEXT_END();
}
if (!strEq(info->group, data->group) && info->group != NULL) // {vm_covered}
{
strFree(data->group);
MEM_CONTEXT_BEGIN(data->memContext)
{
data->group = strDup(info->group);
}
MEM_CONTEXT_END();
}
}
FUNCTION_TEST_RETURN_VOID();
}
void
storageRemoteInfoProtocol(const VariantList *paramList, ProtocolServer *server)
storageRemoteInfoProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
StorageInfo info = storageInterfaceInfoP(
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)),
(StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)), .followLink = varBool(varLstGet(paramList, 2)));
// Get file info
const String *file = pckReadStrP(param);
StorageInfoLevel level = (StorageInfoLevel)pckReadU32P(param);
bool followLink = pckReadBoolP(param);
protocolServerResponse(server, VARBOOL(info.exists));
StorageInfo info = storageInterfaceInfoP(storageRemoteProtocolLocal.driver, file, level, .followLink = followLink);
// Write file info to protocol
PackWrite *write = protocolPackNew();
pckWriteBoolP(write, info.exists, .defaultWrite = true);
if (info.exists)
{
storageRemoteInfoWrite(server, &info);
protocolServerResponse(server, NULL);
storageRemoteInfoProtocolPut(&(StorageRemoteInfoProtocolWriteData){0}, write, &info);
protocolServerDataPut(server, write);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
typedef struct StorageRemoteProtocolInfoListCallbackData
{
ProtocolServer *const server;
StorageRemoteInfoProtocolWriteData writeData;
} StorageRemoteProtocolInfoListCallbackData;
static void
storageRemoteProtocolInfoListCallback(void *dataVoid, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM_P(VOID, dataVoid);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(dataVoid != NULL);
ASSERT(info != NULL);
StorageRemoteProtocolInfoListCallbackData *data = dataVoid;
PackWrite *write = protocolPackNew();
pckWriteStrP(write, info->name);
storageRemoteInfoProtocolPut(&data->writeData, write, info);
protocolServerDataPut(data->server, write);
FUNCTION_TEST_RETURN_VOID();
}
void
storageRemoteInfoListProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
const String *const path = pckReadStrP(param);
const StorageInfoLevel level = (StorageInfoLevel)pckReadU32P(param);
StorageRemoteProtocolInfoListCallbackData data = {.server = server, .writeData = {.memContext = memContextCurrent()}};
const bool result = storageInterfaceInfoListP(
storageRemoteProtocolLocal.driver, path, level, storageRemoteProtocolInfoListCallback, &data);
// Indicate whether or not the path was found
PackWrite *write = protocolPackNew();
pckWriteBoolP(write, result, .defaultWrite = true);
protocolServerDataPut(server, write);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -202,58 +324,34 @@ storageRemoteInfoProtocol(const VariantList *paramList, ProtocolServer *server)
/**********************************************************************************************************************************/
void
storageRemoteInfoListProtocol(const VariantList *paramList, ProtocolServer *server)
storageRemoteOpenReadProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
bool result = storageInterfaceInfoListP(
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)),
(StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)), storageRemoteProtocolInfoListCallback, server);
const String *file = pckReadStrP(param);
bool ignoreMissing = pckReadBoolP(param);
const Variant *limit = jsonToVar(pckReadStrP(param));
const Variant *filter = jsonToVar(pckReadStrP(param));
protocolServerWriteLine(server, NULL);
protocolServerResponse(server, VARBOOL(result));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
void
storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// Create the read object
IoRead *fileRead = storageReadIo(
storageInterfaceNewReadP(
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)),
.limit = varLstGet(paramList, 2)));
storageInterfaceNewReadP(storageRemoteProtocolLocal.driver, file, ignoreMissing, .limit = limit));
// Set filter group based on passed filters
storageRemoteFilterGroup(ioReadFilterGroup(fileRead), varLstGet(paramList, 3));
storageRemoteFilterGroup(ioReadFilterGroup(fileRead), filter);
// Check if the file exists
bool exists = ioReadOpen(fileRead);
protocolServerResponse(server, VARBOOL(exists));
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), exists, .defaultWrite = true));
// Transfer the file if it exists
if (exists)
@ -267,9 +365,9 @@ storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *serv
if (!bufEmpty(buffer))
{
ioWriteStrLine(protocolServerIoWrite(server), strNewFmt(PROTOCOL_BLOCK_HEADER "%zu", bufUsed(buffer)));
ioWrite(protocolServerIoWrite(server), buffer);
ioWriteFlush(protocolServerIoWrite(server));
PackWrite *write = protocolPackNew();
pckWriteBinP(write, buffer);
protocolServerDataPut(server, write);
bufUsedZero(buffer);
}
@ -278,13 +376,12 @@ storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *serv
ioReadClose(fileRead);
// Write a zero block to show file is complete
ioWriteLine(protocolServerIoWrite(server), BUFSTRDEF(PROTOCOL_BLOCK_HEADER "0"));
ioWriteFlush(protocolServerIoWrite(server));
// Push filter results
protocolServerResponse(server, ioFilterGroupResultAll(ioReadFilterGroup(fileRead)));
// Write filter results
protocolServerDataPut(
server, pckWriteStrP(protocolPackNew(), jsonFromVar(ioFilterGroupResultAll(ioReadFilterGroup(fileRead)))));
}
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -293,79 +390,85 @@ storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *serv
/**********************************************************************************************************************************/
void
storageRemoteOpenWriteProtocol(const VariantList *paramList, ProtocolServer *server)
storageRemoteOpenWriteProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
// Create the write object
const String *file = pckReadStrP(param);
mode_t modeFile = pckReadModeP(param);
mode_t modePath = pckReadModeP(param);
const String *user = pckReadStrP(param);
const String *group = pckReadStrP(param);
time_t timeModified = pckReadTimeP(param);
bool createPath = pckReadBoolP(param);
bool syncFile = pckReadBoolP(param);
bool syncPath = pckReadBoolP(param);
bool atomic = pckReadBoolP(param);
const Variant *filter = jsonToVar(pckReadStrP(param));
IoWrite *fileWrite = storageWriteIo(
storageInterfaceNewWriteP(
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)),
.modeFile = (mode_t)varUIntForce(varLstGet(paramList, 1)),
.modePath = (mode_t)varUIntForce(varLstGet(paramList, 2)), .user = varStr(varLstGet(paramList, 3)),
.group = varStr(varLstGet(paramList, 4)), .timeModified = (time_t)varUInt64Force(varLstGet(paramList, 5)),
.createPath = varBool(varLstGet(paramList, 6)), .syncFile = varBool(varLstGet(paramList, 7)),
.syncPath = varBool(varLstGet(paramList, 8)), .atomic = varBool(varLstGet(paramList, 9))));
storageRemoteProtocolLocal.driver, file, .modeFile = modeFile, .modePath = modePath, .user = user, .group = group,
.timeModified = timeModified, .createPath = createPath, .syncFile = syncFile, .syncPath = syncPath,
.atomic = atomic));
// Set filter group based on passed filters
storageRemoteFilterGroup(ioWriteFilterGroup(fileWrite), varLstGet(paramList, 10));
storageRemoteFilterGroup(ioWriteFilterGroup(fileWrite), filter);
// Open file
ioWriteOpen(fileWrite);
protocolServerResponse(server, NULL);
protocolServerDataPut(server, NULL);
// Write data
Buffer *buffer = bufNew(ioBufferSize());
ssize_t remaining;
do
{
// How much data is remaining to write?
remaining = storageRemoteProtocolBlockSize(ioReadLine(protocolServerIoRead(server)));
PackRead *read = protocolServerDataGet(server);
// Write data
if (remaining > 0)
{
size_t bytesToCopy = (size_t)remaining;
do
{
if (bytesToCopy < bufSize(buffer))
bufLimitSet(buffer, bytesToCopy);
bytesToCopy -= ioRead(protocolServerIoRead(server), buffer);
ioWrite(fileWrite, buffer);
bufUsedZero(buffer);
bufLimitClear(buffer);
}
while (bytesToCopy > 0);
}
// Close when all data has been written
else if (remaining == 0)
// Write is complete
if (read == NULL)
{
ioWriteClose(fileWrite);
// Push filter results
protocolServerResponse(server, ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite)));
protocolServerDataPut(
server, pckWriteStrP(protocolPackNew(), jsonFromVar(ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite)))));
break;
}
// Write was aborted so free the file
// Else more data to write
else
{
pckReadNext(read);
// Write data
if (pckReadType(read) == pckTypeBin)
{
Buffer *const buffer = pckReadBinP(read);
ioWrite(fileWrite, buffer);
bufFree(buffer);
}
// Else write terminated unexpectedly
else
{
protocolServerDataGet(server);
ioWriteFree(fileWrite);
protocolServerResponse(server, NULL);
break;
}
}
while (remaining > 0);
}
while (true);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -374,24 +477,26 @@ storageRemoteOpenWriteProtocol(const VariantList *paramList, ProtocolServer *ser
/**********************************************************************************************************************************/
void
storageRemotePathCreateProtocol(const VariantList *paramList, ProtocolServer *server)
storageRemotePathCreateProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
storageInterfacePathCreateP(
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)),
varBool(varLstGet(paramList, 2)), (mode_t)varUIntForce(varLstGet(paramList, 3)));
const String *path = pckReadStrP(param);
bool errorOnExists = pckReadBoolP(param);
bool noParentCreate = pckReadBoolP(param);
mode_t mode = pckReadModeP(param);
protocolServerResponse(server, NULL);
storageInterfacePathCreateP(storageRemoteProtocolLocal.driver, path, errorOnExists, noParentCreate, mode);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -400,24 +505,26 @@ storageRemotePathCreateProtocol(const VariantList *paramList, ProtocolServer *se
/**********************************************************************************************************************************/
void
storageRemotePathRemoveProtocol(const VariantList *paramList, ProtocolServer *server)
storageRemotePathRemoveProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
protocolServerResponse(
server,
VARBOOL(
storageInterfacePathRemoveP(
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)))));
const String *path = pckReadStrP(param);
bool recurse = pckReadBoolP(param);
const bool result = storageInterfacePathRemoveP(storageRemoteProtocolLocal.driver, path, recurse);
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), result, .defaultWrite = true));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -426,21 +533,23 @@ storageRemotePathRemoveProtocol(const VariantList *paramList, ProtocolServer *se
/**********************************************************************************************************************************/
void
storageRemotePathSyncProtocol(const VariantList *paramList, ProtocolServer *server)
storageRemotePathSyncProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
storageInterfacePathSyncP(storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)));
protocolServerResponse(server, NULL);
const String *path = pckReadStrP(param);
storageInterfacePathSyncP(storageRemoteProtocolLocal.driver, path);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
@ -449,54 +558,26 @@ storageRemotePathSyncProtocol(const VariantList *paramList, ProtocolServer *serv
/**********************************************************************************************************************************/
void
storageRemoteRemoveProtocol(const VariantList *paramList, ProtocolServer *server)
storageRemoteRemoveProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PACK_READ, param);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(paramList != NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
ASSERT(storageRemoteProtocolLocal.driver != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
storageInterfaceRemoveP(
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), .errorOnMissing = varBool(varLstGet(paramList, 1)));
protocolServerResponse(server, NULL);
const String *file = pckReadStrP(param);
bool errorOnMissing = pckReadBoolP(param);
storageInterfaceRemoveP(storageRemoteProtocolLocal.driver, file, .errorOnMissing = errorOnMissing);
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
ssize_t
storageRemoteProtocolBlockSize(const String *message)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STRING, message);
FUNCTION_LOG_END();
ASSERT(message != NULL);
// Regular expression to check block messages
static RegExp *blockRegExp = NULL;
// Create block regular expression if it has not been created yet
if (blockRegExp == NULL)
{
MEM_CONTEXT_BEGIN(memContextTop())
{
blockRegExp = regExpNew(BLOCK_REG_EXP_STR);
}
MEM_CONTEXT_END();
}
// Validate the header block size message
if (!regExpMatch(blockRegExp, message))
THROW_FMT(ProtocolError, "'%s' is not a valid block size message", strZ(message));
FUNCTION_LOG_RETURN(SSIZE, (ssize_t)cvtZToInt(strZ(message) + sizeof(PROTOCOL_BLOCK_HEADER) - 1));
}

29
src/storage/remote/protocol.h
View File

@ -4,31 +4,22 @@ Remote Storage Protocol Handler
#ifndef STORAGE_REMOTE_PROTOCOL_H
#define STORAGE_REMOTE_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/variantList.h"
#include "common/type/pack.h"
#include "protocol/server.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define PROTOCOL_BLOCK_HEADER "BRBLOCK"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Get size of the next transfer block
ssize_t storageRemoteProtocolBlockSize(const String *message);
// Process storage protocol requests
void storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *server);
void storageRemoteInfoProtocol(const VariantList *paramList, ProtocolServer *server);
void storageRemoteInfoListProtocol(const VariantList *paramList, ProtocolServer *server);
void storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *server);
void storageRemoteOpenWriteProtocol(const VariantList *paramList, ProtocolServer *server);
void storageRemotePathCreateProtocol(const VariantList *paramList, ProtocolServer *server);
void storageRemotePathRemoveProtocol(const VariantList *paramList, ProtocolServer *server);
void storageRemotePathSyncProtocol(const VariantList *paramList, ProtocolServer *server);
void storageRemoteRemoveProtocol(const VariantList *paramList, ProtocolServer *server);
void storageRemoteFeatureProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteInfoProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteInfoListProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteOpenReadProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteOpenWriteProtocol(PackRead *param, ProtocolServer *server);
void storageRemotePathCreateProtocol(PackRead *param, ProtocolServer *server);
void storageRemotePathRemoveProtocol(PackRead *param, ProtocolServer *server);
void storageRemotePathSyncProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteRemoveProtocol(PackRead *param, ProtocolServer *server);
/***********************************************************************************************************************************
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()

59
src/storage/remote/read.c
View File

@ -12,6 +12,7 @@ Remote Storage Read
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/convert.h"
#include "common/type/json.h"
#include "common/type/object.h"
#include "storage/remote/protocol.h"
#include "storage/remote/read.h"
@ -29,6 +30,7 @@ typedef struct StorageReadRemote
ProtocolClient *client; // Protocol client for requests
size_t remaining; // Bytes remaining to be read in block
Buffer *block; // Block currently being read
bool eof; // Has the file reached eof?
#ifdef DEBUG
@ -70,13 +72,20 @@ storageReadRemoteOpen(THIS_VOID)
}
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ);
protocolCommandParamAdd(command, VARSTR(this->interface.name));
protocolCommandParamAdd(command, VARBOOL(this->interface.ignoreMissing));
protocolCommandParamAdd(command, this->interface.limit);
protocolCommandParamAdd(command, ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read))));
PackWrite *const param = protocolCommandParam(command);
result = varBool(protocolClientExecute(this->client, command, true));
pckWriteStrP(param, this->interface.name);
pckWriteBoolP(param, this->interface.ignoreMissing);
pckWriteStrP(param, jsonFromVar(this->interface.limit));
pckWriteStrP(param, jsonFromVar(ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read)))));
protocolClientCommandPut(this->client, command);
// If the file exists
result = pckReadBoolP(protocolClientDataGet(this->client));
if (result)
{
// Clear filters since they will be run on the remote side
ioFilterGroupClear(ioReadFilterGroup(storageReadIo(this->read)));
@ -84,6 +93,10 @@ storageReadRemoteOpen(THIS_VOID)
if (this->interface.compressible)
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilter(compressTypeGz));
}
// Else nothing to do
else
protocolClientDataEndGet(this->client);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result);
@ -118,13 +131,28 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
{
MEM_CONTEXT_TEMP_BEGIN()
{
this->remaining = (size_t)storageRemoteProtocolBlockSize(ioReadLine(protocolClientIoRead(this->client)));
PackRead *const read = protocolClientDataGet(this->client);
pckReadNext(read);
if (this->remaining == 0)
// If binary then read the next block
if (pckReadType(read) == pckTypeBin)
{
ioFilterGroupResultAllSet(
ioReadFilterGroup(storageReadIo(this->read)), protocolClientReadOutput(this->client, true));
MEM_CONTEXT_BEGIN(this->memContext)
{
this->block = pckReadBinP(read);
this->remaining = bufUsed(this->block);
}
MEM_CONTEXT_END();
}
// Else read is complete and get the filter list
else
{
bufFree(this->block);
ioFilterGroupResultAllSet(ioReadFilterGroup(storageReadIo(this->read)), jsonToVar(pckReadStrP(read)));
this->eof = true;
protocolClientDataEndGet(this->client);
}
#ifdef DEBUG
@ -140,14 +168,19 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
// If the buffer can contain all remaining bytes
if (bufRemains(buffer) >= this->remaining)
{
bufLimitSet(buffer, bufUsed(buffer) + this->remaining);
ioRead(protocolClientIoRead(this->client), buffer);
bufLimitClear(buffer);
bufCatSub(buffer, this->block, bufUsed(this->block) - this->remaining, this->remaining);
this->remaining = 0;
bufFree(this->block);
this->block = NULL;
}
// Else read what we can
else
this->remaining -= ioRead(protocolClientIoRead(this->client), buffer);
{
size_t remains = bufRemains(buffer);
bufCatSub(buffer, this->block, bufUsed(this->block) - this->remaining, remains);
this->remaining -= remains;
}
}
}
while (!this->eof && !bufFull(buffer));

175
src/storage/remote/storage.c
View File

@ -8,6 +8,7 @@ Remote Storage
#include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/object.h"
#include "common/type/pack.h"
#include "storage/remote/protocol.h"
#include "storage/remote/read.h"
#include "storage/remote/storage.intern.h"
@ -25,31 +26,81 @@ struct StorageRemote
};
/**********************************************************************************************************************************/
// Helper to parse storage info from the protocol output
typedef struct StorageRemoteInfoData
{
time_t timeModifiedLast; // timeModified from last call
mode_t modeLast; // mode from last call
uid_t userIdLast; // userId from last call
gid_t groupIdLast; // groupId from last call
String *user; // user from last call
String *group; // group from last call
} StorageRemoteInfoData;
// Helper to get storage info from the protocol output
static void
storageRemoteInfoParse(ProtocolClient *client, StorageInfo *info)
storageRemoteInfoGet(StorageRemoteInfoData *const data, PackRead *const read, StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PROTOCOL_CLIENT, client);
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(PACK_READ, read);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
info->type = jsonToUInt(protocolClientReadLine(client));
info->timeModified = (time_t)jsonToUInt64(protocolClientReadLine(client));
ASSERT(data != NULL);
ASSERT(read != NULL);
ASSERT(info != NULL);
// Read type and time modified
info->type = pckReadU32P(read);
info->timeModified = pckReadTimeP(read) + data->timeModifiedLast;
// Read size for files
if (info->type == storageTypeFile)
info->size = jsonToUInt64(protocolClientReadLine(client));
info->size = pckReadU64P(read);
// Read fields needed for detail level
if (info->level >= storageInfoLevelDetail)
{
info->userId = jsonToUInt(protocolClientReadLine(client));
info->user = jsonToStr(protocolClientReadLine(client));
info->groupId = jsonToUInt(protocolClientReadLine(client));
info->group = jsonToStr(protocolClientReadLine(client));
info->mode = (mode_t)jsonToUInt(protocolClientReadLine(client));
// Read mode
info->mode = pckReadModeP(read, .defaultValue = data->modeLast);
// Read user id/name
info->userId = pckReadU32P(read, .defaultValue = data->userIdLast);
if (pckReadBoolP(read)) // {vm_covered}
info->user = NULL; // {vm_covered}
else
info->user = pckReadStrP(read, .defaultValue = data->user);
// Read group id/name
info->groupId = pckReadU32P(read, .defaultValue = data->groupIdLast);
if (pckReadBoolP(read)) // {vm_covered}
info->group = NULL; // {vm_covered}
else
info->group = pckReadStrP(read, .defaultValue = data->group);
// Read link destination
if (info->type == storageTypeLink)
info->linkDestination = jsonToStr(protocolClientReadLine(client));
info->linkDestination = pckReadStrP(read);
}
// Store defaults to use for the next call
data->timeModifiedLast = info->timeModified;
data->modeLast = info->mode;
data->userIdLast = info->userId;
data->groupIdLast = info->groupId;
if (!strEq(info->user, data->user) && info->user != NULL) // {vm_covered}
{
strFree(data->user);
data->user = strDup(info->user);
}
if (!strEq(info->group, data->group) && info->group != NULL) // {vm_covered}
{
strFree(data->group);
data->group = strDup(info->group);
}
FUNCTION_TEST_RETURN_VOID();
@ -74,25 +125,31 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, Storage
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO);
protocolCommandParamAdd(command, VARSTR(file));
protocolCommandParamAdd(command, VARUINT(level));
protocolCommandParamAdd(command, VARBOOL(param.followLink));
PackWrite *const commandParam = protocolCommandParam(command);
result.exists = varBool(protocolClientExecute(this->client, command, true));
pckWriteStrP(commandParam, file);
pckWriteU32P(commandParam, level);
pckWriteBoolP(commandParam, param.followLink);
// Put command
protocolClientCommandPut(this->client, command);
// Read info from protocol
PackRead *read = protocolClientDataGet(this->client);
result.exists = pckReadBoolP(read);
if (result.exists)
{
// Read info from protocol into prior context
MEM_CONTEXT_PRIOR_BEGIN()
{
result.name = strDup(result.name);
storageRemoteInfoParse(this->client, &result);
storageRemoteInfoGet(&(StorageRemoteInfoData){0}, read, &result);
}
MEM_CONTEXT_PRIOR_END();
// Acknowledge command completed
protocolClientReadOutput(this->client, false);
}
protocolClientDataEndGet(this->client);
}
MEM_CONTEXT_TEMP_END();
@ -125,36 +182,41 @@ storageRemoteInfoList(
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_LIST);
protocolCommandParamAdd(command, VARSTR(path));
protocolCommandParamAdd(command, VARUINT(level));
PackWrite *const commandParam = protocolCommandParam(command);
// Send command
protocolClientWriteCommand(this->client, command);
pckWriteStrP(commandParam, path);
pckWriteU32P(commandParam, level);
// Put command
protocolClientCommandPut(this->client, command);
// Read list
StorageRemoteInfoData parseData = {0};
// Read list. The list ends when there is a blank line -- this is safe even for file systems that allow blank filenames
// since the filename is json-encoded so will always include quotes.
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
const String *name = protocolClientReadLine(this->client);
PackRead *read = protocolClientDataGet(this->client);
pckReadNext(read);
while (strSize(name) != 0)
while (pckReadType(read) == pckTypeStr)
{
StorageInfo info = {.exists = true, .level = level, .name = jsonToStr(name)};
StorageInfo info = {.exists = true, .level = level, .name = pckReadStrP(read)};
storageRemoteInfoParse(this->client, &info);
storageRemoteInfoGet(&parseData, read, &info);
callback(callbackData, &info);
// Reset the memory context occasionally so we don't use too much memory or slow down processing
MEM_CONTEXT_TEMP_RESET(1000);
// Read the next item
name = protocolClientReadLine(this->client);
read = protocolClientDataGet(this->client);
pckReadNext(read);
}
result = pckReadBoolP(read);
}
MEM_CONTEXT_TEMP_END();
// Acknowledge command completed
result = varBool(protocolClientReadOutput(this->client, true));
protocolClientDataEndGet(this->client);
}
MEM_CONTEXT_TEMP_END();
@ -240,10 +302,12 @@ storageRemotePathCreate(
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_CREATE);
protocolCommandParamAdd(command, VARSTR(path));
protocolCommandParamAdd(command, VARBOOL(errorOnExists));
protocolCommandParamAdd(command, VARBOOL(noParentCreate));
protocolCommandParamAdd(command, VARUINT(mode));
PackWrite *const commandParam = protocolCommandParam(command);
pckWriteStrP(commandParam, path);
pckWriteBoolP(commandParam, errorOnExists);
pckWriteBoolP(commandParam, noParentCreate);
pckWriteModeP(commandParam, mode);
protocolClientExecute(this->client, command, false);
}
@ -273,10 +337,12 @@ storageRemotePathRemove(THIS_VOID, const String *path, bool recurse, StorageInte
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE);
protocolCommandParamAdd(command, VARSTR(path));
protocolCommandParamAdd(command, VARBOOL(recurse));
PackWrite *const commandParam = protocolCommandParam(command);
result = varBool(protocolClientExecute(this->client, command, true));
pckWriteStrP(commandParam, path);
pckWriteBoolP(commandParam, recurse);
result = pckReadBoolP(protocolClientExecute(this->client, command, true));
}
MEM_CONTEXT_TEMP_END();
@ -301,7 +367,7 @@ storageRemotePathSync(THIS_VOID, const String *path, StorageInterfacePathSyncPar
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_SYNC);
protocolCommandParamAdd(command, VARSTR(path));
pckWriteStrP(protocolCommandParam(command), path);
protocolClientExecute(this->client, command, false);
}
@ -328,8 +394,10 @@ storageRemoteRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam p
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_REMOVE);
protocolCommandParamAdd(command, VARSTR(file));
protocolCommandParamAdd(command, VARBOOL(param.errorOnMissing));
PackWrite *const commandParam = protocolCommandParam(command);
pckWriteStrP(commandParam, file);
pckWriteBoolP(commandParam, param.errorOnMissing);
protocolClientExecute(this->client, command, false);
}
@ -388,22 +456,17 @@ storageRemoteNew(
// Get storage features from the remote
MEM_CONTEXT_TEMP_BEGIN()
{
// Send command
protocolClientWriteCommand(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE));
// Execute command and get result
PackRead *result = protocolClientExecute(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE), true);
// Read values
path = jsonToStr(protocolClientReadLine(driver->client));
driver->interface.feature = jsonToUInt64(protocolClientReadLine(driver->client));
// Acknowledge command completed
protocolClientReadOutput(driver->client, false);
// Dup path into parent context
// Get path in parent context
MEM_CONTEXT_PRIOR_BEGIN()
{
path = strDup(path);
path = pckReadStrP(result);
}
MEM_CONTEXT_PRIOR_END();
driver->interface.feature = pckReadU64P(result);
}
MEM_CONTEXT_TEMP_END();

47
src/storage/remote/write.c
View File

@ -8,6 +8,7 @@ Remote Storage File write
#include "common/io/write.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "common/type/object.h"
#include "storage/remote/protocol.h"
#include "storage/remote/write.h"
@ -51,9 +52,9 @@ storageWriteRemoteFreeResource(THIS_VOID)
ASSERT(this != NULL);
ioWriteLine(protocolClientIoWrite(this->client), BUFSTRDEF(PROTOCOL_BLOCK_HEADER "-1"));
ioWriteFlush(protocolClientIoWrite(this->client));
protocolClientReadOutput(this->client, false);
protocolClientDataPut(this->client, pckWriteBoolP(protocolPackNew(), false));
protocolClientDataPut(this->client, NULL);
protocolClientDataEndGet(this->client);
FUNCTION_LOG_RETURN_VOID();
}
@ -79,19 +80,22 @@ storageWriteRemoteOpen(THIS_VOID)
ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilter(compressTypeGz));
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE);
protocolCommandParamAdd(command, VARSTR(this->interface.name));
protocolCommandParamAdd(command, VARUINT(this->interface.modeFile));
protocolCommandParamAdd(command, VARUINT(this->interface.modePath));
protocolCommandParamAdd(command, VARSTR(this->interface.user));
protocolCommandParamAdd(command, VARSTR(this->interface.group));
protocolCommandParamAdd(command, VARINT64(this->interface.timeModified));
protocolCommandParamAdd(command, VARBOOL(this->interface.createPath));
protocolCommandParamAdd(command, VARBOOL(this->interface.syncFile));
protocolCommandParamAdd(command, VARBOOL(this->interface.syncPath));
protocolCommandParamAdd(command, VARBOOL(this->interface.atomic));
protocolCommandParamAdd(command, ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write))));
PackWrite *const param = protocolCommandParam(command);
protocolClientExecute(this->client, command, false);
pckWriteStrP(param, this->interface.name);
pckWriteModeP(param, this->interface.modeFile);
pckWriteModeP(param, this->interface.modePath);
pckWriteStrP(param, this->interface.user);
pckWriteStrP(param, this->interface.group);
pckWriteTimeP(param, this->interface.timeModified);
pckWriteBoolP(param, this->interface.createPath);
pckWriteBoolP(param, this->interface.syncFile);
pckWriteBoolP(param, this->interface.syncPath);
pckWriteBoolP(param, this->interface.atomic);
pckWriteStrP(param, jsonFromVar(ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write)))));
protocolClientCommandPut(this->client, command);
protocolClientDataGet(this->client);
// Clear filters since they will be run on the remote side
ioFilterGroupClear(ioWriteFilterGroup(storageWriteIo(this->write)));
@ -128,9 +132,7 @@ storageWriteRemote(THIS_VOID, const Buffer *buffer)
ASSERT(this != NULL);
ASSERT(buffer != NULL);
ioWriteStrLine(protocolClientIoWrite(this->client), strNewFmt(PROTOCOL_BLOCK_HEADER "%zu", bufUsed(buffer)));
ioWrite(protocolClientIoWrite(this->client), buffer);
ioWriteFlush(protocolClientIoWrite(this->client));
protocolClientDataPut(this->client, pckWriteBinP(protocolPackNew(), buffer));
#ifdef DEBUG
this->protocolWriteBytes += bufUsed(buffer);
@ -156,11 +158,12 @@ storageWriteRemoteClose(THIS_VOID)
// Close if the file has not already been closed
if (this->client != NULL)
{
ioWriteLine(protocolClientIoWrite(this->client), BUFSTRDEF(PROTOCOL_BLOCK_HEADER "0"));
ioWriteFlush(protocolClientIoWrite(this->client));
ioFilterGroupResultAllSet(ioWriteFilterGroup(storageWriteIo(this->write)), protocolClientReadOutput(this->client, true));
this->client = NULL;
protocolClientDataPut(this->client, NULL);
ioFilterGroupResultAllSet(
ioWriteFilterGroup(storageWriteIo(this->write)), jsonToVar(pckReadStrP(protocolClientDataGet(this->client))));
protocolClientDataEndGet(this->client);
this->client = NULL;
memContextCallbackClear(this->memContext);
}

2
test/define.yaml
View File

@ -454,7 +454,7 @@ unit:
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: protocol
total: 9
total: 7
harness:
name: protocol
shim:

15
test/src/module/command/backupTest.c
View File

@ -1322,14 +1322,15 @@ testRun(void)
// Create job that skips file
job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ(stringIdBit5, "x")));
VariantList *result = varLstNew();
varLstAdd(result, varNewUInt64(backupCopyResultNoOp));
varLstAdd(result, varNewUInt64(0));
varLstAdd(result, varNewUInt64(0));
varLstAdd(result, NULL);
varLstAdd(result, NULL);
PackWrite *const resultPack = protocolPackNew();
pckWriteU32P(resultPack, backupCopyResultNoOp);
pckWriteU64P(resultPack, 0);
pckWriteU64P(resultPack, 0);
pckWriteStrP(resultPack, NULL);
pckWriteStrP(resultPack, NULL);
pckWriteEndP(resultPack);
protocolParallelJobResultSet(job, varNewVarLst(result));
protocolParallelJobResultSet(job, pckReadNewBuf(pckWriteBuf(resultPack)));
// Create manifest with file
Manifest *manifest = manifestNewInternal();

594
test/src/module/protocol/protocolTest.c
View File

@ -11,102 +11,143 @@ Test Protocol
#include "version.h"
#include "common/harnessConfig.h"
#include "common/harnessError.h"
#include "common/harnessFork.h"
#include "common/harnessPack.h"
/***********************************************************************************************************************************
Test protocol request handler
Test protocol server command handlers
***********************************************************************************************************************************/
static unsigned int testServerProtocolErrorTotal = 0;
#define TEST_PROTOCOL_COMMAND_ASSERT STRID5("assert", 0x2922ce610)
static void
testServerAssertProtocol(const VariantList *paramList, ProtocolServer *server)
__attribute__((__noreturn__)) static void
testCommandAssertProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(paramList == NULL);
ASSERT(param == NULL);
ASSERT(server != NULL);
hrnErrorThrowP();
// No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
}
#define TEST_PROTOCOL_COMMAND_ERROR STRID5("error", 0x127ca450)
__attribute__((__noreturn__)) static void
testCommandErrorProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(server != NULL);
hrnErrorThrowP(.errorType = &FormatError);
// No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
}
#define TEST_PROTOCOL_COMMAND_SIMPLE STRID5("c-simple", 0x2b20d4cf630)
static void
testCommandRequestSimpleProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(param == NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
THROW(AssertError, "test assert");
protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), STRDEF("output")));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID();
}
#define TEST_PROTOCOL_COMMAND_COMPLEX STRID5("c-complex", 0x182b20d78f630)
static void
testServerRequestSimpleProtocol(const VariantList *paramList, ProtocolServer *server)
testCommandRequestComplexProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(paramList == NULL);
ASSERT(param != NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
protocolServerResponse(server, varNewBool(true));
TEST_RESULT_UINT(pckReadU32P(param), 87, "param check");
TEST_RESULT_STR_Z(pckReadStrP(param), "data", "param check");
TEST_RESULT_VOID(protocolServerDataPut(server, NULL), "sync");
TEST_RESULT_BOOL(pckReadBoolP(protocolServerDataGet(server)), true, "data get");
TEST_RESULT_UINT(pckReadModeP(protocolServerDataGet(server)), 0644, "data get");
TEST_RESULT_PTR(protocolServerDataGet(server), NULL, "data end get");
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true)), "data put");
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteI32P(protocolPackNew(), -1)), "data put");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
}
MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID();
}
#define TEST_PROTOCOL_COMMAND_RETRY STRID5("retry", 0x19950b20)
static unsigned int testCommandRetryTotal = 1;
static void
testServerRequestComplexProtocol(const VariantList *paramList, ProtocolServer *server)
testCommandRetryProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
FUNCTION_HARNESS_PARAM(PACK_READ, param);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(paramList == NULL);
ASSERT(param == NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
protocolServerResponse(server, varNewBool(false));
protocolServerWriteLine(server, STRDEF("LINEOFTEXT"));
protocolServerWriteLine(server, NULL);
ioWriteFlush(protocolServerIoWrite(server));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID();
}
static void
testServerErrorUntil0Protocol(const VariantList *paramList, ProtocolServer *server)
if (testCommandRetryTotal > 0)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
FUNCTION_HARNESS_END();
ASSERT(paramList == NULL);
ASSERT(server != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
if (testServerProtocolErrorTotal > 0)
{
testServerProtocolErrorTotal--;
testCommandRetryTotal--;
THROW(FormatError, "error-until-0");
}
protocolServerResponse(server, varNewBool(true));
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true));
protocolServerDataEndPut(server);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_HARNESS_RETURN_VOID();
}
#define TEST_PROTOCOL_SERVER_HANDLER_LIST \
{.command = TEST_PROTOCOL_COMMAND_ASSERT, .handler = testCommandAssertProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_ERROR, .handler = testCommandErrorProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_SIMPLE, .handler = testCommandRequestSimpleProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_COMPLEX, .handler = testCommandRequestComplexProtocol}, \
{.command = TEST_PROTOCOL_COMMAND_RETRY, .handler = testCommandRetryProtocol},
/***********************************************************************************************************************************
Test ParallelJobCallback
***********************************************************************************************************************************/
@ -390,35 +431,7 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("ProtocolCommand"))
{
ProtocolCommand *command = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(command, protocolCommandNew(strIdFromZ(stringIdBit5, "cmd-one")), "create command");
TEST_RESULT_PTR(protocolCommandParamAdd(command, VARSTRDEF("param1")), command, "add param");
TEST_RESULT_PTR(protocolCommandParamAdd(command, VARSTRDEF("param2")), command, "add param");
TEST_RESULT_PTR(protocolCommandMove(command, memContextPrior()), command, "move protocol command");
TEST_RESULT_PTR(protocolCommandMove(NULL, memContextPrior()), NULL, "move null protocol command");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_STR_Z(protocolCommandToLog(command), "{command: cmd-one}", "check log");
TEST_RESULT_STR_Z(protocolCommandJson(command), "{\"cmd\":\"cmd-one\",\"param\":[\"param1\",\"param2\"]}", "check json");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(command, protocolCommandNew(strIdFromZ(stringIdBit5, "cmd2")), "create command");
TEST_RESULT_STR_Z(protocolCommandToLog(command), "{command: cmd2}", "check log");
TEST_RESULT_STR_Z(protocolCommandJson(command), "{\"cmd\":\"cmd2\"}", "check json");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(protocolCommandFree(command), "free command");
}
// *****************************************************************************************************************************
if (testBegin("ProtocolClient"))
if (testBegin("ProtocolClient, ProtocolCommand, and ProtocolServer"))
{
HARNESS_FORK_BEGIN()
{
@ -430,6 +443,7 @@ testRun(void)
ioWriteOpen(write);
// Various bogus greetings
// -----------------------------------------------------------------------------------------------------------------
ioWriteStrLine(write, STRDEF("bogus greeting"));
ioWriteFlush(write);
ioWriteStrLine(write, STRDEF("{\"name\":999}"));
@ -443,56 +457,38 @@ testRun(void)
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"bogus\"}"));
ioWriteFlush(write);
// Correct greeting with noop
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}"));
ioWriteFlush(write);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("server");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop");
ioWriteStrLine(write, STRDEF("{}"));
ioWriteFlush(write);
ProtocolServer *server = NULL;
// Throw errors
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop with error text");
ioWriteStrLine(write, STRDEF("{\"err\":25,\"out\":\"sample error message\",\"errStack\":\"stack data\"}"));
ioWriteFlush(write);
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(
server,
protocolServerMove(
protocolServerNew(STRDEF("test server"), STRDEF("test"), read, write), memContextPrior()),
"new server");
TEST_RESULT_VOID(protocolServerMove(NULL, memContextPrior()), "move null server");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop with no error text");
ioWriteStrLine(write, STRDEF("{\"err\":255}"));
ioWriteFlush(write);
const ProtocolServerHandler commandHandler[] = {TEST_PROTOCOL_SERVER_HANDLER_LIST};
// No output expected
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop with parameters returned");
ioWriteStrLine(write, STRDEF("{\"out\":[\"bogus\"]}"));
ioWriteFlush(write);
// This cannot run in a TEST* macro because tests are run by the command handlers
protocolServerProcess(server, NULL, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler));
// Send output
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"test\"}", "test command");
ioWriteStrLine(write, STRDEF(".OUTPUT"));
ioWriteStrLine(write, STRDEF("{\"out\":[\"value1\",\"value2\"]}"));
ioWriteFlush(write);
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("server with retries");
// invalid line
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"invalid-line\"}", "invalid line command");
ioWrite(write, LF_BUF);
ioWriteFlush(write);
VariantList *retryList = varLstNew();
varLstAdd(retryList, varNewUInt64(0));
// error instead of output
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"err-i-o\"}", "error instead of output command");
ioWriteStrLine(write, STRDEF("{\"err\":255}"));
ioWriteFlush(write);
TEST_ASSIGN(
server, protocolServerNew(STRDEF("test server"), STRDEF("test"), read, write), "new server");
// unexpected output
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"unexp-output\"}", "unexpected output");
ioWriteStrLine(write, STRDEF("{}"));
ioWriteFlush(write);
// invalid prefix
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"i-pr\"}", "invalid prefix");
ioWriteStrLine(write, STRDEF("~line"));
ioWriteFlush(write);
// Wait for exit
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"exit\"}", "exit command");
// This cannot run in a TEST* macro because tests are run by the command handlers
protocolServerProcess(server, retryList, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler));
}
HARNESS_FORK_CHILD_END();
@ -503,7 +499,9 @@ testRun(void)
IoWrite *write = ioFdWriteNew(STRDEF("client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000);
ioWriteOpen(write);
// Various bogus greetings
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("bogus greetings");
TEST_ERROR(
protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), JsonFormatError,
"expected '{' at 'bogus greeting'");
@ -526,7 +524,9 @@ testRun(void)
"expected value '" PROJECT_VERSION "' for greeting key 'version' but got 'bogus'\n"
"HINT: is the same version of " PROJECT_NAME " installed on the local and remote host?");
// Correct greeting
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("new client with successful handshake");
ProtocolClient *client = NULL;
MEM_CONTEXT_TEMP_BEGIN()
@ -535,193 +535,96 @@ testRun(void)
client,
protocolClientMove(
protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), memContextPrior()),
"create client");
"new client");
TEST_RESULT_VOID(protocolClientMove(NULL, memContextPrior()), "move null client");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_PTR(protocolClientIoRead(client), client->pub.read, "get read io");
TEST_RESULT_PTR(protocolClientIoWrite(client), client->pub.write, "get write io");
TEST_RESULT_INT(protocolClientIoReadFd(client), ioReadFd(client->pub.read), "get read fd");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("invalid command");
// Throw errors
TEST_ERROR(
protocolClientNoOp(client), AssertError,
"raised from test client: sample error message\nstack data");
protocolClientExecute(client, protocolCommandNew(strIdFromZ(stringIdBit6, "BOGUS")), false), ProtocolError,
"raised from test client: invalid command 'BOGUS' (0x38eacd271)");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command throws assert");
TEST_ERROR(
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ASSERT), false), AssertError,
"raised from test client: ERR_MESSAGE\nERR_STACK_TRACE");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command throws error");
TEST_ERROR(
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ERROR), false), FormatError,
"raised from test client: ERR_MESSAGE");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command throws error in debug log level");
harnessLogLevelSet(logLevelDebug);
TEST_ERROR(
protocolClientNoOp(client), UnknownError,
"raised from test client: no details available\nno stack trace available");
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ERROR), false), FormatError,
"raised from test client: ERR_MESSAGE\nERR_STACK_TRACE");
harnessLogLevelReset();
// No output expected
TEST_ERROR(protocolClientNoOp(client), AssertError, "no output required by command");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("simple command");
// Get command output
const VariantList *output = NULL;
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "test"))),
"execute command with output");
TEST_RESULT_STR_Z(protocolClientReadLine(client), "OUTPUT", "check output");
TEST_ASSIGN(output, varVarLst(protocolClientReadOutput(client, true)), "execute command with output");
TEST_RESULT_UINT(varLstSize(output), 2, "check output size");
TEST_RESULT_STR_Z(varStr(varLstGet(output, 0)), "value1", "check value1");
TEST_RESULT_STR_Z(varStr(varLstGet(output, 1)), "value2", "check value2");
// Invalid line
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "invalid-line"))),
"execute command that returns invalid line");
TEST_ERROR(protocolClientReadLine(client), FormatError, "unexpected empty line");
// Error instead of output
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "err-i-o"))),
"execute command that returns error instead of output");
TEST_ERROR(protocolClientReadLine(client), UnknownError, "raised from test client: no details available");
// Unexpected output
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "unexp-output"))),
"execute command that returns unexpected output");
TEST_ERROR(protocolClientReadLine(client), FormatError, "expected error but got output");
// Invalid prefix
TEST_RESULT_VOID(
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "i-pr"))),
"execute command that returns an invalid prefix");
TEST_ERROR(protocolClientReadLine(client), FormatError, "invalid prefix in '~line'");
// Free client
TEST_RESULT_VOID(protocolClientFree(client), "free client");
}
HARNESS_FORK_PARENT_END();
}
HARNESS_FORK_END();
}
// *****************************************************************************************************************************
if (testBegin("ProtocolServer"))
{
HARNESS_FORK_BEGIN()
{
HARNESS_FORK_CHILD_BEGIN(0, true)
{
IoRead *read = ioFdReadNew(STRDEF("client read"), HARNESS_FORK_CHILD_READ(), 2000);
ioReadOpen(read);
IoWrite *write = ioFdWriteNew(STRDEF("client write"), HARNESS_FORK_CHILD_WRITE(), 2000);
ioWriteOpen(write);
// Check greeting
TEST_RESULT_STR_Z(
ioReadLine(read), "{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}",
"check greeting");
// Noop
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"noop\"}")), "write noop");
TEST_RESULT_VOID(ioWriteFlush(write), "flush noop");
TEST_RESULT_STR_Z(ioReadLine(read), "{}", "noop result");
// Invalid command
KeyValue *result = NULL;
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"bogus\"}")), "write bogus");
TEST_RESULT_VOID(ioWriteFlush(write), "flush bogus");
TEST_ASSIGN(result, varKv(jsonToVar(ioReadLine(read))), "parse error result");
TEST_RESULT_INT(varIntForce(kvGet(result, VARSTRDEF("err"))), 39, " check code");
TEST_RESULT_STR_Z(
varStr(kvGet(result, VARSTRDEF("out"))), "invalid command 'bogus' (0x13a9de20)", " check message");
TEST_RESULT_BOOL(kvGet(result, VARSTRDEF("errStack")) != NULL, true, " check stack exists");
// Simple request
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"r-s\"}")), "write simple request");
TEST_RESULT_VOID(ioWriteFlush(write), "flush simple request");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"out\":true}", "simple request result");
// Throw an assert error which will include a stack trace
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"assert\"}")), "write assert");
TEST_RESULT_VOID(ioWriteFlush(write), "flush assert error");
TEST_ASSIGN(result, varKv(jsonToVar(ioReadLine(read))), "parse error result");
TEST_RESULT_INT(varIntForce(kvGet(result, VARSTRDEF("err"))), 25, " check code");
TEST_RESULT_STR_Z(varStr(kvGet(result, VARSTRDEF("out"))), "test assert", " check message");
TEST_RESULT_BOOL(kvGet(result, VARSTRDEF("errStack")) != NULL, true, " check stack exists");
// Complex request -- after process loop has been restarted
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"r-c\"}")), "write complex request");
TEST_RESULT_VOID(ioWriteFlush(write), "flush complex request");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"out\":false}", "complex request result");
TEST_RESULT_STR_Z(ioReadLine(read), ".LINEOFTEXT", "complex request result");
TEST_RESULT_STR_Z(ioReadLine(read), ".", "complex request result");
// Exit
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"exit\"}")), "write exit");
TEST_RESULT_VOID(ioWriteFlush(write), "flush exit");
// Retry errors until success
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"ezero\"}")), "write error-until-0");
TEST_RESULT_VOID(ioWriteFlush(write), "flush error-until-0");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"out\":true}", "error-until-0 result");
// Exit
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"exit\"}")), "write exit");
TEST_RESULT_VOID(ioWriteFlush(write), "flush exit");
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN()
{
IoRead *read = ioFdReadNew(STRDEF("server read"), HARNESS_FORK_PARENT_READ_PROCESS(0), 2000);
ioReadOpen(read);
IoWrite *write = ioFdWriteNew(STRDEF("server write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000);
ioWriteOpen(write);
// Send greeting
ProtocolServer *server = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(
server,
protocolServerMove(
protocolServerNew(STRDEF("test server"), STRDEF("test"), read, write), memContextPrior()),
"create server");
TEST_RESULT_VOID(protocolServerMove(NULL, memContextPrior()), "move null server");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_PTR(protocolServerIoRead(server), server->pub.read, "get read io");
TEST_RESULT_PTR(protocolServerIoWrite(server), server->pub.write, "get write io");
ProtocolServerHandler commandHandler[] =
{
{.command = strIdFromZ(stringIdBit5, "assert"), .handler = testServerAssertProtocol},
{.command = strIdFromZ(stringIdBit5, "r-s"), .handler = testServerRequestSimpleProtocol},
{.command = strIdFromZ(stringIdBit5, "r-c"), .handler = testServerRequestComplexProtocol},
{.command = strIdFromZ(stringIdBit5, "ezero"), .handler = testServerErrorUntil0Protocol},
};
TEST_RESULT_VOID(
protocolServerProcess(server, NULL, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler)),
"run process loop");
pckReadStrP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_SIMPLE), true)), "output",
"execute");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("run process loop with retries");
TEST_TITLE("complex command");
VariantList *retryInterval = varLstNew();
varLstAdd(retryInterval, varNewUInt64(0));
varLstAdd(retryInterval, varNewUInt64(50));
// Put the command to the server
ProtocolCommand *command = NULL;
TEST_ASSIGN(command, protocolCommandNew(TEST_PROTOCOL_COMMAND_COMPLEX), "command");
TEST_RESULT_VOID(pckWriteU32P(protocolCommandParam(command), 87), "param");
TEST_RESULT_VOID(pckWriteStrP(protocolCommandParam(command), STRDEF("data")), "param");
TEST_RESULT_VOID(protocolClientCommandPut(client, command), "command put");
testServerProtocolErrorTotal = 2;
// Read null data to indicate that the server has started the command and is read to receive data
TEST_RESULT_PTR(protocolClientDataGet(client), NULL, "command started and ready for data");
TEST_RESULT_VOID(
protocolServerProcess(server, retryInterval, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler)),
"run process loop");
// Write data to the server
TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteBoolP(protocolPackNew(), true)), "data put");
TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteModeP(protocolPackNew(), 0644)), "data put");
TEST_RESULT_VOID(protocolClientDataPut(client, NULL), "data end put");
// Get data from the server
TEST_RESULT_BOOL(pckReadBoolP(protocolClientDataGet(client)), true, "data get");
TEST_RESULT_INT(pckReadI32P(protocolClientDataGet(client)), -1, "data get");
TEST_RESULT_VOID(protocolClientDataEndGet(client), "data end get");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("free server");
TEST_TITLE("free client");
TEST_RESULT_VOID(protocolServerFree(server), "free");
TEST_RESULT_VOID(protocolClientFree(client), "free");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("new client with server retries");
TEST_ASSIGN(client, protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), "new client");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("command with retry");
TEST_RESULT_BOOL(
pckReadBoolP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_RETRY), true)), true,
"execute");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("free client");
TEST_RESULT_VOID(protocolClientFree(client), "free");
}
HARNESS_FORK_PARENT_END();
}
@ -731,6 +634,8 @@ testRun(void)
// *****************************************************************************************************************************
if (testBegin("ProtocolParallel and ProtocolParallelJob"))
{
TEST_TITLE("job state transitions");
ProtocolParallelJob *job = NULL;
MEM_CONTEXT_TEMP_BEGIN()
@ -760,63 +665,63 @@ testRun(void)
// Local 1
HARNESS_FORK_CHILD_BEGIN(0, true)
{
IoRead *read = ioFdReadNew(STRDEF("server read"), HARNESS_FORK_CHILD_READ(), 10000);
IoRead *read = ioFdReadNew(STRDEF("local server 1 read"), HARNESS_FORK_CHILD_READ(), 10000);
ioReadOpen(read);
IoWrite *write = ioFdWriteNew(STRDEF("server write"), HARNESS_FORK_CHILD_WRITE(), 2000);
IoWrite *write = ioFdWriteNew(STRDEF("local server 1 write"), HARNESS_FORK_CHILD_WRITE(), 2000);
ioWriteOpen(write);
// Greeting with noop
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}"));
ioWriteFlush(write);
ProtocolServer *server = NULL;
TEST_ASSIGN(server, protocolServerNew(STRDEF("local server 1"), STRDEF("test"), read, write), "local server 1");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop");
ioWriteStrLine(write, STRDEF("{}"));
ioWriteFlush(write);
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_NOOP, "noop command get");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
// Command with output
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c-one"), "c-one command get");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"c-one\",\"param\":[\"param1\",\"param2\"]}", "command1");
sleepMSec(4000);
ioWriteStrLine(write, STRDEF("{\"out\":1}"));
ioWriteFlush(write);
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 1)), "data end put");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
// Wait for exit
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"exit\"}", "exit command");
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "noop command get");
}
HARNESS_FORK_CHILD_END();
// Local 2
HARNESS_FORK_CHILD_BEGIN(0, true)
{
IoRead *read = ioFdReadNew(STRDEF("server read"), HARNESS_FORK_CHILD_READ(), 10000);
IoRead *read = ioFdReadNew(STRDEF("local server 2 read"), HARNESS_FORK_CHILD_READ(), 10000);
ioReadOpen(read);
IoWrite *write = ioFdWriteNew(STRDEF("server write"), HARNESS_FORK_CHILD_WRITE(), 2000);
IoWrite *write = ioFdWriteNew(STRDEF("local server 2 write"), HARNESS_FORK_CHILD_WRITE(), 2000);
ioWriteOpen(write);
// Greeting with noop
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}"));
ioWriteFlush(write);
ProtocolServer *server = NULL;
TEST_ASSIGN(server, protocolServerNew(STRDEF("local server 2"), STRDEF("test"), read, write), "local server 2");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop");
ioWriteStrLine(write, STRDEF("{}"));
ioWriteFlush(write);
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_NOOP, "noop command get");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
// Command with output
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c2"), "c2 command get");
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"c2\",\"param\":[\"param1\"]}", "command2");
sleepMSec(1000);
ioWriteStrLine(write, STRDEF("{\"out\":2}"));
ioWriteFlush(write);
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"c-three\",\"param\":[\"param1\"]}", "command3");
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 2)), "data end put");
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
ioWriteStrLine(write, STRDEF("{\"err\":39,\"out\":\"very serious error\"}"));
ioWriteFlush(write);
// Command with error
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c-three"), "c-three command get");
TEST_RESULT_VOID(protocolServerError(server, 39, STRDEF("very serious error"), STRDEF("stack")), "error put");
// Wait for exit
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"exit\"}", "exit command");
CHECK(protocolServerCommandGet(server).id == PROTOCOL_COMMAND_EXIT);
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN()
{
// -----------------------------------------------------------------------------------------------------------------
TestParallelJobCallback data = {.jobList = lstNewP(sizeof(ProtocolParallelJob *))};
ProtocolParallel *parallel = NULL;
TEST_ASSIGN(parallel, protocolParallelNew(2000, testParallelJobCallback, &data), "create parallel");
@ -829,57 +734,59 @@ testRun(void)
for (unsigned int clientIdx = 0; clientIdx < clientTotal; clientIdx++)
{
IoRead *read = ioFdReadNew(
strNewFmt("client %u read", clientIdx), HARNESS_FORK_PARENT_READ_PROCESS(clientIdx), 2000);
strNewFmt("local client %u read", clientIdx), HARNESS_FORK_PARENT_READ_PROCESS(clientIdx), 2000);
ioReadOpen(read);
IoWrite *write = ioFdWriteNew(
strNewFmt("client %u write", clientIdx), HARNESS_FORK_PARENT_WRITE_PROCESS(clientIdx), 2000);
strNewFmt("local client %u write", clientIdx), HARNESS_FORK_PARENT_WRITE_PROCESS(clientIdx), 2000);
ioWriteOpen(write);
TEST_ASSIGN(
client[clientIdx],
protocolClientNew(strNewFmt("test client %u", clientIdx), STRDEF("test"), read, write),
strZ(strNewFmt("create client %u", clientIdx)));
protocolClientNew(strNewFmt("local client %u", clientIdx), STRDEF("test"), read, write),
strZ(strNewFmt("local client %u new", clientIdx)));
TEST_RESULT_VOID(
protocolParallelClientAdd(parallel, client[clientIdx]), strZ(strNewFmt("add client %u", clientIdx)));
protocolParallelClientAdd(parallel, client[clientIdx]), strZ(strNewFmt("local client %u add", clientIdx)));
}
// Attempt to add client without an fd
const String *protocolString = STRDEF(
"{\"name\":\"pgBackRest\",\"service\":\"error\",\"version\":\"" PROJECT_VERSION "\"}\n"
"{}\n");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on add without an fd");
IoRead *read = ioBufferReadNew(BUFSTR(protocolString));
ioReadOpen(read);
IoWrite *write = ioBufferWriteNew(bufNew(1024));
ioWriteOpen(write);
// Fake a client without a read fd
ProtocolClient clientError = {.pub = {.read = ioBufferReadNew(bufNew(0))}, .name = STRDEF("test")};
ProtocolClient *clientError = protocolClientNew(STRDEF("error"), STRDEF("error"), read, write);
TEST_ERROR(protocolParallelClientAdd(parallel, clientError), AssertError, "client with read fd is required");
protocolClientFree(clientError);
TEST_ERROR(protocolParallelClientAdd(parallel, &clientError), AssertError, "client with read fd is required");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("add jobs");
// Add jobs
ProtocolCommand *command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-one"));
protocolCommandParamAdd(command, VARSTRDEF("param1"));
protocolCommandParamAdd(command, VARSTRDEF("param2"));
ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("job1"), command);
pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
pckWriteStrP(protocolCommandParam(command), STRDEF("param2"));
ProtocolParallelJob *job = protocolParallelJobNew(varNewStr(STRDEF("job1")), command);
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
command = protocolCommandNew(strIdFromZ(stringIdBit5, "c2"));
protocolCommandParamAdd(command, VARSTRDEF("param1"));
job = protocolParallelJobNew(VARSTRDEF("job2"), command);
pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
job = protocolParallelJobNew(varNewStr(STRDEF("job2")), command);
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-three"));
protocolCommandParamAdd(command, VARSTRDEF("param1"));
job = protocolParallelJobNew(VARSTRDEF("job3"), command);
pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
job = protocolParallelJobNew(varNewStr(STRDEF("job3")), command);
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
// Process jobs
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("process jobs with no result");
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
// Process jobs
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("result for job 2");
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
@ -887,35 +794,40 @@ testRun(void)
TEST_RESULT_BOOL(
protocolParallelJobProcessId(job) >= 1 && protocolParallelJobProcessId(job) <= 2, true,
"check process id is valid");
TEST_RESULT_INT(varIntForce(protocolParallelJobResult(job)), 2, "check result is 2");
TEST_RESULT_UINT(pckReadU32P(protocolParallelJobResult(job)), 2, "check result is 2");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
// Process jobs
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("error for job 3");
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job3", "check key is job3");
TEST_RESULT_INT(protocolParallelJobErrorCode(job), 39, "check error code");
TEST_RESULT_STR_Z(
protocolParallelJobErrorMessage(job), "raised from test client 1: very serious error",
protocolParallelJobErrorMessage(job), "raised from local client 1: very serious error",
"check error message");
TEST_RESULT_PTR(protocolParallelJobResult(job), NULL, "check result is null");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
// Process jobs
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("process jobs with no result");
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
TEST_RESULT_BOOL(protocolParallelDone(parallel), false, "check not done");
// Process jobs
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("result for job 1");
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job1", "check key is job1");
TEST_RESULT_INT(varIntForce(protocolParallelJobResult(job)), 1, "check result is 1");
TEST_RESULT_UINT(pckReadU32P(protocolParallelJobResult(job)), 1, "check result is 1");
TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check done");
TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check still done");

3
test/src/module/storage/remoteTest.c
View File

@ -279,9 +279,6 @@ testRun(void)
((StorageReadRemote *)fileRead->driver)->protocolReadBytes < bufSize(contentBuf), true,
" check compressed read size");
TEST_ERROR(
storageRemoteProtocolBlockSize(STRDEF("bogus")), ProtocolError, "'bogus' is not a valid block size message");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("file missing");