You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-15 01:04:37 +02:00
Binary protocol.
Switch from JSON-based to binary protocol for communicating with local and remote process. The pack type is used to implement the binary protocol. There are a number advantages: * The pack type is more compact than JSON and are more efficient to render/parse. * Packs are more strictly typed than JSON. * Each protocol message is written entirely within ProtocolServer/ProtocolClient so is less likely to get interrupted by an error and leave the protocol in a bad state. * There is no limit on message size. Previously this was limited by buffer size without a custom implementation, as was done for read/writing files. Some cruft from the Perl days was removed, specifically allowing NULL messages and stack traces. This is no longer possible in C. There is room for improvement here, in particular locking down the allowed sequence of protocol messages and building a state machine to enforce it. This will be useful for resetting the protocol when it gets in a bad state.
This commit is contained in:
@ -15,6 +15,17 @@
|
|||||||
<release date="XXXX-XX-XX" version="2.35dev" title="UNDER DEVELOPMENT">
|
<release date="XXXX-XX-XX" version="2.35dev" title="UNDER DEVELOPMENT">
|
||||||
<release-core-list>
|
<release-core-list>
|
||||||
<release-development-list>
|
<release-development-list>
|
||||||
|
<release-item>
|
||||||
|
<github-pull-request id="1424"/>
|
||||||
|
|
||||||
|
<release-item-contributor-list>
|
||||||
|
<release-item-contributor id="david.steele"/>
|
||||||
|
<release-item-reviewer id="cynthia.shang"/>
|
||||||
|
</release-item-contributor-list>
|
||||||
|
|
||||||
|
<p>Binary protocol.</p>
|
||||||
|
</release-item>
|
||||||
|
|
||||||
<release-item>
|
<release-item>
|
||||||
<github-pull-request id="1421"/>
|
<github-pull-request id="1421"/>
|
||||||
|
|
||||||
|
@ -851,19 +851,21 @@ static ProtocolParallelJob *archiveGetAsyncCallback(void *data, unsigned int cli
|
|||||||
const ArchiveFileMap *archiveFileMap = lstGet(jobData->archiveFileMapList, jobData->archiveFileIdx);
|
const ArchiveFileMap *archiveFileMap = lstGet(jobData->archiveFileMapList, jobData->archiveFileIdx);
|
||||||
jobData->archiveFileIdx++;
|
jobData->archiveFileIdx++;
|
||||||
|
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_GET_FILE);
|
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_GET_FILE);
|
||||||
protocolCommandParamAdd(command, VARSTR(archiveFileMap->request));
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
|
|
||||||
|
pckWriteStrP(param, archiveFileMap->request);
|
||||||
|
|
||||||
// Add actual files to get
|
// Add actual files to get
|
||||||
for (unsigned int actualIdx = 0; actualIdx < lstSize(archiveFileMap->actualList); actualIdx++)
|
for (unsigned int actualIdx = 0; actualIdx < lstSize(archiveFileMap->actualList); actualIdx++)
|
||||||
{
|
{
|
||||||
const ArchiveGetFile *actual = lstGet(archiveFileMap->actualList, actualIdx);
|
const ArchiveGetFile *const actual = lstGet(archiveFileMap->actualList, actualIdx);
|
||||||
|
|
||||||
protocolCommandParamAdd(command, VARSTR(actual->file));
|
pckWriteStrP(param, actual->file);
|
||||||
protocolCommandParamAdd(command, VARUINT(actual->repoIdx));
|
pckWriteU32P(param, actual->repoIdx);
|
||||||
protocolCommandParamAdd(command, VARSTR(actual->archiveId));
|
pckWriteStrP(param, actual->archiveId);
|
||||||
protocolCommandParamAdd(command, VARUINT64(actual->cipherType));
|
pckWriteU64P(param, actual->cipherType);
|
||||||
protocolCommandParamAdd(command, VARSTR(actual->cipherPassArchive));
|
pckWriteStrP(param, actual->cipherPassArchive);
|
||||||
}
|
}
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(archiveFileMap->request), command));
|
FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(archiveFileMap->request), command));
|
||||||
@ -941,12 +943,12 @@ cmdArchiveGetAsync(void)
|
|||||||
if (protocolParallelJobErrorCode(job) == 0)
|
if (protocolParallelJobErrorCode(job) == 0)
|
||||||
{
|
{
|
||||||
// Get the actual file retrieved
|
// Get the actual file retrieved
|
||||||
const VariantList *fileResult = varVarLst(protocolParallelJobResult(job));
|
PackRead *const fileResult = protocolParallelJobResult(job);
|
||||||
ArchiveGetFile *file = lstGet(fileMap->actualList, varUIntForce(varLstGet(fileResult, 0)));
|
ArchiveGetFile *file = lstGet(fileMap->actualList, pckReadU32P(fileResult));
|
||||||
ASSERT(file != NULL);
|
ASSERT(file != NULL);
|
||||||
|
|
||||||
// Output file warnings
|
// Output file warnings
|
||||||
StringList *fileWarnList = strLstNewVarLst(varVarLst(varLstGet(fileResult, 1)));
|
StringList *fileWarnList = pckReadStrLstP(fileResult);
|
||||||
|
|
||||||
for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++)
|
for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++)
|
||||||
LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx)));
|
LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx)));
|
||||||
|
@ -15,54 +15,47 @@ Archive Get Protocol Handler
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
archiveGetFileProtocol(const VariantList *paramList, ProtocolServer *server)
|
archiveGetFileProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
const String *request = varStr(varLstGet(paramList, 0));
|
// Get request
|
||||||
|
const String *const request = pckReadStrP(param);
|
||||||
const unsigned int paramFixed = 1; // Fixed params before the actual list
|
|
||||||
const unsigned int paramActual = 5; // Parameters in each index of the actual list
|
|
||||||
|
|
||||||
// Check that the correct number of list parameters were passed
|
|
||||||
CHECK((varLstSize(paramList) - paramFixed) % paramActual == 0);
|
|
||||||
|
|
||||||
// Build the actual list
|
// Build the actual list
|
||||||
List *actualList = lstNewP(sizeof(ArchiveGetFile));
|
List *actualList = lstNewP(sizeof(ArchiveGetFile));
|
||||||
unsigned int actualListSize = (varLstSize(paramList) - paramFixed) / paramActual;
|
|
||||||
|
|
||||||
for (unsigned int actualIdx = 0; actualIdx < actualListSize; actualIdx++)
|
while (!pckReadNullP(param))
|
||||||
{
|
{
|
||||||
lstAdd(
|
ArchiveGetFile actual = {.file = pckReadStrP(param)};
|
||||||
actualList,
|
actual.repoIdx = pckReadU32P(param);
|
||||||
&(ArchiveGetFile)
|
actual.archiveId = pckReadStrP(param);
|
||||||
{
|
actual.cipherType = pckReadU64P(param);
|
||||||
.file = varStr(varLstGet(paramList, paramFixed + (actualIdx * paramActual))),
|
actual.cipherPassArchive = pckReadStrP(param);
|
||||||
.repoIdx = varUIntForce(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 1)),
|
|
||||||
.archiveId = varStr(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 2)),
|
lstAdd(actualList, &actual);
|
||||||
.cipherType = varUInt64(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 3)),
|
|
||||||
.cipherPassArchive = varStr(varLstGet(paramList, paramFixed + (actualIdx * paramActual) + 4)),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return result
|
// Get file
|
||||||
ArchiveGetFileResult fileResult = archiveGetFile(
|
ArchiveGetFileResult fileResult = archiveGetFile(
|
||||||
storageSpoolWrite(), request, actualList,
|
storageSpoolWrite(), request, actualList,
|
||||||
strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s." STORAGE_FILE_TEMP_EXT, strZ(request)));
|
strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s." STORAGE_FILE_TEMP_EXT, strZ(request)));
|
||||||
|
|
||||||
VariantList *result = varLstNew();
|
// Return result
|
||||||
varLstAdd(result, varNewUInt(fileResult.actualIdx));
|
PackWrite *const resultPack = protocolPackNew();
|
||||||
varLstAdd(result, varNewVarLst(varLstNewStrLst(fileResult.warnList)));
|
pckWriteU32P(resultPack, fileResult.actualIdx);
|
||||||
|
pckWriteStrLstP(resultPack, fileResult.warnList);
|
||||||
|
|
||||||
protocolServerResponse(server, varNewVarLst(result));
|
protocolServerDataPut(server, resultPack);
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
@ -4,16 +4,14 @@ Archive Get Protocol Handler
|
|||||||
#ifndef COMMAND_ARCHIVE_GET_PROTOCOL_H
|
#ifndef COMMAND_ARCHIVE_GET_PROTOCOL_H
|
||||||
#define COMMAND_ARCHIVE_GET_PROTOCOL_H
|
#define COMMAND_ARCHIVE_GET_PROTOCOL_H
|
||||||
|
|
||||||
#include "common/type/string.h"
|
#include "common/type/pack.h"
|
||||||
#include "common/type/stringId.h"
|
|
||||||
#include "common/type/variantList.h"
|
|
||||||
#include "protocol/server.h"
|
#include "protocol/server.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
// Process protocol requests
|
// Process protocol requests
|
||||||
void archiveGetFileProtocol(const VariantList *paramList, ProtocolServer *server);
|
void archiveGetFileProtocol(PackRead *param, ProtocolServer *server);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
||||||
|
@ -14,50 +14,55 @@ Archive Push Protocol Handler
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server)
|
archivePushFileProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
// Build the repo data list
|
// Read parameters
|
||||||
|
const String *const walSource = pckReadStrP(param);
|
||||||
|
const bool headerCheck = pckReadBoolP(param);
|
||||||
|
const unsigned int pgVersion = pckReadU32P(param);
|
||||||
|
const uint64_t pgSystemId = pckReadU64P(param);
|
||||||
|
const String *const archiveFile = pckReadStrP(param);
|
||||||
|
const CompressType compressType = pckReadU32P(param);
|
||||||
|
const int compressLevel = pckReadI32P(param);
|
||||||
|
const StringList *const priorErrorList = pckReadStrLstP(param);
|
||||||
|
|
||||||
|
// Read repo data
|
||||||
List *repoList = lstNewP(sizeof(ArchivePushFileRepoData));
|
List *repoList = lstNewP(sizeof(ArchivePushFileRepoData));
|
||||||
unsigned int repoListSize = varUIntForce(varLstGet(paramList, 8));
|
|
||||||
unsigned int paramIdx = 9;
|
|
||||||
|
|
||||||
for (unsigned int repoListIdx = 0; repoListIdx < repoListSize; repoListIdx++)
|
pckReadArrayBeginP(param);
|
||||||
|
|
||||||
|
while (!pckReadNullP(param))
|
||||||
{
|
{
|
||||||
lstAdd(
|
pckReadObjBeginP(param);
|
||||||
repoList,
|
|
||||||
&(ArchivePushFileRepoData)
|
|
||||||
{
|
|
||||||
.repoIdx = varUIntForce(varLstGet(paramList, paramIdx)),
|
|
||||||
.archiveId = varStr(varLstGet(paramList, paramIdx + 1)),
|
|
||||||
.cipherType = varUInt64(varLstGet(paramList, paramIdx + 2)),
|
|
||||||
.cipherPass = varStr(varLstGet(paramList, paramIdx + 3)),
|
|
||||||
});
|
|
||||||
|
|
||||||
paramIdx += 4;
|
ArchivePushFileRepoData repo = {.repoIdx = pckReadU32P(param)};
|
||||||
|
repo.archiveId = pckReadStrP(param);
|
||||||
|
repo.cipherType = pckReadU64P(param);
|
||||||
|
repo.cipherPass = pckReadStrP(param);
|
||||||
|
pckReadObjEndP(param);
|
||||||
|
|
||||||
|
lstAdd(repoList, &repo);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Push the file
|
pckReadArrayEndP(param);
|
||||||
ArchivePushFileResult fileResult = archivePushFile(
|
|
||||||
varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUIntForce(varLstGet(paramList, 2)),
|
// Push file
|
||||||
varUInt64(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)),
|
const ArchivePushFileResult fileResult = archivePushFile(
|
||||||
(CompressType)varUIntForce(varLstGet(paramList, 5)), varIntForce(varLstGet(paramList, 6)), repoList,
|
walSource, headerCheck, pgVersion, pgSystemId, archiveFile, compressType, compressLevel, repoList, priorErrorList);
|
||||||
strLstNewVarLst(varVarLst(varLstGet(paramList, 7))));
|
|
||||||
|
|
||||||
// Return result
|
// Return result
|
||||||
VariantList *result = varLstNew();
|
protocolServerDataPut(server, pckWriteStrLstP(protocolPackNew(), fileResult.warnList));
|
||||||
varLstAdd(result, varNewVarLst(varLstNewStrLst(fileResult.warnList)));
|
protocolServerDataEndPut(server);
|
||||||
|
|
||||||
protocolServerResponse(server, varNewVarLst(result));
|
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
@ -4,15 +4,14 @@ Archive Push Protocol Handler
|
|||||||
#ifndef COMMAND_ARCHIVE_PUSH_PROTOCOL_H
|
#ifndef COMMAND_ARCHIVE_PUSH_PROTOCOL_H
|
||||||
#define COMMAND_ARCHIVE_PUSH_PROTOCOL_H
|
#define COMMAND_ARCHIVE_PUSH_PROTOCOL_H
|
||||||
|
|
||||||
#include "common/type/string.h"
|
#include "common/type/pack.h"
|
||||||
#include "common/type/variantList.h"
|
|
||||||
#include "protocol/server.h"
|
#include "protocol/server.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
// Process protocol requests
|
// Process protocol requests
|
||||||
void archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server);
|
void archivePushFileProtocol(PackRead *param, ProtocolServer *server);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
||||||
|
@ -460,28 +460,35 @@ archivePushAsyncCallback(void *data, unsigned int clientIdx)
|
|||||||
const String *walFile = strLstGet(jobData->walFileList, jobData->walFileIdx);
|
const String *walFile = strLstGet(jobData->walFileList, jobData->walFileIdx);
|
||||||
jobData->walFileIdx++;
|
jobData->walFileIdx++;
|
||||||
|
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE);
|
ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE);
|
||||||
protocolCommandParamAdd(command, VARSTR(strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile))));
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptArchiveHeaderCheck)));
|
|
||||||
protocolCommandParamAdd(command, VARUINT(jobData->archiveInfo.pgVersion));
|
pckWriteStrP(param, strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile)));
|
||||||
protocolCommandParamAdd(command, VARUINT64(jobData->archiveInfo.pgSystemId));
|
pckWriteBoolP(param, cfgOptionBool(cfgOptArchiveHeaderCheck));
|
||||||
protocolCommandParamAdd(command, VARSTR(walFile));
|
pckWriteU32P(param, jobData->archiveInfo.pgVersion);
|
||||||
protocolCommandParamAdd(command, VARUINT(jobData->compressType));
|
pckWriteU64P(param, jobData->archiveInfo.pgSystemId);
|
||||||
protocolCommandParamAdd(command, VARINT(jobData->compressLevel));
|
pckWriteStrP(param, walFile);
|
||||||
protocolCommandParamAdd(command, varNewVarLst(varLstNewStrLst(jobData->archiveInfo.errorList)));
|
pckWriteU32P(param, jobData->compressType);
|
||||||
protocolCommandParamAdd(command, VARUINT(lstSize(jobData->archiveInfo.repoList)));
|
pckWriteI32P(param, jobData->compressLevel);
|
||||||
|
pckWriteStrLstP(param, jobData->archiveInfo.errorList);
|
||||||
|
|
||||||
// Add data for each repo to push to
|
// Add data for each repo to push to
|
||||||
|
pckWriteArrayBeginP(param);
|
||||||
|
|
||||||
for (unsigned int repoListIdx = 0; repoListIdx < lstSize(jobData->archiveInfo.repoList); repoListIdx++)
|
for (unsigned int repoListIdx = 0; repoListIdx < lstSize(jobData->archiveInfo.repoList); repoListIdx++)
|
||||||
{
|
{
|
||||||
ArchivePushFileRepoData *data = lstGet(jobData->archiveInfo.repoList, repoListIdx);
|
ArchivePushFileRepoData *data = lstGet(jobData->archiveInfo.repoList, repoListIdx);
|
||||||
|
|
||||||
protocolCommandParamAdd(command, VARUINT(data->repoIdx));
|
pckWriteObjBeginP(param);
|
||||||
protocolCommandParamAdd(command, VARSTR(data->archiveId));
|
pckWriteU32P(param, data->repoIdx);
|
||||||
protocolCommandParamAdd(command, VARUINT64(data->cipherType));
|
pckWriteStrP(param, data->archiveId);
|
||||||
protocolCommandParamAdd(command, VARSTR(data->cipherPass));
|
pckWriteU64P(param, data->cipherType);
|
||||||
|
pckWriteStrP(param, data->cipherPass);
|
||||||
|
pckWriteObjEndP(param);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pckWriteArrayEndP(param);
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(walFile), command));
|
FUNCTION_TEST_RETURN(protocolParallelJobNew(VARSTR(walFile), command));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -572,11 +579,8 @@ cmdArchivePushAsync(void)
|
|||||||
// The job was successful
|
// The job was successful
|
||||||
if (protocolParallelJobErrorCode(job) == 0)
|
if (protocolParallelJobErrorCode(job) == 0)
|
||||||
{
|
{
|
||||||
// Get job result
|
|
||||||
const VariantList *fileResult = varVarLst(protocolParallelJobResult(job));
|
|
||||||
|
|
||||||
// Output file warnings
|
// Output file warnings
|
||||||
StringList *fileWarnList = strLstNewVarLst(varVarLst(varLstGet(fileResult, 0)));
|
StringList *fileWarnList = pckReadStrLstP(protocolParallelJobResult(job));
|
||||||
|
|
||||||
for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++)
|
for (unsigned int warnIdx = 0; warnIdx < strLstSize(fileWarnList); warnIdx++)
|
||||||
LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx)));
|
LOG_WARN_PID(processId, strZ(strLstGet(fileWarnList, warnIdx)));
|
||||||
|
@ -23,6 +23,7 @@ Backup Command
|
|||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/time.h"
|
#include "common/time.h"
|
||||||
#include "common/type/convert.h"
|
#include "common/type/convert.h"
|
||||||
|
#include "common/type/json.h"
|
||||||
#include "config/config.h"
|
#include "config/config.h"
|
||||||
#include "db/helper.h"
|
#include "db/helper.h"
|
||||||
#include "info/infoArchive.h"
|
#include "info/infoArchive.h"
|
||||||
@ -1043,12 +1044,12 @@ backupJobResult(
|
|||||||
const ManifestFile *const file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
|
const ManifestFile *const file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
|
||||||
const unsigned int processId = protocolParallelJobProcessId(job);
|
const unsigned int processId = protocolParallelJobProcessId(job);
|
||||||
|
|
||||||
const VariantList *const jobResult = varVarLst(protocolParallelJobResult(job));
|
PackRead *const jobResult = protocolParallelJobResult(job);
|
||||||
const BackupCopyResult copyResult = (BackupCopyResult)varUIntForce(varLstGet(jobResult, 0));
|
const BackupCopyResult copyResult = (BackupCopyResult)pckReadU32P(jobResult);
|
||||||
const uint64_t copySize = varUInt64(varLstGet(jobResult, 1));
|
const uint64_t copySize = pckReadU64P(jobResult);
|
||||||
const uint64_t repoSize = varUInt64(varLstGet(jobResult, 2));
|
const uint64_t repoSize = pckReadU64P(jobResult);
|
||||||
const String *const copyChecksum = varStr(varLstGet(jobResult, 3));
|
const String *const copyChecksum = pckReadStrP(jobResult);
|
||||||
const KeyValue *const checksumPageResult = varKv(varLstGet(jobResult, 4));
|
const KeyValue *const checksumPageResult = varKv(jsonToVar(pckReadStrP(jobResult, .defaultValue = NULL_STR)));
|
||||||
|
|
||||||
// Increment backup copy progress
|
// Increment backup copy progress
|
||||||
sizeCopied += copySize;
|
sizeCopied += copySize;
|
||||||
@ -1439,23 +1440,23 @@ static ProtocolParallelJob *backupJobCallback(void *data, unsigned int clientIdx
|
|||||||
|
|
||||||
// Create backup job
|
// Create backup job
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
|
||||||
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
|
|
||||||
protocolCommandParamAdd(command, VARSTR(manifestPathPg(file->name)));
|
pckWriteStrP(param, manifestPathPg(file->name));
|
||||||
protocolCommandParamAdd(
|
pckWriteBoolP(param, !strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)));
|
||||||
command, VARBOOL(!strEq(file->name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))));
|
pckWriteU64P(param, file->size);
|
||||||
protocolCommandParamAdd(command, VARUINT64(file->size));
|
pckWriteBoolP(param, !file->primary);
|
||||||
protocolCommandParamAdd(command, VARBOOL(!file->primary));
|
pckWriteStrP(param, file->checksumSha1[0] != 0 ? STR(file->checksumSha1) : NULL);
|
||||||
protocolCommandParamAdd(command, file->checksumSha1[0] != 0 ? VARSTRZ(file->checksumSha1) : NULL);
|
pckWriteBoolP(param, file->checksumPage);
|
||||||
protocolCommandParamAdd(command, VARBOOL(file->checksumPage));
|
pckWriteU64P(param, jobData->lsnStart);
|
||||||
protocolCommandParamAdd(command, VARUINT64(jobData->lsnStart));
|
pckWriteStrP(param, file->name);
|
||||||
protocolCommandParamAdd(command, VARSTR(file->name));
|
pckWriteBoolP(param, file->reference != NULL);
|
||||||
protocolCommandParamAdd(command, VARBOOL(file->reference != NULL));
|
pckWriteU32P(param, jobData->compressType);
|
||||||
protocolCommandParamAdd(command, VARUINT(jobData->compressType));
|
pckWriteI32P(param, jobData->compressLevel);
|
||||||
protocolCommandParamAdd(command, VARINT(jobData->compressLevel));
|
pckWriteStrP(param, jobData->backupLabel);
|
||||||
protocolCommandParamAdd(command, VARSTR(jobData->backupLabel));
|
pckWriteBoolP(param, jobData->delta);
|
||||||
protocolCommandParamAdd(command, VARBOOL(jobData->delta));
|
pckWriteU64P(param, jobData->cipherSubPass == NULL ? cipherTypeNone : cipherTypeAes256Cbc);
|
||||||
protocolCommandParamAdd(command, VARUINT64(jobData->cipherType));
|
pckWriteStrP(param, jobData->cipherSubPass);
|
||||||
protocolCommandParamAdd(command, VARSTR(jobData->cipherSubPass));
|
|
||||||
|
|
||||||
// Remove job from the queue
|
// Remove job from the queue
|
||||||
lstRemoveIdx(queue, 0);
|
lstRemoveIdx(queue, 0);
|
||||||
|
@ -9,41 +9,56 @@ Backup Protocol Handler
|
|||||||
#include "common/io/io.h"
|
#include "common/io/io.h"
|
||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
|
#include "common/type/json.h"
|
||||||
#include "config/config.h"
|
#include "config/config.h"
|
||||||
#include "storage/helper.h"
|
#include "storage/helper.h"
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
backupFileProtocol(const VariantList *paramList, ProtocolServer *server)
|
backupFileProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
// Backup the file
|
// Backup file
|
||||||
BackupFileResult result = backupFile(
|
const String *const pgFile = pckReadStrP(param);
|
||||||
varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)),
|
const bool pgFileIgnoreMissing = pckReadBoolP(param);
|
||||||
varBool(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)), varBool(varLstGet(paramList, 5)),
|
const uint64_t pgFileSize = pckReadU64P(param);
|
||||||
varUInt64(varLstGet(paramList, 6)), varStr(varLstGet(paramList, 7)), varBool(varLstGet(paramList, 8)),
|
const bool pgFileCopyExactSize = pckReadBoolP(param);
|
||||||
(CompressType)varUIntForce(varLstGet(paramList, 9)), varIntForce(varLstGet(paramList, 10)),
|
const String *const pgFileChecksum = pckReadStrP(param);
|
||||||
varStr(varLstGet(paramList, 11)), varBool(varLstGet(paramList, 12)), varUInt64(varLstGet(paramList, 13)),
|
const bool pgFileChecksumPage = pckReadBoolP(param);
|
||||||
varStr(varLstGet(paramList, 14)));
|
const uint64_t pgFileChecksumPageLsnLimit = pckReadU64P(param);
|
||||||
|
const String *const repoFile = pckReadStrP(param);
|
||||||
|
const bool repoFileHasReference = pckReadBoolP(param);
|
||||||
|
const CompressType repoFileCompressType = (CompressType)pckReadU32P(param);
|
||||||
|
const int repoFileCompressLevel = pckReadI32P(param);
|
||||||
|
const String *const backupLabel = pckReadStrP(param);
|
||||||
|
const bool delta = pckReadBoolP(param);
|
||||||
|
const CipherType cipherType = (CipherType)pckReadU64P(param);
|
||||||
|
const String *const cipherPass = pckReadStrP(param);
|
||||||
|
|
||||||
// Return backup result
|
const BackupFileResult result = backupFile(
|
||||||
VariantList *resultList = varLstNew();
|
pgFile, pgFileIgnoreMissing, pgFileSize, pgFileCopyExactSize, pgFileChecksum, pgFileChecksumPage,
|
||||||
varLstAdd(resultList, varNewUInt(result.backupCopyResult));
|
pgFileChecksumPageLsnLimit, repoFile, repoFileHasReference, repoFileCompressType, repoFileCompressLevel,
|
||||||
varLstAdd(resultList, varNewUInt64(result.copySize));
|
backupLabel, delta, cipherType, cipherPass);
|
||||||
varLstAdd(resultList, varNewUInt64(result.repoSize));
|
|
||||||
varLstAdd(resultList, varNewStr(result.copyChecksum));
|
|
||||||
varLstAdd(resultList, result.pageChecksumResult != NULL ? varNewKv(result.pageChecksumResult) : NULL);
|
|
||||||
|
|
||||||
protocolServerResponse(server, varNewVarLst(resultList));
|
// Return result
|
||||||
|
PackWrite *const resultPack = protocolPackNew();
|
||||||
|
pckWriteU32P(resultPack, result.backupCopyResult);
|
||||||
|
pckWriteU64P(resultPack, result.copySize);
|
||||||
|
pckWriteU64P(resultPack, result.repoSize);
|
||||||
|
pckWriteStrP(resultPack, result.copyChecksum);
|
||||||
|
pckWriteStrP(resultPack, result.pageChecksumResult != NULL ? jsonFromKv(result.pageChecksumResult) : NULL);
|
||||||
|
|
||||||
|
protocolServerDataPut(server, resultPack);
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
@ -4,15 +4,14 @@ Backup Protocol Handler
|
|||||||
#ifndef COMMAND_BACKUP_PROTOCOL_H
|
#ifndef COMMAND_BACKUP_PROTOCOL_H
|
||||||
#define COMMAND_BACKUP_PROTOCOL_H
|
#define COMMAND_BACKUP_PROTOCOL_H
|
||||||
|
|
||||||
#include "common/type/string.h"
|
#include "common/type/pack.h"
|
||||||
#include "common/type/variantList.h"
|
|
||||||
#include "protocol/server.h"
|
#include "protocol/server.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
// Process protocol requests
|
// Process protocol requests
|
||||||
void backupFileProtocol(const VariantList *paramList, ProtocolServer *server);
|
void backupFileProtocol(PackRead *param, ProtocolServer *server);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
||||||
|
@ -50,8 +50,8 @@ cmdRemote(int fdRead, int fdWrite)
|
|||||||
|
|
||||||
TRY_BEGIN()
|
TRY_BEGIN()
|
||||||
{
|
{
|
||||||
// Read the command. No need to parse it since we know this is the first noop.
|
// Get the command. No need to check parameters since we know this is the first noop.
|
||||||
ioReadLine(read);
|
CHECK(protocolServerCommandGet(server).id == PROTOCOL_COMMAND_NOOP);
|
||||||
|
|
||||||
// Only try the lock if this is process 0, i.e. the remote started from the main process
|
// Only try the lock if this is process 0, i.e. the remote started from the main process
|
||||||
if (cfgOptionUInt(cfgOptProcess) == 0)
|
if (cfgOptionUInt(cfgOptProcess) == 0)
|
||||||
@ -70,7 +70,7 @@ cmdRemote(int fdRead, int fdWrite)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Notify the client of success
|
// Notify the client of success
|
||||||
protocolServerResponse(server, NULL);
|
protocolServerDataEndPut(server);
|
||||||
success = true;
|
success = true;
|
||||||
}
|
}
|
||||||
CATCH_ANY()
|
CATCH_ANY()
|
||||||
|
@ -14,30 +14,43 @@ Restore Protocol Handler
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
restoreFileProtocol(const VariantList *paramList, ProtocolServer *server)
|
restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
protocolServerResponse(
|
// Restore file
|
||||||
server,
|
const String *const repoFile = pckReadStrP(param);
|
||||||
VARBOOL(
|
const unsigned int repoIdx = pckReadU32P(param);
|
||||||
restoreFile(
|
const String *const repoFileReference = pckReadStrP(param);
|
||||||
varStr(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)), varStr(varLstGet(paramList, 2)),
|
const CompressType repoFileCompressType = (CompressType)pckReadU32P(param);
|
||||||
(CompressType)varUIntForce(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)),
|
const String *const pgFile = pckReadStrP(param);
|
||||||
varStr(varLstGet(paramList, 5)), varBoolForce(varLstGet(paramList, 6)), varUInt64(varLstGet(paramList, 7)),
|
const String *const pgFileChecksum = pckReadStrP(param);
|
||||||
(time_t)varInt64Force(varLstGet(paramList, 8)),
|
const bool pgFileZero = pckReadBoolP(param);
|
||||||
(mode_t)cvtZToUIntBase(strZ(varStr(varLstGet(paramList, 9))), 8),
|
const uint64_t pgFileSize = pckReadU64P(param);
|
||||||
varStr(varLstGet(paramList, 10)), varStr(varLstGet(paramList, 11)),
|
const time_t pgFileModified = pckReadTimeP(param);
|
||||||
(time_t)varInt64Force(varLstGet(paramList, 12)), varBoolForce(varLstGet(paramList, 13)),
|
const mode_t pgFileMode = pckReadModeP(param);
|
||||||
varBoolForce(varLstGet(paramList, 14)), varStr(varLstGet(paramList, 15)))));
|
const String *const pgFileUser = pckReadStrP(param);
|
||||||
|
const String *const pgFileGroup = pckReadStrP(param);
|
||||||
|
const time_t copyTimeBegin = pckReadTimeP(param);
|
||||||
|
const bool delta = pckReadBoolP(param);
|
||||||
|
const bool deltaForce = pckReadBoolP(param);
|
||||||
|
const String *const cipherPass = pckReadStrP(param);
|
||||||
|
|
||||||
|
const bool result = restoreFile(
|
||||||
|
repoFile, repoIdx, repoFileReference, repoFileCompressType, pgFile, pgFileChecksum, pgFileZero, pgFileSize,
|
||||||
|
pgFileModified, pgFileMode, pgFileUser, pgFileGroup, copyTimeBegin, delta, deltaForce, cipherPass);
|
||||||
|
|
||||||
|
// Return result
|
||||||
|
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), result));
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
@ -4,15 +4,14 @@ Restore Protocol Handler
|
|||||||
#ifndef COMMAND_RESTORE_PROTOCOL_H
|
#ifndef COMMAND_RESTORE_PROTOCOL_H
|
||||||
#define COMMAND_RESTORE_PROTOCOL_H
|
#define COMMAND_RESTORE_PROTOCOL_H
|
||||||
|
|
||||||
#include "common/type/string.h"
|
#include "common/type/pack.h"
|
||||||
#include "common/type/variantList.h"
|
|
||||||
#include "protocol/server.h"
|
#include "protocol/server.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
// Process protocol requests
|
// Process protocol requests
|
||||||
void restoreFileProtocol(const VariantList *paramList, ProtocolServer *server);
|
void restoreFileProtocol(PackRead *param, ProtocolServer *server);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
||||||
|
@ -2015,7 +2015,7 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
|
|||||||
{
|
{
|
||||||
const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
|
const ManifestFile *file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job)));
|
||||||
bool zeroed = restoreFileZeroed(file->name, zeroExp);
|
bool zeroed = restoreFileZeroed(file->name, zeroExp);
|
||||||
bool copy = varBool(protocolParallelJobResult(job));
|
bool copy = pckReadBoolP(protocolParallelJobResult(job));
|
||||||
|
|
||||||
String *log = strNewZ("restore");
|
String *log = strNewZ("restore");
|
||||||
|
|
||||||
@ -2141,24 +2141,24 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
|
|||||||
|
|
||||||
// Create restore job
|
// Create restore job
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
|
||||||
protocolCommandParamAdd(command, VARSTR(file->name));
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARUINT(jobData->repoIdx));
|
|
||||||
protocolCommandParamAdd(
|
pckWriteStrP(param, file->name);
|
||||||
command, file->reference != NULL ?
|
pckWriteU32P(param, jobData->repoIdx);
|
||||||
VARSTR(file->reference) : VARSTR(manifestData(jobData->manifest)->backupLabel));
|
pckWriteStrP(param, file->reference != NULL ? file->reference : manifestData(jobData->manifest)->backupLabel);
|
||||||
protocolCommandParamAdd(command, VARUINT(manifestData(jobData->manifest)->backupOptionCompressType));
|
pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
|
||||||
protocolCommandParamAdd(command, VARSTR(restoreFilePgPath(jobData->manifest, file->name)));
|
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file->name));
|
||||||
protocolCommandParamAdd(command, VARSTRZ(file->checksumSha1));
|
pckWriteStrP(param, STR(file->checksumSha1));
|
||||||
protocolCommandParamAdd(command, VARBOOL(restoreFileZeroed(file->name, jobData->zeroExp)));
|
pckWriteBoolP(param, restoreFileZeroed(file->name, jobData->zeroExp));
|
||||||
protocolCommandParamAdd(command, VARUINT64(file->size));
|
pckWriteU64P(param, file->size);
|
||||||
protocolCommandParamAdd(command, VARUINT64((uint64_t)file->timestamp));
|
pckWriteTimeP(param, file->timestamp);
|
||||||
protocolCommandParamAdd(command, VARSTR(strNewFmt("%04o", file->mode)));
|
pckWriteModeP(param, file->mode);
|
||||||
protocolCommandParamAdd(command, VARSTR(file->user));
|
pckWriteStrP(param, file->user);
|
||||||
protocolCommandParamAdd(command, VARSTR(file->group));
|
pckWriteStrP(param, file->group);
|
||||||
protocolCommandParamAdd(command, VARUINT64((uint64_t)manifestData(jobData->manifest)->backupTimestampCopyStart));
|
pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
|
||||||
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptDelta)));
|
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
|
||||||
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce)));
|
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
|
||||||
protocolCommandParamAdd(command, VARSTR(jobData->cipherSubPass));
|
pckWriteStrP(param, jobData->cipherSubPass);
|
||||||
|
|
||||||
// Remove job from the queue
|
// Remove job from the queue
|
||||||
lstRemoveIdx(queue, 0);
|
lstRemoveIdx(queue, 0);
|
||||||
|
@ -14,25 +14,29 @@ Verify Protocol Handler
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
verifyFileProtocol(const VariantList *paramList, ProtocolServer *server)
|
verifyFileProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
VerifyResult result = verifyFile(
|
// Verify file
|
||||||
varStr(varLstGet(paramList, 0)), // Full filename
|
const String *const filePathName = pckReadStrP(param);
|
||||||
varStr(varLstGet(paramList, 1)), // Checksum
|
const String *const fileChecksum = pckReadStrP(param);
|
||||||
varUInt64(varLstGet(paramList, 2)), // File size
|
const uint64_t fileSize = pckReadU64P(param);
|
||||||
varStr(varLstGet(paramList, 3))); // Cipher pass
|
const String *const cipherPass = pckReadStrP(param);
|
||||||
|
|
||||||
protocolServerResponse(server, VARUINT(result));
|
const VerifyResult result = verifyFile(filePathName, fileChecksum, fileSize, cipherPass);
|
||||||
|
|
||||||
|
// Return result
|
||||||
|
protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), result));
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
@ -4,15 +4,14 @@ Verify Protocol Handler
|
|||||||
#ifndef COMMAND_VERIFY_PROTOCOL_H
|
#ifndef COMMAND_VERIFY_PROTOCOL_H
|
||||||
#define COMMAND_VERIFY_PROTOCOL_H
|
#define COMMAND_VERIFY_PROTOCOL_H
|
||||||
|
|
||||||
#include "common/type/string.h"
|
#include "common/type/pack.h"
|
||||||
#include "common/type/variantList.h"
|
|
||||||
#include "protocol/server.h"
|
#include "protocol/server.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
// Process protocol requests
|
// Process protocol requests
|
||||||
void verifyFileProtocol(const VariantList *paramList, ProtocolServer *server);
|
void verifyFileProtocol(PackRead *param, ProtocolServer *server);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
||||||
|
@ -755,10 +755,12 @@ verifyArchive(void *data)
|
|||||||
|
|
||||||
// Set up the job
|
// Set up the job
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
|
||||||
protocolCommandParamAdd(command, VARSTR(filePathName));
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARSTR(checksum));
|
|
||||||
protocolCommandParamAdd(command, VARUINT64(archiveResult->pgWalInfo.size));
|
pckWriteStrP(param, filePathName);
|
||||||
protocolCommandParamAdd(command, VARSTR(jobData->walCipherPass));
|
pckWriteStrP(param, checksum);
|
||||||
|
pckWriteU64P(param, archiveResult->pgWalInfo.size);
|
||||||
|
pckWriteStrP(param, jobData->walCipherPass);
|
||||||
|
|
||||||
// Assign job to result, prepending the archiveId to the key for consistency with backup processing
|
// Assign job to result, prepending the archiveId to the key for consistency with backup processing
|
||||||
result = protocolParallelJobNew(
|
result = protocolParallelJobNew(
|
||||||
@ -978,12 +980,13 @@ verifyBackup(void *data)
|
|||||||
{
|
{
|
||||||
// Set up the job
|
// Set up the job
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
|
||||||
protocolCommandParamAdd(command, VARSTR(filePathName));
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
|
|
||||||
|
pckWriteStrP(param, filePathName);
|
||||||
// If the checksum is not present in the manifest, it will be calculated by manifest load
|
// If the checksum is not present in the manifest, it will be calculated by manifest load
|
||||||
protocolCommandParamAdd(command, VARSTRZ(fileData->checksumSha1));
|
pckWriteStrP(param, STR(fileData->checksumSha1));
|
||||||
protocolCommandParamAdd(command, VARUINT64(fileData->size));
|
pckWriteU64P(param, fileData->size);
|
||||||
protocolCommandParamAdd(command, VARSTR(jobData->backupCipherPass));
|
pckWriteStrP(param, jobData->backupCipherPass);
|
||||||
|
|
||||||
// Assign job to result (prepend backup label being processed to the key since some files are in a prior backup)
|
// Assign job to result (prepend backup label being processed to the key since some files are in a prior backup)
|
||||||
result = protocolParallelJobNew(
|
result = protocolParallelJobNew(
|
||||||
@ -1535,7 +1538,7 @@ verifyProcess(unsigned int *errorTotal)
|
|||||||
// The job was successful
|
// The job was successful
|
||||||
if (protocolParallelJobErrorCode(job) == 0)
|
if (protocolParallelJobErrorCode(job) == 0)
|
||||||
{
|
{
|
||||||
const VerifyResult verifyResult = (VerifyResult)varUIntForce(protocolParallelJobResult(job));
|
const VerifyResult verifyResult = (VerifyResult)pckReadU32P(protocolParallelJobResult(job));
|
||||||
|
|
||||||
// Update the result set for the type of file being processed
|
// Update the result set for the type of file being processed
|
||||||
if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR))
|
if (strEq(fileType, STORAGE_REPO_ARCHIVE_STR))
|
||||||
|
@ -17,13 +17,18 @@ Fields in a pack are identified by IDs. A field ID is stored as a delta from the
|
|||||||
that reading from the middle is generally not practical. The size of the gap between field IDs is important -- a gap of 1 never
|
that reading from the middle is generally not practical. The size of the gap between field IDs is important -- a gap of 1 never
|
||||||
incurs extra cost, but depending on the field type larger gaps may require additional bytes to store the field ID delta.
|
incurs extra cost, but depending on the field type larger gaps may require additional bytes to store the field ID delta.
|
||||||
|
|
||||||
|
The standard default is the C default for that type (e.g. bool = false, int = 0) but can be changed with the .defaultValue
|
||||||
|
parameter. For example, pckWriteBoolP(write, false, .defaultWrite = true) will write a 0 (i.e. false) with a field ID into the pack,
|
||||||
|
but pckWriteBoolP(write, false) will not write to the pack, it will simply skip the ID. Note that
|
||||||
|
pckWriteStrP(packWrite, NULL, .defaultWrite = true) is not valid since there is no way to explicitly write a NULL.
|
||||||
|
|
||||||
NULLs are not stored in a pack and are therefore not typed. A NULL is essentially just a gap in the field IDs. Fields that are
|
NULLs are not stored in a pack and are therefore not typed. A NULL is essentially just a gap in the field IDs. Fields that are
|
||||||
frequently NULL are best stored at the end of an object. When using .defaultWrite = false in write functions a NULL will be written
|
frequently NULL are best stored at the end of an object. When using read functions the default will always be returned
|
||||||
(by making a gap in the IDs) if the value matches the default. When using read functions the default will always be returned
|
when the field is NULL (i.e. missing). There are times when NULL must be explicitly passed, for example:
|
||||||
when the field is NULL (i.e. missing). The standard default is the C default for that type (e.g. bool = false, int = 0) but can be
|
pckWriteStrP(resultPack, result.pageChecksumResult != NULL ? jsonFromKv(result.pageChecksumResult) : NULL);
|
||||||
changed with the .defaultValue parameter. For example, pckWriteBoolP(write, false, .defaultWrite = true) will write a 0 with an ID
|
In this case, NULL is declared since jsonFromKv() does not accept a NULL parameter and, following the rules for NULLs the field ID
|
||||||
into the pack, but pckWriteBoolP(write, false) will not write to the pack, it will simply skip the ID. Note that
|
is skipped when result.pageChecksumResult == NULL. Upon reading, we can declare a NULL_STR when a NULL (field ID gap) is
|
||||||
pckWriteStrP(packWrite, NULL, .defaultWrite = true) is not valid since there is no way to explcitly write a NULL.
|
encountered, e.g. jsonToVar(pckReadStrP(jobResult, .defaultValue = NULL_STR)).
|
||||||
|
|
||||||
A pack is an object by default. Objects can store fields, objects, or arrays. Objects and arrays will be referred to collectively as
|
A pack is an object by default. Objects can store fields, objects, or arrays. Objects and arrays will be referred to collectively as
|
||||||
containers. Fields contain data to be stored, e.g. integers, strings, etc.
|
containers. Fields contain data to be stored, e.g. integers, strings, etc.
|
||||||
@ -38,8 +43,8 @@ pckWriteStringP(write, STRDEF("sample"));
|
|||||||
pckWriteEndP();
|
pckWriteEndP();
|
||||||
|
|
||||||
A string representation of this pack is `1:uint64:77,2:bool:false,4:str:sample`. The boolean was stored even though it was the
|
A string representation of this pack is `1:uint64:77,2:bool:false,4:str:sample`. The boolean was stored even though it was the
|
||||||
default because a write was explcitly requested. The int32 field was not stored because the value matched the expicitly set default.
|
default because a write was explicitly requested. The int32 field was not stored because the value matched the explicitly set
|
||||||
Note that there is a gap in the ID stream, which represents the NULL/default value.
|
default. Note that there is a gap in the ID stream, which represents the NULL/default value.
|
||||||
|
|
||||||
This pack can be read with:
|
This pack can be read with:
|
||||||
|
|
||||||
@ -50,8 +55,8 @@ pckReadI32P(read, .defaultValue = -1);
|
|||||||
pckReadStringP(read);
|
pckReadStringP(read);
|
||||||
pckReadEndP();
|
pckReadEndP();
|
||||||
|
|
||||||
Note that defaults are not stored in the pack so any defaults that were applied when writing (by setting .defaulWrite and
|
Note that defaults are not stored in the pack so any defaults that were applied when writing (by setting .defaultValue) must be
|
||||||
optionally .defaultValue) must be applied again when reading (by optionally setting .defaultValue).
|
applied again when reading by setting .defaultValue if the default value is not a standard C default.
|
||||||
|
|
||||||
If we don't care about the NULL/default, another way to read is:
|
If we don't care about the NULL/default, another way to read is:
|
||||||
|
|
||||||
|
@ -7,35 +7,37 @@ Configuration Protocol Handler
|
|||||||
#include "common/io/io.h"
|
#include "common/io/io.h"
|
||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
|
#include "common/type/json.h"
|
||||||
#include "config/config.intern.h"
|
#include "config/config.intern.h"
|
||||||
#include "config/parse.h"
|
#include "config/parse.h"
|
||||||
#include "config/protocol.h"
|
#include "config/protocol.h"
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
configOptionProtocol(const VariantList *paramList, ProtocolServer *server)
|
configOptionProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
VariantList *optionList = varLstNew();
|
VariantList *optionList = varLstNew();
|
||||||
|
|
||||||
for (unsigned int optionIdx = 0; optionIdx < varLstSize(paramList); optionIdx++)
|
while (pckReadNext(param))
|
||||||
{
|
{
|
||||||
CfgParseOptionResult option = cfgParseOptionP(varStr(varLstGet(paramList, optionIdx)));
|
CfgParseOptionResult option = cfgParseOptionP(pckReadStrP(param));
|
||||||
CHECK(option.found);
|
CHECK(option.found);
|
||||||
|
|
||||||
varLstAdd(optionList, varDup(cfgOptionIdx(option.id, cfgOptionKeyToIdx(option.id, option.keyIdx + 1))));
|
varLstAdd(optionList, varDup(cfgOptionIdx(option.id, cfgOptionKeyToIdx(option.id, option.keyIdx + 1))));
|
||||||
}
|
}
|
||||||
|
|
||||||
protocolServerResponse(server, varNewVarLst(optionList));
|
protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), jsonFromVar(varNewVarLst(optionList))));
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -56,13 +58,14 @@ configOptionRemote(ProtocolClient *client, const VariantList *paramList)
|
|||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_CONFIG_OPTION);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_CONFIG_OPTION);
|
||||||
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
|
|
||||||
for (unsigned int paramIdx = 0; paramIdx < varLstSize(paramList); paramIdx++)
|
for (unsigned int paramIdx = 0; paramIdx < varLstSize(paramList); paramIdx++)
|
||||||
protocolCommandParamAdd(command, varLstGet(paramList, paramIdx));
|
pckWriteStrP(param, varStr(varLstGet(paramList, paramIdx)));
|
||||||
|
|
||||||
MEM_CONTEXT_PRIOR_BEGIN()
|
MEM_CONTEXT_PRIOR_BEGIN()
|
||||||
{
|
{
|
||||||
result = varVarLst(protocolClientExecute(client, command, true));
|
result = varVarLst(jsonToVar(pckReadStrP(protocolClientExecute(client, command, true))));
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_PRIOR_END();
|
MEM_CONTEXT_PRIOR_END();
|
||||||
}
|
}
|
||||||
|
@ -4,8 +4,7 @@ Configuration Protocol Handler
|
|||||||
#ifndef CONFIG_PROTOCOL_H
|
#ifndef CONFIG_PROTOCOL_H
|
||||||
#define CONFIG_PROTOCOL_H
|
#define CONFIG_PROTOCOL_H
|
||||||
|
|
||||||
#include "common/type/string.h"
|
#include "common/type/pack.h"
|
||||||
#include "common/type/variantList.h"
|
|
||||||
#include "protocol/client.h"
|
#include "protocol/client.h"
|
||||||
#include "protocol/server.h"
|
#include "protocol/server.h"
|
||||||
|
|
||||||
@ -13,7 +12,7 @@ Configuration Protocol Handler
|
|||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
// Process config protocol requests
|
// Process config protocol requests
|
||||||
void configOptionProtocol(const VariantList *paramList, ProtocolServer *server);
|
void configOptionProtocol(PackRead *param, ProtocolServer *server);
|
||||||
|
|
||||||
// Get option values from a remote process
|
// Get option values from a remote process
|
||||||
VariantList *configOptionRemote(ProtocolClient *client, const VariantList *paramList);
|
VariantList *configOptionRemote(ProtocolClient *client, const VariantList *paramList);
|
||||||
|
13
src/db/db.c
13
src/db/db.c
@ -6,6 +6,7 @@ Database Client
|
|||||||
#include "common/debug.h"
|
#include "common/debug.h"
|
||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
|
#include "common/type/json.h"
|
||||||
#include "common/wait.h"
|
#include "common/wait.h"
|
||||||
#include "db/db.h"
|
#include "db/db.h"
|
||||||
#include "db/protocol.h"
|
#include "db/protocol.h"
|
||||||
@ -46,7 +47,7 @@ dbFreeResource(THIS_VOID)
|
|||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
|
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_CLOSE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_CLOSE);
|
||||||
protocolCommandParamAdd(command, VARUINT(this->remoteIdx));
|
pckWriteU32P(protocolCommandParam(command), this->remoteIdx);
|
||||||
|
|
||||||
protocolClientExecute(this->remoteClient, command, false);
|
protocolClientExecute(this->remoteClient, command, false);
|
||||||
|
|
||||||
@ -109,10 +110,12 @@ dbQuery(Db *this, const String *query)
|
|||||||
if (this->remoteClient != NULL)
|
if (this->remoteClient != NULL)
|
||||||
{
|
{
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_QUERY);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_QUERY);
|
||||||
protocolCommandParamAdd(command, VARUINT(this->remoteIdx));
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARSTR(query));
|
|
||||||
|
|
||||||
result = varVarLst(protocolClientExecute(this->remoteClient, command, true));
|
pckWriteU32P(param, this->remoteIdx);
|
||||||
|
pckWriteStrP(param, query);
|
||||||
|
|
||||||
|
result = varVarLst(jsonToVar(pckReadStrP(protocolClientExecute(this->remoteClient, command, true))));
|
||||||
}
|
}
|
||||||
// Else locally
|
// Else locally
|
||||||
else
|
else
|
||||||
@ -199,7 +202,7 @@ dbOpen(Db *this)
|
|||||||
if (this->remoteClient != NULL)
|
if (this->remoteClient != NULL)
|
||||||
{
|
{
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_OPEN);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_OPEN);
|
||||||
this->remoteIdx = varUIntForce(protocolClientExecute(this->remoteClient, command, true));
|
this->remoteIdx = pckReadU32P(protocolClientExecute(this->remoteClient, command, true));
|
||||||
|
|
||||||
// Set a callback to notify the remote when a connection is closed
|
// Set a callback to notify the remote when a connection is closed
|
||||||
memContextCallbackSet(this->pub.memContext, dbFreeResource, this);
|
memContextCallbackSet(this->pub.memContext, dbFreeResource, this);
|
||||||
|
@ -7,6 +7,7 @@ Db Protocol Handler
|
|||||||
#include "common/io/io.h"
|
#include "common/io/io.h"
|
||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
|
#include "common/type/json.h"
|
||||||
#include "common/type/list.h"
|
#include "common/type/list.h"
|
||||||
#include "config/config.h"
|
#include "config/config.h"
|
||||||
#include "db/protocol.h"
|
#include "db/protocol.h"
|
||||||
@ -23,14 +24,14 @@ static struct
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
|
dbOpenProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList == NULL);
|
ASSERT(param == NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
@ -46,8 +47,6 @@ dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add db to the list
|
// Add db to the list
|
||||||
unsigned int dbIdx = lstSize(dbProtocolLocal.pgClientList);
|
|
||||||
|
|
||||||
MEM_CONTEXT_BEGIN(lstMemContext(dbProtocolLocal.pgClientList))
|
MEM_CONTEXT_BEGIN(lstMemContext(dbProtocolLocal.pgClientList))
|
||||||
{
|
{
|
||||||
// Only a single db is passed to the remote
|
// Only a single db is passed to the remote
|
||||||
@ -61,7 +60,8 @@ dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
|
|||||||
MEM_CONTEXT_END();
|
MEM_CONTEXT_END();
|
||||||
|
|
||||||
// Return db index which should be included in subsequent calls
|
// Return db index which should be included in subsequent calls
|
||||||
protocolServerResponse(server, VARUINT(dbIdx));
|
protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), lstSize(dbProtocolLocal.pgClientList) - 1));
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -70,24 +70,23 @@ dbOpenProtocol(const VariantList *paramList, ProtocolServer *server)
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
dbQueryProtocol(const VariantList *paramList, ProtocolServer *server)
|
dbQueryProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
protocolServerResponse(
|
PgClient *const pgClient = *(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param));
|
||||||
server,
|
const String *const query = pckReadStrP(param);
|
||||||
varNewVarLst(
|
|
||||||
pgClientQuery(
|
protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), jsonFromVar(varNewVarLst(pgClientQuery(pgClient, query)))));
|
||||||
*(PgClient **)lstGet(dbProtocolLocal.pgClientList, varUIntForce(varLstGet(paramList, 0))),
|
protocolServerDataEndPut(server);
|
||||||
varStr(varLstGet(paramList, 1)))));
|
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -96,20 +95,20 @@ dbQueryProtocol(const VariantList *paramList, ProtocolServer *server)
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
dbCloseProtocol(const VariantList *paramList, ProtocolServer *server)
|
dbCloseProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
pgClientClose(*(PgClient **)lstGet(dbProtocolLocal.pgClientList, varUIntForce(varLstGet(paramList, 0))));
|
pgClientClose(*(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param)));
|
||||||
protocolServerResponse(server, NULL);
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
@ -4,18 +4,16 @@ Db Protocol Handler
|
|||||||
#ifndef DB_PROTOCOL_H
|
#ifndef DB_PROTOCOL_H
|
||||||
#define DB_PROTOCOL_H
|
#define DB_PROTOCOL_H
|
||||||
|
|
||||||
#include "common/type/string.h"
|
#include "common/type/pack.h"
|
||||||
#include "common/type/variantList.h"
|
|
||||||
#include "protocol/client.h"
|
|
||||||
#include "protocol/server.h"
|
#include "protocol/server.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
// Process db protocol requests
|
// Process db protocol requests
|
||||||
void dbOpenProtocol(const VariantList *paramList, ProtocolServer *server);
|
void dbOpenProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void dbQueryProtocol(const VariantList *paramList, ProtocolServer *server);
|
void dbQueryProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void dbCloseProtocol(const VariantList *paramList, ProtocolServer *server);
|
void dbCloseProtocol(PackRead *param, ProtocolServer *server);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
||||||
|
@ -10,6 +10,7 @@ Protocol Client
|
|||||||
#include "common/type/json.h"
|
#include "common/type/json.h"
|
||||||
#include "common/type/keyValue.h"
|
#include "common/type/keyValue.h"
|
||||||
#include "protocol/client.h"
|
#include "protocol/client.h"
|
||||||
|
#include "protocol/server.h"
|
||||||
#include "version.h"
|
#include "version.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
@ -19,20 +20,16 @@ STRING_EXTERN(PROTOCOL_GREETING_NAME_STR, PROTOCOL_GRE
|
|||||||
STRING_EXTERN(PROTOCOL_GREETING_SERVICE_STR, PROTOCOL_GREETING_SERVICE);
|
STRING_EXTERN(PROTOCOL_GREETING_SERVICE_STR, PROTOCOL_GREETING_SERVICE);
|
||||||
STRING_EXTERN(PROTOCOL_GREETING_VERSION_STR, PROTOCOL_GREETING_VERSION);
|
STRING_EXTERN(PROTOCOL_GREETING_VERSION_STR, PROTOCOL_GREETING_VERSION);
|
||||||
|
|
||||||
STRING_EXTERN(PROTOCOL_ERROR_STR, PROTOCOL_ERROR);
|
|
||||||
STRING_EXTERN(PROTOCOL_ERROR_STACK_STR, PROTOCOL_ERROR_STACK);
|
|
||||||
|
|
||||||
STRING_EXTERN(PROTOCOL_OUTPUT_STR, PROTOCOL_OUTPUT);
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Object type
|
Object type
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
struct ProtocolClient
|
struct ProtocolClient
|
||||||
{
|
{
|
||||||
ProtocolClientPub pub; // Publicly accessible variables
|
ProtocolClientPub pub; // Publicly accessible variables
|
||||||
const String *name;
|
IoWrite *write; // Write interface
|
||||||
const String *errorPrefix;
|
const String *name; // Name displayed in logging
|
||||||
TimeMSec keepAliveTime;
|
const String *errorPrefix; // Prefix used when throwing error
|
||||||
|
TimeMSec keepAliveTime; // Last time data was put to the server
|
||||||
};
|
};
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
@ -52,7 +49,7 @@ protocolClientFreeResource(THIS_VOID)
|
|||||||
// Send an exit command but don't wait to see if it succeeds
|
// Send an exit command but don't wait to see if it succeeds
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
protocolClientWriteCommand(this, protocolCommandNew(PROTOCOL_COMMAND_EXIT));
|
protocolClientCommandPut(this, protocolCommandNew(PROTOCOL_COMMAND_EXIT));
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -86,8 +83,8 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
|
|||||||
{
|
{
|
||||||
.memContext = memContextCurrent(),
|
.memContext = memContextCurrent(),
|
||||||
.read = read,
|
.read = read,
|
||||||
.write = write,
|
|
||||||
},
|
},
|
||||||
|
.write = write,
|
||||||
.name = strDup(name),
|
.name = strDup(name),
|
||||||
.errorPrefix = strNewFmt("raised from %s", strZ(name)),
|
.errorPrefix = strNewFmt("raised from %s", strZ(name)),
|
||||||
.keepAliveTime = timeMSec(),
|
.keepAliveTime = timeMSec(),
|
||||||
@ -96,7 +93,7 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
|
|||||||
// Read, parse, and check the protocol greeting
|
// Read, parse, and check the protocol greeting
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
String *greeting = ioReadLine(protocolClientIoRead(this));
|
String *greeting = ioReadLine(this->pub.read);
|
||||||
KeyValue *greetingKv = jsonToKv(greeting);
|
KeyValue *greetingKv = jsonToKv(greeting);
|
||||||
|
|
||||||
const String *expected[] =
|
const String *expected[] =
|
||||||
@ -143,93 +140,127 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
// Helper to process errors
|
void
|
||||||
static void
|
protocolClientDataPut(ProtocolClient *const this, PackWrite *const data)
|
||||||
protocolClientProcessError(ProtocolClient *this, KeyValue *errorKv)
|
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
||||||
FUNCTION_LOG_PARAM(KEY_VALUE, errorKv);
|
FUNCTION_LOG_PARAM(PACK_WRITE, data);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
ASSERT(errorKv != NULL);
|
|
||||||
|
// End the pack
|
||||||
|
if (data != NULL)
|
||||||
|
pckWriteEndP(data);
|
||||||
|
|
||||||
|
// Write the data
|
||||||
|
PackWrite *dataMessage = pckWriteNew(this->write);
|
||||||
|
pckWriteU32P(dataMessage, protocolMessageTypeData, .defaultWrite = true);
|
||||||
|
pckWritePackP(dataMessage, data);
|
||||||
|
pckWriteEndP(dataMessage);
|
||||||
|
|
||||||
|
// Flush when there is no more data to put
|
||||||
|
if (data == NULL)
|
||||||
|
ioWriteFlush(this->write);
|
||||||
|
|
||||||
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**********************************************************************************************************************************/
|
||||||
|
// Helper to process errors
|
||||||
|
static void
|
||||||
|
protocolClientError(ProtocolClient *const this, const ProtocolMessageType type, PackRead *const error)
|
||||||
|
{
|
||||||
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
|
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
||||||
|
FUNCTION_LOG_PARAM(ENUM, type);
|
||||||
|
FUNCTION_LOG_PARAM(PACK_READ, error);
|
||||||
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
|
if (type == protocolMessageTypeError)
|
||||||
|
{
|
||||||
|
const ErrorType *type = errorTypeFromCode(pckReadI32P(error));
|
||||||
|
String *const message = strNewFmt("%s: %s", strZ(this->errorPrefix), strZ(pckReadStrP(error)));
|
||||||
|
const String *const stack = pckReadStrP(error);
|
||||||
|
pckReadEndP(error);
|
||||||
|
|
||||||
|
CHECK(message != NULL);
|
||||||
|
|
||||||
|
// Add stack trace if the error is an assertion or debug-level logging is enabled
|
||||||
|
if (type == &AssertError || logAny(logLevelDebug))
|
||||||
|
{
|
||||||
|
CHECK(stack != NULL);
|
||||||
|
|
||||||
|
strCat(message, LF_STR);
|
||||||
|
strCat(message, stack);
|
||||||
|
}
|
||||||
|
|
||||||
|
THROWP(type, strZ(message));
|
||||||
|
}
|
||||||
|
|
||||||
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**********************************************************************************************************************************/
|
||||||
|
PackRead *
|
||||||
|
protocolClientDataGet(ProtocolClient *const this)
|
||||||
|
{
|
||||||
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
|
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
||||||
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
|
PackRead *result = NULL;
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
// Process error if any
|
PackRead *response = pckReadNew(this->pub.read);
|
||||||
const Variant *error = kvGet(errorKv, VARSTR(PROTOCOL_ERROR_STR));
|
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
|
||||||
|
|
||||||
if (error != NULL)
|
protocolClientError(this, type, response);
|
||||||
|
|
||||||
|
CHECK(type == protocolMessageTypeData);
|
||||||
|
|
||||||
|
MEM_CONTEXT_PRIOR_BEGIN()
|
||||||
{
|
{
|
||||||
const ErrorType *type = errorTypeFromCode(varIntForce(error));
|
result = pckReadPackP(response);
|
||||||
const String *message = varStr(kvGet(errorKv, VARSTR(PROTOCOL_OUTPUT_STR)));
|
|
||||||
|
|
||||||
// Required part of the message
|
|
||||||
String *throwMessage = strNewFmt(
|
|
||||||
"%s: %s", strZ(this->errorPrefix), message == NULL ? "no details available" : strZ(message));
|
|
||||||
|
|
||||||
// Add stack trace if the error is an assertion or debug-level logging is enabled
|
|
||||||
if (type == &AssertError || logAny(logLevelDebug))
|
|
||||||
{
|
|
||||||
const String *stack = varStr(kvGet(errorKv, VARSTR(PROTOCOL_ERROR_STACK_STR)));
|
|
||||||
|
|
||||||
strCat(throwMessage, LF_STR);
|
|
||||||
strCat(throwMessage, stack == NULL ? STRDEF("no stack trace available") : stack);
|
|
||||||
}
|
|
||||||
|
|
||||||
THROWP(type, strZ(throwMessage));
|
|
||||||
}
|
}
|
||||||
|
MEM_CONTEXT_PRIOR_END();
|
||||||
|
|
||||||
|
pckReadEndP(response);
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
FUNCTION_LOG_RETURN(PACK_READ, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**********************************************************************************************************************************/
|
||||||
|
void
|
||||||
|
protocolClientDataEndGet(ProtocolClient *const this)
|
||||||
|
{
|
||||||
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
|
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
||||||
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
|
{
|
||||||
|
PackRead *response = pckReadNew(this->pub.read);
|
||||||
|
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
|
||||||
|
|
||||||
|
protocolClientError(this, type, response);
|
||||||
|
|
||||||
|
CHECK(type == protocolMessageTypeDataEnd);
|
||||||
|
|
||||||
|
pckReadEndP(response);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN_VOID();
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
const Variant *
|
|
||||||
protocolClientReadOutput(ProtocolClient *this, bool outputRequired)
|
|
||||||
{
|
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
|
||||||
FUNCTION_LOG_PARAM(BOOL, outputRequired);
|
|
||||||
FUNCTION_LOG_END();
|
|
||||||
|
|
||||||
ASSERT(this != NULL);
|
|
||||||
|
|
||||||
const Variant *result = NULL;
|
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
|
||||||
{
|
|
||||||
// Read the response
|
|
||||||
String *response = ioReadLine(protocolClientIoRead(this));
|
|
||||||
KeyValue *responseKv = varKv(jsonToVar(response));
|
|
||||||
|
|
||||||
// Process error if any
|
|
||||||
protocolClientProcessError(this, responseKv);
|
|
||||||
|
|
||||||
// Get output
|
|
||||||
result = kvGet(responseKv, VARSTR(PROTOCOL_OUTPUT_STR));
|
|
||||||
|
|
||||||
if (outputRequired)
|
|
||||||
{
|
|
||||||
// Just move the entire response kv since the output is the largest part if it
|
|
||||||
kvMove(responseKv, memContextPrior());
|
|
||||||
}
|
|
||||||
// Else if no output is required then there should not be any
|
|
||||||
else if (result != NULL)
|
|
||||||
THROW(AssertError, "no output required by command");
|
|
||||||
|
|
||||||
// Reset the keep alive time
|
|
||||||
this->keepAliveTime = timeMSec();
|
|
||||||
}
|
|
||||||
MEM_CONTEXT_TEMP_END();
|
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN_CONST(VARIANT, result);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command)
|
protocolClientCommandPut(ProtocolClient *const this, ProtocolCommand *const command)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
||||||
@ -239,9 +270,8 @@ protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command)
|
|||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
ASSERT(command != NULL);
|
ASSERT(command != NULL);
|
||||||
|
|
||||||
// Write out the command
|
// Put command
|
||||||
ioWriteStrLine(protocolClientIoWrite(this), protocolCommandJson(command));
|
protocolCommandPut(command, this->write);
|
||||||
ioWriteFlush(protocolClientIoWrite(this));
|
|
||||||
|
|
||||||
// Reset the keep alive time
|
// Reset the keep alive time
|
||||||
this->keepAliveTime = timeMSec();
|
this->keepAliveTime = timeMSec();
|
||||||
@ -250,21 +280,31 @@ protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
const Variant *
|
PackRead *
|
||||||
protocolClientExecute(ProtocolClient *this, const ProtocolCommand *command, bool outputRequired)
|
protocolClientExecute(ProtocolClient *const this, ProtocolCommand *const command, const bool resultRequired)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
|
FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
|
||||||
FUNCTION_LOG_PARAM(BOOL, outputRequired);
|
FUNCTION_LOG_PARAM(BOOL, resultRequired);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
ASSERT(command != NULL);
|
ASSERT(command != NULL);
|
||||||
|
|
||||||
protocolClientWriteCommand(this, command);
|
// Put command
|
||||||
|
protocolClientCommandPut(this, command);
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN_CONST(VARIANT, protocolClientReadOutput(this, outputRequired));
|
// Read result if required
|
||||||
|
PackRead *result = NULL;
|
||||||
|
|
||||||
|
if (resultRequired)
|
||||||
|
result = protocolClientDataGet(this);
|
||||||
|
|
||||||
|
// Read response
|
||||||
|
protocolClientDataEndGet(this);
|
||||||
|
|
||||||
|
FUNCTION_LOG_RETURN(PACK_READ, result);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
@ -286,50 +326,6 @@ protocolClientNoOp(ProtocolClient *this)
|
|||||||
FUNCTION_LOG_RETURN_VOID();
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
|
||||||
String *
|
|
||||||
protocolClientReadLine(ProtocolClient *this)
|
|
||||||
{
|
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
|
|
||||||
FUNCTION_LOG_END();
|
|
||||||
|
|
||||||
ASSERT(this != NULL);
|
|
||||||
|
|
||||||
String *result = NULL;
|
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
|
||||||
{
|
|
||||||
result = ioReadLine(protocolClientIoRead(this));
|
|
||||||
|
|
||||||
if (strSize(result) == 0)
|
|
||||||
{
|
|
||||||
THROW(FormatError, "unexpected empty line");
|
|
||||||
}
|
|
||||||
else if (strZ(result)[0] == '{')
|
|
||||||
{
|
|
||||||
KeyValue *responseKv = varKv(jsonToVar(result));
|
|
||||||
|
|
||||||
// Process expected error
|
|
||||||
protocolClientProcessError(this, responseKv);
|
|
||||||
|
|
||||||
// If not an error then there is probably a protocol bug
|
|
||||||
THROW(FormatError, "expected error but got output");
|
|
||||||
}
|
|
||||||
else if (strZ(result)[0] != '.')
|
|
||||||
THROW_FMT(FormatError, "invalid prefix in '%s'", strZ(result));
|
|
||||||
|
|
||||||
MEM_CONTEXT_PRIOR_BEGIN()
|
|
||||||
{
|
|
||||||
result = strSub(result, 1);
|
|
||||||
}
|
|
||||||
MEM_CONTEXT_PRIOR_END();
|
|
||||||
}
|
|
||||||
MEM_CONTEXT_TEMP_END();
|
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STRING, result);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
String *
|
String *
|
||||||
protocolClientToLog(const ProtocolClient *this)
|
protocolClientToLog(const ProtocolClient *this)
|
||||||
|
@ -4,6 +4,24 @@ Protocol Client
|
|||||||
#ifndef PROTOCOL_CLIENT_H
|
#ifndef PROTOCOL_CLIENT_H
|
||||||
#define PROTOCOL_CLIENT_H
|
#define PROTOCOL_CLIENT_H
|
||||||
|
|
||||||
|
/***********************************************************************************************************************************
|
||||||
|
Message types used by the protocol
|
||||||
|
***********************************************************************************************************************************/
|
||||||
|
typedef enum
|
||||||
|
{
|
||||||
|
// Data passed between client and server in either direction. This can be used as many times as needed.
|
||||||
|
protocolMessageTypeData = 0,
|
||||||
|
|
||||||
|
// Indicates no more data for the server to return to the client and ends the command
|
||||||
|
protocolMessageTypeDataEnd = 1,
|
||||||
|
|
||||||
|
// Command sent from the client to the server
|
||||||
|
protocolMessageTypeCommand = 2,
|
||||||
|
|
||||||
|
// An error occurred on the server and the command ended abnormally. protocolMessageTypeDataEnd will not be sent to the client.
|
||||||
|
protocolMessageTypeError = 3,
|
||||||
|
} ProtocolMessageType;
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Object type
|
Object type
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
@ -27,13 +45,18 @@ Constants
|
|||||||
#define PROTOCOL_COMMAND_EXIT STRID5("exit", 0xa27050)
|
#define PROTOCOL_COMMAND_EXIT STRID5("exit", 0xa27050)
|
||||||
#define PROTOCOL_COMMAND_NOOP STRID5("noop", 0x83dee0)
|
#define PROTOCOL_COMMAND_NOOP STRID5("noop", 0x83dee0)
|
||||||
|
|
||||||
#define PROTOCOL_ERROR "err"
|
/***********************************************************************************************************************************
|
||||||
STRING_DECLARE(PROTOCOL_ERROR_STR);
|
This size should be safe for most pack data without wasting a lot of space. If binary data is being transferred then this size can
|
||||||
#define PROTOCOL_ERROR_STACK "errStack"
|
be added to the expected binary size to account for overhead.
|
||||||
STRING_DECLARE(PROTOCOL_ERROR_STACK_STR);
|
***********************************************************************************************************************************/
|
||||||
|
#define PROTOCOL_PACK_DEFAULT_SIZE 1024
|
||||||
|
|
||||||
#define PROTOCOL_OUTPUT "out"
|
// Pack large enough for standard data. Note that the buffer will automatically resize when required.
|
||||||
STRING_DECLARE(PROTOCOL_OUTPUT_STR);
|
__attribute__((always_inline)) static inline PackWrite *
|
||||||
|
protocolPackNew(void)
|
||||||
|
{
|
||||||
|
return pckWriteNewBuf(bufNew(PROTOCOL_PACK_DEFAULT_SIZE));
|
||||||
|
}
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Constructors
|
Constructors
|
||||||
@ -47,28 +70,20 @@ typedef struct ProtocolClientPub
|
|||||||
{
|
{
|
||||||
MemContext *memContext; // Mem context
|
MemContext *memContext; // Mem context
|
||||||
IoRead *read; // Read interface
|
IoRead *read; // Read interface
|
||||||
IoWrite *write; // Write interface
|
|
||||||
} ProtocolClientPub;
|
} ProtocolClientPub;
|
||||||
|
|
||||||
// Read interface
|
// Read file descriptor
|
||||||
__attribute__((always_inline)) static inline IoRead *
|
__attribute__((always_inline)) static inline int
|
||||||
protocolClientIoRead(ProtocolClient *const this)
|
protocolClientIoReadFd(ProtocolClient *const this)
|
||||||
{
|
{
|
||||||
return THIS_PUB(ProtocolClient)->read;
|
return ioReadFd(THIS_PUB(ProtocolClient)->read);
|
||||||
}
|
|
||||||
|
|
||||||
// Write interface
|
|
||||||
__attribute__((always_inline)) static inline IoWrite *
|
|
||||||
protocolClientIoWrite(ProtocolClient *const this)
|
|
||||||
{
|
|
||||||
return THIS_PUB(ProtocolClient)->write;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
// Execute a protocol command and get the output
|
// Execute a command and get the result
|
||||||
const Variant *protocolClientExecute(ProtocolClient *this, const ProtocolCommand *command, bool outputRequired);
|
PackRead *protocolClientExecute(ProtocolClient *this, ProtocolCommand *command, bool resultRequired);
|
||||||
|
|
||||||
// Move to a new parent mem context
|
// Move to a new parent mem context
|
||||||
__attribute__((always_inline)) static inline ProtocolClient *
|
__attribute__((always_inline)) static inline ProtocolClient *
|
||||||
@ -80,14 +95,15 @@ protocolClientMove(ProtocolClient *const this, MemContext *const parentNew)
|
|||||||
// Send noop to test connection or keep it alive
|
// Send noop to test connection or keep it alive
|
||||||
void protocolClientNoOp(ProtocolClient *this);
|
void protocolClientNoOp(ProtocolClient *this);
|
||||||
|
|
||||||
// Read a line
|
// Get data put by the server
|
||||||
String *protocolClientReadLine(ProtocolClient *this);
|
PackRead *protocolClientDataGet(ProtocolClient *this);
|
||||||
|
void protocolClientDataEndGet(ProtocolClient *this);
|
||||||
|
|
||||||
// Read the command output
|
// Put command to the server
|
||||||
const Variant *protocolClientReadOutput(ProtocolClient *this, bool outputRequired);
|
void protocolClientCommandPut(ProtocolClient *this, ProtocolCommand *command);
|
||||||
|
|
||||||
// Write the protocol command
|
// Put data to the server
|
||||||
void protocolClientWriteCommand(ProtocolClient *this, const ProtocolCommand *command);
|
void protocolClientDataPut(ProtocolClient *this, PackWrite *data);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Destructor
|
Destructor
|
||||||
|
@ -6,15 +6,9 @@ Protocol Command
|
|||||||
#include "common/debug.h"
|
#include "common/debug.h"
|
||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
#include "common/type/json.h"
|
|
||||||
#include "common/type/keyValue.h"
|
#include "common/type/keyValue.h"
|
||||||
#include "protocol/command.h"
|
#include "protocol/command.h"
|
||||||
|
#include "protocol/client.h"
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Constants
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
STRING_EXTERN(PROTOCOL_KEY_COMMAND_STR, PROTOCOL_KEY_COMMAND);
|
|
||||||
STRING_EXTERN(PROTOCOL_KEY_PARAMETER_STR, PROTOCOL_KEY_PARAMETER);
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Object type
|
Object type
|
||||||
@ -23,7 +17,7 @@ struct ProtocolCommand
|
|||||||
{
|
{
|
||||||
MemContext *memContext;
|
MemContext *memContext;
|
||||||
StringId command;
|
StringId command;
|
||||||
Variant *parameterList;
|
PackWrite *pack;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
@ -54,33 +48,39 @@ protocolCommandNew(const StringId command)
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
ProtocolCommand *
|
void
|
||||||
protocolCommandParamAdd(ProtocolCommand *this, const Variant *param)
|
protocolCommandPut(ProtocolCommand *const this, IoWrite *const write)
|
||||||
{
|
{
|
||||||
FUNCTION_TEST_BEGIN();
|
FUNCTION_TEST_BEGIN();
|
||||||
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
|
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
|
||||||
FUNCTION_TEST_PARAM(VARIANT, param);
|
|
||||||
FUNCTION_TEST_END();
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
|
ASSERT(write != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_BEGIN(this->memContext)
|
// Write the command and flush to be sure the command gets sent immediately
|
||||||
|
PackWrite *commandPack = pckWriteNew(write);
|
||||||
|
pckWriteU32P(commandPack, protocolMessageTypeCommand, .defaultWrite = true);
|
||||||
|
pckWriteStrIdP(commandPack, this->command);
|
||||||
|
|
||||||
|
// Only write params if there were any
|
||||||
|
if (this->pack != NULL)
|
||||||
{
|
{
|
||||||
// Create parameter list if not already created
|
pckWriteEndP(this->pack);
|
||||||
if (this->parameterList == NULL)
|
pckWritePackP(commandPack, this->pack);
|
||||||
this->parameterList = varNewVarLst(varLstNew());
|
|
||||||
|
|
||||||
// Add parameter to the list
|
|
||||||
varLstAdd(varVarLst(this->parameterList), varDup(param));
|
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_END();
|
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN(this);
|
pckWriteEndP(commandPack);
|
||||||
|
|
||||||
|
// Flush to send command immediately
|
||||||
|
ioWriteFlush(write);
|
||||||
|
|
||||||
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
String *
|
PackWrite *
|
||||||
protocolCommandJson(const ProtocolCommand *this)
|
protocolCommandParam(ProtocolCommand *this)
|
||||||
{
|
{
|
||||||
FUNCTION_TEST_BEGIN();
|
FUNCTION_TEST_BEGIN();
|
||||||
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
|
FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
|
||||||
@ -88,27 +88,16 @@ protocolCommandJson(const ProtocolCommand *this)
|
|||||||
|
|
||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
|
|
||||||
String *result = NULL;
|
if (this->pack == NULL)
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
|
||||||
{
|
{
|
||||||
char commandStrId[STRID_MAX + 1];
|
MEM_CONTEXT_BEGIN(this->memContext)
|
||||||
strIdToZ(this->command, commandStrId);
|
|
||||||
|
|
||||||
KeyValue *command = kvPut(kvNew(), VARSTR(PROTOCOL_KEY_COMMAND_STR), VARSTRZ(commandStrId));
|
|
||||||
|
|
||||||
if (this->parameterList != NULL)
|
|
||||||
kvPut(command, VARSTR(PROTOCOL_KEY_PARAMETER_STR), this->parameterList);
|
|
||||||
|
|
||||||
MEM_CONTEXT_PRIOR_BEGIN()
|
|
||||||
{
|
{
|
||||||
result = jsonFromKv(command);
|
this->pack = protocolPackNew();
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_PRIOR_END();
|
MEM_CONTEXT_END();
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN(result);
|
FUNCTION_TEST_RETURN(this->pack);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
|
@ -10,28 +10,13 @@ Object type
|
|||||||
typedef struct ProtocolCommand ProtocolCommand;
|
typedef struct ProtocolCommand ProtocolCommand;
|
||||||
|
|
||||||
#include "common/type/object.h"
|
#include "common/type/object.h"
|
||||||
#include "common/type/stringId.h"
|
#include "common/type/pack.h"
|
||||||
#include "common/type/variant.h"
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Constants
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
#define PROTOCOL_KEY_COMMAND "cmd"
|
|
||||||
STRING_DECLARE(PROTOCOL_KEY_COMMAND_STR);
|
|
||||||
#define PROTOCOL_KEY_PARAMETER "param"
|
|
||||||
STRING_DECLARE(PROTOCOL_KEY_PARAMETER_STR);
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Constructors
|
Constructors
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
ProtocolCommand *protocolCommandNew(const StringId command);
|
ProtocolCommand *protocolCommandNew(const StringId command);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Getters/Setters
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
// Command JSON
|
|
||||||
String *protocolCommandJson(const ProtocolCommand *this);
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
@ -44,7 +29,10 @@ protocolCommandMove(ProtocolCommand *const this, MemContext *const parentNew)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Read the command output
|
// Read the command output
|
||||||
ProtocolCommand *protocolCommandParamAdd(ProtocolCommand *this, const Variant *param);
|
PackWrite *protocolCommandParam(ProtocolCommand *this);
|
||||||
|
|
||||||
|
// Write protocol command
|
||||||
|
void protocolCommandPut(ProtocolCommand *this, IoWrite *write);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Destructor
|
Destructor
|
||||||
|
@ -10,7 +10,6 @@ Protocol Parallel Executor
|
|||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/macro.h"
|
#include "common/macro.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
#include "common/type/json.h"
|
|
||||||
#include "common/type/keyValue.h"
|
#include "common/type/keyValue.h"
|
||||||
#include "common/type/list.h"
|
#include "common/type/list.h"
|
||||||
#include "protocol/command.h"
|
#include "protocol/command.h"
|
||||||
@ -83,7 +82,7 @@ protocolParallelClientAdd(ProtocolParallel *this, ProtocolClient *client)
|
|||||||
ASSERT(client != NULL);
|
ASSERT(client != NULL);
|
||||||
ASSERT(this->state == protocolParallelJobStatePending);
|
ASSERT(this->state == protocolParallelJobStatePending);
|
||||||
|
|
||||||
if (ioReadFd(protocolClientIoRead(client)) == -1)
|
if (protocolClientIoReadFd(client) == -1)
|
||||||
THROW(AssertError, "client with read fd is required");
|
THROW(AssertError, "client with read fd is required");
|
||||||
|
|
||||||
lstAdd(this->clientList, &client);
|
lstAdd(this->clientList, &client);
|
||||||
@ -128,7 +127,7 @@ protocolParallelProcess(ProtocolParallel *this)
|
|||||||
{
|
{
|
||||||
if (this->clientJobList[clientIdx] != NULL)
|
if (this->clientJobList[clientIdx] != NULL)
|
||||||
{
|
{
|
||||||
int fd = ioReadFd(protocolClientIoRead(*(ProtocolClient **)lstGet(this->clientList, clientIdx)));
|
int fd = protocolClientIoReadFd(*(ProtocolClient **)lstGet(this->clientList, clientIdx));
|
||||||
FD_SET((unsigned int)fd, &selectSet);
|
FD_SET((unsigned int)fd, &selectSet);
|
||||||
|
|
||||||
// Find the max file descriptor needed for select()
|
// Find the max file descriptor needed for select()
|
||||||
@ -159,15 +158,17 @@ protocolParallelProcess(ProtocolParallel *this)
|
|||||||
|
|
||||||
if (job != NULL &&
|
if (job != NULL &&
|
||||||
FD_ISSET(
|
FD_ISSET(
|
||||||
(unsigned int)ioReadFd(protocolClientIoRead(*(ProtocolClient **)lstGet(this->clientList, clientIdx))),
|
(unsigned int)protocolClientIoReadFd(*(ProtocolClient **)lstGet(this->clientList, clientIdx)),
|
||||||
&selectSet))
|
&selectSet))
|
||||||
{
|
{
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
TRY_BEGIN()
|
TRY_BEGIN()
|
||||||
{
|
{
|
||||||
protocolParallelJobResultSet(
|
ProtocolClient *const client = *(ProtocolClient **)lstGet(this->clientList, clientIdx);
|
||||||
job, protocolClientReadOutput(*(ProtocolClient **)lstGet(this->clientList, clientIdx), true));
|
|
||||||
|
protocolParallelJobResultSet(job, protocolClientDataGet(client));
|
||||||
|
protocolClientDataEndGet(client);
|
||||||
}
|
}
|
||||||
CATCH_ANY()
|
CATCH_ANY()
|
||||||
{
|
{
|
||||||
@ -207,9 +208,8 @@ protocolParallelProcess(ProtocolParallel *this)
|
|||||||
// Add to the job list
|
// Add to the job list
|
||||||
lstAdd(this->jobList, &job);
|
lstAdd(this->jobList, &job);
|
||||||
|
|
||||||
// Send the job to the client
|
// Put command
|
||||||
protocolClientWriteCommand(
|
protocolClientCommandPut(*(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job));
|
||||||
*(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job));
|
|
||||||
|
|
||||||
// Set client id and running state
|
// Set client id and running state
|
||||||
protocolParallelJobProcessIdSet(job, clientIdx + 1);
|
protocolParallelJobProcessIdSet(job, clientIdx + 1);
|
||||||
|
@ -92,21 +92,17 @@ protocolParallelJobProcessIdSet(ProtocolParallelJob *this, unsigned int processI
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
protocolParallelJobResultSet(ProtocolParallelJob *this, const Variant *result)
|
protocolParallelJobResultSet(ProtocolParallelJob *const this, PackRead *const result)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, this);
|
FUNCTION_LOG_PARAM(PROTOCOL_PARALLEL_JOB, this);
|
||||||
FUNCTION_LOG_PARAM(VARIANT, result);
|
FUNCTION_LOG_PARAM(PACK_READ, result);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
ASSERT(protocolParallelJobErrorCode(this) == 0);
|
ASSERT(protocolParallelJobErrorCode(this) == 0);
|
||||||
|
|
||||||
MEM_CONTEXT_BEGIN(this->pub.memContext)
|
this->pub.result = pckReadMove(result, this->pub.memContext);
|
||||||
{
|
|
||||||
this->pub.result = varDup(result);
|
|
||||||
}
|
|
||||||
MEM_CONTEXT_END();
|
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN_VOID();
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
}
|
}
|
||||||
@ -143,5 +139,6 @@ protocolParallelJobToLog(const ProtocolParallelJob *this)
|
|||||||
"{state: %s, key: %s, command: %s, code: %d, message: %s, result: %s}",
|
"{state: %s, key: %s, command: %s, code: %d, message: %s, result: %s}",
|
||||||
strZ(strIdToStr(protocolParallelJobState(this))), strZ(varToLog(protocolParallelJobKey(this))),
|
strZ(strIdToStr(protocolParallelJobState(this))), strZ(varToLog(protocolParallelJobKey(this))),
|
||||||
strZ(protocolCommandToLog(protocolParallelJobCommand(this))), protocolParallelJobErrorCode(this),
|
strZ(protocolCommandToLog(protocolParallelJobCommand(this))), protocolParallelJobErrorCode(this),
|
||||||
strZ(strToLog(protocolParallelJobErrorMessage(this))), strZ(varToLog(protocolParallelJobResult(this))));
|
strZ(strToLog(protocolParallelJobErrorMessage(this))),
|
||||||
|
protocolParallelJobResult(this) == NULL ? NULL_Z : strZ(pckReadToLog(protocolParallelJobResult(this))));
|
||||||
}
|
}
|
||||||
|
@ -23,6 +23,7 @@ typedef enum
|
|||||||
|
|
||||||
#include "common/time.h"
|
#include "common/time.h"
|
||||||
#include "common/type/object.h"
|
#include "common/type/object.h"
|
||||||
|
#include "common/type/pack.h"
|
||||||
#include "protocol/client.h"
|
#include "protocol/client.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
@ -37,16 +38,16 @@ typedef struct ProtocolParallelJobPub
|
|||||||
{
|
{
|
||||||
MemContext *memContext; // Mem context
|
MemContext *memContext; // Mem context
|
||||||
const Variant *key; // Unique key used to identify the job
|
const Variant *key; // Unique key used to identify the job
|
||||||
const ProtocolCommand *command; // Command to be executed
|
ProtocolCommand *command; // Command to be executed
|
||||||
unsigned int processId; // Process that executed this job
|
unsigned int processId; // Process that executed this job
|
||||||
ProtocolParallelJobState state; // Current state of the job
|
ProtocolParallelJobState state; // Current state of the job
|
||||||
int code; // Non-zero result indicates an error
|
int code; // Non-zero result indicates an error
|
||||||
String *message; // Message if there was a error
|
String *message; // Message if there was a error
|
||||||
const Variant *result; // Result if job was successful
|
PackRead *result; // Result if job was successful
|
||||||
} ProtocolParallelJobPub;
|
} ProtocolParallelJobPub;
|
||||||
|
|
||||||
// Job command
|
// Job command
|
||||||
__attribute__((always_inline)) static inline const ProtocolCommand *
|
__attribute__((always_inline)) static inline ProtocolCommand *
|
||||||
protocolParallelJobCommand(const ProtocolParallelJob *const this)
|
protocolParallelJobCommand(const ProtocolParallelJob *const this)
|
||||||
{
|
{
|
||||||
return THIS_PUB(ProtocolParallelJob)->command;
|
return THIS_PUB(ProtocolParallelJob)->command;
|
||||||
@ -84,13 +85,13 @@ protocolParallelJobProcessId(const ProtocolParallelJob *const this)
|
|||||||
void protocolParallelJobProcessIdSet(ProtocolParallelJob *this, unsigned int processId);
|
void protocolParallelJobProcessIdSet(ProtocolParallelJob *this, unsigned int processId);
|
||||||
|
|
||||||
// Job result
|
// Job result
|
||||||
__attribute__((always_inline)) static inline const Variant *
|
__attribute__((always_inline)) static inline PackRead *
|
||||||
protocolParallelJobResult(const ProtocolParallelJob *const this)
|
protocolParallelJobResult(const ProtocolParallelJob *const this)
|
||||||
{
|
{
|
||||||
return THIS_PUB(ProtocolParallelJob)->result;
|
return THIS_PUB(ProtocolParallelJob)->result;
|
||||||
}
|
}
|
||||||
|
|
||||||
void protocolParallelJobResultSet(ProtocolParallelJob *this, const Variant *result);
|
void protocolParallelJobResultSet(ProtocolParallelJob *const this, PackRead *const result);
|
||||||
|
|
||||||
// Job state
|
// Job state
|
||||||
__attribute__((always_inline)) static inline ProtocolParallelJobState
|
__attribute__((always_inline)) static inline ProtocolParallelJobState
|
||||||
|
@ -12,7 +12,6 @@ Protocol Server
|
|||||||
#include "common/type/json.h"
|
#include "common/type/json.h"
|
||||||
#include "common/type/keyValue.h"
|
#include "common/type/keyValue.h"
|
||||||
#include "common/type/list.h"
|
#include "common/type/list.h"
|
||||||
#include "protocol/client.h"
|
|
||||||
#include "protocol/helper.h"
|
#include "protocol/helper.h"
|
||||||
#include "protocol/server.h"
|
#include "protocol/server.h"
|
||||||
#include "version.h"
|
#include "version.h"
|
||||||
@ -22,8 +21,10 @@ Object type
|
|||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
struct ProtocolServer
|
struct ProtocolServer
|
||||||
{
|
{
|
||||||
ProtocolServerPub pub; // Publicly accessible variables
|
MemContext *memContext; // Mem context
|
||||||
const String *name;
|
IoRead *read; // Read interface
|
||||||
|
IoWrite *write; // Write interface
|
||||||
|
const String *name; // Name displayed in logging
|
||||||
};
|
};
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
@ -49,12 +50,9 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
|
|||||||
|
|
||||||
*this = (ProtocolServer)
|
*this = (ProtocolServer)
|
||||||
{
|
{
|
||||||
.pub =
|
.memContext = memContextCurrent(),
|
||||||
{
|
.read = read,
|
||||||
.memContext = memContextCurrent(),
|
.write = write,
|
||||||
.read = read,
|
|
||||||
.write = write,
|
|
||||||
},
|
|
||||||
.name = strDup(name),
|
.name = strDup(name),
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -66,8 +64,8 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
|
|||||||
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_SERVICE_STR), VARSTR(service));
|
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_SERVICE_STR), VARSTR(service));
|
||||||
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_VERSION_STR), VARSTRZ(PROJECT_VERSION));
|
kvPut(greetingKv, VARSTR(PROTOCOL_GREETING_VERSION_STR), VARSTRZ(PROJECT_VERSION));
|
||||||
|
|
||||||
ioWriteStrLine(protocolServerIoWrite(this), jsonFromKv(greetingKv));
|
ioWriteStrLine(this->write, jsonFromKv(greetingKv));
|
||||||
ioWriteFlush(protocolServerIoWrite(this));
|
ioWriteFlush(this->write);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
}
|
}
|
||||||
@ -92,17 +90,50 @@ protocolServerError(ProtocolServer *this, int code, const String *message, const
|
|||||||
ASSERT(message != NULL);
|
ASSERT(message != NULL);
|
||||||
ASSERT(stack != NULL);
|
ASSERT(stack != NULL);
|
||||||
|
|
||||||
KeyValue *error = kvNew();
|
// Write the error and flush to be sure it gets sent immediately
|
||||||
kvPut(error, VARSTR(PROTOCOL_ERROR_STR), VARINT(code));
|
PackWrite *error = pckWriteNew(this->write);
|
||||||
kvPut(error, VARSTR(PROTOCOL_OUTPUT_STR), VARSTR(message));
|
pckWriteU32P(error, protocolMessageTypeError);
|
||||||
kvPut(error, VARSTR(PROTOCOL_ERROR_STACK_STR), VARSTR(stack));
|
pckWriteI32P(error, code);
|
||||||
|
pckWriteStrP(error, message);
|
||||||
|
pckWriteStrP(error, stack);
|
||||||
|
pckWriteEndP(error);
|
||||||
|
|
||||||
ioWriteStrLine(protocolServerIoWrite(this), jsonFromKv(error));
|
ioWriteFlush(this->write);
|
||||||
ioWriteFlush(protocolServerIoWrite(this));
|
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN_VOID();
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**********************************************************************************************************************************/
|
||||||
|
ProtocolServerCommandGetResult
|
||||||
|
protocolServerCommandGet(ProtocolServer *const this)
|
||||||
|
{
|
||||||
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
|
||||||
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
|
ProtocolServerCommandGetResult result = {0};
|
||||||
|
|
||||||
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
|
{
|
||||||
|
PackRead *const command = pckReadNew(this->read);
|
||||||
|
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(command);
|
||||||
|
|
||||||
|
CHECK(type == protocolMessageTypeCommand);
|
||||||
|
|
||||||
|
MEM_CONTEXT_PRIOR_BEGIN()
|
||||||
|
{
|
||||||
|
result.id = pckReadStrIdP(command);
|
||||||
|
result.param = pckReadPackBufP(command);
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_PRIOR_END();
|
||||||
|
|
||||||
|
pckReadEndP(command);
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
FUNCTION_LOG_RETURN_STRUCT(result);
|
||||||
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
protocolServerProcess(
|
protocolServerProcess(
|
||||||
@ -129,17 +160,15 @@ protocolServerProcess(
|
|||||||
{
|
{
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
// Read command
|
// Get command
|
||||||
KeyValue *commandKv = jsonToKv(ioReadLine(protocolServerIoRead(this)));
|
ProtocolServerCommandGetResult command = protocolServerCommandGet(this);
|
||||||
const StringId command = strIdFromStr(stringIdBit5, varStr(kvGet(commandKv, VARSTR(PROTOCOL_KEY_COMMAND_STR))));
|
|
||||||
VariantList *paramList = varVarLst(kvGet(commandKv, VARSTR(PROTOCOL_KEY_PARAMETER_STR)));
|
|
||||||
|
|
||||||
// Find the handler
|
// Find the handler
|
||||||
ProtocolServerCommandHandler handler = NULL;
|
ProtocolServerCommandHandler handler = NULL;
|
||||||
|
|
||||||
for (unsigned int handlerIdx = 0; handlerIdx < handlerListSize; handlerIdx++)
|
for (unsigned int handlerIdx = 0; handlerIdx < handlerListSize; handlerIdx++)
|
||||||
{
|
{
|
||||||
if (command == handlerList[handlerIdx].command)
|
if (command.id == handlerList[handlerIdx].command)
|
||||||
{
|
{
|
||||||
handler = handlerList[handlerIdx].handler;
|
handler = handlerList[handlerIdx].handler;
|
||||||
break;
|
break;
|
||||||
@ -151,7 +180,7 @@ protocolServerProcess(
|
|||||||
{
|
{
|
||||||
// Send the command to the handler. Run the handler in the server's memory context in case any persistent data
|
// Send the command to the handler. Run the handler in the server's memory context in case any persistent data
|
||||||
// needs to be stored by the handler.
|
// needs to be stored by the handler.
|
||||||
MEM_CONTEXT_BEGIN(this->pub.memContext)
|
MEM_CONTEXT_BEGIN(this->memContext)
|
||||||
{
|
{
|
||||||
// Initialize retries in case of command failure
|
// Initialize retries in case of command failure
|
||||||
bool retry = false;
|
bool retry = false;
|
||||||
@ -164,7 +193,7 @@ protocolServerProcess(
|
|||||||
|
|
||||||
TRY_BEGIN()
|
TRY_BEGIN()
|
||||||
{
|
{
|
||||||
handler(paramList, this);
|
handler(pckReadNewBuf(command.param), this);
|
||||||
}
|
}
|
||||||
CATCH_ANY()
|
CATCH_ANY()
|
||||||
{
|
{
|
||||||
@ -180,9 +209,8 @@ protocolServerProcess(
|
|||||||
"retry %s after %" PRIu64 "ms: %s", errorTypeName(errorType()), retrySleepMs,
|
"retry %s after %" PRIu64 "ms: %s", errorTypeName(errorType()), retrySleepMs,
|
||||||
errorMessage());
|
errorMessage());
|
||||||
|
|
||||||
// Sleep if there is an interval
|
// Sleep for interval
|
||||||
if (retrySleepMs > 0)
|
sleepMSec(retrySleepMs);
|
||||||
sleepMSec(retrySleepMs);
|
|
||||||
|
|
||||||
// Decrement retries remaining and retry
|
// Decrement retries remaining and retry
|
||||||
retryRemaining--;
|
retryRemaining--;
|
||||||
@ -204,18 +232,19 @@ protocolServerProcess(
|
|||||||
// Else check built-in commands
|
// Else check built-in commands
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
switch (command)
|
switch (command.id)
|
||||||
{
|
{
|
||||||
case PROTOCOL_COMMAND_EXIT:
|
case PROTOCOL_COMMAND_EXIT:
|
||||||
exit = true;
|
exit = true;
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case PROTOCOL_COMMAND_NOOP:
|
case PROTOCOL_COMMAND_NOOP:
|
||||||
protocolServerResponse(this, NULL);
|
protocolServerDataEndPut(this);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
THROW_FMT(ProtocolError, "invalid command '%s' (0x%" PRIx64 ")", strZ(strIdToStr(command)), command);
|
THROW_FMT(
|
||||||
|
ProtocolError, "invalid command '%s' (0x%" PRIx64 ")", strZ(strIdToStr(command.id)), command.id);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -239,45 +268,75 @@ protocolServerProcess(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
PackRead *
|
||||||
protocolServerResponse(ProtocolServer *this, const Variant *output)
|
protocolServerDataGet(ProtocolServer *const this)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
|
||||||
FUNCTION_LOG_PARAM(VARIANT, output);
|
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
KeyValue *result = kvNew();
|
PackRead *result = NULL;
|
||||||
|
|
||||||
if (output != NULL)
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
kvPut(result, VARSTR(PROTOCOL_OUTPUT_STR), output);
|
{
|
||||||
|
PackRead *data = pckReadNew(this->read);
|
||||||
|
ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(data);
|
||||||
|
|
||||||
ioWriteStrLine(protocolServerIoWrite(this), jsonFromKv(result));
|
CHECK(type == protocolMessageTypeData);
|
||||||
ioWriteFlush(protocolServerIoWrite(this));
|
|
||||||
|
MEM_CONTEXT_PRIOR_BEGIN()
|
||||||
|
{
|
||||||
|
result = pckReadPackP(data);
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_PRIOR_END();
|
||||||
|
|
||||||
|
pckReadEndP(data);
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
FUNCTION_LOG_RETURN(PACK_READ, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**********************************************************************************************************************************/
|
||||||
|
void
|
||||||
|
protocolServerDataPut(ProtocolServer *const this, PackWrite *const data)
|
||||||
|
{
|
||||||
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
|
||||||
|
FUNCTION_LOG_PARAM(PACK_WRITE, data);
|
||||||
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
|
// End the pack
|
||||||
|
if (data != NULL)
|
||||||
|
pckWriteEndP(data);
|
||||||
|
|
||||||
|
// Write the result
|
||||||
|
PackWrite *resultMessage = pckWriteNew(this->write);
|
||||||
|
pckWriteU32P(resultMessage, protocolMessageTypeData, .defaultWrite = true);
|
||||||
|
pckWritePackP(resultMessage, data);
|
||||||
|
pckWriteEndP(resultMessage);
|
||||||
|
|
||||||
|
// Flush on NULL result since it might be used to synchronize
|
||||||
|
if (data == NULL)
|
||||||
|
ioWriteFlush(this->write);
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN_VOID();
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
protocolServerWriteLine(ProtocolServer *this, const String *line)
|
protocolServerDataEndPut(ProtocolServer *const this)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
|
||||||
FUNCTION_LOG_PARAM(STRING, line);
|
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(this != NULL);
|
// Write the response and flush to be sure it gets sent immediately
|
||||||
|
PackWrite *response = pckWriteNew(this->write);
|
||||||
|
pckWriteU32P(response, protocolMessageTypeDataEnd, .defaultWrite = true);
|
||||||
|
pckWriteEndP(response);
|
||||||
|
|
||||||
// Dot indicates the start of an lf-terminated line
|
ioWriteFlush(this->write);
|
||||||
ioWrite(protocolServerIoWrite(this), DOT_BUF);
|
|
||||||
|
|
||||||
// Write the line if it exists
|
|
||||||
if (line != NULL)
|
|
||||||
ioWriteStr(protocolServerIoWrite(this), line);
|
|
||||||
|
|
||||||
// Terminate with a linefeed
|
|
||||||
ioWrite(protocolServerIoWrite(this), LF_BUF);
|
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN_VOID();
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
@ -12,7 +12,9 @@ typedef struct ProtocolServer ProtocolServer;
|
|||||||
#include "common/io/read.h"
|
#include "common/io/read.h"
|
||||||
#include "common/io/write.h"
|
#include "common/io/write.h"
|
||||||
#include "common/type/object.h"
|
#include "common/type/object.h"
|
||||||
|
#include "common/type/pack.h"
|
||||||
#include "common/type/stringId.h"
|
#include "common/type/stringId.h"
|
||||||
|
#include "protocol/client.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Protocol command handler type and structure
|
Protocol command handler type and structure
|
||||||
@ -20,7 +22,7 @@ Protocol command handler type and structure
|
|||||||
An array of this struct must be passed to protocolServerProcess() for the server to process commands. Each command handler should
|
An array of this struct must be passed to protocolServerProcess() for the server to process commands. Each command handler should
|
||||||
implement a single command, as defined by the command string.
|
implement a single command, as defined by the command string.
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
typedef void (*ProtocolServerCommandHandler)(const VariantList *paramList, ProtocolServer *server);
|
typedef void (*ProtocolServerCommandHandler)(PackRead *param, ProtocolServer *server);
|
||||||
|
|
||||||
typedef struct ProtocolServerHandler
|
typedef struct ProtocolServerHandler
|
||||||
{
|
{
|
||||||
@ -35,44 +37,36 @@ Constructors
|
|||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
ProtocolServer *protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write);
|
ProtocolServer *protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Getters/Setters
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
typedef struct ProtocolServerPub
|
|
||||||
{
|
|
||||||
MemContext *memContext; // Mem context
|
|
||||||
IoRead *read; // Read interface
|
|
||||||
IoWrite *write; // Write interface
|
|
||||||
} ProtocolServerPub;
|
|
||||||
|
|
||||||
// Read interface
|
|
||||||
__attribute__((always_inline)) static inline IoRead *
|
|
||||||
protocolServerIoRead(ProtocolServer *const this)
|
|
||||||
{
|
|
||||||
return THIS_PUB(ProtocolServer)->read;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write interface
|
|
||||||
__attribute__((always_inline)) static inline IoWrite *
|
|
||||||
protocolServerIoWrite(ProtocolServer *const this)
|
|
||||||
{
|
|
||||||
return THIS_PUB(ProtocolServer)->write;
|
|
||||||
}
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
|
// Get command from the client. Outside ProtocolServer, this is used when the first noop command needs to be processed before
|
||||||
|
// running protocolServerProcess(), which allows an error to be returned to the client if initialization fails.
|
||||||
|
typedef struct ProtocolServerCommandGetResult
|
||||||
|
{
|
||||||
|
StringId id; // Command identifier
|
||||||
|
Buffer *param; // Parameter pack
|
||||||
|
} ProtocolServerCommandGetResult;
|
||||||
|
|
||||||
|
ProtocolServerCommandGetResult protocolServerCommandGet(ProtocolServer *this);
|
||||||
|
|
||||||
|
// Get data from the client
|
||||||
|
PackRead *protocolServerDataGet(ProtocolServer *this);
|
||||||
|
|
||||||
|
// Put data to the client
|
||||||
|
void protocolServerDataPut(ProtocolServer *this, PackWrite *data);
|
||||||
|
|
||||||
|
// Put data end to the client. This ends command processing and no more data should be sent.
|
||||||
|
void protocolServerDataEndPut(ProtocolServer *this);
|
||||||
|
|
||||||
// Return an error
|
// Return an error
|
||||||
void protocolServerError(ProtocolServer *this, int code, const String *message, const String *stack);
|
void protocolServerError(ProtocolServer *this, int code, const String *message, const String *stack);
|
||||||
|
|
||||||
// Process requests
|
// Process requests
|
||||||
void protocolServerProcess(
|
void protocolServerProcess(
|
||||||
ProtocolServer *this, const VariantList *retryInterval, const ProtocolServerHandler *const handlerList,
|
ProtocolServer *this, const VariantList *retryInterval, const ProtocolServerHandler *handlerList,
|
||||||
const unsigned int handlerListSize);
|
const unsigned int handlerListSize);
|
||||||
|
|
||||||
// Respond to request with output if provided
|
|
||||||
void protocolServerResponse(ProtocolServer *this, const Variant *output);
|
|
||||||
|
|
||||||
// Move to a new parent mem context
|
// Move to a new parent mem context
|
||||||
__attribute__((always_inline)) static inline ProtocolServer *
|
__attribute__((always_inline)) static inline ProtocolServer *
|
||||||
protocolServerMove(ProtocolServer *const this, MemContext *const parentNew)
|
protocolServerMove(ProtocolServer *const this, MemContext *const parentNew)
|
||||||
@ -80,9 +74,6 @@ protocolServerMove(ProtocolServer *const this, MemContext *const parentNew)
|
|||||||
return objMove(this, parentNew);
|
return objMove(this, parentNew);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write a line
|
|
||||||
void protocolServerWriteLine(ProtocolServer *this, const String *line);
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Destructor
|
Destructor
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
|
@ -14,6 +14,7 @@ Remote Storage Protocol Handler
|
|||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
#include "common/regExp.h"
|
#include "common/regExp.h"
|
||||||
|
#include "common/type/pack.h"
|
||||||
#include "common/type/json.h"
|
#include "common/type/json.h"
|
||||||
#include "config/config.h"
|
#include "config/config.h"
|
||||||
#include "protocol/helper.h"
|
#include "protocol/helper.h"
|
||||||
@ -21,11 +22,6 @@ Remote Storage Protocol Handler
|
|||||||
#include "storage/helper.h"
|
#include "storage/helper.h"
|
||||||
#include "storage/storage.intern.h"
|
#include "storage/storage.intern.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Regular expressions
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
STRING_STATIC(BLOCK_REG_EXP_STR, PROTOCOL_BLOCK_HEADER "-1|[0-9]+");
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Local variables
|
Local variables
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
@ -76,64 +72,16 @@ storageRemoteFilterGroup(IoFilterGroup *filterGroup, const Variant *filterList)
|
|||||||
FUNCTION_TEST_RETURN_VOID();
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Callback to write info list into the protocol
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
// Helper to write storage info into the protocol. This function is not called unless the info exists so no need to write exists or
|
|
||||||
// check for level == storageInfoLevelExists.
|
|
||||||
static void
|
|
||||||
storageRemoteInfoWrite(ProtocolServer *server, const StorageInfo *info)
|
|
||||||
{
|
|
||||||
FUNCTION_TEST_BEGIN();
|
|
||||||
FUNCTION_TEST_PARAM(PROTOCOL_SERVER, server);
|
|
||||||
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
|
|
||||||
FUNCTION_TEST_END();
|
|
||||||
|
|
||||||
protocolServerWriteLine(server, jsonFromUInt(info->type));
|
|
||||||
protocolServerWriteLine(server, jsonFromInt64(info->timeModified));
|
|
||||||
|
|
||||||
if (info->type == storageTypeFile)
|
|
||||||
protocolServerWriteLine(server, jsonFromUInt64(info->size));
|
|
||||||
|
|
||||||
if (info->level >= storageInfoLevelDetail)
|
|
||||||
{
|
|
||||||
protocolServerWriteLine(server, jsonFromUInt(info->userId));
|
|
||||||
protocolServerWriteLine(server, jsonFromStr(info->user));
|
|
||||||
protocolServerWriteLine(server, jsonFromUInt(info->groupId));
|
|
||||||
protocolServerWriteLine(server, jsonFromStr(info->group));
|
|
||||||
protocolServerWriteLine(server, jsonFromUInt(info->mode));
|
|
||||||
|
|
||||||
if (info->type == storageTypeLink)
|
|
||||||
protocolServerWriteLine(server, jsonFromStr(info->linkDestination));
|
|
||||||
}
|
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN_VOID();
|
|
||||||
}
|
|
||||||
|
|
||||||
static void
|
|
||||||
storageRemoteProtocolInfoListCallback(void *server, const StorageInfo *info)
|
|
||||||
{
|
|
||||||
FUNCTION_TEST_BEGIN();
|
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
|
||||||
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
|
|
||||||
FUNCTION_TEST_END();
|
|
||||||
|
|
||||||
protocolServerWriteLine(server, jsonFromStr(info->name));
|
|
||||||
storageRemoteInfoWrite(server, info);
|
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN_VOID();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *server)
|
storageRemoteFeatureProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList == NULL);
|
ASSERT(param == NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
@ -158,10 +106,12 @@ storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *serve
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Return storage features
|
// Return storage features
|
||||||
protocolServerWriteLine(server, jsonFromStr(storagePathP(storage, NULL)));
|
PackWrite *result = protocolPackNew();
|
||||||
protocolServerWriteLine(server, jsonFromUInt64(storageInterface(storage).feature));
|
pckWriteStrP(result, storagePathP(storage, NULL));
|
||||||
|
pckWriteU64P(result, storageInterface(storage).feature);
|
||||||
|
|
||||||
protocolServerResponse(server, NULL);
|
protocolServerDataPut(server, result);
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -169,58 +119,203 @@ storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *serve
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
typedef struct StorageRemoteInfoProcotolWriteData
|
||||||
storageRemoteInfoProtocol(const VariantList *paramList, ProtocolServer *server)
|
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
MemContext *memContext; // Mem context used to store values from last call
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
time_t timeModifiedLast; // timeModified from last call
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
mode_t modeLast; // mode from last call
|
||||||
FUNCTION_LOG_END();
|
uid_t userIdLast; // userId from last call
|
||||||
|
gid_t groupIdLast; // groupId from last call
|
||||||
|
String *user; // user from last call
|
||||||
|
String *group; // group from last call
|
||||||
|
} StorageRemoteInfoProtocolWriteData;
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
// Helper to write storage info into the protocol. This function is not called unless the info exists so no need to write exists or
|
||||||
ASSERT(server != NULL);
|
// check for level == storageInfoLevelExists.
|
||||||
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
//
|
||||||
|
// Fields that do not change from one call to the next are omitted to save bandwidth.
|
||||||
|
static void
|
||||||
|
storageRemoteInfoProtocolPut(
|
||||||
|
StorageRemoteInfoProtocolWriteData *const data, PackWrite *const write, const StorageInfo *const info)
|
||||||
|
{
|
||||||
|
FUNCTION_TEST_BEGIN();
|
||||||
|
FUNCTION_TEST_PARAM_P(VOID, data);
|
||||||
|
FUNCTION_TEST_PARAM(PACK_WRITE, write);
|
||||||
|
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
|
||||||
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
ASSERT(data != NULL);
|
||||||
|
ASSERT(write != NULL);
|
||||||
|
ASSERT(info != NULL);
|
||||||
|
|
||||||
|
// Write type and time
|
||||||
|
pckWriteU32P(write, info->type);
|
||||||
|
pckWriteTimeP(write, info->timeModified - data->timeModifiedLast);
|
||||||
|
|
||||||
|
// Write size for files
|
||||||
|
if (info->type == storageTypeFile)
|
||||||
|
pckWriteU64P(write, info->size);
|
||||||
|
|
||||||
|
// Write fields needed for detail level
|
||||||
|
if (info->level >= storageInfoLevelDetail)
|
||||||
{
|
{
|
||||||
StorageInfo info = storageInterfaceInfoP(
|
// Write mode
|
||||||
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)),
|
pckWriteModeP(write, info->mode, .defaultValue = data->modeLast);
|
||||||
(StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)), .followLink = varBool(varLstGet(paramList, 2)));
|
|
||||||
|
|
||||||
protocolServerResponse(server, VARBOOL(info.exists));
|
// Write user id/name
|
||||||
|
pckWriteU32P(write, info->userId, .defaultValue = data->userIdLast);
|
||||||
|
|
||||||
if (info.exists)
|
if (info->user == NULL) // {vm_covered}
|
||||||
|
pckWriteBoolP(write, true); // {vm_covered}
|
||||||
|
else
|
||||||
{
|
{
|
||||||
storageRemoteInfoWrite(server, &info);
|
pckWriteBoolP(write, false);
|
||||||
protocolServerResponse(server, NULL);
|
pckWriteStrP(write, info->user, .defaultValue = data->user);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write group id/name
|
||||||
|
pckWriteU32P(write, info->groupId, .defaultValue = data->groupIdLast);
|
||||||
|
|
||||||
|
if (info->group == NULL) // {vm_covered}
|
||||||
|
pckWriteBoolP(write, true); // {vm_covered}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
pckWriteBoolP(write, false);
|
||||||
|
pckWriteStrP(write, info->group, .defaultValue = data->group);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write link destination
|
||||||
|
if (info->type == storageTypeLink)
|
||||||
|
pckWriteStrP(write, info->linkDestination);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store defaults to use for the next call. If memContext is NULL this function is only being called one time so there is no
|
||||||
|
// point in storing defaults.
|
||||||
|
if (data->memContext != NULL)
|
||||||
|
{
|
||||||
|
data->timeModifiedLast = info->timeModified;
|
||||||
|
data->modeLast = info->mode;
|
||||||
|
data->userIdLast = info->userId;
|
||||||
|
data->groupIdLast = info->groupId;
|
||||||
|
|
||||||
|
if (!strEq(info->user, data->user) && info->user != NULL) // {vm_covered}
|
||||||
|
{
|
||||||
|
strFree(data->user);
|
||||||
|
|
||||||
|
MEM_CONTEXT_BEGIN(data->memContext)
|
||||||
|
{
|
||||||
|
data->user = strDup(info->user);
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_END();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!strEq(info->group, data->group) && info->group != NULL) // {vm_covered}
|
||||||
|
{
|
||||||
|
strFree(data->group);
|
||||||
|
|
||||||
|
MEM_CONTEXT_BEGIN(data->memContext)
|
||||||
|
{
|
||||||
|
data->group = strDup(info->group);
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_END();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
storageRemoteInfoProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
|
{
|
||||||
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
|
ASSERT(param != NULL);
|
||||||
|
ASSERT(server != NULL);
|
||||||
|
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
||||||
|
|
||||||
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
|
{
|
||||||
|
// Get file info
|
||||||
|
const String *file = pckReadStrP(param);
|
||||||
|
StorageInfoLevel level = (StorageInfoLevel)pckReadU32P(param);
|
||||||
|
bool followLink = pckReadBoolP(param);
|
||||||
|
|
||||||
|
StorageInfo info = storageInterfaceInfoP(storageRemoteProtocolLocal.driver, file, level, .followLink = followLink);
|
||||||
|
|
||||||
|
// Write file info to protocol
|
||||||
|
PackWrite *write = protocolPackNew();
|
||||||
|
pckWriteBoolP(write, info.exists, .defaultWrite = true);
|
||||||
|
|
||||||
|
if (info.exists)
|
||||||
|
storageRemoteInfoProtocolPut(&(StorageRemoteInfoProtocolWriteData){0}, write, &info);
|
||||||
|
|
||||||
|
protocolServerDataPut(server, write);
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN_VOID();
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
|
typedef struct StorageRemoteProtocolInfoListCallbackData
|
||||||
|
{
|
||||||
|
ProtocolServer *const server;
|
||||||
|
StorageRemoteInfoProtocolWriteData writeData;
|
||||||
|
} StorageRemoteProtocolInfoListCallbackData;
|
||||||
|
|
||||||
|
static void
|
||||||
|
storageRemoteProtocolInfoListCallback(void *dataVoid, const StorageInfo *info)
|
||||||
|
{
|
||||||
|
FUNCTION_TEST_BEGIN();
|
||||||
|
FUNCTION_LOG_PARAM_P(VOID, dataVoid);
|
||||||
|
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
|
||||||
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
|
ASSERT(dataVoid != NULL);
|
||||||
|
ASSERT(info != NULL);
|
||||||
|
|
||||||
|
StorageRemoteProtocolInfoListCallbackData *data = dataVoid;
|
||||||
|
|
||||||
|
PackWrite *write = protocolPackNew();
|
||||||
|
pckWriteStrP(write, info->name);
|
||||||
|
storageRemoteInfoProtocolPut(&data->writeData, write, info);
|
||||||
|
protocolServerDataPut(data->server, write);
|
||||||
|
|
||||||
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
storageRemoteInfoListProtocol(const VariantList *paramList, ProtocolServer *server)
|
storageRemoteInfoListProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
bool result = storageInterfaceInfoListP(
|
const String *const path = pckReadStrP(param);
|
||||||
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)),
|
const StorageInfoLevel level = (StorageInfoLevel)pckReadU32P(param);
|
||||||
(StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)), storageRemoteProtocolInfoListCallback, server);
|
|
||||||
|
|
||||||
protocolServerWriteLine(server, NULL);
|
StorageRemoteProtocolInfoListCallbackData data = {.server = server, .writeData = {.memContext = memContextCurrent()}};
|
||||||
protocolServerResponse(server, VARBOOL(result));
|
|
||||||
|
const bool result = storageInterfaceInfoListP(
|
||||||
|
storageRemoteProtocolLocal.driver, path, level, storageRemoteProtocolInfoListCallback, &data);
|
||||||
|
|
||||||
|
// Indicate whether or not the path was found
|
||||||
|
PackWrite *write = protocolPackNew();
|
||||||
|
pckWriteBoolP(write, result, .defaultWrite = true);
|
||||||
|
protocolServerDataPut(server, write);
|
||||||
|
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -229,31 +324,34 @@ storageRemoteInfoListProtocol(const VariantList *paramList, ProtocolServer *serv
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *server)
|
storageRemoteOpenReadProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
|
const String *file = pckReadStrP(param);
|
||||||
|
bool ignoreMissing = pckReadBoolP(param);
|
||||||
|
const Variant *limit = jsonToVar(pckReadStrP(param));
|
||||||
|
const Variant *filter = jsonToVar(pckReadStrP(param));
|
||||||
|
|
||||||
// Create the read object
|
// Create the read object
|
||||||
IoRead *fileRead = storageReadIo(
|
IoRead *fileRead = storageReadIo(
|
||||||
storageInterfaceNewReadP(
|
storageInterfaceNewReadP(storageRemoteProtocolLocal.driver, file, ignoreMissing, .limit = limit));
|
||||||
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)),
|
|
||||||
.limit = varLstGet(paramList, 2)));
|
|
||||||
|
|
||||||
// Set filter group based on passed filters
|
// Set filter group based on passed filters
|
||||||
storageRemoteFilterGroup(ioReadFilterGroup(fileRead), varLstGet(paramList, 3));
|
storageRemoteFilterGroup(ioReadFilterGroup(fileRead), filter);
|
||||||
|
|
||||||
// Check if the file exists
|
// Check if the file exists
|
||||||
bool exists = ioReadOpen(fileRead);
|
bool exists = ioReadOpen(fileRead);
|
||||||
protocolServerResponse(server, VARBOOL(exists));
|
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), exists, .defaultWrite = true));
|
||||||
|
|
||||||
// Transfer the file if it exists
|
// Transfer the file if it exists
|
||||||
if (exists)
|
if (exists)
|
||||||
@ -267,9 +365,9 @@ storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *serv
|
|||||||
|
|
||||||
if (!bufEmpty(buffer))
|
if (!bufEmpty(buffer))
|
||||||
{
|
{
|
||||||
ioWriteStrLine(protocolServerIoWrite(server), strNewFmt(PROTOCOL_BLOCK_HEADER "%zu", bufUsed(buffer)));
|
PackWrite *write = protocolPackNew();
|
||||||
ioWrite(protocolServerIoWrite(server), buffer);
|
pckWriteBinP(write, buffer);
|
||||||
ioWriteFlush(protocolServerIoWrite(server));
|
protocolServerDataPut(server, write);
|
||||||
|
|
||||||
bufUsedZero(buffer);
|
bufUsedZero(buffer);
|
||||||
}
|
}
|
||||||
@ -278,13 +376,12 @@ storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *serv
|
|||||||
|
|
||||||
ioReadClose(fileRead);
|
ioReadClose(fileRead);
|
||||||
|
|
||||||
// Write a zero block to show file is complete
|
// Write filter results
|
||||||
ioWriteLine(protocolServerIoWrite(server), BUFSTRDEF(PROTOCOL_BLOCK_HEADER "0"));
|
protocolServerDataPut(
|
||||||
ioWriteFlush(protocolServerIoWrite(server));
|
server, pckWriteStrP(protocolPackNew(), jsonFromVar(ioFilterGroupResultAll(ioReadFilterGroup(fileRead)))));
|
||||||
|
|
||||||
// Push filter results
|
|
||||||
protocolServerResponse(server, ioFilterGroupResultAll(ioReadFilterGroup(fileRead)));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -293,79 +390,85 @@ storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *serv
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
storageRemoteOpenWriteProtocol(const VariantList *paramList, ProtocolServer *server)
|
storageRemoteOpenWriteProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
// Create the write object
|
// Create the write object
|
||||||
|
const String *file = pckReadStrP(param);
|
||||||
|
mode_t modeFile = pckReadModeP(param);
|
||||||
|
mode_t modePath = pckReadModeP(param);
|
||||||
|
const String *user = pckReadStrP(param);
|
||||||
|
const String *group = pckReadStrP(param);
|
||||||
|
time_t timeModified = pckReadTimeP(param);
|
||||||
|
bool createPath = pckReadBoolP(param);
|
||||||
|
bool syncFile = pckReadBoolP(param);
|
||||||
|
bool syncPath = pckReadBoolP(param);
|
||||||
|
bool atomic = pckReadBoolP(param);
|
||||||
|
const Variant *filter = jsonToVar(pckReadStrP(param));
|
||||||
|
|
||||||
IoWrite *fileWrite = storageWriteIo(
|
IoWrite *fileWrite = storageWriteIo(
|
||||||
storageInterfaceNewWriteP(
|
storageInterfaceNewWriteP(
|
||||||
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)),
|
storageRemoteProtocolLocal.driver, file, .modeFile = modeFile, .modePath = modePath, .user = user, .group = group,
|
||||||
.modeFile = (mode_t)varUIntForce(varLstGet(paramList, 1)),
|
.timeModified = timeModified, .createPath = createPath, .syncFile = syncFile, .syncPath = syncPath,
|
||||||
.modePath = (mode_t)varUIntForce(varLstGet(paramList, 2)), .user = varStr(varLstGet(paramList, 3)),
|
.atomic = atomic));
|
||||||
.group = varStr(varLstGet(paramList, 4)), .timeModified = (time_t)varUInt64Force(varLstGet(paramList, 5)),
|
|
||||||
.createPath = varBool(varLstGet(paramList, 6)), .syncFile = varBool(varLstGet(paramList, 7)),
|
|
||||||
.syncPath = varBool(varLstGet(paramList, 8)), .atomic = varBool(varLstGet(paramList, 9))));
|
|
||||||
|
|
||||||
// Set filter group based on passed filters
|
// Set filter group based on passed filters
|
||||||
storageRemoteFilterGroup(ioWriteFilterGroup(fileWrite), varLstGet(paramList, 10));
|
storageRemoteFilterGroup(ioWriteFilterGroup(fileWrite), filter);
|
||||||
|
|
||||||
// Open file
|
// Open file
|
||||||
ioWriteOpen(fileWrite);
|
ioWriteOpen(fileWrite);
|
||||||
protocolServerResponse(server, NULL);
|
protocolServerDataPut(server, NULL);
|
||||||
|
|
||||||
// Write data
|
// Write data
|
||||||
Buffer *buffer = bufNew(ioBufferSize());
|
|
||||||
ssize_t remaining;
|
|
||||||
|
|
||||||
do
|
do
|
||||||
{
|
{
|
||||||
// How much data is remaining to write?
|
PackRead *read = protocolServerDataGet(server);
|
||||||
remaining = storageRemoteProtocolBlockSize(ioReadLine(protocolServerIoRead(server)));
|
|
||||||
|
|
||||||
// Write data
|
// Write is complete
|
||||||
if (remaining > 0)
|
if (read == NULL)
|
||||||
{
|
|
||||||
size_t bytesToCopy = (size_t)remaining;
|
|
||||||
|
|
||||||
do
|
|
||||||
{
|
|
||||||
if (bytesToCopy < bufSize(buffer))
|
|
||||||
bufLimitSet(buffer, bytesToCopy);
|
|
||||||
|
|
||||||
bytesToCopy -= ioRead(protocolServerIoRead(server), buffer);
|
|
||||||
ioWrite(fileWrite, buffer);
|
|
||||||
|
|
||||||
bufUsedZero(buffer);
|
|
||||||
bufLimitClear(buffer);
|
|
||||||
}
|
|
||||||
while (bytesToCopy > 0);
|
|
||||||
}
|
|
||||||
// Close when all data has been written
|
|
||||||
else if (remaining == 0)
|
|
||||||
{
|
{
|
||||||
ioWriteClose(fileWrite);
|
ioWriteClose(fileWrite);
|
||||||
|
|
||||||
// Push filter results
|
// Push filter results
|
||||||
protocolServerResponse(server, ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite)));
|
protocolServerDataPut(
|
||||||
|
server, pckWriteStrP(protocolPackNew(), jsonFromVar(ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite)))));
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
// Write was aborted so free the file
|
// Else more data to write
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
ioWriteFree(fileWrite);
|
pckReadNext(read);
|
||||||
protocolServerResponse(server, NULL);
|
|
||||||
|
// Write data
|
||||||
|
if (pckReadType(read) == pckTypeBin)
|
||||||
|
{
|
||||||
|
Buffer *const buffer = pckReadBinP(read);
|
||||||
|
|
||||||
|
ioWrite(fileWrite, buffer);
|
||||||
|
bufFree(buffer);
|
||||||
|
}
|
||||||
|
// Else write terminated unexpectedly
|
||||||
|
else
|
||||||
|
{
|
||||||
|
protocolServerDataGet(server);
|
||||||
|
ioWriteFree(fileWrite);
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
while (remaining > 0);
|
while (true);
|
||||||
|
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -374,24 +477,26 @@ storageRemoteOpenWriteProtocol(const VariantList *paramList, ProtocolServer *ser
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
storageRemotePathCreateProtocol(const VariantList *paramList, ProtocolServer *server)
|
storageRemotePathCreateProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
storageInterfacePathCreateP(
|
const String *path = pckReadStrP(param);
|
||||||
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)),
|
bool errorOnExists = pckReadBoolP(param);
|
||||||
varBool(varLstGet(paramList, 2)), (mode_t)varUIntForce(varLstGet(paramList, 3)));
|
bool noParentCreate = pckReadBoolP(param);
|
||||||
|
mode_t mode = pckReadModeP(param);
|
||||||
|
|
||||||
protocolServerResponse(server, NULL);
|
storageInterfacePathCreateP(storageRemoteProtocolLocal.driver, path, errorOnExists, noParentCreate, mode);
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -400,24 +505,26 @@ storageRemotePathCreateProtocol(const VariantList *paramList, ProtocolServer *se
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
storageRemotePathRemoveProtocol(const VariantList *paramList, ProtocolServer *server)
|
storageRemotePathRemoveProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
protocolServerResponse(
|
const String *path = pckReadStrP(param);
|
||||||
server,
|
bool recurse = pckReadBoolP(param);
|
||||||
VARBOOL(
|
|
||||||
storageInterfacePathRemoveP(
|
const bool result = storageInterfacePathRemoveP(storageRemoteProtocolLocal.driver, path, recurse);
|
||||||
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)))));
|
|
||||||
|
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), result, .defaultWrite = true));
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -426,21 +533,23 @@ storageRemotePathRemoveProtocol(const VariantList *paramList, ProtocolServer *se
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
storageRemotePathSyncProtocol(const VariantList *paramList, ProtocolServer *server)
|
storageRemotePathSyncProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
storageInterfacePathSyncP(storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)));
|
const String *path = pckReadStrP(param);
|
||||||
protocolServerResponse(server, NULL);
|
|
||||||
|
storageInterfacePathSyncP(storageRemoteProtocolLocal.driver, path);
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -449,54 +558,26 @@ storageRemotePathSyncProtocol(const VariantList *paramList, ProtocolServer *serv
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
void
|
void
|
||||||
storageRemoteRemoveProtocol(const VariantList *paramList, ProtocolServer *server)
|
storageRemoteRemoveProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_LOG_PARAM(PACK_READ, param);
|
||||||
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(paramList != NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
ASSERT(storageRemoteProtocolLocal.driver != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
storageInterfaceRemoveP(
|
const String *file = pckReadStrP(param);
|
||||||
storageRemoteProtocolLocal.driver, varStr(varLstGet(paramList, 0)), .errorOnMissing = varBool(varLstGet(paramList, 1)));
|
bool errorOnMissing = pckReadBoolP(param);
|
||||||
protocolServerResponse(server, NULL);
|
|
||||||
|
storageInterfaceRemoveP(storageRemoteProtocolLocal.driver, file, .errorOnMissing = errorOnMissing);
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN_VOID();
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
|
||||||
ssize_t
|
|
||||||
storageRemoteProtocolBlockSize(const String *message)
|
|
||||||
{
|
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
|
||||||
FUNCTION_LOG_PARAM(STRING, message);
|
|
||||||
FUNCTION_LOG_END();
|
|
||||||
|
|
||||||
ASSERT(message != NULL);
|
|
||||||
|
|
||||||
// Regular expression to check block messages
|
|
||||||
static RegExp *blockRegExp = NULL;
|
|
||||||
|
|
||||||
// Create block regular expression if it has not been created yet
|
|
||||||
if (blockRegExp == NULL)
|
|
||||||
{
|
|
||||||
MEM_CONTEXT_BEGIN(memContextTop())
|
|
||||||
{
|
|
||||||
blockRegExp = regExpNew(BLOCK_REG_EXP_STR);
|
|
||||||
}
|
|
||||||
MEM_CONTEXT_END();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate the header block size message
|
|
||||||
if (!regExpMatch(blockRegExp, message))
|
|
||||||
THROW_FMT(ProtocolError, "'%s' is not a valid block size message", strZ(message));
|
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(SSIZE, (ssize_t)cvtZToInt(strZ(message) + sizeof(PROTOCOL_BLOCK_HEADER) - 1));
|
|
||||||
}
|
|
||||||
|
@ -4,31 +4,22 @@ Remote Storage Protocol Handler
|
|||||||
#ifndef STORAGE_REMOTE_PROTOCOL_H
|
#ifndef STORAGE_REMOTE_PROTOCOL_H
|
||||||
#define STORAGE_REMOTE_PROTOCOL_H
|
#define STORAGE_REMOTE_PROTOCOL_H
|
||||||
|
|
||||||
#include "common/type/string.h"
|
#include "common/type/pack.h"
|
||||||
#include "common/type/variantList.h"
|
|
||||||
#include "protocol/server.h"
|
#include "protocol/server.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Constants
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
#define PROTOCOL_BLOCK_HEADER "BRBLOCK"
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
// Get size of the next transfer block
|
|
||||||
ssize_t storageRemoteProtocolBlockSize(const String *message);
|
|
||||||
|
|
||||||
// Process storage protocol requests
|
// Process storage protocol requests
|
||||||
void storageRemoteFeatureProtocol(const VariantList *paramList, ProtocolServer *server);
|
void storageRemoteFeatureProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void storageRemoteInfoProtocol(const VariantList *paramList, ProtocolServer *server);
|
void storageRemoteInfoProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void storageRemoteInfoListProtocol(const VariantList *paramList, ProtocolServer *server);
|
void storageRemoteInfoListProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void storageRemoteOpenReadProtocol(const VariantList *paramList, ProtocolServer *server);
|
void storageRemoteOpenReadProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void storageRemoteOpenWriteProtocol(const VariantList *paramList, ProtocolServer *server);
|
void storageRemoteOpenWriteProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void storageRemotePathCreateProtocol(const VariantList *paramList, ProtocolServer *server);
|
void storageRemotePathCreateProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void storageRemotePathRemoveProtocol(const VariantList *paramList, ProtocolServer *server);
|
void storageRemotePathRemoveProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void storageRemotePathSyncProtocol(const VariantList *paramList, ProtocolServer *server);
|
void storageRemotePathSyncProtocol(PackRead *param, ProtocolServer *server);
|
||||||
void storageRemoteRemoveProtocol(const VariantList *paramList, ProtocolServer *server);
|
void storageRemoteRemoveProtocol(PackRead *param, ProtocolServer *server);
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
|
||||||
|
@ -12,6 +12,7 @@ Remote Storage Read
|
|||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
#include "common/type/convert.h"
|
#include "common/type/convert.h"
|
||||||
|
#include "common/type/json.h"
|
||||||
#include "common/type/object.h"
|
#include "common/type/object.h"
|
||||||
#include "storage/remote/protocol.h"
|
#include "storage/remote/protocol.h"
|
||||||
#include "storage/remote/read.h"
|
#include "storage/remote/read.h"
|
||||||
@ -29,6 +30,7 @@ typedef struct StorageReadRemote
|
|||||||
|
|
||||||
ProtocolClient *client; // Protocol client for requests
|
ProtocolClient *client; // Protocol client for requests
|
||||||
size_t remaining; // Bytes remaining to be read in block
|
size_t remaining; // Bytes remaining to be read in block
|
||||||
|
Buffer *block; // Block currently being read
|
||||||
bool eof; // Has the file reached eof?
|
bool eof; // Has the file reached eof?
|
||||||
|
|
||||||
#ifdef DEBUG
|
#ifdef DEBUG
|
||||||
@ -70,19 +72,30 @@ storageReadRemoteOpen(THIS_VOID)
|
|||||||
}
|
}
|
||||||
|
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ);
|
||||||
protocolCommandParamAdd(command, VARSTR(this->interface.name));
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARBOOL(this->interface.ignoreMissing));
|
|
||||||
protocolCommandParamAdd(command, this->interface.limit);
|
|
||||||
protocolCommandParamAdd(command, ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read))));
|
|
||||||
|
|
||||||
result = varBool(protocolClientExecute(this->client, command, true));
|
pckWriteStrP(param, this->interface.name);
|
||||||
|
pckWriteBoolP(param, this->interface.ignoreMissing);
|
||||||
|
pckWriteStrP(param, jsonFromVar(this->interface.limit));
|
||||||
|
pckWriteStrP(param, jsonFromVar(ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read)))));
|
||||||
|
|
||||||
// Clear filters since they will be run on the remote side
|
protocolClientCommandPut(this->client, command);
|
||||||
ioFilterGroupClear(ioReadFilterGroup(storageReadIo(this->read)));
|
|
||||||
|
|
||||||
// If the file is compressible add decompression filter locally
|
// If the file exists
|
||||||
if (this->interface.compressible)
|
result = pckReadBoolP(protocolClientDataGet(this->client));
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilter(compressTypeGz));
|
|
||||||
|
if (result)
|
||||||
|
{
|
||||||
|
// Clear filters since they will be run on the remote side
|
||||||
|
ioFilterGroupClear(ioReadFilterGroup(storageReadIo(this->read)));
|
||||||
|
|
||||||
|
// If the file is compressible add decompression filter locally
|
||||||
|
if (this->interface.compressible)
|
||||||
|
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilter(compressTypeGz));
|
||||||
|
}
|
||||||
|
// Else nothing to do
|
||||||
|
else
|
||||||
|
protocolClientDataEndGet(this->client);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -118,13 +131,28 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
|
|||||||
{
|
{
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
this->remaining = (size_t)storageRemoteProtocolBlockSize(ioReadLine(protocolClientIoRead(this->client)));
|
PackRead *const read = protocolClientDataGet(this->client);
|
||||||
|
pckReadNext(read);
|
||||||
|
|
||||||
if (this->remaining == 0)
|
// If binary then read the next block
|
||||||
|
if (pckReadType(read) == pckTypeBin)
|
||||||
{
|
{
|
||||||
ioFilterGroupResultAllSet(
|
MEM_CONTEXT_BEGIN(this->memContext)
|
||||||
ioReadFilterGroup(storageReadIo(this->read)), protocolClientReadOutput(this->client, true));
|
{
|
||||||
|
this->block = pckReadBinP(read);
|
||||||
|
this->remaining = bufUsed(this->block);
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_END();
|
||||||
|
}
|
||||||
|
// Else read is complete and get the filter list
|
||||||
|
else
|
||||||
|
{
|
||||||
|
bufFree(this->block);
|
||||||
|
|
||||||
|
ioFilterGroupResultAllSet(ioReadFilterGroup(storageReadIo(this->read)), jsonToVar(pckReadStrP(read)));
|
||||||
this->eof = true;
|
this->eof = true;
|
||||||
|
|
||||||
|
protocolClientDataEndGet(this->client);
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef DEBUG
|
#ifdef DEBUG
|
||||||
@ -140,14 +168,19 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
|
|||||||
// If the buffer can contain all remaining bytes
|
// If the buffer can contain all remaining bytes
|
||||||
if (bufRemains(buffer) >= this->remaining)
|
if (bufRemains(buffer) >= this->remaining)
|
||||||
{
|
{
|
||||||
bufLimitSet(buffer, bufUsed(buffer) + this->remaining);
|
bufCatSub(buffer, this->block, bufUsed(this->block) - this->remaining, this->remaining);
|
||||||
ioRead(protocolClientIoRead(this->client), buffer);
|
|
||||||
bufLimitClear(buffer);
|
|
||||||
this->remaining = 0;
|
this->remaining = 0;
|
||||||
|
bufFree(this->block);
|
||||||
|
this->block = NULL;
|
||||||
}
|
}
|
||||||
// Else read what we can
|
// Else read what we can
|
||||||
else
|
else
|
||||||
this->remaining -= ioRead(protocolClientIoRead(this->client), buffer);
|
{
|
||||||
|
size_t remains = bufRemains(buffer);
|
||||||
|
bufCatSub(buffer, this->block, bufUsed(this->block) - this->remaining, remains);
|
||||||
|
this->remaining -= remains;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
while (!this->eof && !bufFull(buffer));
|
while (!this->eof && !bufFull(buffer));
|
||||||
|
@ -8,6 +8,7 @@ Remote Storage
|
|||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
#include "common/type/json.h"
|
#include "common/type/json.h"
|
||||||
#include "common/type/object.h"
|
#include "common/type/object.h"
|
||||||
|
#include "common/type/pack.h"
|
||||||
#include "storage/remote/protocol.h"
|
#include "storage/remote/protocol.h"
|
||||||
#include "storage/remote/read.h"
|
#include "storage/remote/read.h"
|
||||||
#include "storage/remote/storage.intern.h"
|
#include "storage/remote/storage.intern.h"
|
||||||
@ -25,31 +26,81 @@ struct StorageRemote
|
|||||||
};
|
};
|
||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
// Helper to parse storage info from the protocol output
|
typedef struct StorageRemoteInfoData
|
||||||
|
{
|
||||||
|
time_t timeModifiedLast; // timeModified from last call
|
||||||
|
mode_t modeLast; // mode from last call
|
||||||
|
uid_t userIdLast; // userId from last call
|
||||||
|
gid_t groupIdLast; // groupId from last call
|
||||||
|
String *user; // user from last call
|
||||||
|
String *group; // group from last call
|
||||||
|
} StorageRemoteInfoData;
|
||||||
|
|
||||||
|
// Helper to get storage info from the protocol output
|
||||||
static void
|
static void
|
||||||
storageRemoteInfoParse(ProtocolClient *client, StorageInfo *info)
|
storageRemoteInfoGet(StorageRemoteInfoData *const data, PackRead *const read, StorageInfo *const info)
|
||||||
{
|
{
|
||||||
FUNCTION_TEST_BEGIN();
|
FUNCTION_TEST_BEGIN();
|
||||||
FUNCTION_TEST_PARAM(PROTOCOL_CLIENT, client);
|
FUNCTION_TEST_PARAM_P(VOID, data);
|
||||||
|
FUNCTION_TEST_PARAM(PACK_READ, read);
|
||||||
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
|
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
|
||||||
FUNCTION_TEST_END();
|
FUNCTION_TEST_END();
|
||||||
|
|
||||||
info->type = jsonToUInt(protocolClientReadLine(client));
|
ASSERT(data != NULL);
|
||||||
info->timeModified = (time_t)jsonToUInt64(protocolClientReadLine(client));
|
ASSERT(read != NULL);
|
||||||
|
ASSERT(info != NULL);
|
||||||
|
|
||||||
|
// Read type and time modified
|
||||||
|
info->type = pckReadU32P(read);
|
||||||
|
info->timeModified = pckReadTimeP(read) + data->timeModifiedLast;
|
||||||
|
|
||||||
|
// Read size for files
|
||||||
if (info->type == storageTypeFile)
|
if (info->type == storageTypeFile)
|
||||||
info->size = jsonToUInt64(protocolClientReadLine(client));
|
info->size = pckReadU64P(read);
|
||||||
|
|
||||||
|
// Read fields needed for detail level
|
||||||
if (info->level >= storageInfoLevelDetail)
|
if (info->level >= storageInfoLevelDetail)
|
||||||
{
|
{
|
||||||
info->userId = jsonToUInt(protocolClientReadLine(client));
|
// Read mode
|
||||||
info->user = jsonToStr(protocolClientReadLine(client));
|
info->mode = pckReadModeP(read, .defaultValue = data->modeLast);
|
||||||
info->groupId = jsonToUInt(protocolClientReadLine(client));
|
|
||||||
info->group = jsonToStr(protocolClientReadLine(client));
|
|
||||||
info->mode = (mode_t)jsonToUInt(protocolClientReadLine(client));
|
|
||||||
|
|
||||||
|
// Read user id/name
|
||||||
|
info->userId = pckReadU32P(read, .defaultValue = data->userIdLast);
|
||||||
|
|
||||||
|
if (pckReadBoolP(read)) // {vm_covered}
|
||||||
|
info->user = NULL; // {vm_covered}
|
||||||
|
else
|
||||||
|
info->user = pckReadStrP(read, .defaultValue = data->user);
|
||||||
|
|
||||||
|
// Read group id/name
|
||||||
|
info->groupId = pckReadU32P(read, .defaultValue = data->groupIdLast);
|
||||||
|
|
||||||
|
if (pckReadBoolP(read)) // {vm_covered}
|
||||||
|
info->group = NULL; // {vm_covered}
|
||||||
|
else
|
||||||
|
info->group = pckReadStrP(read, .defaultValue = data->group);
|
||||||
|
|
||||||
|
// Read link destination
|
||||||
if (info->type == storageTypeLink)
|
if (info->type == storageTypeLink)
|
||||||
info->linkDestination = jsonToStr(protocolClientReadLine(client));
|
info->linkDestination = pckReadStrP(read);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store defaults to use for the next call
|
||||||
|
data->timeModifiedLast = info->timeModified;
|
||||||
|
data->modeLast = info->mode;
|
||||||
|
data->userIdLast = info->userId;
|
||||||
|
data->groupIdLast = info->groupId;
|
||||||
|
|
||||||
|
if (!strEq(info->user, data->user) && info->user != NULL) // {vm_covered}
|
||||||
|
{
|
||||||
|
strFree(data->user);
|
||||||
|
data->user = strDup(info->user);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!strEq(info->group, data->group) && info->group != NULL) // {vm_covered}
|
||||||
|
{
|
||||||
|
strFree(data->group);
|
||||||
|
data->group = strDup(info->group);
|
||||||
}
|
}
|
||||||
|
|
||||||
FUNCTION_TEST_RETURN_VOID();
|
FUNCTION_TEST_RETURN_VOID();
|
||||||
@ -74,25 +125,31 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, Storage
|
|||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO);
|
||||||
protocolCommandParamAdd(command, VARSTR(file));
|
PackWrite *const commandParam = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARUINT(level));
|
|
||||||
protocolCommandParamAdd(command, VARBOOL(param.followLink));
|
|
||||||
|
|
||||||
result.exists = varBool(protocolClientExecute(this->client, command, true));
|
pckWriteStrP(commandParam, file);
|
||||||
|
pckWriteU32P(commandParam, level);
|
||||||
|
pckWriteBoolP(commandParam, param.followLink);
|
||||||
|
|
||||||
|
// Put command
|
||||||
|
protocolClientCommandPut(this->client, command);
|
||||||
|
|
||||||
|
// Read info from protocol
|
||||||
|
PackRead *read = protocolClientDataGet(this->client);
|
||||||
|
|
||||||
|
result.exists = pckReadBoolP(read);
|
||||||
|
|
||||||
if (result.exists)
|
if (result.exists)
|
||||||
{
|
{
|
||||||
// Read info from protocol into prior context
|
// Read info from protocol into prior context
|
||||||
MEM_CONTEXT_PRIOR_BEGIN()
|
MEM_CONTEXT_PRIOR_BEGIN()
|
||||||
{
|
{
|
||||||
result.name = strDup(result.name);
|
storageRemoteInfoGet(&(StorageRemoteInfoData){0}, read, &result);
|
||||||
storageRemoteInfoParse(this->client, &result);
|
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_PRIOR_END();
|
MEM_CONTEXT_PRIOR_END();
|
||||||
|
|
||||||
// Acknowledge command completed
|
|
||||||
protocolClientReadOutput(this->client, false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
protocolClientDataEndGet(this->client);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -125,36 +182,41 @@ storageRemoteInfoList(
|
|||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_LIST);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_LIST);
|
||||||
protocolCommandParamAdd(command, VARSTR(path));
|
PackWrite *const commandParam = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARUINT(level));
|
|
||||||
|
|
||||||
// Send command
|
pckWriteStrP(commandParam, path);
|
||||||
protocolClientWriteCommand(this->client, command);
|
pckWriteU32P(commandParam, level);
|
||||||
|
|
||||||
|
// Put command
|
||||||
|
protocolClientCommandPut(this->client, command);
|
||||||
|
|
||||||
|
// Read list
|
||||||
|
StorageRemoteInfoData parseData = {0};
|
||||||
|
|
||||||
// Read list. The list ends when there is a blank line -- this is safe even for file systems that allow blank filenames
|
|
||||||
// since the filename is json-encoded so will always include quotes.
|
|
||||||
MEM_CONTEXT_TEMP_RESET_BEGIN()
|
MEM_CONTEXT_TEMP_RESET_BEGIN()
|
||||||
{
|
{
|
||||||
const String *name = protocolClientReadLine(this->client);
|
PackRead *read = protocolClientDataGet(this->client);
|
||||||
|
pckReadNext(read);
|
||||||
|
|
||||||
while (strSize(name) != 0)
|
while (pckReadType(read) == pckTypeStr)
|
||||||
{
|
{
|
||||||
StorageInfo info = {.exists = true, .level = level, .name = jsonToStr(name)};
|
StorageInfo info = {.exists = true, .level = level, .name = pckReadStrP(read)};
|
||||||
|
|
||||||
storageRemoteInfoParse(this->client, &info);
|
storageRemoteInfoGet(&parseData, read, &info);
|
||||||
callback(callbackData, &info);
|
callback(callbackData, &info);
|
||||||
|
|
||||||
// Reset the memory context occasionally so we don't use too much memory or slow down processing
|
// Reset the memory context occasionally so we don't use too much memory or slow down processing
|
||||||
MEM_CONTEXT_TEMP_RESET(1000);
|
MEM_CONTEXT_TEMP_RESET(1000);
|
||||||
|
|
||||||
// Read the next item
|
read = protocolClientDataGet(this->client);
|
||||||
name = protocolClientReadLine(this->client);
|
pckReadNext(read);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
result = pckReadBoolP(read);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
// Acknowledge command completed
|
protocolClientDataEndGet(this->client);
|
||||||
result = varBool(protocolClientReadOutput(this->client, true));
|
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -240,10 +302,12 @@ storageRemotePathCreate(
|
|||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_CREATE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_CREATE);
|
||||||
protocolCommandParamAdd(command, VARSTR(path));
|
PackWrite *const commandParam = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARBOOL(errorOnExists));
|
|
||||||
protocolCommandParamAdd(command, VARBOOL(noParentCreate));
|
pckWriteStrP(commandParam, path);
|
||||||
protocolCommandParamAdd(command, VARUINT(mode));
|
pckWriteBoolP(commandParam, errorOnExists);
|
||||||
|
pckWriteBoolP(commandParam, noParentCreate);
|
||||||
|
pckWriteModeP(commandParam, mode);
|
||||||
|
|
||||||
protocolClientExecute(this->client, command, false);
|
protocolClientExecute(this->client, command, false);
|
||||||
}
|
}
|
||||||
@ -273,10 +337,12 @@ storageRemotePathRemove(THIS_VOID, const String *path, bool recurse, StorageInte
|
|||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE);
|
||||||
protocolCommandParamAdd(command, VARSTR(path));
|
PackWrite *const commandParam = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARBOOL(recurse));
|
|
||||||
|
|
||||||
result = varBool(protocolClientExecute(this->client, command, true));
|
pckWriteStrP(commandParam, path);
|
||||||
|
pckWriteBoolP(commandParam, recurse);
|
||||||
|
|
||||||
|
result = pckReadBoolP(protocolClientExecute(this->client, command, true));
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
@ -301,7 +367,7 @@ storageRemotePathSync(THIS_VOID, const String *path, StorageInterfacePathSyncPar
|
|||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_SYNC);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_SYNC);
|
||||||
protocolCommandParamAdd(command, VARSTR(path));
|
pckWriteStrP(protocolCommandParam(command), path);
|
||||||
|
|
||||||
protocolClientExecute(this->client, command, false);
|
protocolClientExecute(this->client, command, false);
|
||||||
}
|
}
|
||||||
@ -328,8 +394,10 @@ storageRemoteRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam p
|
|||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_REMOVE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_REMOVE);
|
||||||
protocolCommandParamAdd(command, VARSTR(file));
|
PackWrite *const commandParam = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARBOOL(param.errorOnMissing));
|
|
||||||
|
pckWriteStrP(commandParam, file);
|
||||||
|
pckWriteBoolP(commandParam, param.errorOnMissing);
|
||||||
|
|
||||||
protocolClientExecute(this->client, command, false);
|
protocolClientExecute(this->client, command, false);
|
||||||
}
|
}
|
||||||
@ -388,22 +456,17 @@ storageRemoteNew(
|
|||||||
// Get storage features from the remote
|
// Get storage features from the remote
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
// Send command
|
// Execute command and get result
|
||||||
protocolClientWriteCommand(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE));
|
PackRead *result = protocolClientExecute(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE), true);
|
||||||
|
|
||||||
// Read values
|
// Get path in parent context
|
||||||
path = jsonToStr(protocolClientReadLine(driver->client));
|
|
||||||
driver->interface.feature = jsonToUInt64(protocolClientReadLine(driver->client));
|
|
||||||
|
|
||||||
// Acknowledge command completed
|
|
||||||
protocolClientReadOutput(driver->client, false);
|
|
||||||
|
|
||||||
// Dup path into parent context
|
|
||||||
MEM_CONTEXT_PRIOR_BEGIN()
|
MEM_CONTEXT_PRIOR_BEGIN()
|
||||||
{
|
{
|
||||||
path = strDup(path);
|
path = pckReadStrP(result);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_PRIOR_END();
|
MEM_CONTEXT_PRIOR_END();
|
||||||
|
|
||||||
|
driver->interface.feature = pckReadU64P(result);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
@ -8,6 +8,7 @@ Remote Storage File write
|
|||||||
#include "common/io/write.h"
|
#include "common/io/write.h"
|
||||||
#include "common/log.h"
|
#include "common/log.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
|
#include "common/type/json.h"
|
||||||
#include "common/type/object.h"
|
#include "common/type/object.h"
|
||||||
#include "storage/remote/protocol.h"
|
#include "storage/remote/protocol.h"
|
||||||
#include "storage/remote/write.h"
|
#include "storage/remote/write.h"
|
||||||
@ -51,9 +52,9 @@ storageWriteRemoteFreeResource(THIS_VOID)
|
|||||||
|
|
||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
|
|
||||||
ioWriteLine(protocolClientIoWrite(this->client), BUFSTRDEF(PROTOCOL_BLOCK_HEADER "-1"));
|
protocolClientDataPut(this->client, pckWriteBoolP(protocolPackNew(), false));
|
||||||
ioWriteFlush(protocolClientIoWrite(this->client));
|
protocolClientDataPut(this->client, NULL);
|
||||||
protocolClientReadOutput(this->client, false);
|
protocolClientDataEndGet(this->client);
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN_VOID();
|
FUNCTION_LOG_RETURN_VOID();
|
||||||
}
|
}
|
||||||
@ -79,19 +80,22 @@ storageWriteRemoteOpen(THIS_VOID)
|
|||||||
ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilter(compressTypeGz));
|
ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilter(compressTypeGz));
|
||||||
|
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE);
|
||||||
protocolCommandParamAdd(command, VARSTR(this->interface.name));
|
PackWrite *const param = protocolCommandParam(command);
|
||||||
protocolCommandParamAdd(command, VARUINT(this->interface.modeFile));
|
|
||||||
protocolCommandParamAdd(command, VARUINT(this->interface.modePath));
|
|
||||||
protocolCommandParamAdd(command, VARSTR(this->interface.user));
|
|
||||||
protocolCommandParamAdd(command, VARSTR(this->interface.group));
|
|
||||||
protocolCommandParamAdd(command, VARINT64(this->interface.timeModified));
|
|
||||||
protocolCommandParamAdd(command, VARBOOL(this->interface.createPath));
|
|
||||||
protocolCommandParamAdd(command, VARBOOL(this->interface.syncFile));
|
|
||||||
protocolCommandParamAdd(command, VARBOOL(this->interface.syncPath));
|
|
||||||
protocolCommandParamAdd(command, VARBOOL(this->interface.atomic));
|
|
||||||
protocolCommandParamAdd(command, ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write))));
|
|
||||||
|
|
||||||
protocolClientExecute(this->client, command, false);
|
pckWriteStrP(param, this->interface.name);
|
||||||
|
pckWriteModeP(param, this->interface.modeFile);
|
||||||
|
pckWriteModeP(param, this->interface.modePath);
|
||||||
|
pckWriteStrP(param, this->interface.user);
|
||||||
|
pckWriteStrP(param, this->interface.group);
|
||||||
|
pckWriteTimeP(param, this->interface.timeModified);
|
||||||
|
pckWriteBoolP(param, this->interface.createPath);
|
||||||
|
pckWriteBoolP(param, this->interface.syncFile);
|
||||||
|
pckWriteBoolP(param, this->interface.syncPath);
|
||||||
|
pckWriteBoolP(param, this->interface.atomic);
|
||||||
|
pckWriteStrP(param, jsonFromVar(ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write)))));
|
||||||
|
|
||||||
|
protocolClientCommandPut(this->client, command);
|
||||||
|
protocolClientDataGet(this->client);
|
||||||
|
|
||||||
// Clear filters since they will be run on the remote side
|
// Clear filters since they will be run on the remote side
|
||||||
ioFilterGroupClear(ioWriteFilterGroup(storageWriteIo(this->write)));
|
ioFilterGroupClear(ioWriteFilterGroup(storageWriteIo(this->write)));
|
||||||
@ -128,9 +132,7 @@ storageWriteRemote(THIS_VOID, const Buffer *buffer)
|
|||||||
ASSERT(this != NULL);
|
ASSERT(this != NULL);
|
||||||
ASSERT(buffer != NULL);
|
ASSERT(buffer != NULL);
|
||||||
|
|
||||||
ioWriteStrLine(protocolClientIoWrite(this->client), strNewFmt(PROTOCOL_BLOCK_HEADER "%zu", bufUsed(buffer)));
|
protocolClientDataPut(this->client, pckWriteBinP(protocolPackNew(), buffer));
|
||||||
ioWrite(protocolClientIoWrite(this->client), buffer);
|
|
||||||
ioWriteFlush(protocolClientIoWrite(this->client));
|
|
||||||
|
|
||||||
#ifdef DEBUG
|
#ifdef DEBUG
|
||||||
this->protocolWriteBytes += bufUsed(buffer);
|
this->protocolWriteBytes += bufUsed(buffer);
|
||||||
@ -156,11 +158,12 @@ storageWriteRemoteClose(THIS_VOID)
|
|||||||
// Close if the file has not already been closed
|
// Close if the file has not already been closed
|
||||||
if (this->client != NULL)
|
if (this->client != NULL)
|
||||||
{
|
{
|
||||||
ioWriteLine(protocolClientIoWrite(this->client), BUFSTRDEF(PROTOCOL_BLOCK_HEADER "0"));
|
protocolClientDataPut(this->client, NULL);
|
||||||
ioWriteFlush(protocolClientIoWrite(this->client));
|
ioFilterGroupResultAllSet(
|
||||||
ioFilterGroupResultAllSet(ioWriteFilterGroup(storageWriteIo(this->write)), protocolClientReadOutput(this->client, true));
|
ioWriteFilterGroup(storageWriteIo(this->write)), jsonToVar(pckReadStrP(protocolClientDataGet(this->client))));
|
||||||
this->client = NULL;
|
protocolClientDataEndGet(this->client);
|
||||||
|
|
||||||
|
this->client = NULL;
|
||||||
memContextCallbackClear(this->memContext);
|
memContextCallbackClear(this->memContext);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -454,7 +454,7 @@ unit:
|
|||||||
test:
|
test:
|
||||||
# ----------------------------------------------------------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------------------------------------------------------
|
||||||
- name: protocol
|
- name: protocol
|
||||||
total: 9
|
total: 7
|
||||||
harness:
|
harness:
|
||||||
name: protocol
|
name: protocol
|
||||||
shim:
|
shim:
|
||||||
|
@ -1322,14 +1322,15 @@ testRun(void)
|
|||||||
// Create job that skips file
|
// Create job that skips file
|
||||||
job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ(stringIdBit5, "x")));
|
job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ(stringIdBit5, "x")));
|
||||||
|
|
||||||
VariantList *result = varLstNew();
|
PackWrite *const resultPack = protocolPackNew();
|
||||||
varLstAdd(result, varNewUInt64(backupCopyResultNoOp));
|
pckWriteU32P(resultPack, backupCopyResultNoOp);
|
||||||
varLstAdd(result, varNewUInt64(0));
|
pckWriteU64P(resultPack, 0);
|
||||||
varLstAdd(result, varNewUInt64(0));
|
pckWriteU64P(resultPack, 0);
|
||||||
varLstAdd(result, NULL);
|
pckWriteStrP(resultPack, NULL);
|
||||||
varLstAdd(result, NULL);
|
pckWriteStrP(resultPack, NULL);
|
||||||
|
pckWriteEndP(resultPack);
|
||||||
|
|
||||||
protocolParallelJobResultSet(job, varNewVarLst(result));
|
protocolParallelJobResultSet(job, pckReadNewBuf(pckWriteBuf(resultPack)));
|
||||||
|
|
||||||
// Create manifest with file
|
// Create manifest with file
|
||||||
Manifest *manifest = manifestNewInternal();
|
Manifest *manifest = manifestNewInternal();
|
||||||
|
@ -11,102 +11,143 @@ Test Protocol
|
|||||||
#include "version.h"
|
#include "version.h"
|
||||||
|
|
||||||
#include "common/harnessConfig.h"
|
#include "common/harnessConfig.h"
|
||||||
|
#include "common/harnessError.h"
|
||||||
#include "common/harnessFork.h"
|
#include "common/harnessFork.h"
|
||||||
|
#include "common/harnessPack.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Test protocol request handler
|
Test protocol server command handlers
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
static unsigned int testServerProtocolErrorTotal = 0;
|
#define TEST_PROTOCOL_COMMAND_ASSERT STRID5("assert", 0x2922ce610)
|
||||||
|
|
||||||
static void
|
__attribute__((__noreturn__)) static void
|
||||||
testServerAssertProtocol(const VariantList *paramList, ProtocolServer *server)
|
testCommandAssertProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_HARNESS_BEGIN();
|
FUNCTION_HARNESS_BEGIN();
|
||||||
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_HARNESS_PARAM(PACK_READ, param);
|
||||||
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_HARNESS_END();
|
FUNCTION_HARNESS_END();
|
||||||
|
|
||||||
ASSERT(paramList == NULL);
|
ASSERT(param == NULL);
|
||||||
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
|
hrnErrorThrowP();
|
||||||
|
|
||||||
|
// No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
|
||||||
|
}
|
||||||
|
|
||||||
|
#define TEST_PROTOCOL_COMMAND_ERROR STRID5("error", 0x127ca450)
|
||||||
|
|
||||||
|
__attribute__((__noreturn__)) static void
|
||||||
|
testCommandErrorProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
|
{
|
||||||
|
FUNCTION_HARNESS_BEGIN();
|
||||||
|
FUNCTION_HARNESS_PARAM(PACK_READ, param);
|
||||||
|
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
|
||||||
|
FUNCTION_HARNESS_END();
|
||||||
|
|
||||||
|
ASSERT(param == NULL);
|
||||||
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
|
hrnErrorThrowP(.errorType = &FormatError);
|
||||||
|
|
||||||
|
// No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
|
||||||
|
}
|
||||||
|
|
||||||
|
#define TEST_PROTOCOL_COMMAND_SIMPLE STRID5("c-simple", 0x2b20d4cf630)
|
||||||
|
|
||||||
|
static void
|
||||||
|
testCommandRequestSimpleProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
|
{
|
||||||
|
FUNCTION_HARNESS_BEGIN();
|
||||||
|
FUNCTION_HARNESS_PARAM(PACK_READ, param);
|
||||||
|
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
|
||||||
|
FUNCTION_HARNESS_END();
|
||||||
|
|
||||||
|
ASSERT(param == NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
THROW(AssertError, "test assert");
|
protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), STRDEF("output")));
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
FUNCTION_HARNESS_RETURN_VOID();
|
FUNCTION_HARNESS_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define TEST_PROTOCOL_COMMAND_COMPLEX STRID5("c-complex", 0x182b20d78f630)
|
||||||
|
|
||||||
static void
|
static void
|
||||||
testServerRequestSimpleProtocol(const VariantList *paramList, ProtocolServer *server)
|
testCommandRequestComplexProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_HARNESS_BEGIN();
|
FUNCTION_HARNESS_BEGIN();
|
||||||
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_HARNESS_PARAM(PACK_READ, param);
|
||||||
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_HARNESS_END();
|
FUNCTION_HARNESS_END();
|
||||||
|
|
||||||
ASSERT(paramList == NULL);
|
ASSERT(param != NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
protocolServerResponse(server, varNewBool(true));
|
TEST_RESULT_UINT(pckReadU32P(param), 87, "param check");
|
||||||
|
TEST_RESULT_STR_Z(pckReadStrP(param), "data", "param check");
|
||||||
|
|
||||||
|
TEST_RESULT_VOID(protocolServerDataPut(server, NULL), "sync");
|
||||||
|
|
||||||
|
TEST_RESULT_BOOL(pckReadBoolP(protocolServerDataGet(server)), true, "data get");
|
||||||
|
TEST_RESULT_UINT(pckReadModeP(protocolServerDataGet(server)), 0644, "data get");
|
||||||
|
TEST_RESULT_PTR(protocolServerDataGet(server), NULL, "data end get");
|
||||||
|
|
||||||
|
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true)), "data put");
|
||||||
|
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteI32P(protocolPackNew(), -1)), "data put");
|
||||||
|
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
FUNCTION_HARNESS_RETURN_VOID();
|
FUNCTION_HARNESS_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define TEST_PROTOCOL_COMMAND_RETRY STRID5("retry", 0x19950b20)
|
||||||
|
|
||||||
|
static unsigned int testCommandRetryTotal = 1;
|
||||||
|
|
||||||
static void
|
static void
|
||||||
testServerRequestComplexProtocol(const VariantList *paramList, ProtocolServer *server)
|
testCommandRetryProtocol(PackRead *const param, ProtocolServer *const server)
|
||||||
{
|
{
|
||||||
FUNCTION_HARNESS_BEGIN();
|
FUNCTION_HARNESS_BEGIN();
|
||||||
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
|
FUNCTION_HARNESS_PARAM(PACK_READ, param);
|
||||||
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
|
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
|
||||||
FUNCTION_HARNESS_END();
|
FUNCTION_HARNESS_END();
|
||||||
|
|
||||||
ASSERT(paramList == NULL);
|
ASSERT(param == NULL);
|
||||||
ASSERT(server != NULL);
|
ASSERT(server != NULL);
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
protocolServerResponse(server, varNewBool(false));
|
if (testCommandRetryTotal > 0)
|
||||||
protocolServerWriteLine(server, STRDEF("LINEOFTEXT"));
|
|
||||||
protocolServerWriteLine(server, NULL);
|
|
||||||
ioWriteFlush(protocolServerIoWrite(server));
|
|
||||||
}
|
|
||||||
MEM_CONTEXT_TEMP_END();
|
|
||||||
|
|
||||||
FUNCTION_HARNESS_RETURN_VOID();
|
|
||||||
}
|
|
||||||
|
|
||||||
static void
|
|
||||||
testServerErrorUntil0Protocol(const VariantList *paramList, ProtocolServer *server)
|
|
||||||
{
|
|
||||||
FUNCTION_HARNESS_BEGIN();
|
|
||||||
FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
|
|
||||||
FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
|
|
||||||
FUNCTION_HARNESS_END();
|
|
||||||
|
|
||||||
ASSERT(paramList == NULL);
|
|
||||||
ASSERT(server != NULL);
|
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
|
||||||
{
|
|
||||||
if (testServerProtocolErrorTotal > 0)
|
|
||||||
{
|
{
|
||||||
testServerProtocolErrorTotal--;
|
testCommandRetryTotal--;
|
||||||
THROW(FormatError, "error-until-0");
|
THROW(FormatError, "error-until-0");
|
||||||
}
|
}
|
||||||
|
|
||||||
protocolServerResponse(server, varNewBool(true));
|
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true));
|
||||||
|
protocolServerDataEndPut(server);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
FUNCTION_HARNESS_RETURN_VOID();
|
FUNCTION_HARNESS_RETURN_VOID();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define TEST_PROTOCOL_SERVER_HANDLER_LIST \
|
||||||
|
{.command = TEST_PROTOCOL_COMMAND_ASSERT, .handler = testCommandAssertProtocol}, \
|
||||||
|
{.command = TEST_PROTOCOL_COMMAND_ERROR, .handler = testCommandErrorProtocol}, \
|
||||||
|
{.command = TEST_PROTOCOL_COMMAND_SIMPLE, .handler = testCommandRequestSimpleProtocol}, \
|
||||||
|
{.command = TEST_PROTOCOL_COMMAND_COMPLEX, .handler = testCommandRequestComplexProtocol}, \
|
||||||
|
{.command = TEST_PROTOCOL_COMMAND_RETRY, .handler = testCommandRetryProtocol},
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Test ParallelJobCallback
|
Test ParallelJobCallback
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
@ -390,35 +431,7 @@ testRun(void)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// *****************************************************************************************************************************
|
// *****************************************************************************************************************************
|
||||||
if (testBegin("ProtocolCommand"))
|
if (testBegin("ProtocolClient, ProtocolCommand, and ProtocolServer"))
|
||||||
{
|
|
||||||
ProtocolCommand *command = NULL;
|
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
|
||||||
{
|
|
||||||
TEST_ASSIGN(command, protocolCommandNew(strIdFromZ(stringIdBit5, "cmd-one")), "create command");
|
|
||||||
TEST_RESULT_PTR(protocolCommandParamAdd(command, VARSTRDEF("param1")), command, "add param");
|
|
||||||
TEST_RESULT_PTR(protocolCommandParamAdd(command, VARSTRDEF("param2")), command, "add param");
|
|
||||||
|
|
||||||
TEST_RESULT_PTR(protocolCommandMove(command, memContextPrior()), command, "move protocol command");
|
|
||||||
TEST_RESULT_PTR(protocolCommandMove(NULL, memContextPrior()), NULL, "move null protocol command");
|
|
||||||
}
|
|
||||||
MEM_CONTEXT_TEMP_END();
|
|
||||||
|
|
||||||
TEST_RESULT_STR_Z(protocolCommandToLog(command), "{command: cmd-one}", "check log");
|
|
||||||
TEST_RESULT_STR_Z(protocolCommandJson(command), "{\"cmd\":\"cmd-one\",\"param\":[\"param1\",\"param2\"]}", "check json");
|
|
||||||
|
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
|
||||||
TEST_ASSIGN(command, protocolCommandNew(strIdFromZ(stringIdBit5, "cmd2")), "create command");
|
|
||||||
TEST_RESULT_STR_Z(protocolCommandToLog(command), "{command: cmd2}", "check log");
|
|
||||||
TEST_RESULT_STR_Z(protocolCommandJson(command), "{\"cmd\":\"cmd2\"}", "check json");
|
|
||||||
|
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
|
||||||
TEST_RESULT_VOID(protocolCommandFree(command), "free command");
|
|
||||||
}
|
|
||||||
|
|
||||||
// *****************************************************************************************************************************
|
|
||||||
if (testBegin("ProtocolClient"))
|
|
||||||
{
|
{
|
||||||
HARNESS_FORK_BEGIN()
|
HARNESS_FORK_BEGIN()
|
||||||
{
|
{
|
||||||
@ -430,6 +443,7 @@ testRun(void)
|
|||||||
ioWriteOpen(write);
|
ioWriteOpen(write);
|
||||||
|
|
||||||
// Various bogus greetings
|
// Various bogus greetings
|
||||||
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
ioWriteStrLine(write, STRDEF("bogus greeting"));
|
ioWriteStrLine(write, STRDEF("bogus greeting"));
|
||||||
ioWriteFlush(write);
|
ioWriteFlush(write);
|
||||||
ioWriteStrLine(write, STRDEF("{\"name\":999}"));
|
ioWriteStrLine(write, STRDEF("{\"name\":999}"));
|
||||||
@ -443,56 +457,38 @@ testRun(void)
|
|||||||
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"bogus\"}"));
|
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"bogus\"}"));
|
||||||
ioWriteFlush(write);
|
ioWriteFlush(write);
|
||||||
|
|
||||||
// Correct greeting with noop
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}"));
|
TEST_TITLE("server");
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop");
|
ProtocolServer *server = NULL;
|
||||||
ioWriteStrLine(write, STRDEF("{}"));
|
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
// Throw errors
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop with error text");
|
{
|
||||||
ioWriteStrLine(write, STRDEF("{\"err\":25,\"out\":\"sample error message\",\"errStack\":\"stack data\"}"));
|
TEST_ASSIGN(
|
||||||
ioWriteFlush(write);
|
server,
|
||||||
|
protocolServerMove(
|
||||||
|
protocolServerNew(STRDEF("test server"), STRDEF("test"), read, write), memContextPrior()),
|
||||||
|
"new server");
|
||||||
|
TEST_RESULT_VOID(protocolServerMove(NULL, memContextPrior()), "move null server");
|
||||||
|
}
|
||||||
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop with no error text");
|
const ProtocolServerHandler commandHandler[] = {TEST_PROTOCOL_SERVER_HANDLER_LIST};
|
||||||
ioWriteStrLine(write, STRDEF("{\"err\":255}"));
|
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
// No output expected
|
// This cannot run in a TEST* macro because tests are run by the command handlers
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop with parameters returned");
|
protocolServerProcess(server, NULL, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler));
|
||||||
ioWriteStrLine(write, STRDEF("{\"out\":[\"bogus\"]}"));
|
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
// Send output
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"test\"}", "test command");
|
TEST_TITLE("server with retries");
|
||||||
ioWriteStrLine(write, STRDEF(".OUTPUT"));
|
|
||||||
ioWriteStrLine(write, STRDEF("{\"out\":[\"value1\",\"value2\"]}"));
|
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
// invalid line
|
VariantList *retryList = varLstNew();
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"invalid-line\"}", "invalid line command");
|
varLstAdd(retryList, varNewUInt64(0));
|
||||||
ioWrite(write, LF_BUF);
|
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
// error instead of output
|
TEST_ASSIGN(
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"err-i-o\"}", "error instead of output command");
|
server, protocolServerNew(STRDEF("test server"), STRDEF("test"), read, write), "new server");
|
||||||
ioWriteStrLine(write, STRDEF("{\"err\":255}"));
|
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
// unexpected output
|
// This cannot run in a TEST* macro because tests are run by the command handlers
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"unexp-output\"}", "unexpected output");
|
protocolServerProcess(server, retryList, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler));
|
||||||
ioWriteStrLine(write, STRDEF("{}"));
|
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
// invalid prefix
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"i-pr\"}", "invalid prefix");
|
|
||||||
ioWriteStrLine(write, STRDEF("~line"));
|
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
// Wait for exit
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"exit\"}", "exit command");
|
|
||||||
}
|
}
|
||||||
HARNESS_FORK_CHILD_END();
|
HARNESS_FORK_CHILD_END();
|
||||||
|
|
||||||
@ -503,7 +499,9 @@ testRun(void)
|
|||||||
IoWrite *write = ioFdWriteNew(STRDEF("client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000);
|
IoWrite *write = ioFdWriteNew(STRDEF("client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000);
|
||||||
ioWriteOpen(write);
|
ioWriteOpen(write);
|
||||||
|
|
||||||
// Various bogus greetings
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("bogus greetings");
|
||||||
|
|
||||||
TEST_ERROR(
|
TEST_ERROR(
|
||||||
protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), JsonFormatError,
|
protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), JsonFormatError,
|
||||||
"expected '{' at 'bogus greeting'");
|
"expected '{' at 'bogus greeting'");
|
||||||
@ -526,7 +524,9 @@ testRun(void)
|
|||||||
"expected value '" PROJECT_VERSION "' for greeting key 'version' but got 'bogus'\n"
|
"expected value '" PROJECT_VERSION "' for greeting key 'version' but got 'bogus'\n"
|
||||||
"HINT: is the same version of " PROJECT_NAME " installed on the local and remote host?");
|
"HINT: is the same version of " PROJECT_NAME " installed on the local and remote host?");
|
||||||
|
|
||||||
// Correct greeting
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("new client with successful handshake");
|
||||||
|
|
||||||
ProtocolClient *client = NULL;
|
ProtocolClient *client = NULL;
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
@ -535,193 +535,96 @@ testRun(void)
|
|||||||
client,
|
client,
|
||||||
protocolClientMove(
|
protocolClientMove(
|
||||||
protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), memContextPrior()),
|
protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), memContextPrior()),
|
||||||
"create client");
|
"new client");
|
||||||
TEST_RESULT_VOID(protocolClientMove(NULL, memContextPrior()), "move null client");
|
TEST_RESULT_VOID(protocolClientMove(NULL, memContextPrior()), "move null client");
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
TEST_RESULT_PTR(protocolClientIoRead(client), client->pub.read, "get read io");
|
TEST_RESULT_INT(protocolClientIoReadFd(client), ioReadFd(client->pub.read), "get read fd");
|
||||||
TEST_RESULT_PTR(protocolClientIoWrite(client), client->pub.write, "get write io");
|
|
||||||
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("invalid command");
|
||||||
|
|
||||||
// Throw errors
|
|
||||||
TEST_ERROR(
|
TEST_ERROR(
|
||||||
protocolClientNoOp(client), AssertError,
|
protocolClientExecute(client, protocolCommandNew(strIdFromZ(stringIdBit6, "BOGUS")), false), ProtocolError,
|
||||||
"raised from test client: sample error message\nstack data");
|
"raised from test client: invalid command 'BOGUS' (0x38eacd271)");
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("command throws assert");
|
||||||
|
|
||||||
|
TEST_ERROR(
|
||||||
|
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ASSERT), false), AssertError,
|
||||||
|
"raised from test client: ERR_MESSAGE\nERR_STACK_TRACE");
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("command throws error");
|
||||||
|
|
||||||
|
TEST_ERROR(
|
||||||
|
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ERROR), false), FormatError,
|
||||||
|
"raised from test client: ERR_MESSAGE");
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("command throws error in debug log level");
|
||||||
|
|
||||||
harnessLogLevelSet(logLevelDebug);
|
harnessLogLevelSet(logLevelDebug);
|
||||||
|
|
||||||
TEST_ERROR(
|
TEST_ERROR(
|
||||||
protocolClientNoOp(client), UnknownError,
|
protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ERROR), false), FormatError,
|
||||||
"raised from test client: no details available\nno stack trace available");
|
"raised from test client: ERR_MESSAGE\nERR_STACK_TRACE");
|
||||||
|
|
||||||
harnessLogLevelReset();
|
harnessLogLevelReset();
|
||||||
|
|
||||||
// No output expected
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
TEST_ERROR(protocolClientNoOp(client), AssertError, "no output required by command");
|
TEST_TITLE("simple command");
|
||||||
|
|
||||||
// Get command output
|
|
||||||
const VariantList *output = NULL;
|
|
||||||
|
|
||||||
TEST_RESULT_VOID(
|
|
||||||
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "test"))),
|
|
||||||
"execute command with output");
|
|
||||||
TEST_RESULT_STR_Z(protocolClientReadLine(client), "OUTPUT", "check output");
|
|
||||||
TEST_ASSIGN(output, varVarLst(protocolClientReadOutput(client, true)), "execute command with output");
|
|
||||||
TEST_RESULT_UINT(varLstSize(output), 2, "check output size");
|
|
||||||
TEST_RESULT_STR_Z(varStr(varLstGet(output, 0)), "value1", "check value1");
|
|
||||||
TEST_RESULT_STR_Z(varStr(varLstGet(output, 1)), "value2", "check value2");
|
|
||||||
|
|
||||||
// Invalid line
|
|
||||||
TEST_RESULT_VOID(
|
|
||||||
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "invalid-line"))),
|
|
||||||
"execute command that returns invalid line");
|
|
||||||
TEST_ERROR(protocolClientReadLine(client), FormatError, "unexpected empty line");
|
|
||||||
|
|
||||||
// Error instead of output
|
|
||||||
TEST_RESULT_VOID(
|
|
||||||
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "err-i-o"))),
|
|
||||||
"execute command that returns error instead of output");
|
|
||||||
TEST_ERROR(protocolClientReadLine(client), UnknownError, "raised from test client: no details available");
|
|
||||||
|
|
||||||
// Unexpected output
|
|
||||||
TEST_RESULT_VOID(
|
|
||||||
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "unexp-output"))),
|
|
||||||
"execute command that returns unexpected output");
|
|
||||||
TEST_ERROR(protocolClientReadLine(client), FormatError, "expected error but got output");
|
|
||||||
|
|
||||||
// Invalid prefix
|
|
||||||
TEST_RESULT_VOID(
|
|
||||||
protocolClientWriteCommand(client, protocolCommandNew(strIdFromZ(stringIdBit5, "i-pr"))),
|
|
||||||
"execute command that returns an invalid prefix");
|
|
||||||
TEST_ERROR(protocolClientReadLine(client), FormatError, "invalid prefix in '~line'");
|
|
||||||
|
|
||||||
// Free client
|
|
||||||
TEST_RESULT_VOID(protocolClientFree(client), "free client");
|
|
||||||
}
|
|
||||||
HARNESS_FORK_PARENT_END();
|
|
||||||
}
|
|
||||||
HARNESS_FORK_END();
|
|
||||||
}
|
|
||||||
|
|
||||||
// *****************************************************************************************************************************
|
|
||||||
if (testBegin("ProtocolServer"))
|
|
||||||
{
|
|
||||||
HARNESS_FORK_BEGIN()
|
|
||||||
{
|
|
||||||
HARNESS_FORK_CHILD_BEGIN(0, true)
|
|
||||||
{
|
|
||||||
IoRead *read = ioFdReadNew(STRDEF("client read"), HARNESS_FORK_CHILD_READ(), 2000);
|
|
||||||
ioReadOpen(read);
|
|
||||||
IoWrite *write = ioFdWriteNew(STRDEF("client write"), HARNESS_FORK_CHILD_WRITE(), 2000);
|
|
||||||
ioWriteOpen(write);
|
|
||||||
|
|
||||||
// Check greeting
|
|
||||||
TEST_RESULT_STR_Z(
|
TEST_RESULT_STR_Z(
|
||||||
ioReadLine(read), "{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}",
|
pckReadStrP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_SIMPLE), true)), "output",
|
||||||
"check greeting");
|
"execute");
|
||||||
|
|
||||||
// Noop
|
|
||||||
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"noop\"}")), "write noop");
|
|
||||||
TEST_RESULT_VOID(ioWriteFlush(write), "flush noop");
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{}", "noop result");
|
|
||||||
|
|
||||||
// Invalid command
|
|
||||||
KeyValue *result = NULL;
|
|
||||||
|
|
||||||
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"bogus\"}")), "write bogus");
|
|
||||||
TEST_RESULT_VOID(ioWriteFlush(write), "flush bogus");
|
|
||||||
TEST_ASSIGN(result, varKv(jsonToVar(ioReadLine(read))), "parse error result");
|
|
||||||
TEST_RESULT_INT(varIntForce(kvGet(result, VARSTRDEF("err"))), 39, " check code");
|
|
||||||
TEST_RESULT_STR_Z(
|
|
||||||
varStr(kvGet(result, VARSTRDEF("out"))), "invalid command 'bogus' (0x13a9de20)", " check message");
|
|
||||||
TEST_RESULT_BOOL(kvGet(result, VARSTRDEF("errStack")) != NULL, true, " check stack exists");
|
|
||||||
|
|
||||||
// Simple request
|
|
||||||
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"r-s\"}")), "write simple request");
|
|
||||||
TEST_RESULT_VOID(ioWriteFlush(write), "flush simple request");
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"out\":true}", "simple request result");
|
|
||||||
|
|
||||||
// Throw an assert error which will include a stack trace
|
|
||||||
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"assert\"}")), "write assert");
|
|
||||||
TEST_RESULT_VOID(ioWriteFlush(write), "flush assert error");
|
|
||||||
TEST_ASSIGN(result, varKv(jsonToVar(ioReadLine(read))), "parse error result");
|
|
||||||
TEST_RESULT_INT(varIntForce(kvGet(result, VARSTRDEF("err"))), 25, " check code");
|
|
||||||
TEST_RESULT_STR_Z(varStr(kvGet(result, VARSTRDEF("out"))), "test assert", " check message");
|
|
||||||
TEST_RESULT_BOOL(kvGet(result, VARSTRDEF("errStack")) != NULL, true, " check stack exists");
|
|
||||||
|
|
||||||
// Complex request -- after process loop has been restarted
|
|
||||||
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"r-c\"}")), "write complex request");
|
|
||||||
TEST_RESULT_VOID(ioWriteFlush(write), "flush complex request");
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"out\":false}", "complex request result");
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), ".LINEOFTEXT", "complex request result");
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), ".", "complex request result");
|
|
||||||
|
|
||||||
// Exit
|
|
||||||
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"exit\"}")), "write exit");
|
|
||||||
TEST_RESULT_VOID(ioWriteFlush(write), "flush exit");
|
|
||||||
|
|
||||||
// Retry errors until success
|
|
||||||
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"ezero\"}")), "write error-until-0");
|
|
||||||
TEST_RESULT_VOID(ioWriteFlush(write), "flush error-until-0");
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"out\":true}", "error-until-0 result");
|
|
||||||
|
|
||||||
// Exit
|
|
||||||
TEST_RESULT_VOID(ioWriteStrLine(write, STRDEF("{\"cmd\":\"exit\"}")), "write exit");
|
|
||||||
TEST_RESULT_VOID(ioWriteFlush(write), "flush exit");
|
|
||||||
}
|
|
||||||
HARNESS_FORK_CHILD_END();
|
|
||||||
|
|
||||||
HARNESS_FORK_PARENT_BEGIN()
|
|
||||||
{
|
|
||||||
IoRead *read = ioFdReadNew(STRDEF("server read"), HARNESS_FORK_PARENT_READ_PROCESS(0), 2000);
|
|
||||||
ioReadOpen(read);
|
|
||||||
IoWrite *write = ioFdWriteNew(STRDEF("server write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0), 2000);
|
|
||||||
ioWriteOpen(write);
|
|
||||||
|
|
||||||
// Send greeting
|
|
||||||
ProtocolServer *server = NULL;
|
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
|
||||||
{
|
|
||||||
TEST_ASSIGN(
|
|
||||||
server,
|
|
||||||
protocolServerMove(
|
|
||||||
protocolServerNew(STRDEF("test server"), STRDEF("test"), read, write), memContextPrior()),
|
|
||||||
"create server");
|
|
||||||
TEST_RESULT_VOID(protocolServerMove(NULL, memContextPrior()), "move null server");
|
|
||||||
}
|
|
||||||
MEM_CONTEXT_TEMP_END();
|
|
||||||
|
|
||||||
TEST_RESULT_PTR(protocolServerIoRead(server), server->pub.read, "get read io");
|
|
||||||
TEST_RESULT_PTR(protocolServerIoWrite(server), server->pub.write, "get write io");
|
|
||||||
|
|
||||||
ProtocolServerHandler commandHandler[] =
|
|
||||||
{
|
|
||||||
{.command = strIdFromZ(stringIdBit5, "assert"), .handler = testServerAssertProtocol},
|
|
||||||
{.command = strIdFromZ(stringIdBit5, "r-s"), .handler = testServerRequestSimpleProtocol},
|
|
||||||
{.command = strIdFromZ(stringIdBit5, "r-c"), .handler = testServerRequestComplexProtocol},
|
|
||||||
{.command = strIdFromZ(stringIdBit5, "ezero"), .handler = testServerErrorUntil0Protocol},
|
|
||||||
};
|
|
||||||
|
|
||||||
TEST_RESULT_VOID(
|
|
||||||
protocolServerProcess(server, NULL, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler)),
|
|
||||||
"run process loop");
|
|
||||||
|
|
||||||
// -----------------------------------------------------------------------------------------------------------------
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
TEST_TITLE("run process loop with retries");
|
TEST_TITLE("complex command");
|
||||||
|
|
||||||
VariantList *retryInterval = varLstNew();
|
// Put the command to the server
|
||||||
varLstAdd(retryInterval, varNewUInt64(0));
|
ProtocolCommand *command = NULL;
|
||||||
varLstAdd(retryInterval, varNewUInt64(50));
|
TEST_ASSIGN(command, protocolCommandNew(TEST_PROTOCOL_COMMAND_COMPLEX), "command");
|
||||||
|
TEST_RESULT_VOID(pckWriteU32P(protocolCommandParam(command), 87), "param");
|
||||||
|
TEST_RESULT_VOID(pckWriteStrP(protocolCommandParam(command), STRDEF("data")), "param");
|
||||||
|
TEST_RESULT_VOID(protocolClientCommandPut(client, command), "command put");
|
||||||
|
|
||||||
testServerProtocolErrorTotal = 2;
|
// Read null data to indicate that the server has started the command and is read to receive data
|
||||||
|
TEST_RESULT_PTR(protocolClientDataGet(client), NULL, "command started and ready for data");
|
||||||
|
|
||||||
TEST_RESULT_VOID(
|
// Write data to the server
|
||||||
protocolServerProcess(server, retryInterval, commandHandler, PROTOCOL_SERVER_HANDLER_LIST_SIZE(commandHandler)),
|
TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteBoolP(protocolPackNew(), true)), "data put");
|
||||||
"run process loop");
|
TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteModeP(protocolPackNew(), 0644)), "data put");
|
||||||
|
TEST_RESULT_VOID(protocolClientDataPut(client, NULL), "data end put");
|
||||||
|
|
||||||
|
// Get data from the server
|
||||||
|
TEST_RESULT_BOOL(pckReadBoolP(protocolClientDataGet(client)), true, "data get");
|
||||||
|
TEST_RESULT_INT(pckReadI32P(protocolClientDataGet(client)), -1, "data get");
|
||||||
|
TEST_RESULT_VOID(protocolClientDataEndGet(client), "data end get");
|
||||||
|
|
||||||
// -----------------------------------------------------------------------------------------------------------------
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
TEST_TITLE("free server");
|
TEST_TITLE("free client");
|
||||||
|
|
||||||
TEST_RESULT_VOID(protocolServerFree(server), "free");
|
TEST_RESULT_VOID(protocolClientFree(client), "free");
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("new client with server retries");
|
||||||
|
|
||||||
|
TEST_ASSIGN(client, protocolClientNew(STRDEF("test client"), STRDEF("test"), read, write), "new client");
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("command with retry");
|
||||||
|
|
||||||
|
TEST_RESULT_BOOL(
|
||||||
|
pckReadBoolP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_RETRY), true)), true,
|
||||||
|
"execute");
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("free client");
|
||||||
|
|
||||||
|
TEST_RESULT_VOID(protocolClientFree(client), "free");
|
||||||
}
|
}
|
||||||
HARNESS_FORK_PARENT_END();
|
HARNESS_FORK_PARENT_END();
|
||||||
}
|
}
|
||||||
@ -731,6 +634,8 @@ testRun(void)
|
|||||||
// *****************************************************************************************************************************
|
// *****************************************************************************************************************************
|
||||||
if (testBegin("ProtocolParallel and ProtocolParallelJob"))
|
if (testBegin("ProtocolParallel and ProtocolParallelJob"))
|
||||||
{
|
{
|
||||||
|
TEST_TITLE("job state transitions");
|
||||||
|
|
||||||
ProtocolParallelJob *job = NULL;
|
ProtocolParallelJob *job = NULL;
|
||||||
|
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
@ -760,63 +665,63 @@ testRun(void)
|
|||||||
// Local 1
|
// Local 1
|
||||||
HARNESS_FORK_CHILD_BEGIN(0, true)
|
HARNESS_FORK_CHILD_BEGIN(0, true)
|
||||||
{
|
{
|
||||||
IoRead *read = ioFdReadNew(STRDEF("server read"), HARNESS_FORK_CHILD_READ(), 10000);
|
IoRead *read = ioFdReadNew(STRDEF("local server 1 read"), HARNESS_FORK_CHILD_READ(), 10000);
|
||||||
ioReadOpen(read);
|
ioReadOpen(read);
|
||||||
IoWrite *write = ioFdWriteNew(STRDEF("server write"), HARNESS_FORK_CHILD_WRITE(), 2000);
|
IoWrite *write = ioFdWriteNew(STRDEF("local server 1 write"), HARNESS_FORK_CHILD_WRITE(), 2000);
|
||||||
ioWriteOpen(write);
|
ioWriteOpen(write);
|
||||||
|
|
||||||
// Greeting with noop
|
ProtocolServer *server = NULL;
|
||||||
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}"));
|
TEST_ASSIGN(server, protocolServerNew(STRDEF("local server 1"), STRDEF("test"), read, write), "local server 1");
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop");
|
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_NOOP, "noop command get");
|
||||||
ioWriteStrLine(write, STRDEF("{}"));
|
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
|
||||||
ioWriteFlush(write);
|
|
||||||
|
// Command with output
|
||||||
|
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c-one"), "c-one command get");
|
||||||
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"c-one\",\"param\":[\"param1\",\"param2\"]}", "command1");
|
|
||||||
sleepMSec(4000);
|
sleepMSec(4000);
|
||||||
ioWriteStrLine(write, STRDEF("{\"out\":1}"));
|
|
||||||
ioWriteFlush(write);
|
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 1)), "data end put");
|
||||||
|
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
|
||||||
|
|
||||||
// Wait for exit
|
// Wait for exit
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"exit\"}", "exit command");
|
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "noop command get");
|
||||||
}
|
}
|
||||||
HARNESS_FORK_CHILD_END();
|
HARNESS_FORK_CHILD_END();
|
||||||
|
|
||||||
// Local 2
|
// Local 2
|
||||||
HARNESS_FORK_CHILD_BEGIN(0, true)
|
HARNESS_FORK_CHILD_BEGIN(0, true)
|
||||||
{
|
{
|
||||||
IoRead *read = ioFdReadNew(STRDEF("server read"), HARNESS_FORK_CHILD_READ(), 10000);
|
IoRead *read = ioFdReadNew(STRDEF("local server 2 read"), HARNESS_FORK_CHILD_READ(), 10000);
|
||||||
ioReadOpen(read);
|
ioReadOpen(read);
|
||||||
IoWrite *write = ioFdWriteNew(STRDEF("server write"), HARNESS_FORK_CHILD_WRITE(), 2000);
|
IoWrite *write = ioFdWriteNew(STRDEF("local server 2 write"), HARNESS_FORK_CHILD_WRITE(), 2000);
|
||||||
ioWriteOpen(write);
|
ioWriteOpen(write);
|
||||||
|
|
||||||
// Greeting with noop
|
ProtocolServer *server = NULL;
|
||||||
ioWriteStrLine(write, STRDEF("{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}"));
|
TEST_ASSIGN(server, protocolServerNew(STRDEF("local server 2"), STRDEF("test"), read, write), "local server 2");
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"noop\"}", "noop");
|
TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_NOOP, "noop command get");
|
||||||
ioWriteStrLine(write, STRDEF("{}"));
|
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
|
||||||
ioWriteFlush(write);
|
|
||||||
|
// Command with output
|
||||||
|
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c2"), "c2 command get");
|
||||||
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"c2\",\"param\":[\"param1\"]}", "command2");
|
|
||||||
sleepMSec(1000);
|
sleepMSec(1000);
|
||||||
ioWriteStrLine(write, STRDEF("{\"out\":2}"));
|
|
||||||
ioWriteFlush(write);
|
|
||||||
|
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"c-three\",\"param\":[\"param1\"]}", "command3");
|
TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 2)), "data end put");
|
||||||
|
TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
|
||||||
|
|
||||||
ioWriteStrLine(write, STRDEF("{\"err\":39,\"out\":\"very serious error\"}"));
|
// Command with error
|
||||||
ioWriteFlush(write);
|
TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ(stringIdBit5, "c-three"), "c-three command get");
|
||||||
|
TEST_RESULT_VOID(protocolServerError(server, 39, STRDEF("very serious error"), STRDEF("stack")), "error put");
|
||||||
|
|
||||||
// Wait for exit
|
// Wait for exit
|
||||||
TEST_RESULT_STR_Z(ioReadLine(read), "{\"cmd\":\"exit\"}", "exit command");
|
CHECK(protocolServerCommandGet(server).id == PROTOCOL_COMMAND_EXIT);
|
||||||
}
|
}
|
||||||
HARNESS_FORK_CHILD_END();
|
HARNESS_FORK_CHILD_END();
|
||||||
|
|
||||||
HARNESS_FORK_PARENT_BEGIN()
|
HARNESS_FORK_PARENT_BEGIN()
|
||||||
{
|
{
|
||||||
// -----------------------------------------------------------------------------------------------------------------
|
|
||||||
TestParallelJobCallback data = {.jobList = lstNewP(sizeof(ProtocolParallelJob *))};
|
TestParallelJobCallback data = {.jobList = lstNewP(sizeof(ProtocolParallelJob *))};
|
||||||
ProtocolParallel *parallel = NULL;
|
ProtocolParallel *parallel = NULL;
|
||||||
TEST_ASSIGN(parallel, protocolParallelNew(2000, testParallelJobCallback, &data), "create parallel");
|
TEST_ASSIGN(parallel, protocolParallelNew(2000, testParallelJobCallback, &data), "create parallel");
|
||||||
@ -829,57 +734,59 @@ testRun(void)
|
|||||||
for (unsigned int clientIdx = 0; clientIdx < clientTotal; clientIdx++)
|
for (unsigned int clientIdx = 0; clientIdx < clientTotal; clientIdx++)
|
||||||
{
|
{
|
||||||
IoRead *read = ioFdReadNew(
|
IoRead *read = ioFdReadNew(
|
||||||
strNewFmt("client %u read", clientIdx), HARNESS_FORK_PARENT_READ_PROCESS(clientIdx), 2000);
|
strNewFmt("local client %u read", clientIdx), HARNESS_FORK_PARENT_READ_PROCESS(clientIdx), 2000);
|
||||||
ioReadOpen(read);
|
ioReadOpen(read);
|
||||||
IoWrite *write = ioFdWriteNew(
|
IoWrite *write = ioFdWriteNew(
|
||||||
strNewFmt("client %u write", clientIdx), HARNESS_FORK_PARENT_WRITE_PROCESS(clientIdx), 2000);
|
strNewFmt("local client %u write", clientIdx), HARNESS_FORK_PARENT_WRITE_PROCESS(clientIdx), 2000);
|
||||||
ioWriteOpen(write);
|
ioWriteOpen(write);
|
||||||
|
|
||||||
TEST_ASSIGN(
|
TEST_ASSIGN(
|
||||||
client[clientIdx],
|
client[clientIdx],
|
||||||
protocolClientNew(strNewFmt("test client %u", clientIdx), STRDEF("test"), read, write),
|
protocolClientNew(strNewFmt("local client %u", clientIdx), STRDEF("test"), read, write),
|
||||||
strZ(strNewFmt("create client %u", clientIdx)));
|
strZ(strNewFmt("local client %u new", clientIdx)));
|
||||||
TEST_RESULT_VOID(
|
TEST_RESULT_VOID(
|
||||||
protocolParallelClientAdd(parallel, client[clientIdx]), strZ(strNewFmt("add client %u", clientIdx)));
|
protocolParallelClientAdd(parallel, client[clientIdx]), strZ(strNewFmt("local client %u add", clientIdx)));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Attempt to add client without an fd
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
const String *protocolString = STRDEF(
|
TEST_TITLE("error on add without an fd");
|
||||||
"{\"name\":\"pgBackRest\",\"service\":\"error\",\"version\":\"" PROJECT_VERSION "\"}\n"
|
|
||||||
"{}\n");
|
|
||||||
|
|
||||||
IoRead *read = ioBufferReadNew(BUFSTR(protocolString));
|
// Fake a client without a read fd
|
||||||
ioReadOpen(read);
|
ProtocolClient clientError = {.pub = {.read = ioBufferReadNew(bufNew(0))}, .name = STRDEF("test")};
|
||||||
IoWrite *write = ioBufferWriteNew(bufNew(1024));
|
|
||||||
ioWriteOpen(write);
|
|
||||||
|
|
||||||
ProtocolClient *clientError = protocolClientNew(STRDEF("error"), STRDEF("error"), read, write);
|
TEST_ERROR(protocolParallelClientAdd(parallel, &clientError), AssertError, "client with read fd is required");
|
||||||
TEST_ERROR(protocolParallelClientAdd(parallel, clientError), AssertError, "client with read fd is required");
|
|
||||||
protocolClientFree(clientError);
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("add jobs");
|
||||||
|
|
||||||
// Add jobs
|
|
||||||
ProtocolCommand *command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-one"));
|
ProtocolCommand *command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-one"));
|
||||||
protocolCommandParamAdd(command, VARSTRDEF("param1"));
|
pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
|
||||||
protocolCommandParamAdd(command, VARSTRDEF("param2"));
|
pckWriteStrP(protocolCommandParam(command), STRDEF("param2"));
|
||||||
ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("job1"), command);
|
|
||||||
|
ProtocolParallelJob *job = protocolParallelJobNew(varNewStr(STRDEF("job1")), command);
|
||||||
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
|
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
|
||||||
|
|
||||||
command = protocolCommandNew(strIdFromZ(stringIdBit5, "c2"));
|
command = protocolCommandNew(strIdFromZ(stringIdBit5, "c2"));
|
||||||
protocolCommandParamAdd(command, VARSTRDEF("param1"));
|
pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
|
||||||
job = protocolParallelJobNew(VARSTRDEF("job2"), command);
|
|
||||||
|
job = protocolParallelJobNew(varNewStr(STRDEF("job2")), command);
|
||||||
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
|
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
|
||||||
|
|
||||||
command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-three"));
|
command = protocolCommandNew(strIdFromZ(stringIdBit5, "c-three"));
|
||||||
protocolCommandParamAdd(command, VARSTRDEF("param1"));
|
pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
|
||||||
job = protocolParallelJobNew(VARSTRDEF("job3"), command);
|
|
||||||
|
job = protocolParallelJobNew(varNewStr(STRDEF("job3")), command);
|
||||||
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
|
TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
|
||||||
|
|
||||||
// Process jobs
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
|
TEST_TITLE("process jobs with no result");
|
||||||
|
|
||||||
|
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
|
||||||
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
|
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
|
||||||
|
|
||||||
// Process jobs
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("result for job 2");
|
||||||
|
|
||||||
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
|
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
|
||||||
|
|
||||||
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
|
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
|
||||||
@ -887,35 +794,40 @@ testRun(void)
|
|||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
protocolParallelJobProcessId(job) >= 1 && protocolParallelJobProcessId(job) <= 2, true,
|
protocolParallelJobProcessId(job) >= 1 && protocolParallelJobProcessId(job) <= 2, true,
|
||||||
"check process id is valid");
|
"check process id is valid");
|
||||||
TEST_RESULT_INT(varIntForce(protocolParallelJobResult(job)), 2, "check result is 2");
|
TEST_RESULT_UINT(pckReadU32P(protocolParallelJobResult(job)), 2, "check result is 2");
|
||||||
|
|
||||||
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
|
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
|
||||||
|
|
||||||
// Process jobs
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("error for job 3");
|
||||||
|
|
||||||
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
|
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
|
||||||
|
|
||||||
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
|
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
|
||||||
TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job3", "check key is job3");
|
TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job3", "check key is job3");
|
||||||
TEST_RESULT_INT(protocolParallelJobErrorCode(job), 39, "check error code");
|
TEST_RESULT_INT(protocolParallelJobErrorCode(job), 39, "check error code");
|
||||||
TEST_RESULT_STR_Z(
|
TEST_RESULT_STR_Z(
|
||||||
protocolParallelJobErrorMessage(job), "raised from test client 1: very serious error",
|
protocolParallelJobErrorMessage(job), "raised from local client 1: very serious error",
|
||||||
"check error message");
|
"check error message");
|
||||||
TEST_RESULT_PTR(protocolParallelJobResult(job), NULL, "check result is null");
|
TEST_RESULT_PTR(protocolParallelJobResult(job), NULL, "check result is null");
|
||||||
|
|
||||||
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
|
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no more results");
|
||||||
|
|
||||||
// Process jobs
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
|
TEST_TITLE("process jobs with no result");
|
||||||
|
|
||||||
|
TEST_RESULT_INT(protocolParallelProcess(parallel), 0, "process jobs");
|
||||||
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
|
TEST_RESULT_PTR(protocolParallelResult(parallel), NULL, "check no result");
|
||||||
TEST_RESULT_BOOL(protocolParallelDone(parallel), false, "check not done");
|
TEST_RESULT_BOOL(protocolParallelDone(parallel), false, "check not done");
|
||||||
|
|
||||||
// Process jobs
|
// -----------------------------------------------------------------------------------------------------------------
|
||||||
|
TEST_TITLE("result for job 1");
|
||||||
|
|
||||||
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
|
TEST_RESULT_INT(protocolParallelProcess(parallel), 1, "process jobs");
|
||||||
|
|
||||||
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
|
TEST_ASSIGN(job, protocolParallelResult(parallel), "get result");
|
||||||
TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job1", "check key is job1");
|
TEST_RESULT_STR_Z(varStr(protocolParallelJobKey(job)), "job1", "check key is job1");
|
||||||
TEST_RESULT_INT(varIntForce(protocolParallelJobResult(job)), 1, "check result is 1");
|
TEST_RESULT_UINT(pckReadU32P(protocolParallelJobResult(job)), 1, "check result is 1");
|
||||||
|
|
||||||
TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check done");
|
TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check done");
|
||||||
TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check still done");
|
TEST_RESULT_BOOL(protocolParallelDone(parallel), true, "check still done");
|
||||||
|
@ -279,9 +279,6 @@ testRun(void)
|
|||||||
((StorageReadRemote *)fileRead->driver)->protocolReadBytes < bufSize(contentBuf), true,
|
((StorageReadRemote *)fileRead->driver)->protocolReadBytes < bufSize(contentBuf), true,
|
||||||
" check compressed read size");
|
" check compressed read size");
|
||||||
|
|
||||||
TEST_ERROR(
|
|
||||||
storageRemoteProtocolBlockSize(STRDEF("bogus")), ProtocolError, "'bogus' is not a valid block size message");
|
|
||||||
|
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
// -------------------------------------------------------------------------------------------------------------------------
|
||||||
TEST_TITLE("file missing");
|
TEST_TITLE("file missing");
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user