Protocol command multiplexing.

Previously it was not possible to read or write two files at the same time on the same remote because the protocol was entirely taken over by the read or write command. Multiple reads are required to make restores efficient when a list of bundled files is being read but blocks need to be retrieved from a separate file or a different part of the same file.

Improve that situation with sessions that allow related commands to be run with shared state. Also break read/write into separate requests (rather than pushing all data at once) so they can be multiplexed.

The disadvantage for read/write is that they now require more back and forth to transfer a file. This is mitigated by sending asynchronous read/write requests to keep both server and client as busy as possible. Reads that can fit into a single buffer are optimized to transfer in a single command. Reads that transfer the entire file can also skip the close command since it is implicit on end-of-file.

These changes allow the protocol to be simplified to provide one response per request, which makes the data end message obsolete. Any data sent for the request is now added to the parameters so no data needs to be sent separately to the server outside the request parameters.

Also update the Db protocol to use the new sessions. Previously this code had tracked its own sessions.

doc/xml/release/2024/2.54.xml

@@ -1,2 +1,16 @@
 <release date="XXXX-XX-XX" version="2.54dev" title="UNDER DEVELOPMENT">
+    <release-core-list>
+        <release-development-list>
+            <release-item>
+                <github-pull-request id="2108"/>
+
+                <release-item-contributor-list>
+                    <release-item-contributor id="david.steele"/>
+                    <release-item-reviewer id="david.christensen"/>
+                </release-item-contributor-list>
+
+                <p>Protocol command multiplexing.</p>
+            </release-item>
+        </release-development-list>
+    </release-core-list>
 </release>

src/Makefile.in

@@ -169,7 +169,6 @@ SRCS = \
 	postgres/interface/crc32.c \
 	postgres/interface/page.c \
 	protocol/client.c \
-	protocol/command.c \
 	protocol/helper.c \
 	protocol/parallel.c \
 	protocol/parallelJob.c \

src/command/archive/get/get.c

@@ -878,8 +878,7 @@ archiveGetAsyncCallback(void *const data, const unsigned int clientIdx)
             const ArchiveFileMap *const archiveFileMap = lstGet(jobData->archiveFileMapList, jobData->archiveFileIdx);
             jobData->archiveFileIdx++;
 
-            ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_GET_FILE);
+            PackWrite *const param = protocolPackNew();
-            PackWrite *const param = protocolCommandParam(command);
 
             pckWriteStrP(param, archiveFileMap->request);
 
@@ -897,7 +896,7 @@ archiveGetAsyncCallback(void *const data, const unsigned int clientIdx)
 
             MEM_CONTEXT_PRIOR_BEGIN()
             {
-                result = protocolParallelJobNew(VARSTR(archiveFileMap->request), command);
+                result = protocolParallelJobNew(VARSTR(archiveFileMap->request), PROTOCOL_COMMAND_ARCHIVE_GET_FILE, param);
             }
             MEM_CONTEXT_PRIOR_END();
         }

src/command/archive/get/protocol.c

@@ -14,16 +14,16 @@ Archive Get Protocol Handler
 #include "storage/write.intern.h"
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-archiveGetFileProtocol(PackRead *const param, ProtocolServer *const server)
+archiveGetFileProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
@@ -50,14 +50,11 @@ archiveGetFileProtocol(PackRead *const param, ProtocolServer *const server)
             strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s." STORAGE_FILE_TEMP_EXT, strZ(request)));
 
         // Return result
-        PackWrite *const resultPack = protocolPackNew();
+        PackWrite *const data = protocolServerResultData(result);
-        pckWriteU32P(resultPack, fileResult.actualIdx);
+        pckWriteU32P(data, fileResult.actualIdx);
-        pckWriteStrLstP(resultPack, fileResult.warnList);
+        pckWriteStrLstP(data, fileResult.warnList);
-
-        protocolServerDataPut(server, resultPack);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }

src/command/archive/get/protocol.h

@@ -11,7 +11,7 @@ Archive Get Protocol Handler
 Functions
 ***********************************************************************************************************************************/
 // Process protocol requests
-FN_EXTERN void archiveGetFileProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *archiveGetFileProtocol(PackRead *param);
 
 /***********************************************************************************************************************************
 Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
 #define PROTOCOL_COMMAND_ARCHIVE_GET_FILE                           STRID5("ag-f", 0x36ce10)
 
 #define PROTOCOL_SERVER_HANDLER_ARCHIVE_GET_LIST                                                                                   \
-    {.command = PROTOCOL_COMMAND_ARCHIVE_GET_FILE, .handler = archiveGetFileProtocol},
+    {.command = PROTOCOL_COMMAND_ARCHIVE_GET_FILE, .process = archiveGetFileProtocol},
 
 #endif

src/command/archive/push/protocol.c

@@ -13,16 +13,16 @@ Archive Push Protocol Handler
 #include "storage/helper.h"
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-archivePushFileProtocol(PackRead *const param, ProtocolServer *const server)
+archivePushFileProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
@@ -63,10 +63,9 @@ archivePushFileProtocol(PackRead *const param, ProtocolServer *const server)
             priorErrorList);
 
         // Return result
-        protocolServerDataPut(server, pckWriteStrLstP(protocolPackNew(), fileResult.warnList));
+        pckWriteStrLstP(protocolServerResultData(result), fileResult.warnList);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }

src/command/archive/push/protocol.h

@@ -11,7 +11,7 @@ Archive Push Protocol Handler
 Functions
 ***********************************************************************************************************************************/
 // Process protocol requests
-FN_EXTERN void archivePushFileProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *archivePushFileProtocol(PackRead *param);
 
 /***********************************************************************************************************************************
 Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
 #define PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE                          STRID5("ap-f", 0x36e010)
 
 #define PROTOCOL_SERVER_HANDLER_ARCHIVE_PUSH_LIST                                                                                  \
-    {.command = PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE, .handler = archivePushFileProtocol},
+    {.command = PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE, .process = archivePushFileProtocol},
 
 #endif

src/command/archive/push/push.c

@@ -464,8 +464,7 @@ archivePushAsyncCallback(void *const data, const unsigned int clientIdx)
             const String *const walFile = strLstGet(jobData->walFileList, jobData->walFileIdx);
             jobData->walFileIdx++;
 
-            ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE);
+            PackWrite *const param = protocolPackNew();
-            PackWrite *const param = protocolCommandParam(command);
 
             pckWriteStrP(param, strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile)));
             pckWriteBoolP(param, cfgOptionBool(cfgOptArchiveHeaderCheck));
@@ -496,7 +495,7 @@ archivePushAsyncCallback(void *const data, const unsigned int clientIdx)
 
             MEM_CONTEXT_PRIOR_BEGIN()
             {
-                result = protocolParallelJobNew(VARSTR(walFile), command);
+                result = protocolParallelJobNew(VARSTR(walFile), PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE, param);
             }
             MEM_CONTEXT_PRIOR_END();
         }

src/command/backup/backup.c

@@ -1951,7 +1951,6 @@ backupJobCallback(void *const data, const unsigned int clientIdx)
         const int queueEnd = queueIdx;
 
         // Create backup job
-        ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_BACKUP_FILE);
         PackWrite *param = NULL;
         uint64_t fileTotal = 0;
         uint64_t fileSize = 0;
@@ -1980,7 +1979,7 @@ backupJobCallback(void *const data, const unsigned int clientIdx)
                 // Add common parameters before first file
                 if (param == NULL)
                 {
-                    param = protocolCommandParam(command);
+                    param = protocolPackNew();
 
                     if (bundle && file.size <= jobData->bundleLimit)
                     {
@@ -2068,7 +2067,8 @@ backupJobCallback(void *const data, const unsigned int clientIdx)
                 // Assign job to result
                 MEM_CONTEXT_PRIOR_BEGIN()
                 {
-                    result = protocolParallelJobNew(bundle ? VARUINT64(jobData->bundleId) : VARSTR(fileName), command);
+                    result = protocolParallelJobNew(
+                        bundle ? VARUINT64(jobData->bundleId) : VARSTR(fileName), PROTOCOL_COMMAND_BACKUP_FILE, param);
 
                     if (bundle)
                         jobData->bundleId++;

src/command/backup/protocol.c

@@ -15,16 +15,16 @@ Backup Protocol Handler
 #include "storage/helper.h"
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-backupFileProtocol(PackRead *const param, ProtocolServer *const server)
+backupFileProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
@@ -79,37 +79,34 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server)
         }
 
         // Backup file
-        const List *const result = backupFile(
+        const List *const resultList = backupFile(
             repoFile, bundleId, bundleRaw, blockIncrReference, repoFileCompressType, repoFileCompressLevel, cipherType, cipherPass,
             pgVersionForce, pageSize, fileList);
 
         // Return result
-        PackWrite *const resultPack = protocolPackNew();
+        PackWrite *const data = protocolServerResultData(result);
 
-        for (unsigned int resultIdx = 0; resultIdx < lstSize(result); resultIdx++)
+        for (unsigned int resultIdx = 0; resultIdx < lstSize(resultList); resultIdx++)
         {
-            const BackupFileResult *const fileResult = lstGet(result, resultIdx);
+            const BackupFileResult *const fileResult = lstGet(resultList, resultIdx);
 
             ASSERT(
                 fileResult->backupCopyResult == backupCopyResultSkip || fileResult->copySize != 0 ||
                 bufEq(fileResult->copyChecksum, HASH_TYPE_SHA1_ZERO_BUF));
 
-            pckWriteStrP(resultPack, fileResult->manifestFile);
+            pckWriteStrP(data, fileResult->manifestFile);
-            pckWriteU32P(resultPack, fileResult->backupCopyResult);
+            pckWriteU32P(data, fileResult->backupCopyResult);
-            pckWriteBoolP(resultPack, fileResult->repoInvalid);
+            pckWriteBoolP(data, fileResult->repoInvalid);
-            pckWriteU64P(resultPack, fileResult->copySize);
+            pckWriteU64P(data, fileResult->copySize);
-            pckWriteU64P(resultPack, fileResult->bundleOffset);
+            pckWriteU64P(data, fileResult->bundleOffset);
-            pckWriteU64P(resultPack, fileResult->blockIncrMapSize);
+            pckWriteU64P(data, fileResult->blockIncrMapSize);
-            pckWriteU64P(resultPack, fileResult->repoSize);
+            pckWriteU64P(data, fileResult->repoSize);
-            pckWriteBinP(resultPack, fileResult->copyChecksum);
+            pckWriteBinP(data, fileResult->copyChecksum);
-            pckWriteBinP(resultPack, fileResult->repoChecksum);
+            pckWriteBinP(data, fileResult->repoChecksum);
-            pckWritePackP(resultPack, fileResult->pageChecksumResult);
+            pckWritePackP(data, fileResult->pageChecksumResult);
         }
-
-        protocolServerDataPut(server, resultPack);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }

src/command/backup/protocol.h

@@ -11,7 +11,7 @@ Backup Protocol Handler
 Functions
 ***********************************************************************************************************************************/
 // Process protocol requests
-FN_EXTERN void backupFileProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *backupFileProtocol(PackRead *param);
 
 /***********************************************************************************************************************************
 Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
 #define PROTOCOL_COMMAND_BACKUP_FILE                                STRID5("bp-f", 0x36e020)
 
 #define PROTOCOL_SERVER_HANDLER_BACKUP_LIST                                                                                        \
-    {.command = PROTOCOL_COMMAND_BACKUP_FILE, .handler = backupFileProtocol},
+    {.command = PROTOCOL_COMMAND_BACKUP_FILE, .process = backupFileProtocol},
 
 #endif

src/command/remote/remote.c

@@ -66,7 +66,7 @@ cmdRemote(ProtocolServer *const server)
         TRY_BEGIN()
         {
             // Get the command. No need to check parameters since we know this is the first noop.
-            CHECK(FormatError, protocolServerCommandGet(server).id == PROTOCOL_COMMAND_NOOP, "noop expected");
+            CHECK(FormatError, protocolServerRequest(server).id == PROTOCOL_COMMAND_NOOP, "noop expected");
 
             // Only try the lock if this is process 0, i.e. the remote started from the main process
             if (cfgOptionUInt(cfgOptProcess) == 0)
@@ -83,7 +83,7 @@ cmdRemote(ProtocolServer *const server)
             }
 
             // Notify the client of success
-            protocolServerDataEndPut(server);
+            protocolServerResponseP(server);
             success = true;
         }
         CATCH_ANY()

src/command/restore/protocol.c

@@ -13,16 +13,16 @@ Restore Protocol Handler
 #include "storage/helper.h"
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
+restoreFileProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
@@ -72,26 +72,23 @@ restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
         }
 
         // Restore files
-        const List *const result = restoreFile(
+        const List *const resultList = restoreFile(
             repoFile, repoIdx, repoFileCompressType, copyTimeBegin, delta, deltaForce, bundleRaw, cipherPass, referenceList,
             fileList);
 
         // Return result
-        PackWrite *const resultPack = protocolPackNew();
+        PackWrite *const data = protocolServerResultData(result);
 
-        for (unsigned int resultIdx = 0; resultIdx < lstSize(result); resultIdx++)
+        for (unsigned int resultIdx = 0; resultIdx < lstSize(resultList); resultIdx++)
         {
-            const RestoreFileResult *const fileResult = lstGet(result, resultIdx);
+            const RestoreFileResult *const fileResult = lstGet(resultList, resultIdx);
 
-            pckWriteStrP(resultPack, fileResult->manifestFile);
+            pckWriteStrP(data, fileResult->manifestFile);
-            pckWriteU32P(resultPack, fileResult->result);
+            pckWriteU32P(data, fileResult->result);
-            pckWriteU64P(resultPack, fileResult->blockIncrDeltaSize);
+            pckWriteU64P(data, fileResult->blockIncrDeltaSize);
         }
-
-        protocolServerDataPut(server, resultPack);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }

src/command/restore/protocol.h

@@ -11,7 +11,7 @@ Restore Protocol Handler
 Functions
 ***********************************************************************************************************************************/
 // Process protocol requests
-FN_EXTERN void restoreFileProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *restoreFileProtocol(PackRead *param);
 
 /***********************************************************************************************************************************
 Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
 #define PROTOCOL_COMMAND_RESTORE_FILE                               STRID5("rs-f", 0x36e720)
 
 #define PROTOCOL_SERVER_HANDLER_RESTORE_LIST                                                                                       \
-    {.command = PROTOCOL_COMMAND_RESTORE_FILE, .handler = restoreFileProtocol},
+    {.command = PROTOCOL_COMMAND_RESTORE_FILE, .process = restoreFileProtocol},
 
 #endif

src/command/restore/restore.c

@@ -2309,7 +2309,6 @@ restoreJobCallback(void *const data, const unsigned int clientIdx)
         RestoreJobData *const jobData = data;
 
         // Determine where to begin scanning the queue (we'll stop when we get back here)
-        ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
         PackWrite *param = NULL;
         int queueIdx = (int)(clientIdx % lstSize(jobData->queueList));
         const int queueEnd = queueIdx;
@@ -2334,7 +2333,7 @@ restoreJobCallback(void *const data, const unsigned int clientIdx)
                 // Add common parameters before first file
                 if (param == NULL)
                 {
-                    param = protocolCommandParam(command);
+                    param = protocolPackNew();
 
                     if (file.bundleId != 0)
                     {
@@ -2414,7 +2413,8 @@ restoreJobCallback(void *const data, const unsigned int clientIdx)
                 // Assign job to result
                 MEM_CONTEXT_PRIOR_BEGIN()
                 {
-                    result = protocolParallelJobNew(bundleId != 0 ? VARUINT64(bundleId) : VARSTR(fileName), command);
+                    result = protocolParallelJobNew(
+                        bundleId != 0 ? VARUINT64(bundleId) : VARSTR(fileName), PROTOCOL_COMMAND_RESTORE_FILE, param);
                 }
                 MEM_CONTEXT_PRIOR_END();
 

src/command/verify/protocol.c

@@ -13,16 +13,16 @@ Verify Protocol Handler
 #include "storage/helper.h"
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-verifyFileProtocol(PackRead *const param, ProtocolServer *const server)
+verifyFileProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
@@ -43,13 +43,12 @@ verifyFileProtocol(PackRead *const param, ProtocolServer *const server)
         const uint64_t fileSize = pckReadU64P(param);
         const String *const cipherPass = pckReadStrP(param);
 
-        const VerifyResult result = verifyFile(filePathName, offset, limit, compressType, fileChecksum, fileSize, cipherPass);
-
         // Return result
-        protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), result));
+        pckWriteU32P(
-        protocolServerDataEndPut(server);
+            protocolServerResultData(result),
+            verifyFile(filePathName, offset, limit, compressType, fileChecksum, fileSize, cipherPass));
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }

src/command/verify/protocol.h

@@ -11,7 +11,7 @@ Verify Protocol Handler
 Functions
 ***********************************************************************************************************************************/
 // Process protocol requests
-FN_EXTERN void verifyFileProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *verifyFileProtocol(PackRead *param);
 
 /***********************************************************************************************************************************
 Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@@ -19,6 +19,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
 #define PROTOCOL_COMMAND_VERIFY_FILE                                STRID5("vf-f", 0x36cd60)
 
 #define PROTOCOL_SERVER_HANDLER_VERIFY_LIST                                                                                        \
-    {.command = PROTOCOL_COMMAND_VERIFY_FILE, .handler = verifyFileProtocol},
+    {.command = PROTOCOL_COMMAND_VERIFY_FILE, .process = verifyFileProtocol},
 
 #endif

src/command/verify/verify.c

@@ -754,8 +754,7 @@ verifyArchive(VerifyJobData *const jobData)
                             encodingHex, strSubN(fileName, WAL_SEGMENT_NAME_SIZE + 1, HASH_TYPE_SHA1_SIZE_HEX));
 
                         // Set up the job
-                        ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
+                        PackWrite *const param = protocolPackNew();
-                        PackWrite *const param = protocolCommandParam(command);
 
                         pckWriteStrP(param, filePathName);
                         pckWriteBoolP(param, false);
@@ -769,7 +768,7 @@ verifyArchive(VerifyJobData *const jobData)
 
                         MEM_CONTEXT_PRIOR_BEGIN()
                         {
-                            result = protocolParallelJobNew(VARSTR(jobKey), command);
+                            result = protocolParallelJobNew(VARSTR(jobKey), PROTOCOL_COMMAND_VERIFY_FILE, param);
                         }
                         MEM_CONTEXT_PRIOR_END();
 
@@ -980,8 +979,8 @@ verifyBackup(VerifyJobData *const jobData)
                         if (fileBackupLabel != NULL)
                         {
                             // Set up the job
-                            ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_VERIFY_FILE);
+                            PackWrite *const param = protocolPackNew();
-                            PackWrite *const param = protocolCommandParam(command);
+
                             const String *const filePathName = backupFileRepoPathP(
                                 fileBackupLabel, .manifestName = fileData.name, .bundleId = fileData.bundleId,
                                 .compressType = manifestData(jobData->manifest)->backupOptionCompressType,
@@ -1021,7 +1020,7 @@ verifyBackup(VerifyJobData *const jobData)
 
                             MEM_CONTEXT_PRIOR_BEGIN()
                             {
-                                result = protocolParallelJobNew(VARSTR(jobKey), command);
+                                result = protocolParallelJobNew(VARSTR(jobKey), PROTOCOL_COMMAND_VERIFY_FILE, param);
                             }
                             MEM_CONTEXT_PRIOR_END();
                         }

src/config/protocol.c

@@ -13,16 +13,16 @@ Configuration Protocol Handler
 #include "config/protocol.h"
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-configOptionProtocol(PackRead *const param, ProtocolServer *const server)
+configOptionProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
@@ -36,12 +36,11 @@ configOptionProtocol(PackRead *const param, ProtocolServer *const server)
             varLstAdd(optionList, varDup(cfgOptionIdxVar(option.id, cfgOptionKeyToIdx(option.id, option.keyIdx + 1))));
         }
 
-        protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), jsonFromVar(varNewVarLst(optionList))));
+        pckWriteStrP(protocolServerResultData(result), jsonFromVar(varNewVarLst(optionList)));
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
 /**********************************************************************************************************************************/
@@ -57,13 +56,13 @@ configOptionRemote(ProtocolClient *const client, const VariantList *const paramL
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_CONFIG_OPTION);
+        PackWrite *const param = protocolPackNew();
-        PackWrite *const param = protocolCommandParam(command);
 
         for (unsigned int paramIdx = 0; paramIdx < varLstSize(paramList); paramIdx++)
             pckWriteStrP(param, varStr(varLstGet(paramList, paramIdx)));
 
-        const VariantList *const list = varVarLst(jsonToVar(pckReadStrP(protocolClientExecute(client, command, true))));
+        const VariantList *const list = varVarLst(
+            jsonToVar(pckReadStrP(protocolClientRequestP(client, PROTOCOL_COMMAND_CONFIG_OPTION, .param = param))));
 
         MEM_CONTEXT_PRIOR_BEGIN()
         {

src/config/protocol.h

@@ -12,7 +12,7 @@ Configuration Protocol Handler
 Functions
 ***********************************************************************************************************************************/
 // Process config protocol requests
-FN_EXTERN void configOptionProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *configOptionProtocol(PackRead *param);
 
 // Get option values from a remote process
 FN_EXTERN VariantList *configOptionRemote(ProtocolClient *client, const VariantList *paramList);
@@ -23,6 +23,6 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
 #define PROTOCOL_COMMAND_CONFIG_OPTION                              STRID5("opt-g", 0x7dd20f0)
 
 #define PROTOCOL_SERVER_HANDLER_OPTION_LIST                                                                                        \
-    {.command = PROTOCOL_COMMAND_CONFIG_OPTION, .handler = configOptionProtocol},
+    {.command = PROTOCOL_COMMAND_CONFIG_OPTION, .process = configOptionProtocol},
 
 #endif

src/db/db.c

@@ -30,7 +30,7 @@ struct Db
     DbPub pub;                                                      // Publicly accessible variables
     PgClient *client;                                               // Local PostgreSQL client
     ProtocolClient *remoteClient;                                   // Protocol client for remote db queries
-    unsigned int remoteIdx;                                         // Index provided by the remote on open for subsequent calls
+    ProtocolClientSession *session;                                 // Protocol session for requests
     const Storage *storage;                                         // PostgreSQL storage
     const String *applicationName;                                  // Used to identify this connection in PostgreSQL
     time_t pingTimeLast;                                            // Last time cluster was pinged
@@ -50,11 +50,7 @@ dbFreeResource(THIS_VOID)
 
     ASSERT(this != NULL);
 
-    ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_CLOSE);
+    protocolClientSessionFree(this->session);
-    pckWriteU32P(protocolCommandParam(command), this->remoteIdx);
-
-    protocolClientExecute(this->remoteClient, command, false);
-    protocolCommandFree(command);
 
     FUNCTION_LOG_RETURN_VOID();
 }
@@ -83,6 +79,7 @@ dbNew(PgClient *const client, ProtocolClient *const remoteClient, const Storage
                 .memContext = memContextCurrent(),
             },
             .remoteClient = remoteClient,
+            .session = remoteClient != NULL ? protocolClientSessionNewP(remoteClient, PROTOCOL_COMMAND_DB) : NULL,
             .storage = storage,
             .applicationName = strDup(applicationName),
         };
@@ -117,14 +114,12 @@ dbQuery(Db *const this, const PgClientQueryResult resultType, const String *cons
     {
         MEM_CONTEXT_TEMP_BEGIN()
         {
-            ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_DB_QUERY);
+            PackWrite *const param = protocolPackNew();
-            PackWrite *const param = protocolCommandParam(command);
 
-            pckWriteU32P(param, this->remoteIdx);
             pckWriteStrIdP(param, resultType);
             pckWriteStrP(param, query);
 
-            PackRead *const read = protocolClientExecute(this->remoteClient, command, true);
+            PackRead *const read = protocolClientSessionRequestP(this->session, .param = param);
 
             MEM_CONTEXT_PRIOR_BEGIN()
             {
@@ -240,8 +235,7 @@ dbOpen(Db *const this)
         // Open the connection
         if (this->remoteClient != NULL)
         {
-            ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_DB_OPEN);
+            protocolClientSessionOpenP(this->session);
-            this->remoteIdx = pckReadU32P(protocolClientExecute(this->remoteClient, command, true));
 
             // Set a callback to notify the remote when a connection is closed
             memContextCallbackSet(this->pub.memContext, dbFreeResource, this);

src/db/protocol.c

@@ -14,104 +14,57 @@ Db Protocol Handler
 #include "postgres/client.h"
 #include "postgres/interface.h"
 
-/***********************************************************************************************************************************
-Local variables
-***********************************************************************************************************************************/
-static struct
-{
-    List *pgClientList;                                             // List of db objects
-} dbProtocolLocal;
-
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-dbOpenProtocol(PackRead *const param, ProtocolServer *const server)
+dbOpenProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param == NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        // If the db list does not exist then create it in the top context
+        PgClient *const pgClient = pgClientNew(
-        if (dbProtocolLocal.pgClientList == NULL)
+            cfgOptionStrNull(cfgOptPgSocketPath), cfgOptionUInt(cfgOptPgPort), cfgOptionStr(cfgOptPgDatabase),
-        {
+            cfgOptionStrNull(cfgOptPgUser), cfgOptionUInt64(cfgOptDbTimeout));
-            MEM_CONTEXT_BEGIN(memContextTop())
+        pgClientOpen(pgClient);
-            {
-                dbProtocolLocal.pgClientList = lstNewP(sizeof(PgClient *));
-            }
-            MEM_CONTEXT_END();
-        }
 
-        // Add db to the list
+        // Set session data
-        MEM_CONTEXT_BEGIN(lstMemContext(dbProtocolLocal.pgClientList))
+        protocolServerResultSessionDataSet(result, pgClient);
-        {
-            // Only a single db is passed to the remote
-            PgClient *const pgClient = pgClientNew(
-                cfgOptionStrNull(cfgOptPgSocketPath), cfgOptionUInt(cfgOptPgPort), cfgOptionStr(cfgOptPgDatabase),
-                cfgOptionStrNull(cfgOptPgUser), cfgOptionUInt64(cfgOptDbTimeout));
-            pgClientOpen(pgClient);
-
-            lstAdd(dbProtocolLocal.pgClientList, &pgClient);
-        }
-        MEM_CONTEXT_END();
-
-        // Return db index which should be included in subsequent calls
-        protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), lstSize(dbProtocolLocal.pgClientList) - 1));
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-dbQueryProtocol(PackRead *const param, ProtocolServer *const server)
+dbQueryProtocol(PackRead *const param, void *const pgClient)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
+        FUNCTION_LOG_PARAM(PG_CLIENT, pgClient);
     FUNCTION_LOG_END();
 
+    FUNCTION_AUDIT_STRUCT();
+
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
+    ASSERT(pgClient != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        PgClient *const pgClient = *(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param));
         const PgClientQueryResult resultType = (PgClientQueryResult)pckReadStrIdP(param);
         const String *const query = pckReadStrP(param);
 
-        protocolServerDataPut(server, pckWritePackP(protocolPackNew(), pgClientQuery(pgClient, query, resultType)));
+        pckWritePackP(protocolServerResultData(result), pgClientQuery(pgClient, query, resultType));
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
-}
-
-/**********************************************************************************************************************************/
-FN_EXTERN void
-dbCloseProtocol(PackRead *const param, ProtocolServer *const server)
-{
-    FUNCTION_LOG_BEGIN(logLevelDebug);
-        FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
-    FUNCTION_LOG_END();
-
-    ASSERT(param != NULL);
-    ASSERT(server != NULL);
-
-    MEM_CONTEXT_TEMP_BEGIN()
-    {
-        pgClientClose(*(PgClient **)lstGet(dbProtocolLocal.pgClientList, pckReadU32P(param)));
-        protocolServerDataEndPut(server);
-    }
-    MEM_CONTEXT_TEMP_END();
-
-    FUNCTION_LOG_RETURN_VOID();
 }

src/db/protocol.h

@@ -11,20 +11,15 @@ Db Protocol Handler
 Functions
 ***********************************************************************************************************************************/
 // Process db protocol requests
-FN_EXTERN void dbOpenProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *dbOpenProtocol(PackRead *param);
-FN_EXTERN void dbQueryProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *dbQueryProtocol(PackRead *param, void *sessionData);
-FN_EXTERN void dbCloseProtocol(PackRead *param, ProtocolServer *server);
 
 /***********************************************************************************************************************************
 Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
 ***********************************************************************************************************************************/
-#define PROTOCOL_COMMAND_DB_OPEN                                    STRID5("db-o", 0x7ec440)
+#define PROTOCOL_COMMAND_DB                                         STRID5("db", 0x440)
-#define PROTOCOL_COMMAND_DB_QUERY                                   STRID5("db-q", 0x8ec440)
-#define PROTOCOL_COMMAND_DB_CLOSE                                   STRID5("db-c", 0x1ec440)
 
 #define PROTOCOL_SERVER_HANDLER_DB_LIST                                                                                            \
-    {.command = PROTOCOL_COMMAND_DB_OPEN, .handler = dbOpenProtocol},                                                              \
+    {.command = PROTOCOL_COMMAND_DB, .open = dbOpenProtocol, .processSession = dbQueryProtocol},
-    {.command = PROTOCOL_COMMAND_DB_QUERY, .handler = dbQueryProtocol},                                                            \
-    {.command = PROTOCOL_COMMAND_DB_CLOSE, .handler = dbCloseProtocol},
 
 #endif

src/meson.build

@@ -238,7 +238,6 @@ src_pgbackrest = [
     'postgres/interface/crc32.c',
     'postgres/interface/page.c',
     'protocol/client.c',
-    'protocol/command.c',
     'protocol/helper.c',
     'protocol/parallel.c',
     'protocol/parallelJob.c',

src/postgres/client.c

@@ -354,26 +354,6 @@ pgClientQuery(PgClient *const this, const String *const query, const PgClientQue
     FUNCTION_LOG_RETURN(PACK, result);
 }
 
-/**********************************************************************************************************************************/
-FN_EXTERN void
-pgClientClose(PgClient *const this)
-{
-    FUNCTION_LOG_BEGIN(logLevelDebug);
-        FUNCTION_LOG_PARAM(PG_CLIENT, this);
-    FUNCTION_LOG_END();
-
-    ASSERT(this != NULL);
-
-    if (this->connection != NULL)
-    {
-        memContextCallbackClear(objMemContext(this));
-        PQfinish(this->connection);
-        this->connection = NULL;
-    }
-
-    FUNCTION_LOG_RETURN_VOID();
-}
-
 /**********************************************************************************************************************************/
 FN_EXTERN void
 pgClientToLog(const PgClient *const this, StringStatic *const debugLog)

src/postgres/client.h

@@ -98,9 +98,6 @@ pgClientMove(PgClient *const this, MemContext *const parentNew)
 // Execute a query and return results
 FN_EXTERN Pack *pgClientQuery(PgClient *this, const String *query, PgClientQueryResult resultType);
 
-// Close connection to PostgreSQL
-FN_EXTERN void pgClientClose(PgClient *this);
-
 /***********************************************************************************************************************************
 Destructor
 ***********************************************************************************************************************************/

src/protocol/client.c

@@ -17,20 +17,14 @@ Client state enum
 ***********************************************************************************************************************************/
 typedef enum
 {
-    // Client is waiting for a command
+    // Client is waiting for a response / ready to send a request
     protocolClientStateIdle = STRID5("idle", 0x2b0890),
 
-    // Command put is in progress
+    // Request is in progress
-    protocolClientStateCommandPut = STRID5("cmd-put", 0x52b0d91a30),
+    protocolClientStateRequest = STRID5("request", 0x5265ac4b20),
 
-    // Waiting for command data from server. Only used when dataPut is true in protocolClientCommandPut().
+    // Response is in progress
-    protocolClientStateCommandDataGet = STRID5("cmd-data-get", 0xa14fb0d024d91a30),
+    protocolClientStateResponse = STRID5("response", 0x2cdcf84cb20),
-
-    // Putting data to server. Only used when dataPut is true in protocolClientCommandPut().
-    protocolClientStateDataPut = STRID5("data-put", 0xa561b0d0240),
-
-    // Getting data from server
-    protocolClientStateDataGet = STRID5("data-get", 0xa14fb0d0240),
 } ProtocolClientState;
 
 /***********************************************************************************************************************************
@@ -44,8 +38,246 @@ struct ProtocolClient
     const String *name;                                             // Name displayed in logging
     const String *errorPrefix;                                      // Prefix used when throwing error
     TimeMSec keepAliveTime;                                         // Last time data was put to the server
+    uint64_t sessionTotal;                                          // Total sessions (used to generate session ids)
+    List *sessionList;                                              // Session list
 };
 
+struct ProtocolClientSession
+{
+    ProtocolClientSessionPub pub;                                   // Publicly accessible variables
+    ProtocolClient *client;                                         // Protocol client
+    StringId command;                                               // Command
+    uint64_t sessionId;                                             // Session id
+    bool async;                                                     // Async requests allowed?
+
+    // Stored message (read by another session and stored for later retrieval)
+    bool stored;                                                    // Is a message currently stored?
+    bool close;                                                     // Should the session be closed?
+    ProtocolMessageType type;                                       // Type of last message received
+    PackRead *packRead;                                             // Last message received (if any)
+};
+
+/***********************************************************************************************************************************
+Check protocol state
+***********************************************************************************************************************************/
+static void
+protocolClientStateExpectIdle(const ProtocolClient *const this)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(PROTOCOL_CLIENT, this);
+    FUNCTION_TEST_END();
+
+    if (this->state != protocolClientStateIdle)
+    {
+        THROW_FMT(
+            ProtocolError, "client state is '%s' but expected '%s'", strZ(strIdToStr(this->state)),
+            strZ(strIdToStr(protocolClientStateIdle)));
+    }
+
+    FUNCTION_TEST_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Find a client session
+***********************************************************************************************************************************/
+static unsigned int
+protocolClientSessionFindIdx(const ProtocolClient *const this, const uint64_t sessionId)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(PROTOCOL_CLIENT, this);
+        FUNCTION_TEST_PARAM(UINT64, sessionId);
+    FUNCTION_TEST_END();
+
+    unsigned int result = 0;
+
+    for (; result < lstSize(this->sessionList); result++)
+    {
+        if ((*(ProtocolClientSession **)lstGet(this->sessionList, result))->sessionId == sessionId)
+            break;
+    }
+
+    CHECK_FMT(ProtocolError, result < lstSize(this->sessionList), "unable to find protocol client session %" PRIu64, sessionId);
+
+    FUNCTION_TEST_RETURN(UINT, result);
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN void
+protocolClientRequestInternal(ProtocolClientSession *const this, const ProtocolCommandType type, PackWrite *const param)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
+        FUNCTION_LOG_PARAM(STRING_ID, type);
+        FUNCTION_LOG_PARAM(PACK_WRITE, param);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(!protocolClientSessionQueued(this));
+
+    // Expect idle state before request
+    protocolClientStateExpectIdle(this->client);
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        // Switch state to request
+        this->client->state = protocolClientStateRequest;
+
+        // Write request
+        PackWrite *const request = pckWriteNewIo(this->client->write);
+        pckWriteU32P(request, protocolMessageTypeRequest, .defaultWrite = true);
+        pckWriteStrIdP(request, this->command);
+        pckWriteStrIdP(request, type);
+        pckWriteU64P(request, this->sessionId);
+        pckWriteBoolP(request, this->pub.open);
+
+        // Write request parameters
+        if (param != NULL)
+        {
+            pckWriteEndP(param);
+            pckWritePackP(request, pckWriteResult(param));
+        }
+
+        pckWriteEndP(request);
+
+        // Flush to send request immediately
+        ioWriteFlush(this->client->write);
+
+        this->pub.queued = true;
+
+        // Switch state back to idle after successful request
+        this->client->state = protocolClientStateIdle;
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Process errors
+***********************************************************************************************************************************/
+static void
+protocolClientError(ProtocolClient *const this, const ProtocolMessageType type, PackRead *const error)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
+        FUNCTION_LOG_PARAM(ENUM, type);
+        FUNCTION_LOG_PARAM(PACK_READ, error);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+
+    if (type == protocolMessageTypeError)
+    {
+        ASSERT(error != NULL);
+
+        MEM_CONTEXT_TEMP_BEGIN()
+        {
+            const ErrorType *const type = errorTypeFromCode(pckReadI32P(error));
+            const String *const message = strNewFmt("%s: %s", strZ(this->errorPrefix), strZ(pckReadStrP(error)));
+            const String *const stack = pckReadStrP(error);
+            pckReadEndP(error);
+
+            CHECK(FormatError, message != NULL && stack != NULL, "invalid error data");
+
+            errorInternalThrow(type, __FILE__, __func__, __LINE__, strZ(message), strZ(stack));
+        }
+        MEM_CONTEXT_TEMP_END();
+    }
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/**********************************************************************************************************************************/
+static PackRead *
+protocolClientResponseInternal(ProtocolClientSession *const this)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+
+    // Expect data-get state before data get
+    protocolClientStateExpectIdle(this->client);
+
+    PackRead *result = NULL;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        // Check the session for a stored response
+        bool found = false;
+        ProtocolMessageType type = protocolMessageTypeResponse;
+        bool close = false;
+        PackRead *packRead = NULL;
+
+        if (this->stored)
+        {
+            found = true;
+            type = this->type;
+            close = this->close;
+            packRead = pckReadMove(this->packRead, memContextCurrent());
+
+            this->stored = false;
+        }
+
+        // If not stored then read from protocol
+        while (!found)
+        {
+            // Switch state to response
+            this->client->state = protocolClientStateResponse;
+
+            // Read response
+            PackRead *const response = pckReadNewIo(this->client->pub.read);
+            const uint64_t sessionId = pckReadU64P(response);
+            type = (ProtocolMessageType)pckReadU32P(response);
+            close = pckReadBoolP(response);
+            packRead = pckReadPackReadP(response);
+            pckReadEndP(response);
+
+            // If this response is for another session then store it with that session. Session id 0 indicates a fatal error on the
+            // server that should be reported by the first session that sees it.
+            ASSERT(sessionId != 0 || type == protocolMessageTypeError);
+
+            if (sessionId != 0 && sessionId != this->sessionId)
+            {
+                ProtocolClientSession *const sessionOther = *(ProtocolClientSession **)lstGet(
+                    this->client->sessionList, protocolClientSessionFindIdx(this->client, sessionId));
+
+                CHECK_FMT(
+                    ProtocolError, !sessionOther->stored, "protocol client session %" PRIu64 " already has a stored response",
+                    sessionOther->sessionId);
+
+                sessionOther->stored = true;
+                sessionOther->type = type;
+                sessionOther->close = close;
+                sessionOther->packRead = objMove(packRead, objMemContext(sessionOther));
+            }
+            else
+                found = true;
+
+            // Switch state back to idle after successful response
+            this->client->state = protocolClientStateIdle;
+        }
+
+        // The result is no longer queued -- either it will generate an error or be returned to the user
+        this->pub.queued = false;
+
+        // Close the session if requested
+        if (close)
+            this->pub.open = false;
+
+        // Check if this result is an error and if so throw it
+        protocolClientError(this->client, type, packRead);
+        CHECK(FormatError, type == protocolMessageTypeResponse, "expected response message");
+
+        // Return result the the caller
+        result = objMove(packRead, memContextPrior());
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN(PACK_READ, result);
+}
+
 /***********************************************************************************************************************************
 Close protocol connection
 ***********************************************************************************************************************************/
@@ -60,13 +292,18 @@ protocolClientFreeResource(THIS_VOID)
 
     ASSERT(this != NULL);
 
-    // Switch state to idle so the command is sent no matter the current state
+    // Stop client sessions from sending cancel requests
-    this->state = protocolClientStateIdle;
+    for (unsigned int sessionIdx = 0; sessionIdx < lstSize(this->sessionList); sessionIdx++)
+        memContextCallbackClear(objMemContext(*(ProtocolClientSession **)lstGet(this->sessionList, sessionIdx)));
 
-    // Send an exit command but don't wait to see if it succeeds
+    // Send an exit request but don't wait to see if it succeeds
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        protocolClientCommandPut(this, protocolCommandNew(PROTOCOL_COMMAND_EXIT), false);
+        // Switch state to idle so the request is sent no matter the current state
+        this->state = protocolClientStateIdle;
+
+        // Send exit request
+        protocolClientRequestInternal(protocolClientSessionNewP(this, PROTOCOL_COMMAND_EXIT), protocolCommandTypeProcess, NULL);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -101,6 +338,7 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
             .name = strDup(name),
             .errorPrefix = strNewFmt("raised from %s", strZ(name)),
             .keepAliveTime = timeMSec(),
+            .sessionList = lstNewP(sizeof(ProtocolClientSession)),
         };
 
         // Read, parse, and check the protocol greeting
@@ -153,232 +391,37 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
     FUNCTION_LOG_RETURN(PROTOCOL_CLIENT, this);
 }
 
-/***********************************************************************************************************************************
-Check protocol state
-***********************************************************************************************************************************/
-static void
-protocolClientStateExpect(const ProtocolClient *const this, const ProtocolClientState expect)
-{
-    FUNCTION_TEST_BEGIN();
-        FUNCTION_TEST_PARAM(PROTOCOL_CLIENT, this);
-        FUNCTION_TEST_PARAM(STRING_ID, expect);
-    FUNCTION_TEST_END();
-
-    if (this->state != expect)
-        THROW_FMT(ProtocolError, "client state is '%s' but expected '%s'", strZ(strIdToStr(this->state)), strZ(strIdToStr(expect)));
-
-    FUNCTION_TEST_RETURN_VOID();
-}
-
-/**********************************************************************************************************************************/
-FN_EXTERN void
-protocolClientDataPut(ProtocolClient *const this, PackWrite *const data)
-{
-    FUNCTION_LOG_BEGIN(logLevelTrace);
-        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
-        FUNCTION_LOG_PARAM(PACK_WRITE, data);
-    FUNCTION_LOG_END();
-
-    ASSERT(this != NULL);
-
-    // Expect data-put state before data put
-    protocolClientStateExpect(this, protocolClientStateDataPut);
-
-    MEM_CONTEXT_TEMP_BEGIN()
-    {
-        // End the pack
-        if (data != NULL)
-            pckWriteEndP(data);
-
-        // Write the data
-        PackWrite *dataMessage = pckWriteNewIo(this->write);
-        pckWriteU32P(dataMessage, protocolMessageTypeData, .defaultWrite = true);
-        pckWritePackP(dataMessage, pckWriteResult(data));
-        pckWriteEndP(dataMessage);
-
-        // Flush when there is no more data to put
-        if (data == NULL)
-        {
-            ioWriteFlush(this->write);
-
-            // Switch state to data-get after successful data end put
-            this->state = protocolClientStateDataGet;
-        }
-    }
-    MEM_CONTEXT_TEMP_END();
-
-    FUNCTION_LOG_RETURN_VOID();
-}
-
-/**********************************************************************************************************************************/
-// Helper to process errors
-static void
-protocolClientError(ProtocolClient *const this, const ProtocolMessageType type, PackRead *const error)
-{
-    FUNCTION_LOG_BEGIN(logLevelTrace);
-        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
-        FUNCTION_LOG_PARAM(ENUM, type);
-        FUNCTION_LOG_PARAM(PACK_READ, error);
-    FUNCTION_LOG_END();
-
-    ASSERT(this != NULL);
-    ASSERT(error != NULL);
-
-    if (type == protocolMessageTypeError)
-    {
-        MEM_CONTEXT_TEMP_BEGIN()
-        {
-            const ErrorType *const type = errorTypeFromCode(pckReadI32P(error));
-            const String *const message = strNewFmt("%s: %s", strZ(this->errorPrefix), strZ(pckReadStrP(error)));
-            const String *const stack = pckReadStrP(error);
-            pckReadEndP(error);
-
-            // Switch state to idle after error (server will do the same)
-            this->state = protocolClientStateIdle;
-
-            CHECK(FormatError, message != NULL && stack != NULL, "invalid error data");
-
-            errorInternalThrow(type, __FILE__, __func__, __LINE__, strZ(message), strZ(stack));
-        }
-        MEM_CONTEXT_TEMP_END();
-    }
-
-    FUNCTION_LOG_RETURN_VOID();
-}
-
 /**********************************************************************************************************************************/
 FN_EXTERN PackRead *
-protocolClientDataGet(ProtocolClient *const this)
+protocolClientRequest(ProtocolClient *const this, const StringId command, const ProtocolClientRequestParam param)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
+        FUNCTION_LOG_PARAM(STRING_ID, command);
+        FUNCTION_LOG_PARAM(PACK_WRITE, param.param);
     FUNCTION_LOG_END();
 
     ASSERT(this != NULL);
 
-    // Expect data-get state before data get
-    protocolClientStateExpect(
-        this, this->state == protocolClientStateCommandDataGet ? protocolClientStateCommandDataGet : protocolClientStateDataGet);
-
     PackRead *result = NULL;
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        PackRead *response = pckReadNewIo(this->pub.read);
+        ProtocolClientSession *const session = protocolClientSessionNewP(this, command);
-        ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
 
-        protocolClientError(this, type, response);
+        protocolClientRequestInternal(session, protocolCommandTypeProcess, param.param);
-
-        CHECK(FormatError, type == protocolMessageTypeData, "expected data message");
 
         MEM_CONTEXT_PRIOR_BEGIN()
         {
-            result = pckReadPackReadP(response);
+            result = protocolClientResponseInternal(session);
         }
         MEM_CONTEXT_PRIOR_END();
-
-        pckReadEndP(response);
-
-        // Switch state to data-put after successful command data get
-        if (this->state == protocolClientStateCommandDataGet)
-            this->state = protocolClientStateDataPut;
     }
     MEM_CONTEXT_TEMP_END();
 
     FUNCTION_LOG_RETURN(PACK_READ, result);
 }
 
-/**********************************************************************************************************************************/
-FN_EXTERN void
-protocolClientDataEndGet(ProtocolClient *const this)
-{
-    FUNCTION_LOG_BEGIN(logLevelTrace);
-        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
-    FUNCTION_LOG_END();
-
-    ASSERT(this != NULL);
-
-    // Expect data-get state before data end get
-    protocolClientStateExpect(this, protocolClientStateDataGet);
-
-    MEM_CONTEXT_TEMP_BEGIN()
-    {
-        PackRead *response = pckReadNewIo(this->pub.read);
-        ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(response);
-
-        protocolClientError(this, type, response);
-
-        CHECK(FormatError, type == protocolMessageTypeDataEnd, "expected data end message");
-
-        pckReadEndP(response);
-
-        // Switch state to idle after successful data end get
-        this->state = protocolClientStateIdle;
-    }
-    MEM_CONTEXT_TEMP_END();
-
-    FUNCTION_LOG_RETURN_VOID();
-}
-
-/**********************************************************************************************************************************/
-FN_EXTERN void
-protocolClientCommandPut(ProtocolClient *const this, ProtocolCommand *const command, const bool dataPut)
-{
-    FUNCTION_LOG_BEGIN(logLevelTrace);
-        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
-        FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
-        FUNCTION_LOG_PARAM(BOOL, dataPut);
-    FUNCTION_LOG_END();
-
-    ASSERT(this != NULL);
-    ASSERT(command != NULL);
-
-    // Expect idle state before command put
-    protocolClientStateExpect(this, protocolClientStateIdle);
-
-    // Switch state to cmd-put
-    this->state = protocolClientStateDataPut;
-
-    // Put command
-    protocolCommandPut(command, this->write);
-
-    // Switch state to data-get/data-put after successful command put
-    this->state = dataPut ? protocolClientStateCommandDataGet : protocolClientStateDataGet;
-
-    // Reset the keep alive time
-    this->keepAliveTime = timeMSec();
-
-    FUNCTION_LOG_RETURN_VOID();
-}
-
-/**********************************************************************************************************************************/
-FN_EXTERN PackRead *
-protocolClientExecute(ProtocolClient *const this, ProtocolCommand *const command, const bool resultRequired)
-{
-    FUNCTION_LOG_BEGIN(logLevelTrace);
-        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, this);
-        FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
-        FUNCTION_LOG_PARAM(BOOL, resultRequired);
-    FUNCTION_LOG_END();
-
-    ASSERT(this != NULL);
-    ASSERT(command != NULL);
-
-    // Put command
-    protocolClientCommandPut(this, command, false);
-
-    // Read result if required
-    PackRead *result = NULL;
-
-    if (resultRequired)
-        result = protocolClientDataGet(this);
-
-    // Read response
-    protocolClientDataEndGet(this);
-
-    FUNCTION_LOG_RETURN(PACK_READ, result);
-}
-
 /**********************************************************************************************************************************/
 FN_EXTERN void
 protocolClientNoOp(ProtocolClient *this)
@@ -389,11 +432,193 @@ protocolClientNoOp(ProtocolClient *this)
 
     ASSERT(this != NULL);
 
-    MEM_CONTEXT_TEMP_BEGIN()
+    protocolClientRequestP(this, PROTOCOL_COMMAND_NOOP);
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Free client session
+***********************************************************************************************************************************/
+static void
+protocolClientSessionFreeResource(THIS_VOID)
+{
+    THIS(ProtocolClientSession);
+
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+
+    // If a result is queued then read it before sending cancel
+    if (protocolClientSessionQueued(this))
     {
-        protocolClientExecute(this, protocolCommandNew(PROTOCOL_COMMAND_NOOP), false);
+        // Free stored result
+        if (this->stored)
+        {
+            pckReadFree(this->packRead);
+        }
+        // Else read and free result
+        else
+            pckReadFree(protocolClientResponseInternal(this));
     }
-    MEM_CONTEXT_TEMP_END();
+
+    // If open then cancel
+    if (this->pub.open)
+    {
+        protocolClientRequestInternal(this, protocolCommandTypeCancel, NULL);
+        protocolClientResponseInternal(this);
+        ASSERT(!this->pub.open);
+    }
+
+    // Remove from session list
+    lstRemoveIdx(this->client->sessionList, protocolClientSessionFindIdx(this->client, this->sessionId));
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN ProtocolClientSession *
+protocolClientSessionNew(ProtocolClient *const client, const StringId command, const ProtocolClientSessionNewParam param)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, client);
+        FUNCTION_LOG_PARAM(STRING_ID, command);
+        FUNCTION_LOG_PARAM(BOOL, param.async);
+    FUNCTION_LOG_END();
+
+    ASSERT(client != NULL);
+
+    OBJ_NEW_BEGIN(ProtocolClientSession, .childQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
+    {
+        *this = (ProtocolClientSession)
+        {
+            .client = client,
+            .command = command,
+            .sessionId = ++client->sessionTotal,
+            .async = param.async,
+        };
+    }
+    OBJ_NEW_END();
+
+    // If async then the session will need to be tracked in case another session gets a result for this session
+    if (this->async)
+    {
+        lstAdd(this->client->sessionList, &this);
+
+        // Set a callback to cleanup the session
+        memContextCallbackSet(objMemContext(this), protocolClientSessionFreeResource, this);
+    }
+
+    FUNCTION_LOG_RETURN(PROTOCOL_CLIENT_SESSION, this);
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN PackRead *
+protocolClientSessionOpen(ProtocolClientSession *const this, const ProtocolClientSessionOpenParam param)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
+        FUNCTION_LOG_PARAM(PACK_WRITE, param.param);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(!this->pub.open);
+
+    protocolClientRequestInternal(this, protocolCommandTypeOpen, param.param);
+    this->pub.open = true;
+
+    // Set a callback to cleanup the session if it was not already done for async
+    if (!this->async)
+    {
+        lstAdd(this->client->sessionList, &this);
+        memContextCallbackSet(objMemContext(this), protocolClientSessionFreeResource, this);
+    }
+
+    FUNCTION_LOG_RETURN(PACK_READ, protocolClientResponseInternal(this));
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN PackRead *
+protocolClientSessionRequest(ProtocolClientSession *const this, const ProtocolClientRequestParam param)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
+        FUNCTION_LOG_PARAM(PACK_WRITE, param.param);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+
+    protocolClientRequestInternal(this, protocolCommandTypeProcess, param.param);
+
+    FUNCTION_LOG_RETURN(PACK_READ, protocolClientResponseInternal(this));
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN void
+protocolClientSessionRequestAsync(ProtocolClientSession *const this, const ProtocolClientRequestParam param)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
+        FUNCTION_LOG_PARAM(PACK_WRITE, param.param);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(this->async);
+
+    protocolClientRequestInternal(this, protocolCommandTypeProcess, param.param);
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN PackRead *
+protocolClientSessionResponse(ProtocolClientSession *const this)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(this->async);
+
+    FUNCTION_LOG_RETURN(PACK_READ, protocolClientResponseInternal(this));
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN PackRead *
+protocolClientSessionClose(ProtocolClientSession *const this)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(this->pub.open);
+
+    protocolClientRequestInternal(this, protocolCommandTypeClose, NULL);
+    PackRead *const result = protocolClientResponseInternal(this);
+    ASSERT(!this->pub.open);
+
+    memContextCallbackClear(objMemContext(this));
+    protocolClientSessionFreeResource(this);
+
+    FUNCTION_LOG_RETURN(PACK_READ, result);
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN void
+protocolClientSessionCancel(ProtocolClientSession *const this)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT_SESSION, this);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+
+    memContextCallbackClear(objMemContext(this));
+    protocolClientSessionFreeResource(this);
 
     FUNCTION_LOG_RETURN_VOID();
 }
@@ -406,3 +631,17 @@ protocolClientToLog(const ProtocolClient *const this, StringStatic *const debugL
     strStcResultSizeInc(debugLog, strIdToLog(this->state, strStcRemains(debugLog), strStcRemainsSize(debugLog)));
     strStcCatChr(debugLog, '}');
 }
+
+/**********************************************************************************************************************************/
+FN_EXTERN void
+protocolClientSessionToLog(const ProtocolClientSession *const this, StringStatic *const debugLog)
+{
+    strStcCat(debugLog, "{client: ");
+    protocolClientToLog(this->client, debugLog);
+    strStcCat(debugLog, ", command: ");
+    strStcResultSizeInc(debugLog, strIdToLog(this->command, strStcRemains(debugLog), strStcRemainsSize(debugLog)));
+    strStcFmt(
+        debugLog, ", sessionId: %" PRIu64 ", queued %s, stored: %s", this->sessionId,
+        cvtBoolToConstZ(protocolClientSessionQueued(this)), cvtBoolToConstZ(this->stored));
+    strStcCatChr(debugLog, '}');
+}

src/protocol/client.h

@@ -1,5 +1,28 @@
 /***********************************************************************************************************************************
 Protocol Client
+
+There are three ways to make a request using the client:
+
+1. Synchronous
+
+This is the most common way to make a request. Simply call protocolClientRequestP() and process the result.
+
+2. Synchronous with Session
+
+In some cases it is useful to keep session state between requests. An example of this is queries against a database where it is more
+efficient to maintain the connection to the database between queries. The session is created with protocolClientSessionNewP(),
+opened with protocolClientSessionOpenP(), and closed with protocolClientSessionClose(). Requests are made with
+protocolClientSessionRequest().
+
+3. Asynchronous with Session
+
+Asynchronous requests provide better performance by allowing the client and server to process at the same time. An example of this
+is reading a file where a asynchronous request for the next block of data can be sent before the current block is being processed.
+The next block should be waiting when processing of the current block is complete. The same session functions are used for creation,
+open, and close, except that .async = true is passed to protocolClientSessionNewP(). Asynchronous requests are made with
+protocolClientSessionRequestAsync() and the responses are read with protocolClientSessionResponse(). Note that there can only be one
+outstanding asynchronous request, therefore protocolClientSessionResponse() must be called before
+protocolClientSessionRequestAsync() can be called again.
 ***********************************************************************************************************************************/
 #ifndef PROTOCOL_CLIENT_H
 #define PROTOCOL_CLIENT_H
@@ -9,28 +32,36 @@ Message types used by the protocol
 ***********************************************************************************************************************************/
 typedef enum
 {
-    // Data passed between client and server in either direction. This can be used as many times as needed.
+    // Data passed from server to client
-    protocolMessageTypeData = 0,
+    protocolMessageTypeResponse = 0,
 
-    // Indicates no more data for the server to return to the client and ends the command
+    // Request sent from the client to the server
-    protocolMessageTypeDataEnd = 1,
+    protocolMessageTypeRequest = 1,
 
-    // Command sent from the client to the server
+    // An error occurred on the server and the request ended abnormally. protocolMessageTypeResponse will not be sent to the client.
-    protocolMessageTypeCommand = 2,
+    protocolMessageTypeError = 2,
-
-    // An error occurred on the server and the command ended abnormally. protocolMessageTypeDataEnd will not be sent to the client.
-    protocolMessageTypeError = 3,
 } ProtocolMessageType;
 
+/***********************************************************************************************************************************
+Command types
+***********************************************************************************************************************************/
+typedef enum
+{
+    protocolCommandTypeOpen = STRID5("opn", 0x3a0f0),               // Open command for processing
+    protocolCommandTypeProcess = STRID5("prc", 0xe500),             // Process command
+    protocolCommandTypeClose = STRID5("cls", 0x4d830),              // Close command
+    protocolCommandTypeCancel = STRID5("cnc", 0xdc30),              // Cancel command
+} ProtocolCommandType;
+
 /***********************************************************************************************************************************
 Object type
 ***********************************************************************************************************************************/
 typedef struct ProtocolClient ProtocolClient;
+typedef struct ProtocolClientSession ProtocolClientSession;
 
 #include "common/io/read.h"
 #include "common/io/write.h"
 #include "common/type/object.h"
-#include "protocol/command.h"
 
 /***********************************************************************************************************************************
 Constants
@@ -57,12 +88,12 @@ protocolPackNew(void)
 }
 
 /***********************************************************************************************************************************
-Constructors
+Client Constructors
 ***********************************************************************************************************************************/
 FN_EXTERN ProtocolClient *protocolClientNew(const String *name, const String *service, IoRead *read, IoWrite *write);
 
 /***********************************************************************************************************************************
-Getters/Setters
+Client Getters/Setters
 ***********************************************************************************************************************************/
 typedef struct ProtocolClientPub
 {
@@ -77,11 +108,8 @@ protocolClientIoReadFd(ProtocolClient *const this)
 }
 
 /***********************************************************************************************************************************
-Functions
+Client Functions
 ***********************************************************************************************************************************/
-// Execute a command and get the result
-FN_EXTERN PackRead *protocolClientExecute(ProtocolClient *this, ProtocolCommand *command, bool resultRequired);
-
 // Move to a new parent mem context
 FN_INLINE_ALWAYS ProtocolClient *
 protocolClientMove(ProtocolClient *const this, MemContext *const parentNew)
@@ -99,18 +127,20 @@ protocolClientNoExit(ProtocolClient *const this)
 // Send noop to test connection or keep it alive
 FN_EXTERN void protocolClientNoOp(ProtocolClient *this);
 
-// Get data put by the server
+// Simple request that does not require a session or async
-FN_EXTERN PackRead *protocolClientDataGet(ProtocolClient *this);
+typedef struct ProtocolClientRequestParam
-FN_EXTERN void protocolClientDataEndGet(ProtocolClient *this);
+{
+    VAR_PARAM_HEADER;
+    PackWrite *param;
+} ProtocolClientRequestParam;
 
-// Put command to the server
+#define protocolClientRequestP(this, command, ...)                                                                                          \
-FN_EXTERN void protocolClientCommandPut(ProtocolClient *this, ProtocolCommand *command, const bool dataPut);
+    protocolClientRequest(this, command, (ProtocolClientRequestParam){VAR_PARAM_INIT, __VA_ARGS__})
 
-// Put data to the server
+FN_EXTERN PackRead *protocolClientRequest(ProtocolClient *this, StringId command, ProtocolClientRequestParam param);
-FN_EXTERN void protocolClientDataPut(ProtocolClient *this, PackWrite *data);
 
 /***********************************************************************************************************************************
-Destructor
+Client Destructor
 ***********************************************************************************************************************************/
 FN_INLINE_ALWAYS void
 protocolClientFree(ProtocolClient *const this)
@@ -118,6 +148,90 @@ protocolClientFree(ProtocolClient *const this)
     objFree(this);
 }
 
+/***********************************************************************************************************************************
+Session Constructors
+***********************************************************************************************************************************/
+// New session
+typedef struct ProtocolClientSessionNewParam
+{
+    VAR_PARAM_HEADER;
+    bool async;                                                     // Async requests allowed?
+} ProtocolClientSessionNewParam;
+
+#define protocolClientSessionNewP(client, command, ...)                                                                            \
+    protocolClientSessionNew(client, command, (ProtocolClientSessionNewParam){VAR_PARAM_INIT, __VA_ARGS__})
+
+FN_EXTERN ProtocolClientSession *protocolClientSessionNew(
+    ProtocolClient *client, StringId command, ProtocolClientSessionNewParam param);
+
+/***********************************************************************************************************************************
+Session Getters/Setters
+***********************************************************************************************************************************/
+typedef struct ProtocolClientSessionPub
+{
+    bool open;                                                      // Is the session open?
+    bool queued;                                                    // Is a response currently queued?
+} ProtocolClientSessionPub;
+
+// Is a response currently queued?
+FN_INLINE_ALWAYS bool
+protocolClientSessionQueued(const ProtocolClientSession *const this)
+{
+    return THIS_PUB(ProtocolClientSession)->queued;
+}
+
+// Is the session closed?
+FN_INLINE_ALWAYS bool
+protocolClientSessionClosed(const ProtocolClientSession *const this)
+{
+    return !THIS_PUB(ProtocolClientSession)->open;
+}
+
+/***********************************************************************************************************************************
+Session Functions
+***********************************************************************************************************************************/
+// Session open
+typedef struct ProtocolClientSessionOpenParam
+{
+    VAR_PARAM_HEADER;
+    PackWrite *param;
+} ProtocolClientSessionOpenParam;
+
+#define protocolClientSessionOpenP(this, ...)                                                                                      \
+    protocolClientSessionOpen(this, (ProtocolClientSessionOpenParam){VAR_PARAM_INIT, __VA_ARGS__})
+
+FN_EXTERN PackRead *protocolClientSessionOpen(ProtocolClientSession *const this, ProtocolClientSessionOpenParam param);
+
+// Session request
+#define protocolClientSessionRequestP(this, ...)                                                                                   \
+    protocolClientSessionRequest(this, (ProtocolClientRequestParam){VAR_PARAM_INIT, __VA_ARGS__})
+
+FN_EXTERN PackRead *protocolClientSessionRequest(ProtocolClientSession *const this, ProtocolClientRequestParam param);
+
+// Session async request
+#define protocolClientSessionRequestAsyncP(this, ...)                                                                              \
+    protocolClientSessionRequestAsync(this, (ProtocolClientRequestParam){VAR_PARAM_INIT, __VA_ARGS__})
+
+FN_EXTERN void protocolClientSessionRequestAsync(ProtocolClientSession *const this, ProtocolClientRequestParam param);
+
+// Session response after a call to protocolClientSessionRequestAsyncP()
+FN_EXTERN PackRead *protocolClientSessionResponse(ProtocolClientSession *const this);
+
+// Session close
+FN_EXTERN PackRead *protocolClientSessionClose(ProtocolClientSession *const this);
+
+// Session cancel
+FN_EXTERN void protocolClientSessionCancel(ProtocolClientSession *const this);
+
+/***********************************************************************************************************************************
+Session Destructor
+***********************************************************************************************************************************/
+FN_INLINE_ALWAYS void
+protocolClientSessionFree(ProtocolClientSession *const this)
+{
+    objFree(this);
+}
+
 /***********************************************************************************************************************************
 Macros for function logging
 ***********************************************************************************************************************************/
@@ -128,4 +242,11 @@ FN_EXTERN void protocolClientToLog(const ProtocolClient *this, StringStatic *deb
 #define FUNCTION_LOG_PROTOCOL_CLIENT_FORMAT(value, buffer, bufferSize)                                                             \
     FUNCTION_LOG_OBJECT_FORMAT(value, protocolClientToLog, buffer, bufferSize)
 
+FN_EXTERN void protocolClientSessionToLog(const ProtocolClientSession *this, StringStatic *debugLog);
+
+#define FUNCTION_LOG_PROTOCOL_CLIENT_SESSION_TYPE                                                                                  \
+    ProtocolClientSession *
+#define FUNCTION_LOG_PROTOCOL_CLIENT_SESSION_FORMAT(value, buffer, bufferSize)                                                     \
+    FUNCTION_LOG_OBJECT_FORMAT(value, protocolClientSessionToLog, buffer, bufferSize)
+
 #endif

src/protocol/command.c

@@ -1,107 +0,0 @@
-/***********************************************************************************************************************************
-Protocol Command
-***********************************************************************************************************************************/
-#include "build.auto.h"
-
-#include "common/debug.h"
-#include "common/log.h"
-#include "common/type/keyValue.h"
-#include "protocol/client.h"
-#include "protocol/command.h"
-
-/***********************************************************************************************************************************
-Object type
-***********************************************************************************************************************************/
-struct ProtocolCommand
-{
-    StringId command;
-    PackWrite *pack;
-};
-
-/**********************************************************************************************************************************/
-FN_EXTERN ProtocolCommand *
-protocolCommandNew(const StringId command)
-{
-    FUNCTION_TEST_BEGIN();
-        FUNCTION_TEST_PARAM(STRING_ID, command);
-    FUNCTION_TEST_END();
-
-    ASSERT(command != 0);
-
-    OBJ_NEW_BEGIN(ProtocolCommand, .childQty = MEM_CONTEXT_QTY_MAX)
-    {
-        *this = (ProtocolCommand)
-        {
-            .command = command,
-        };
-    }
-    OBJ_NEW_END();
-
-    FUNCTION_TEST_RETURN(PROTOCOL_COMMAND, this);
-}
-
-/**********************************************************************************************************************************/
-FN_EXTERN void
-protocolCommandPut(ProtocolCommand *const this, IoWrite *const write)
-{
-    FUNCTION_TEST_BEGIN();
-        FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
-    FUNCTION_TEST_END();
-
-    ASSERT(this != NULL);
-    ASSERT(write != NULL);
-
-    MEM_CONTEXT_TEMP_BEGIN()
-    {
-        // Write the command
-        PackWrite *commandPack = pckWriteNewIo(write);
-        pckWriteU32P(commandPack, protocolMessageTypeCommand, .defaultWrite = true);
-        pckWriteStrIdP(commandPack, this->command);
-
-        // Write parameters
-        if (this->pack != NULL)
-        {
-            pckWriteEndP(this->pack);
-            pckWritePackP(commandPack, pckWriteResult(this->pack));
-        }
-
-        pckWriteEndP(commandPack);
-
-        // Flush to send command immediately
-        ioWriteFlush(write);
-    }
-    MEM_CONTEXT_TEMP_END();
-
-    FUNCTION_TEST_RETURN_VOID();
-}
-
-/**********************************************************************************************************************************/
-FN_EXTERN PackWrite *
-protocolCommandParam(ProtocolCommand *this)
-{
-    FUNCTION_TEST_BEGIN();
-        FUNCTION_TEST_PARAM(PROTOCOL_COMMAND, this);
-    FUNCTION_TEST_END();
-
-    ASSERT(this != NULL);
-
-    if (this->pack == NULL)
-    {
-        MEM_CONTEXT_OBJ_BEGIN(this)
-        {
-            this->pack = protocolPackNew();
-        }
-        MEM_CONTEXT_OBJ_END();
-    }
-
-    FUNCTION_TEST_RETURN(PACK_WRITE, this->pack);
-}
-
-/**********************************************************************************************************************************/
-FN_EXTERN void
-protocolCommandToLog(const ProtocolCommand *const this, StringStatic *const debugLog)
-{
-    strStcFmt(debugLog, "{name: ");
-    strStcResultSizeInc(debugLog, strIdToLog(this->command, strStcRemains(debugLog), strStcRemainsSize(debugLog)));
-    strStcCatChr(debugLog, '}');
-}

src/protocol/command.h

@@ -1,55 +0,0 @@
-/***********************************************************************************************************************************
-Protocol Command
-***********************************************************************************************************************************/
-#ifndef PROTOCOL_COMMAND_H
-#define PROTOCOL_COMMAND_H
-
-/***********************************************************************************************************************************
-Object type
-***********************************************************************************************************************************/
-typedef struct ProtocolCommand ProtocolCommand;
-
-#include "common/type/object.h"
-#include "common/type/pack.h"
-
-/***********************************************************************************************************************************
-Constructors
-***********************************************************************************************************************************/
-FN_EXTERN ProtocolCommand *protocolCommandNew(const StringId command);
-
-/***********************************************************************************************************************************
-Functions
-***********************************************************************************************************************************/
-// Move to a new parent mem context
-FN_INLINE_ALWAYS ProtocolCommand *
-protocolCommandMove(ProtocolCommand *const this, MemContext *const parentNew)
-{
-    return objMove(this, parentNew);
-}
-
-// Read the command output
-FN_EXTERN PackWrite *protocolCommandParam(ProtocolCommand *this);
-
-// Write protocol command
-FN_EXTERN void protocolCommandPut(ProtocolCommand *this, IoWrite *write);
-
-/***********************************************************************************************************************************
-Destructor
-***********************************************************************************************************************************/
-FN_INLINE_ALWAYS void
-protocolCommandFree(ProtocolCommand *const this)
-{
-    objFree(this);
-}
-
-/***********************************************************************************************************************************
-Macros for function logging
-***********************************************************************************************************************************/
-FN_EXTERN void protocolCommandToLog(const ProtocolCommand *this, StringStatic *debugLog);
-
-#define FUNCTION_LOG_PROTOCOL_COMMAND_TYPE                                                                                         \
-    ProtocolCommand *
-#define FUNCTION_LOG_PROTOCOL_COMMAND_FORMAT(value, buffer, bufferSize)                                                            \
-    FUNCTION_LOG_OBJECT_FORMAT(value, protocolCommandToLog, buffer, bufferSize)
-
-#endif

src/protocol/helper.c

@@ -413,6 +413,11 @@ protocolServer(IoServer *const tlsServer, IoSession *const socketSession)
         {
             TRY_BEGIN()
             {
+                // Get parameter list from the client. This needs to happen even if we cannot authorize the client so that the
+                // session id gets set for the error.
+                const ProtocolServerRequestResult command = protocolServerRequest(result);
+                CHECK(FormatError, command.id == PROTOCOL_COMMAND_CONFIG, "expected config command");
+
                 // Get list of authorized stanzas for this client
                 CHECK(AssertError, cfgOptionTest(cfgOptTlsServerAuth), "missing auth data");
 
@@ -423,10 +428,7 @@ protocolServer(IoServer *const tlsServer, IoSession *const socketSession)
                 if (clientAuthList == NULL)
                     THROW(AccessError, "access denied");
 
-                // Get parameter list from the client and load it
+                // Load parameter list from the client
-                const ProtocolServerCommandGetResult command = protocolServerCommandGet(result);
-                CHECK(FormatError, command.id == PROTOCOL_COMMAND_CONFIG, "expected config command");
-
                 StringList *const paramList = pckReadStrLstP(pckReadNew(command.param));
                 strLstInsert(paramList, 0, cfgExe());
                 cfgLoad(strLstSize(paramList), strLstPtr(paramList));
@@ -443,7 +445,7 @@ protocolServer(IoServer *const tlsServer, IoSession *const socketSession)
             TRY_END();
 
             // Ack the config command
-            protocolServerDataEndPut(result);
+            protocolServerResponseP(result);
 
             // Move result to prior context and move session into result so there is only one return value
             protocolServerMove(result, memContextPrior());
@@ -452,8 +454,12 @@ protocolServer(IoServer *const tlsServer, IoSession *const socketSession)
         // Else the client can only detect that the server is alive
         else
         {
-            // Send a data end message and return a NULL server. Do not waste time looking at what the client wrote.
+            // A noop command should have been received
-            protocolServerDataEndPut(result);
+            const ProtocolServerRequestResult command = protocolServerRequest(result);
+            CHECK(FormatError, command.id == PROTOCOL_COMMAND_NOOP, "expected config command");
+
+            // Send a data end message and return a NULL server
+            protocolServerResponseP(result);
 
             // Set result to NULL so there is no server for the caller to use. The TLS session will be freed when the temp mem
             // context ends.
@@ -775,10 +781,10 @@ protocolRemoteExec(
                 TRY_BEGIN()
                 {
                     // Pass parameters to server
-                    ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_CONFIG);
+                    PackWrite *const param = protocolPackNew();
-                    pckWriteStrLstP(protocolCommandParam(command), protocolRemoteParam(protocolStorageType, hostIdx));
+
-                    protocolClientExecute(helper->client, command, false);
+                    pckWriteStrLstP(param, protocolRemoteParam(protocolStorageType, hostIdx));
-                    protocolCommandFree(command);
+                    protocolClientRequestP(helper->client, PROTOCOL_COMMAND_CONFIG, .param = param);
                 }
                 CATCH_ANY()
                 {

src/protocol/parallel.c

@@ -11,13 +11,18 @@ Protocol Parallel Executor
 #include "common/macro.h"
 #include "common/type/keyValue.h"
 #include "common/type/list.h"
-#include "protocol/command.h"
 #include "protocol/helper.h"
 #include "protocol/parallel.h"
 
 /***********************************************************************************************************************************
 Object type
 ***********************************************************************************************************************************/
+typedef struct ProtocolParallelJobData
+{
+    ProtocolParallelJob *job;                                       // Job
+    ProtocolClientSession *session;                                 // Protocol session for the job
+} ProtocolParallelJobData;
+
 struct ProtocolParallel
 {
     TimeMSec timeout;                                               // Max time to wait for jobs before returning
@@ -27,7 +32,7 @@ struct ProtocolParallel
     List *clientList;                                               // List of clients to process jobs
     List *jobList;                                                  // List of jobs to be processed
 
-    ProtocolParallelJob **clientJobList;                            // Jobs being processing by each client
+    ProtocolParallelJobData *clientJobList;                         // Jobs being processing by each client
 
     ProtocolParallelJobState state;                                 // Overall state of job processing
 };
@@ -103,7 +108,10 @@ protocolParallelProcess(ProtocolParallel *this)
         {
             MEM_CONTEXT_OBJ_BEGIN(this)
             {
-                this->clientJobList = memNewPtrArray(lstSize(this->clientList));
+                this->clientJobList = memNew(lstSize(this->clientList) * sizeof(ProtocolParallelJobData));
+
+                for (unsigned int jobIdx = 0; jobIdx < lstSize(this->clientList); jobIdx++)
+                    this->clientJobList[jobIdx] = (ProtocolParallelJobData){0};
             }
             MEM_CONTEXT_OBJ_END();
 
@@ -120,7 +128,7 @@ protocolParallelProcess(ProtocolParallel *this)
 
         for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
         {
-            if (this->clientJobList[clientIdx] != NULL)
+            if (this->clientJobList[clientIdx].job != NULL)
             {
                 int fd = protocolClientIoReadFd(*(ProtocolClient **)lstGet(this->clientList, clientIdx));
                 FD_SET(fd, &selectSet);
@@ -149,7 +157,7 @@ protocolParallelProcess(ProtocolParallel *this)
             {
                 for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
                 {
-                    ProtocolParallelJob *job = this->clientJobList[clientIdx];
+                    ProtocolParallelJob *const job = this->clientJobList[clientIdx].job;
 
                     if (job != NULL &&
                         FD_ISSET(
@@ -160,10 +168,8 @@ protocolParallelProcess(ProtocolParallel *this)
                         {
                             TRY_BEGIN()
                             {
-                                ProtocolClient *const client = *(ProtocolClient **)lstGet(this->clientList, clientIdx);
+                                protocolParallelJobResultSet(
-
+                                    job, protocolClientSessionResponse(this->clientJobList[clientIdx].session));
-                                protocolParallelJobResultSet(job, protocolClientDataGet(client));
-                                protocolClientDataEndGet(client);
                             }
                             CATCH_ANY()
                             {
@@ -172,7 +178,8 @@ protocolParallelProcess(ProtocolParallel *this)
                             TRY_END();
 
                             protocolParallelJobStateSet(job, protocolParallelJobStateDone);
-                            this->clientJobList[clientIdx] = NULL;
+                            this->clientJobList[clientIdx].job = NULL;
+                            protocolClientSessionFree(this->clientJobList[clientIdx].session);
                         }
                         MEM_CONTEXT_TEMP_END();
                     }
@@ -186,35 +193,37 @@ protocolParallelProcess(ProtocolParallel *this)
         for (unsigned int clientIdx = 0; clientIdx < lstSize(this->clientList); clientIdx++)
         {
             // If nothing is running for this client
-            if (this->clientJobList[clientIdx] == NULL)
+            if (this->clientJobList[clientIdx].job == NULL)
             {
-                // Get a new job
-                ProtocolParallelJob *job = NULL;
-
                 MEM_CONTEXT_BEGIN(lstMemContext(this->jobList))
                 {
-                    job = this->callbackFunction(this->callbackData, clientIdx);
+                    // Get a new job
+                    ProtocolParallelJob *const job = this->callbackFunction(this->callbackData, clientIdx);
+
+                    // If a new job was found
+                    if (job != NULL)
+                    {
+                        // Add to the job list
+                        lstAdd(this->jobList, &job);
+
+                        // Put command
+                        ProtocolClientSession *const session = protocolClientSessionNewP(
+                            *(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job),
+                            .async = true);
+                        protocolClientSessionRequestAsyncP(session, .param = protocolParallelJobParam(job));
+
+                        // Set client id and running state
+                        protocolParallelJobProcessIdSet(job, clientIdx + 1);
+                        protocolParallelJobStateSet(job, protocolParallelJobStateRunning);
+
+                        this->clientJobList[clientIdx].job = job;
+                        this->clientJobList[clientIdx].session = session;
+                    }
+                    // Else no more jobs for this client so free it
+                    else
+                        protocolLocalFree(clientIdx + 1);
                 }
                 MEM_CONTEXT_END();
-
-                // If a new job was found
-                if (job != NULL)
-                {
-                    // Add to the job list
-                    lstAdd(this->jobList, &job);
-
-                    // Put command
-                    protocolClientCommandPut(
-                        *(ProtocolClient **)lstGet(this->clientList, clientIdx), protocolParallelJobCommand(job), false);
-
-                    // Set client id and running state
-                    protocolParallelJobProcessIdSet(job, clientIdx + 1);
-                    protocolParallelJobStateSet(job, protocolParallelJobStateRunning);
-                    this->clientJobList[clientIdx] = job;
-                }
-                // Else no more jobs for this client so free it
-                else
-                    protocolLocalFree(clientIdx + 1);
             }
         }
     }

src/protocol/parallelJob.c

@@ -5,7 +5,6 @@ Protocol Parallel Job
 
 #include "common/debug.h"
 #include "common/log.h"
-#include "protocol/command.h"
 #include "protocol/parallelJob.h"
 
 /***********************************************************************************************************************************
@@ -18,11 +17,12 @@ struct ProtocolParallelJob
 
 /**********************************************************************************************************************************/
 FN_EXTERN ProtocolParallelJob *
-protocolParallelJobNew(const Variant *key, ProtocolCommand *command)
+protocolParallelJobNew(const Variant *key, const StringId command, PackWrite *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(VARIANT, key);
-        FUNCTION_LOG_PARAM(PROTOCOL_COMMAND, command);
+        FUNCTION_LOG_PARAM(STRING_ID, command);
+        FUNCTION_LOG_PARAM(PACK_WRITE, param);
     FUNCTION_LOG_END();
 
     OBJ_NEW_BEGIN(ProtocolParallelJob, .childQty = MEM_CONTEXT_QTY_MAX)
@@ -33,10 +33,10 @@ protocolParallelJobNew(const Variant *key, ProtocolCommand *command)
             {
                 .state = protocolParallelJobStatePending,
                 .key = varDup(key),
+                .command = command,
+                .param = pckWriteMove(param, objMemContext(this)),
             },
         };
-
-        this->pub.command = protocolCommandMove(command, objMemContext(this));
     }
     OBJ_NEW_END();
 
@@ -136,7 +136,8 @@ protocolParallelJobToLog(const ProtocolParallelJob *const this, StringStatic *co
     varToLog(protocolParallelJobKey(this), debugLog);
 
     strStcCat(debugLog, ", command: ");
-    protocolCommandToLog(protocolParallelJobCommand(this), debugLog);
+    strStcResultSizeInc(
+        debugLog, strIdToLog(protocolParallelJobCommand(this), strStcRemains(debugLog), strStcRemainsSize(debugLog)));
 
     strStcCat(debugLog, ", result: ");
     strStcResultSizeInc(

src/protocol/parallelJob.h

@@ -29,7 +29,7 @@ typedef enum
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN ProtocolParallelJob *protocolParallelJobNew(const Variant *key, ProtocolCommand *command);
+FN_EXTERN ProtocolParallelJob *protocolParallelJobNew(const Variant *key, StringId command, PackWrite *param);
 
 /***********************************************************************************************************************************
 Getters/Setters
@@ -37,7 +37,8 @@ Getters/Setters
 typedef struct ProtocolParallelJobPub
 {
     const Variant *key;                                             // Unique key used to identify the job
-    ProtocolCommand *command;                                       // Command to be executed
+    StringId command;                                               // Command to be executed
+    PackWrite *param;                                               // Command parameters
     unsigned int processId;                                         // Process that executed this job
     ProtocolParallelJobState state;                                 // Current state of the job
     int code;                                                       // Non-zero result indicates an error
@@ -46,12 +47,19 @@ typedef struct ProtocolParallelJobPub
 } ProtocolParallelJobPub;
 
 // Job command
-FN_INLINE_ALWAYS ProtocolCommand *
+FN_INLINE_ALWAYS StringId
 protocolParallelJobCommand(const ProtocolParallelJob *const this)
 {
     return THIS_PUB(ProtocolParallelJob)->command;
 }
 
+// Job command parameters
+FN_INLINE_ALWAYS PackWrite *
+protocolParallelJobParam(const ProtocolParallelJob *const this)
+{
+    return THIS_PUB(ProtocolParallelJob)->param;
+}
+
 // Job error
 FN_INLINE_ALWAYS int
 protocolParallelJobErrorCode(const ProtocolParallelJob *const this)

src/protocol/server.c

@@ -24,8 +24,25 @@ struct ProtocolServer
     IoRead *read;                                                   // Read interface
     IoWrite *write;                                                 // Write interface
     const String *name;                                             // Name displayed in logging
+    List *sessionList;                                              // List of active sessions
+    uint64_t sessionId;                                             // Current session being processed
 };
 
+struct ProtocolServerResult
+{
+    void *sessionData;                                              // Session data
+    PackWrite *data;                                                // Result data
+    size_t extra;                                                   // Extra bytes for initializing data
+    bool close;                                                     // Close session?
+};
+
+// Track server sessions
+typedef struct ProtocolServerSession
+{
+    uint64_t id;                                                    // Session id
+    void *data;                                                     // Data for the session
+} ProtocolServerSession;
+
 /**********************************************************************************************************************************/
 FN_EXTERN ProtocolServer *
 protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write)
@@ -48,6 +65,7 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
             .read = read,
             .write = write,
             .name = strDup(name),
+            .sessionList = lstNewP(sizeof(ProtocolServerSession)),
         };
 
         // Send the protocol greeting
@@ -69,6 +87,40 @@ protocolServerNew(const String *name, const String *service, IoRead *read, IoWri
     FUNCTION_LOG_RETURN(PROTOCOL_SERVER, this);
 }
 
+/**********************************************************************************************************************************/
+FN_EXTERN void
+protocolServerResponse(ProtocolServer *const this, const ProtocolServerResponseParam param)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(PROTOCOL_SERVER, this);
+        FUNCTION_TEST_PARAM(ENUM, param.type);
+        FUNCTION_TEST_PARAM(BOOL, param.close);
+        FUNCTION_TEST_PARAM(PACK_WRITE, param.data);
+    FUNCTION_TEST_END();
+
+    ASSERT(this != NULL);
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        // End the pack
+        if (param.data != NULL)
+            pckWriteEndP(param.data);
+
+        // Write the result
+        PackWrite *const resultMessage = pckWriteNewIo(this->write);
+
+        pckWriteU64P(resultMessage, this->sessionId);
+        pckWriteU32P(resultMessage, param.type, .defaultWrite = true);
+        pckWriteBoolP(resultMessage, param.close);
+        pckWritePackP(resultMessage, pckWriteResult(param.data));
+        pckWriteEndP(resultMessage);
+        ioWriteFlush(this->write);
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_TEST_RETURN_VOID();
+}
+
 /**********************************************************************************************************************************/
 FN_EXTERN void
 protocolServerError(ProtocolServer *this, int code, const String *message, const String *stack)
@@ -87,15 +139,13 @@ protocolServerError(ProtocolServer *this, int code, const String *message, const
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        // Write the error and flush to be sure it gets sent immediately
+        PackWrite *const packWrite = protocolPackNew();
-        PackWrite *error = pckWriteNewIo(this->write);
-        pckWriteU32P(error, protocolMessageTypeError);
-        pckWriteI32P(error, code);
-        pckWriteStrP(error, message);
-        pckWriteStrP(error, stack);
-        pckWriteEndP(error);
 
-        ioWriteFlush(this->write);
+        pckWriteI32P(packWrite, code);
+        pckWriteStrP(packWrite, message);
+        pckWriteStrP(packWrite, stack);
+
+        protocolServerResponseP(this, .type = protocolMessageTypeError, .data = packWrite);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -103,8 +153,8 @@ protocolServerError(ProtocolServer *this, int code, const String *message, const
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN ProtocolServerCommandGetResult
+FN_EXTERN ProtocolServerRequestResult
-protocolServerCommandGet(ProtocolServer *const this)
+protocolServerRequest(ProtocolServer *const this)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
@@ -112,23 +162,28 @@ protocolServerCommandGet(ProtocolServer *const this)
 
     FUNCTION_AUDIT_STRUCT();
 
-    ProtocolServerCommandGetResult result = {0};
+    ProtocolServerRequestResult result = {0};
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        PackRead *const command = pckReadNewIo(this->read);
+        PackRead *const request = pckReadNewIo(this->read);
-        ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(command);
+        ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(request);
 
-        CHECK(FormatError, type == protocolMessageTypeCommand, "expected command message");
+        CHECK(FormatError, type == protocolMessageTypeRequest, "expected request message");
 
         MEM_CONTEXT_PRIOR_BEGIN()
         {
-            result.id = pckReadStrIdP(command);
+            result.id = pckReadStrIdP(request);
-            result.param = pckReadPackP(command);
+            result.type = (ProtocolCommandType)pckReadStrIdP(request);
+            this->sessionId = pckReadU64P(request);
+            result.sessionRequired = pckReadBoolP(request);
+            result.param = pckReadPackP(request);
+
+            ASSERT(this->sessionId != 0);
         }
         MEM_CONTEXT_PRIOR_END();
 
-        pckReadEndP(command);
+        pckReadEndP(request);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -152,7 +207,7 @@ protocolServerProcess(
     ASSERT(handlerList != NULL);
     ASSERT(handlerListSize > 0);
 
-    // Loop until exit command is received
+    // Loop until exit request is received
     bool exit = false;
 
     do
@@ -161,17 +216,17 @@ protocolServerProcess(
         {
             MEM_CONTEXT_TEMP_BEGIN()
             {
-                // Get command
+                // Get request
-                ProtocolServerCommandGetResult command = protocolServerCommandGet(this);
+                ProtocolServerRequestResult request = protocolServerRequest(this);
 
                 // Find the handler
-                ProtocolServerCommandHandler handler = NULL;
+                const ProtocolServerHandler *handler = NULL;
 
                 for (unsigned int handlerIdx = 0; handlerIdx < handlerListSize; handlerIdx++)
                 {
-                    if (command.id == handlerList[handlerIdx].command)
+                    if (request.id == handlerList[handlerIdx].command)
                     {
-                        handler = handlerList[handlerIdx].handler;
+                        handler = &handlerList[handlerIdx];
                         break;
                     }
                 }
@@ -179,14 +234,14 @@ protocolServerProcess(
                 // If handler was found then process
                 if (handler != NULL)
                 {
-                    // Send the command to the handler
+                    // Send the request to the handler
                     MEM_CONTEXT_TEMP_BEGIN()
                     {
                         // Variables to store first error message and retry messages
                         ErrorRetry *const errRetry = errRetryNew();
                         const String *errStackTrace = NULL;
 
-                        // Initialize retries in case of command failure
+                        // Initialize retries in case of request failure
                         bool retry = false;
                         unsigned int retryRemaining = retryInterval != NULL ? varLstSize(retryInterval) : 0;
                         TimeMSec retrySleepMs = 0;
@@ -198,7 +253,166 @@ protocolServerProcess(
 
                             TRY_BEGIN()
                             {
-                                handler(pckReadNew(command.param), this);
+                                // Process request type
+                                switch (request.type)
+                                {
+                                    // Open a protocol session
+                                    case protocolCommandTypeOpen:
+                                    {
+                                        ASSERT(handler->open != NULL);
+
+                                        // Call open handler
+                                        ProtocolServerResult *const openResult = handler->open(pckReadNew(request.param));
+                                        ASSERT(openResult != NULL);
+                                        ASSERT(!openResult->close);
+
+                                        ProtocolServerSession session = {.id = this->sessionId};
+
+                                        // Send data
+                                        protocolServerResponseP(
+                                            this, .data = openResult->data, .close = openResult->sessionData == NULL);
+
+                                        // Create session if open returned session data
+                                        if (openResult->sessionData != NULL)
+                                        {
+                                            session.data = objMove(openResult->sessionData, objMemContext(this->sessionList));
+                                            lstAdd(this->sessionList, &session);
+                                        }
+
+                                        break;
+                                    }
+
+                                    // Process or close/cancel protocol session
+                                    default:
+                                    {
+                                        // Find session data
+                                        void *sessionData = NULL;
+                                        unsigned int sessionListIdx = 0;
+
+                                        if (request.sessionRequired)
+                                        {
+                                            ASSERT(handler->processSession != NULL);
+                                            ASSERT(handler->process == NULL);
+
+                                            for (; sessionListIdx < lstSize(this->sessionList); sessionListIdx++)
+                                            {
+                                                ProtocolServerSession *const session = lstGet(this->sessionList, sessionListIdx);
+
+                                                if (session->id == this->sessionId)
+                                                {
+                                                    sessionData = session->data;
+                                                    break;
+                                                }
+                                            }
+
+                                            // Error when session not found
+                                            if (sessionData == NULL && request.type != protocolCommandTypeCancel)
+                                            {
+                                                THROW_FMT(
+                                                    ProtocolError, "unable to find session id %" PRIu64 " for request %s:%s",
+                                                    this->sessionId, strZ(strIdToStr(request.id)),
+                                                    strZ(strIdToStr(request.type)));
+                                            }
+                                        }
+
+                                        // Process request type
+                                        switch (request.type)
+                                        {
+                                            // Process request
+                                            case protocolCommandTypeProcess:
+                                            {
+                                                // Process session
+                                                ProtocolServerResult *processResult;
+
+                                                if (handler->processSession != NULL)
+                                                {
+                                                    ASSERT(handler->process == NULL);
+                                                    CHECK_FMT(
+                                                        ProtocolError, this->sessionId != 0, "no session id for request %s:%s",
+                                                        strZ(strIdToStr(request.id)), strZ(strIdToStr(request.type)));
+
+                                                    processResult = handler->processSession(pckReadNew(request.param), sessionData);
+                                                }
+                                                // Standalone process
+                                                else
+                                                {
+                                                    ASSERT(handler->process != NULL);
+                                                    ASSERT(!request.sessionRequired);
+
+                                                    processResult = handler->process(pckReadNew(request.param));
+                                                }
+
+                                                if (processResult == NULL)
+                                                    protocolServerResponseP(this);
+                                                else
+                                                {
+                                                    ASSERT(processResult->sessionData == NULL);
+                                                    ASSERT(handler->processSession != NULL || !processResult->close);
+
+                                                    protocolServerResponseP(
+                                                        this, .data = processResult->data, .close = processResult->close);
+
+                                                    // Free session when close is true. This optimization allows an explicit
+                                                    // close to be skipped.
+                                                    if (processResult->close)
+                                                    {
+                                                        objFree(sessionData);
+                                                        lstRemoveIdx(this->sessionList, sessionListIdx);
+                                                    }
+                                                }
+
+                                                break;
+                                            }
+
+                                            // Close protocol session
+                                            case protocolCommandTypeClose:
+                                            {
+                                                // If there is a close handler then call it
+                                                PackWrite *data = NULL;
+
+                                                if (handler->close != NULL)
+                                                {
+                                                    ProtocolServerResult *const closeResult = handler->close(
+                                                        pckReadNew(request.param), sessionData);
+                                                    ASSERT(closeResult != NULL);
+                                                    ASSERT(closeResult->sessionData == NULL);
+                                                    ASSERT(!closeResult->close);
+
+                                                    data = closeResult->data;
+                                                }
+
+                                                // Send data
+                                                protocolServerResponseP(this, .data = data, .close = true);
+
+                                                // Free the session
+                                                objFree(sessionData);
+                                                lstRemoveIdx(this->sessionList, sessionListIdx);
+
+                                                break;
+                                            }
+
+                                            // Cancel protocol session
+                                            default:
+                                            {
+                                                CHECK_FMT(
+                                                    ProtocolError, request.type == protocolCommandTypeCancel,
+                                                    "unknown request type '%s'", strZ(strIdToStr(request.type)));
+
+                                                // Send NULL data
+                                                protocolServerResponseP(this, .close = true);
+
+                                                // Free the session
+                                                if (sessionData != NULL)
+                                                {
+                                                    objFree(sessionData);
+                                                    lstRemoveIdx(this->sessionList, sessionListIdx);
+                                                }
+
+                                                break;
+                                            }
+                                        }
+                                    }
+                                }
                             }
                             CATCH_ANY()
                             {
@@ -228,7 +442,7 @@ protocolServerProcess(
                                     retryRemaining--;
                                     retry = true;
 
-                                    // Send keep-alive to remotes. A retry means the command is taking longer than usual so make
+                                    // Send keep-alive to remotes. A retry means the request is taking longer than usual so make
                                     // sure the remote does not timeout.
                                     protocolKeepAlive();
                                 }
@@ -245,22 +459,22 @@ protocolServerProcess(
                     }
                     MEM_CONTEXT_TEMP_END();
                 }
-                // Else check built-in commands
+                // Else check built-in requests
                 else
                 {
-                    switch (command.id)
+                    switch (request.id)
                     {
                         case PROTOCOL_COMMAND_EXIT:
                             exit = true;
                             break;
 
                         case PROTOCOL_COMMAND_NOOP:
-                            protocolServerDataEndPut(this);
+                            protocolServerResponseP(this);
                             break;
 
                         default:
                             THROW_FMT(
-                                ProtocolError, "invalid command '%s' (0x%" PRIx64 ")", strZ(strIdToStr(command.id)), command.id);
+                                ProtocolError, "invalid request '%s' (0x%" PRIx64 ")", strZ(strIdToStr(request.id)), request.id);
                     }
                 }
 
@@ -273,6 +487,9 @@ protocolServerProcess(
         }
         CATCH_FATAL()
         {
+            // Zero session id so a fatal error will be handled by the first client that sees it
+            this->sessionId = 0;
+
             // Report error to the client
             protocolServerError(this, errorCode(), STR(errorMessage()), STR(errorStackTrace()));
 
@@ -287,85 +504,75 @@ protocolServerProcess(
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN PackRead *
+FN_EXTERN ProtocolServerResult *
-protocolServerDataGet(ProtocolServer *const this)
+protocolServerResultNew(const ProtocolServerResultNewParam param)
 {
-    FUNCTION_LOG_BEGIN(logLevelTrace);
+    FUNCTION_TEST_BEGIN();
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
+        FUNCTION_TEST_PARAM(SIZE, param.extra);
-    FUNCTION_LOG_END();
+    FUNCTION_TEST_END();
 
-    PackRead *result = NULL;
+    OBJ_NEW_BEGIN(ProtocolServerResult, .childQty = MEM_CONTEXT_QTY_MAX)
-
-    MEM_CONTEXT_TEMP_BEGIN()
     {
-        PackRead *data = pckReadNewIo(this->read);
+        *this = (ProtocolServerResult)
-        ProtocolMessageType type = (ProtocolMessageType)pckReadU32P(data);
-
-        CHECK(FormatError, type == protocolMessageTypeData, "expected data message");
-
-        MEM_CONTEXT_PRIOR_BEGIN()
         {
-            result = pckReadPackReadP(data);
+            .extra = param.extra,
-        }
+        };
-        MEM_CONTEXT_PRIOR_END();
-
-        pckReadEndP(data);
     }
-    MEM_CONTEXT_TEMP_END();
+    OBJ_NEW_END();
 
-    FUNCTION_LOG_RETURN(PACK_READ, result);
+    FUNCTION_TEST_RETURN(PROTOCOL_SERVER_RESULT, this);
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN PackWrite *
+protocolServerResultData(ProtocolServerResult *const this)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(PROTOCOL_SERVER_RESULT, this);
+    FUNCTION_TEST_END();
+
+    ASSERT(this != NULL);
+    ASSERT(this->data == NULL);
+
+    MEM_CONTEXT_OBJ_BEGIN(this)
+    {
+        this->data = pckWriteNewP(.size = PROTOCOL_PACK_DEFAULT_SIZE + this->extra);
+    }
+    MEM_CONTEXT_OBJ_END();
+
+    FUNCTION_TEST_RETURN(PACK_WRITE, this->data);
 }
 
 /**********************************************************************************************************************************/
 FN_EXTERN void
-protocolServerDataPut(ProtocolServer *const this, PackWrite *const data)
+protocolServerResultSessionDataSet(ProtocolServerResult *const this, void *const sessionData)
 {
-    FUNCTION_LOG_BEGIN(logLevelTrace);
+    FUNCTION_TEST_BEGIN();
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
+        FUNCTION_TEST_PARAM(PROTOCOL_SERVER_RESULT, this);
-        FUNCTION_LOG_PARAM(PACK_WRITE, data);
+        FUNCTION_TEST_PARAM_P(VOID, sessionData);
-    FUNCTION_LOG_END();
+    FUNCTION_TEST_END();
 
-    MEM_CONTEXT_TEMP_BEGIN()
+    ASSERT(this != NULL);
-    {
+    ASSERT(sessionData != NULL);
-        // End the pack
-        if (data != NULL)
-            pckWriteEndP(data);
 
-        // Write the result
+    this->sessionData = objMove(sessionData, objMemContext(this));
-        PackWrite *resultMessage = pckWriteNewIo(this->write);
-        pckWriteU32P(resultMessage, protocolMessageTypeData, .defaultWrite = true);
-        pckWritePackP(resultMessage, pckWriteResult(data));
-        pckWriteEndP(resultMessage);
 
-        // Flush on NULL result since it might be used to synchronize
+    FUNCTION_TEST_RETURN_VOID();
-        if (data == NULL)
-            ioWriteFlush(this->write);
-    }
-    MEM_CONTEXT_TEMP_END();
-
-    FUNCTION_LOG_RETURN_VOID();
 }
 
 /**********************************************************************************************************************************/
 FN_EXTERN void
-protocolServerDataEndPut(ProtocolServer *const this)
+protocolServerResultCloseSet(ProtocolServerResult *const this)
 {
-    FUNCTION_LOG_BEGIN(logLevelTrace);
+    FUNCTION_TEST_BEGIN();
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
+        FUNCTION_TEST_PARAM(PROTOCOL_SERVER_RESULT, this);
-    FUNCTION_LOG_END();
+    FUNCTION_TEST_END();
 
-    MEM_CONTEXT_TEMP_BEGIN()
+    ASSERT(this != NULL);
-    {
-        // Write the response and flush to be sure it gets sent immediately
-        PackWrite *response = pckWriteNewIo(this->write);
-        pckWriteU32P(response, protocolMessageTypeDataEnd, .defaultWrite = true);
-        pckWriteEndP(response);
-    }
-    MEM_CONTEXT_TEMP_END();
 
-    ioWriteFlush(this->write);
+    this->close = true;
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_TEST_RETURN_VOID();
 }
 
 /**********************************************************************************************************************************/
@@ -374,3 +581,12 @@ protocolServerToLog(const ProtocolServer *const this, StringStatic *const debugL
 {
     strStcFmt(debugLog, "{name: %s}", strZ(this->name));
 }
+
+/**********************************************************************************************************************************/
+FN_EXTERN void
+protocolServerResultToLog(const ProtocolServerResult *const this, StringStatic *const debugLog)
+{
+    strStcFmt(
+        debugLog, "{data: %s, sessionData: %s, close: %s}", cvtBoolToConstZ(this->data != NULL),
+        cvtBoolToConstZ(this->sessionData != NULL), cvtBoolToConstZ(this->close));
+}

src/protocol/server.h

@@ -8,6 +8,7 @@ Protocol Server
 Object type
 ***********************************************************************************************************************************/
 typedef struct ProtocolServer ProtocolServer;
+typedef struct ProtocolServerResult ProtocolServerResult;
 
 #include "common/io/read.h"
 #include "common/io/write.h"
@@ -22,42 +23,55 @@ Protocol command handler type and structure
 An array of this struct must be passed to protocolServerProcess() for the server to process commands. Each command handler should
 implement a single command, as defined by the command string.
 ***********************************************************************************************************************************/
-typedef void (*ProtocolServerCommandHandler)(PackRead *param, ProtocolServer *server);
+typedef ProtocolServerResult *(*ProtocolServerCommandOpenHandler)(PackRead *param);
+typedef ProtocolServerResult *(*ProtocolServerCommandProcessHandler)(PackRead *param);
+typedef ProtocolServerResult *(*ProtocolServerCommandProcessSessionHandler)(PackRead *param, void *sessionData);
+typedef ProtocolServerResult *(*ProtocolServerCommandCloseHandler)(PackRead *param, void *sessionData);
 
 typedef struct ProtocolServerHandler
 {
     StringId command;                                               // 5-bit StringId that identifies the protocol command
-    ProtocolServerCommandHandler handler;                           // Function that handles the protocol command
+    ProtocolServerCommandOpenHandler open;                          // Function that opens the protocol session
+    ProtocolServerCommandProcessHandler process;                    // Function that processes the protocol command
+    ProtocolServerCommandProcessSessionHandler processSession;      // Function that processes the protocol command for a session
+    ProtocolServerCommandCloseHandler close;                        // Function that closes the protocol session
 } ProtocolServerHandler;
 
 /***********************************************************************************************************************************
-Constructors
+Server Constructors
 ***********************************************************************************************************************************/
 FN_EXTERN ProtocolServer *protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write);
 
 /***********************************************************************************************************************************
-Functions
+Server Functions
 ***********************************************************************************************************************************/
 // Get command from the client. Outside ProtocolServer, this is used when the first noop command needs to be processed before
 // running protocolServerProcess(), which allows an error to be returned to the client if initialization fails.
-typedef struct ProtocolServerCommandGetResult
+typedef struct ProtocolServerRequestResult
 {
     StringId id;                                                    // Command identifier
+    ProtocolCommandType type;                                       // Command type
+    bool sessionRequired;                                           // Session with more than one command
     Pack *param;                                                    // Parameter pack
-} ProtocolServerCommandGetResult;
+} ProtocolServerRequestResult;
 
-FN_EXTERN ProtocolServerCommandGetResult protocolServerCommandGet(ProtocolServer *this);
+FN_EXTERN ProtocolServerRequestResult protocolServerRequest(ProtocolServer *this);
 
-// Get data from the client
+// Send response to client
-FN_EXTERN PackRead *protocolServerDataGet(ProtocolServer *this);
+typedef struct ProtocolServerResponseParam
+{
+    VAR_PARAM_HEADER;
+    bool close;                                                     // Has the session been closed?
+    ProtocolMessageType type;                                       // Message type
+    PackWrite *data;                                                // Response data
+} ProtocolServerResponseParam;
 
-// Put data to the client
+#define protocolServerResponseP(this, ...)                                                                                         \
-FN_EXTERN void protocolServerDataPut(ProtocolServer *this, PackWrite *data);
+    protocolServerResponse(this, (ProtocolServerResponseParam){VAR_PARAM_INIT, __VA_ARGS__})
 
-// Put data end to the client. This ends command processing and no more data should be sent.
+FN_EXTERN void protocolServerResponse(ProtocolServer *const this, ProtocolServerResponseParam param);
-FN_EXTERN void protocolServerDataEndPut(ProtocolServer *this);
 
-// Return an error
+// Send an error to client
 FN_EXTERN void protocolServerError(ProtocolServer *this, int code, const String *message, const String *stack);
 
 // Process requests
@@ -73,7 +87,7 @@ protocolServerMove(ProtocolServer *const this, MemContext *const parentNew)
 }
 
 /***********************************************************************************************************************************
-Destructor
+Server Destructor
 ***********************************************************************************************************************************/
 FN_INLINE_ALWAYS void
 protocolServerFree(ProtocolServer *const this)
@@ -81,6 +95,32 @@ protocolServerFree(ProtocolServer *const this)
     objFree(this);
 }
 
+/***********************************************************************************************************************************
+Result Constructors
+***********************************************************************************************************************************/
+typedef struct ProtocolServerResultNewParam
+{
+    VAR_PARAM_HEADER;
+    size_t extra;
+} ProtocolServerResultNewParam;
+
+#define protocolServerResultNewP(...)                                                                                              \
+    protocolServerResultNew((ProtocolServerResultNewParam){VAR_PARAM_INIT, __VA_ARGS__})
+
+FN_EXTERN ProtocolServerResult *protocolServerResultNew(ProtocolServerResultNewParam param);
+
+/***********************************************************************************************************************************
+Result Getters/Setters
+***********************************************************************************************************************************/
+// Create PackWrite object required to send data to the client
+FN_EXTERN PackWrite *protocolServerResultData(ProtocolServerResult *this);
+
+// Set session data
+FN_EXTERN void protocolServerResultSessionDataSet(ProtocolServerResult *const this, void *const sessionData);
+
+// Close session
+FN_EXTERN void protocolServerResultCloseSet(ProtocolServerResult *const this);
+
 /***********************************************************************************************************************************
 Macros for function logging
 ***********************************************************************************************************************************/
@@ -91,4 +131,11 @@ FN_EXTERN void protocolServerToLog(const ProtocolServer *this, StringStatic *deb
 #define FUNCTION_LOG_PROTOCOL_SERVER_FORMAT(value, buffer, bufferSize)                                                             \
     FUNCTION_LOG_OBJECT_FORMAT(value, protocolServerToLog, buffer, bufferSize)
 
+FN_EXTERN void protocolServerResultToLog(const ProtocolServerResult *this, StringStatic *debugLog);
+
+#define FUNCTION_LOG_PROTOCOL_SERVER_RESULT_TYPE                                                                                   \
+    ProtocolServerResult *
+#define FUNCTION_LOG_PROTOCOL_SERVER_RESULT_FORMAT(value, buffer, bufferSize)                                                      \
+    FUNCTION_LOG_OBJECT_FORMAT(value, protocolServerResultToLog, buffer, bufferSize)
+
 #endif

src/storage/remote/protocol.c

@@ -124,18 +124,18 @@ storageRemoteFilterGroup(IoFilterGroup *const filterGroup, const Pack *const fil
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-storageRemoteFeatureProtocol(PackRead *const param, ProtocolServer *const server)
+storageRemoteFeatureProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     FUNCTION_AUDIT_HELPER();
 
     ASSERT(param == NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
@@ -159,16 +159,14 @@ storageRemoteFeatureProtocol(PackRead *const param, ProtocolServer *const server
         }
 
         // Return storage features
-        PackWrite *result = protocolPackNew();
+        PackWrite *const data = protocolServerResultData(result);
-        pckWriteStrP(result, storagePathP(storage, NULL));
-        pckWriteU64P(result, storageInterface(storage).feature);
 
-        protocolServerDataPut(server, result);
+        pckWriteStrP(data, storagePathP(storage, NULL));
-        protocolServerDataEndPut(server);
+        pckWriteU64P(data, storageInterface(storage).feature);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
 /**********************************************************************************************************************************/
@@ -264,18 +262,18 @@ storageRemoteInfoProtocolPut(
     FUNCTION_TEST_RETURN_VOID();
 }
 
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-storageRemoteInfoProtocol(PackRead *const param, ProtocolServer *const server)
+storageRemoteInfoProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
     ASSERT(storageRemoteProtocolLocal.driver != NULL);
 
+    ProtocolServerResult *const result = protocolServerResultNewP();
+
     MEM_CONTEXT_TEMP_BEGIN()
     {
         // Get file info
@@ -286,31 +284,26 @@ storageRemoteInfoProtocol(PackRead *const param, ProtocolServer *const server)
         StorageInfo info = storageInterfaceInfoP(storageRemoteProtocolLocal.driver, file, level, .followLink = followLink);
 
         // Write file info to protocol
-        PackWrite *write = protocolPackNew();
+        PackWrite *const data = protocolServerResultData(result);
-        pckWriteBoolP(write, info.exists, .defaultWrite = true);
+        pckWriteBoolP(data, info.exists, .defaultWrite = true);
 
         if (info.exists)
-            storageRemoteInfoProtocolPut(&(StorageRemoteInfoProtocolWriteData){0}, write, &info);
+            storageRemoteInfoProtocolPut(&(StorageRemoteInfoProtocolWriteData){0}, data, &info);
-
-        protocolServerDataPut(server, write);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-storageRemoteLinkCreateProtocol(PackRead *const param, ProtocolServer *const server)
+storageRemoteLinkCreateProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
     ASSERT(storageRemoteProtocolLocal.driver != NULL);
 
     MEM_CONTEXT_TEMP_BEGIN()
@@ -320,32 +313,35 @@ storageRemoteLinkCreateProtocol(PackRead *const param, ProtocolServer *const ser
         const StorageLinkType linkType = (StorageLinkType)pckReadU32P(param);
 
         storageInterfaceLinkCreateP(storageRemoteProtocolLocal.driver, target, linkPath, .linkType = linkType);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-storageRemoteListProtocol(PackRead *const param, ProtocolServer *const server)
+storageRemoteListProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
     ASSERT(storageRemoteProtocolLocal.driver != NULL);
 
+    ProtocolServerResult *const result = protocolServerResultNewP(.extra = ioBufferSize());
+
     MEM_CONTEXT_TEMP_BEGIN()
     {
         const String *const path = pckReadStrP(param);
         const StorageInfoLevel level = (StorageInfoLevel)pckReadU32P(param);
         StorageRemoteInfoProtocolWriteData writeData = {0};
         StorageList *const list = storageInterfaceListP(storageRemoteProtocolLocal.driver, path, level);
+        PackWrite *const data = protocolServerResultData(result);
+
+        // Indicate whether or not the path was found
+        pckWriteBoolP(data, list != NULL, .defaultWrite = true);
 
         // Put list
         if (list != NULL)
@@ -354,194 +350,235 @@ storageRemoteListProtocol(PackRead *const param, ProtocolServer *const server)
             {
                 const StorageInfo info = storageLstGet(list, listIdx);
 
-                PackWrite *const write = protocolPackNew();
+                pckWriteObjBeginP(data);
-                pckWriteStrP(write, info.name);
+                pckWriteStrP(data, info.name);
-                storageRemoteInfoProtocolPut(&writeData, write, &info);
+                storageRemoteInfoProtocolPut(&writeData, data, &info);
-                protocolServerDataPut(server, write);
+                pckWriteObjEndP(data);
-                pckWriteFree(write);
             }
         }
-
-        // Indicate whether or not the path was found
-        PackWrite *write = protocolPackNew();
-        pckWriteBoolP(write, list != NULL, .defaultWrite = true);
-        protocolServerDataPut(server, write);
-
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+static bool
-storageRemoteOpenReadProtocol(PackRead *const param, ProtocolServer *const server)
+storageRemoteReadInternal(StorageRead *const fileRead, PackWrite *const packWrite)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(STORAGE_READ, fileRead);
+        FUNCTION_LOG_PARAM(PACK_WRITE, packWrite);
+    FUNCTION_LOG_END();
+
+    ASSERT(fileRead != NULL);
+    ASSERT(packWrite != NULL);
+
+    FUNCTION_AUDIT_HELPER();
+
+    // Read block and send to client
+    Buffer *const buffer = bufNew(ioBufferSize());
+
+    ioRead(storageReadIo(fileRead), buffer);
+    pckWriteBoolP(packWrite, bufEmpty(buffer));
+
+    if (!bufEmpty(buffer))
+        pckWriteBinP(packWrite, buffer);
+
+    // On eof
+    bool result = true;
+
+    if (ioReadEof(storageReadIo(fileRead)))
+    {
+        // Close file (needed to get filter results)
+        ioReadClose(storageReadIo(fileRead));
+
+        // Write filter results
+        pckWritePackP(packWrite, ioFilterGroupResultAll(ioReadFilterGroup(storageReadIo(fileRead))));
+
+        // Let the server know to close the session
+        result = false;
+    }
+
+    FUNCTION_LOG_RETURN(BOOL, result);
+}
+
+FN_EXTERN ProtocolServerResult *
+storageRemoteReadOpenProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
     ASSERT(storageRemoteProtocolLocal.driver != NULL);
 
+    ProtocolServerResult *const result = protocolServerResultNewP(.extra = ioBufferSize());
+
     MEM_CONTEXT_TEMP_BEGIN()
     {
         const String *file = pckReadStrP(param);
-        bool ignoreMissing = pckReadBoolP(param);
+        const bool ignoreMissing = pckReadBoolP(param);
         const uint64_t offset = pckReadU64P(param);
         const Variant *const limit = pckReadNullP(param) ? NULL : VARUINT64(pckReadU64P(param));
         const Pack *const filter = pckReadPackP(param);
 
         // Create the read object
-        IoRead *fileRead = storageReadIo(
+        StorageRead *const fileRead = storageInterfaceNewReadP(
-            storageInterfaceNewReadP(storageRemoteProtocolLocal.driver, file, ignoreMissing, .offset = offset, .limit = limit));
+            storageRemoteProtocolLocal.driver, file, ignoreMissing, .offset = offset, .limit = limit);
 
         // Set filter group based on passed filters
-        storageRemoteFilterGroup(ioReadFilterGroup(fileRead), filter);
+        storageRemoteFilterGroup(ioReadFilterGroup(storageReadIo(fileRead)), filter);
 
-        // Check if the file exists
+        // Determine if file exists
-        bool exists = ioReadOpen(fileRead);
+        PackWrite *const data = protocolServerResultData(result);
-        protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), exists, .defaultWrite = true));
+        const bool exists = ioReadOpen(storageReadIo(fileRead));
 
-        // Transfer the file if it exists
+        pckWriteBoolP(data, exists, .defaultWrite = true);
+
+        // If the file exists
         if (exists)
         {
-            Buffer *buffer = bufNew(ioBufferSize());
+            // If there is more to read then set session data
-
+            if (storageRemoteReadInternal(fileRead, data))
-            // Write file out to protocol layer
+                protocolServerResultSessionDataSet(result, fileRead);
-            do
-            {
-                ioRead(fileRead, buffer);
-
-                if (!bufEmpty(buffer))
-                {
-                    MEM_CONTEXT_TEMP_BEGIN()
-                    {
-                        PackWrite *write = pckWriteNewP(.size = ioBufferSize() + PROTOCOL_PACK_DEFAULT_SIZE);
-                        pckWriteBinP(write, buffer);
-                        protocolServerDataPut(server, write);
-                    }
-                    MEM_CONTEXT_TEMP_END();
-
-                    bufUsedZero(buffer);
-                }
-            }
-            while (!ioReadEof(fileRead));
-
-            ioReadClose(fileRead);
-
-            // Write filter results
-            protocolServerDataPut(server, pckWritePackP(protocolPackNew(), ioFilterGroupResultAll(ioReadFilterGroup(fileRead))));
         }
-
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
-/**********************************************************************************************************************************/
+FN_EXTERN ProtocolServerResult *
-FN_EXTERN void
+storageRemoteReadProtocol(PackRead *const param, void *const fileRead)
-storageRemoteOpenWriteProtocol(PackRead *const param, ProtocolServer *const server)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
+        FUNCTION_LOG_PARAM(STORAGE_READ, fileRead);
     FUNCTION_LOG_END();
 
-    ASSERT(param != NULL);
+    FUNCTION_AUDIT_STRUCT();
-    ASSERT(server != NULL);
+
+    ASSERT(param == NULL);
+    ASSERT(fileRead != NULL);
     ASSERT(storageRemoteProtocolLocal.driver != NULL);
 
+    ProtocolServerResult *const result = protocolServerResultNewP(.extra = ioBufferSize());
+
+    if (!storageRemoteReadInternal(fileRead, protocolServerResultData(result)))
+        protocolServerResultCloseSet(result);
+
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN ProtocolServerResult *
+storageRemoteWriteOpenProtocol(PackRead *const param)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(PACK_READ, param);
+    FUNCTION_LOG_END();
+
+    FUNCTION_AUDIT_STRUCT();
+
+    ASSERT(param != NULL);
+    ASSERT(storageRemoteProtocolLocal.driver != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
+
     MEM_CONTEXT_TEMP_BEGIN()
     {
         // Create the write object
-        const String *file = pckReadStrP(param);
+        const String *const file = pckReadStrP(param);
-        mode_t modeFile = pckReadModeP(param);
+        const mode_t modeFile = pckReadModeP(param);
-        mode_t modePath = pckReadModeP(param);
+        const mode_t modePath = pckReadModeP(param);
-        const String *user = pckReadStrP(param);
+        const String *const user = pckReadStrP(param);
-        const String *group = pckReadStrP(param);
+        const String *const group = pckReadStrP(param);
-        time_t timeModified = pckReadTimeP(param);
+        const time_t timeModified = pckReadTimeP(param);
-        bool createPath = pckReadBoolP(param);
+        const bool createPath = pckReadBoolP(param);
-        bool syncFile = pckReadBoolP(param);
+        const bool syncFile = pckReadBoolP(param);
-        bool syncPath = pckReadBoolP(param);
+        const bool syncPath = pckReadBoolP(param);
-        bool atomic = pckReadBoolP(param);
+        const bool atomic = pckReadBoolP(param);
         const Pack *const filter = pckReadPackP(param);
 
-        IoWrite *fileWrite = storageWriteIo(
+        StorageWrite *const fileWrite = storageInterfaceNewWriteP(
-            storageInterfaceNewWriteP(
+            storageRemoteProtocolLocal.driver, file, .modeFile = modeFile, .modePath = modePath, .user = user, .group = group,
-                storageRemoteProtocolLocal.driver, file, .modeFile = modeFile, .modePath = modePath, .user = user, .group = group,
+            .timeModified = timeModified, .createPath = createPath, .syncFile = syncFile, .syncPath = syncPath, .atomic = atomic,
-                .timeModified = timeModified, .createPath = createPath, .syncFile = syncFile, .syncPath = syncPath,
+            .truncate = true);
-                .atomic = atomic, .truncate = true));
 
         // Set filter group based on passed filters
-        storageRemoteFilterGroup(ioWriteFilterGroup(fileWrite), filter);
+        storageRemoteFilterGroup(ioWriteFilterGroup(storageWriteIo(fileWrite)), filter);
 
         // Open file
-        ioWriteOpen(fileWrite);
+        ioWriteOpen(storageWriteIo(fileWrite));
-        protocolServerDataPut(server, NULL);
 
-        // Write data
+        // Set session data
-        do
+        protocolServerResultSessionDataSet(result, fileWrite);
-        {
-            PackRead *read = protocolServerDataGet(server);
-
-            // Write is complete
-            if (read == NULL)
-            {
-                ioWriteClose(fileWrite);
-
-                // Push filter results
-                protocolServerDataPut(
-                    server, pckWritePackP(protocolPackNew(), ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite))));
-                break;
-            }
-            // Else more data to write
-            else
-            {
-                pckReadNext(read);
-
-                // Write data
-                if (pckReadType(read) == pckTypeBin)
-                {
-                    Buffer *const buffer = pckReadBinP(read);
-
-                    ioWrite(fileWrite, buffer);
-                    bufFree(buffer);
-                }
-                // Else write terminated unexpectedly
-                else
-                {
-                    protocolServerDataGet(server);
-                    ioWriteFree(fileWrite);
-                    break;
-                }
-            }
-        }
-        while (true);
-
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
-/**********************************************************************************************************************************/
+FN_EXTERN ProtocolServerResult *
-FN_EXTERN void
+storageRemoteWriteProtocol(PackRead *const param, void *const fileWrite)
-storageRemotePathCreateProtocol(PackRead *const param, ProtocolServer *const server)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(PACK_READ, param);
+        FUNCTION_LOG_PARAM(STORAGE_READ, fileWrite);
+    FUNCTION_LOG_END();
+
+    ASSERT(param != NULL);
+    ASSERT(fileWrite != NULL);
+    ASSERT(storageRemoteProtocolLocal.driver != NULL);
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        Buffer *const buffer = pckReadBinP(param);
+
+        ioWrite(storageWriteIo(fileWrite), buffer);
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
+}
+
+FN_EXTERN ProtocolServerResult *
+storageRemoteWriteCloseProtocol(PackRead *const param, void *const fileWrite)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(PACK_READ, param);
+        FUNCTION_LOG_PARAM(STORAGE_WRITE, fileWrite);
+    FUNCTION_LOG_END();
+
+    FUNCTION_AUDIT_STRUCT();
+
+    ASSERT(param == NULL);
+    ASSERT(fileWrite != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        ioWriteClose(storageWriteIo(fileWrite));
+
+        // Send filter results
+        pckWritePackP(protocolServerResultData(result), ioFilterGroupResultAll(ioWriteFilterGroup(storageWriteIo(fileWrite))));
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
+}
+
+/**********************************************************************************************************************************/
+FN_EXTERN ProtocolServerResult *
+storageRemotePathCreateProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
     ASSERT(storageRemoteProtocolLocal.driver != NULL);
 
     MEM_CONTEXT_TEMP_BEGIN()
@@ -552,52 +589,48 @@ storageRemotePathCreateProtocol(PackRead *const param, ProtocolServer *const ser
         mode_t mode = pckReadModeP(param);
 
         storageInterfacePathCreateP(storageRemoteProtocolLocal.driver, path, errorOnExists, noParentCreate, mode);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-storageRemotePathRemoveProtocol(PackRead *const param, ProtocolServer *const server)
+storageRemotePathRemoveProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
     ASSERT(storageRemoteProtocolLocal.driver != NULL);
 
+    ProtocolServerResult *const result = protocolServerResultNewP();
+
     MEM_CONTEXT_TEMP_BEGIN()
     {
         const String *path = pckReadStrP(param);
         bool recurse = pckReadBoolP(param);
 
-        const bool result = storageInterfacePathRemoveP(storageRemoteProtocolLocal.driver, path, recurse);
+        pckWriteBoolP(
-
+            protocolServerResultData(result), storageInterfacePathRemoveP(storageRemoteProtocolLocal.driver, path, recurse),
-        protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), result, .defaultWrite = true));
+            .defaultWrite = true);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-storageRemotePathSyncProtocol(PackRead *const param, ProtocolServer *const server)
+storageRemotePathSyncProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
     ASSERT(storageRemoteProtocolLocal.driver != NULL);
 
     MEM_CONTEXT_TEMP_BEGIN()
@@ -605,24 +638,21 @@ storageRemotePathSyncProtocol(PackRead *const param, ProtocolServer *const serve
         const String *path = pckReadStrP(param);
 
         storageInterfacePathSyncP(storageRemoteProtocolLocal.driver, path);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
 }
 
 /**********************************************************************************************************************************/
-FN_EXTERN void
+FN_EXTERN ProtocolServerResult *
-storageRemoteRemoveProtocol(PackRead *const param, ProtocolServer *const server)
+storageRemoteRemoveProtocol(PackRead *const param)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(PACK_READ, param);
-        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_LOG_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
     ASSERT(storageRemoteProtocolLocal.driver != NULL);
 
     MEM_CONTEXT_TEMP_BEGIN()
@@ -631,9 +661,8 @@ storageRemoteRemoveProtocol(PackRead *const param, ProtocolServer *const server)
         bool errorOnMissing = pckReadBoolP(param);
 
         storageInterfaceRemoveP(storageRemoteProtocolLocal.driver, file, .errorOnMissing = errorOnMissing);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_VOID();
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER_RESULT, NULL);
 }

src/storage/remote/protocol.h

@@ -11,16 +11,19 @@ Remote Storage Protocol Handler
 Functions
 ***********************************************************************************************************************************/
 // Process storage protocol requests
-FN_EXTERN void storageRemoteFeatureProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemoteFeatureProtocol(PackRead *param);
-FN_EXTERN void storageRemoteInfoProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemoteInfoProtocol(PackRead *param);
-FN_EXTERN void storageRemoteLinkCreateProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemoteLinkCreateProtocol(PackRead *param);
-FN_EXTERN void storageRemoteListProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemoteListProtocol(PackRead *param);
-FN_EXTERN void storageRemoteOpenReadProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemotePathCreateProtocol(PackRead *param);
-FN_EXTERN void storageRemoteOpenWriteProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemotePathRemoveProtocol(PackRead *param);
-FN_EXTERN void storageRemotePathCreateProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemotePathSyncProtocol(PackRead *param);
-FN_EXTERN void storageRemotePathRemoveProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemoteReadOpenProtocol(PackRead *param);
-FN_EXTERN void storageRemotePathSyncProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemoteReadProtocol(PackRead *param, void *fileRead);
-FN_EXTERN void storageRemoteRemoveProtocol(PackRead *param, ProtocolServer *server);
+FN_EXTERN ProtocolServerResult *storageRemoteRemoveProtocol(PackRead *param);
+FN_EXTERN ProtocolServerResult *storageRemoteWriteOpenProtocol(PackRead *param);
+FN_EXTERN ProtocolServerResult *storageRemoteWriteProtocol(PackRead *param, void *fileWrite);
+FN_EXTERN ProtocolServerResult *storageRemoteWriteCloseProtocol(PackRead *param, void *fileWrite);
 
 /***********************************************************************************************************************************
 Protocol commands for ProtocolServerHandler arrays passed to protocolServerProcess()
@@ -29,24 +32,25 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
 #define PROTOCOL_COMMAND_STORAGE_INFO                               STRID5("s-i", 0x27730)
 #define PROTOCOL_COMMAND_STORAGE_LINK_CREATE                        STRID5("s-lc", 0x1b3730)
 #define PROTOCOL_COMMAND_STORAGE_LIST                               STRID5("s-l", 0x33730)
-#define PROTOCOL_COMMAND_STORAGE_OPEN_READ                          STRID5("s-or", 0x93f730)
-#define PROTOCOL_COMMAND_STORAGE_OPEN_WRITE                         STRID5("s-ow", 0xbbf730)
 #define PROTOCOL_COMMAND_STORAGE_PATH_CREATE                        STRID5("s-pc", 0x1c3730)
-#define PROTOCOL_COMMAND_STORAGE_REMOVE                             STRID5("s-r", 0x4b730)
 #define PROTOCOL_COMMAND_STORAGE_PATH_REMOVE                        STRID5("s-pr", 0x943730)
 #define PROTOCOL_COMMAND_STORAGE_PATH_SYNC                          STRID5("s-ps", 0x9c3730)
+#define PROTOCOL_COMMAND_STORAGE_READ                               STRID5("s-rd", 0x24b730)
+#define PROTOCOL_COMMAND_STORAGE_REMOVE                             STRID5("s-r", 0x4b730)
+#define PROTOCOL_COMMAND_STORAGE_WRITE                              STRID5("s-wr", 0x95f730)
 
 #define PROTOCOL_SERVER_HANDLER_STORAGE_REMOTE_LIST                                                                                \
-    {.command = PROTOCOL_COMMAND_STORAGE_FEATURE, .handler = storageRemoteFeatureProtocol},                                        \
+    {.command = PROTOCOL_COMMAND_STORAGE_FEATURE, .process = storageRemoteFeatureProtocol},                                        \
-    {.command = PROTOCOL_COMMAND_STORAGE_INFO, .handler = storageRemoteInfoProtocol},                                              \
+    {.command = PROTOCOL_COMMAND_STORAGE_INFO, .process = storageRemoteInfoProtocol},                                              \
-    {.command = PROTOCOL_COMMAND_STORAGE_LINK_CREATE, .handler = storageRemoteLinkCreateProtocol},                                 \
+    {.command = PROTOCOL_COMMAND_STORAGE_LINK_CREATE, .process = storageRemoteLinkCreateProtocol},                                 \
-    {.command = PROTOCOL_COMMAND_STORAGE_LIST, .handler = storageRemoteListProtocol},                                              \
+    {.command = PROTOCOL_COMMAND_STORAGE_LIST, .process = storageRemoteListProtocol},                                              \
-    {.command = PROTOCOL_COMMAND_STORAGE_OPEN_READ, .handler = storageRemoteOpenReadProtocol},                                     \
+    {.command = PROTOCOL_COMMAND_STORAGE_PATH_CREATE, .process = storageRemotePathCreateProtocol},                                 \
-    {.command = PROTOCOL_COMMAND_STORAGE_OPEN_WRITE, .handler = storageRemoteOpenWriteProtocol},                                   \
+    {.command = PROTOCOL_COMMAND_STORAGE_PATH_REMOVE, .process = storageRemotePathRemoveProtocol},                                 \
-    {.command = PROTOCOL_COMMAND_STORAGE_PATH_CREATE, .handler = storageRemotePathCreateProtocol},                                 \
+    {.command = PROTOCOL_COMMAND_STORAGE_PATH_SYNC, .process = storageRemotePathSyncProtocol},                                     \
-    {.command = PROTOCOL_COMMAND_STORAGE_PATH_REMOVE, .handler = storageRemotePathRemoveProtocol},                                 \
+    {.command = PROTOCOL_COMMAND_STORAGE_READ, .open = storageRemoteReadOpenProtocol, .processSession = storageRemoteReadProtocol},\
-    {.command = PROTOCOL_COMMAND_STORAGE_PATH_SYNC, .handler = storageRemotePathSyncProtocol},                                     \
+    {.command = PROTOCOL_COMMAND_STORAGE_REMOVE, .process = storageRemoteRemoveProtocol},                                          \
-    {.command = PROTOCOL_COMMAND_STORAGE_REMOVE, .handler = storageRemoteRemoveProtocol},
+    {.command = PROTOCOL_COMMAND_STORAGE_WRITE, .open = storageRemoteWriteOpenProtocol,                                            \
+        .processSession = storageRemoteWriteProtocol, .close = storageRemoteWriteCloseProtocol},
 
 /***********************************************************************************************************************************
 Filters that may be passed to a remote

src/storage/remote/read.c

@@ -27,9 +27,11 @@ typedef struct StorageReadRemote
     StorageRead *read;                                              // Storage read interface
 
     ProtocolClient *client;                                         // Protocol client for requests
+    ProtocolClientSession *session;                                 // Protocol session for requests
     size_t remaining;                                               // Bytes remaining to be read in block
     Buffer *block;                                                  // Block currently being read
     bool eof;                                                       // Has the file reached eof?
+    bool eofFound;                                                  // Eof found but a block is remaining to be read
 
 #ifdef DEBUG
     uint64_t protocolReadBytes;                                     // How many bytes were read from the protocol layer?
@@ -45,55 +47,50 @@ Macros for function logging
     objNameToLog(value, "StorageReadRemote", buffer, bufferSize)
 
 /***********************************************************************************************************************************
-Clear protocol if the entire file is not read or an error occurs before the read is complete. This is required to clear the
+Read from a file
-protocol state so a subsequent command can succeed.
 ***********************************************************************************************************************************/
 static void
-storageReadRemoteFreeResource(THIS_VOID)
+storageReadRemoteInternal(StorageReadRemote *const this, PackRead *const packRead)
 {
-    THIS(StorageReadRemote);
-
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(STORAGE_READ_REMOTE, this);
+        FUNCTION_LOG_PARAM(PACK_READ, packRead);
     FUNCTION_LOG_END();
 
     ASSERT(this != NULL);
+    ASSERT(packRead != NULL);
 
-    // Read if eof has not been reached
+    FUNCTION_AUDIT_HELPER();
-    if (!this->eof)
+
+    // If not done then read the next block
+    if (!pckReadBoolP(packRead))
     {
-        do
+        MEM_CONTEXT_OBJ_BEGIN(this)
         {
-            MEM_CONTEXT_TEMP_BEGIN()
+            this->block = pckReadBinP(packRead);
-            {
+            this->remaining = bufUsed(this->block);
-                PackRead *const read = protocolClientDataGet(this->client);
-                pckReadNext(read);
-
-                // If binary then discard
-                if (pckReadType(read) == pckTypeBin)
-                {
-                    pckReadBinP(read);
-                }
-                // Else read is complete so discard the filter list
-                else
-                {
-                    pckReadPackP(read);
-                    protocolClientDataEndGet(this->client);
-
-                    this->eof = true;
-                }
-            }
-            MEM_CONTEXT_TEMP_END();
         }
-        while (!this->eof);
+        MEM_CONTEXT_OBJ_END();
     }
 
+    // If eof then get results
+    if (protocolClientSessionClosed(this->session))
+    {
+        ioFilterGroupResultAllSet(ioReadFilterGroup(storageReadIo(this->read)), pckReadPackP(packRead));
+
+        this->eofFound = true;
+
+        if (this->remaining == 0)
+            this->eof = true;
+    }
+
+#ifdef DEBUG
+    this->protocolReadBytes += this->remaining;
+#endif
+
     FUNCTION_LOG_RETURN_VOID();
 }
 
-/***********************************************************************************************************************************
-Read from a file
-***********************************************************************************************************************************/
 static size_t
 storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
 {
@@ -120,33 +117,13 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
             {
                 MEM_CONTEXT_TEMP_BEGIN()
                 {
-                    PackRead *const read = protocolClientDataGet(this->client);
+                    if (!protocolClientSessionQueued(this->session))
-                    pckReadNext(read);
+                        protocolClientSessionRequestAsyncP(this->session);
 
-                    // If binary then read the next block
+                    storageReadRemoteInternal(this, protocolClientSessionResponse(this->session));
-                    if (pckReadType(read) == pckTypeBin)
-                    {
-                        MEM_CONTEXT_OBJ_BEGIN(this)
-                        {
-                            this->block = pckReadBinP(read);
-                            this->remaining = bufUsed(this->block);
-                        }
-                        MEM_CONTEXT_OBJ_END();
-                    }
-                    // Else read is complete and get the filter list
-                    else
-                    {
-                        bufFree(this->block);
 
-                        ioFilterGroupResultAllSet(ioReadFilterGroup(storageReadIo(this->read)), pckReadPackP(read));
+                    if (!this->eofFound)
-                        this->eof = true;
+                        protocolClientSessionRequestAsyncP(this->session);
-
-                        protocolClientDataEndGet(this->client);
-                    }
-
-#ifdef DEBUG
-                    this->protocolReadBytes += this->remaining;
-#endif
                 }
                 MEM_CONTEXT_TEMP_END();
             }
@@ -167,6 +144,9 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
                 {
                     bufFree(this->block);
                     this->block = NULL;
+
+                    if (this->eofFound)
+                        this->eof = true;
                 }
             }
         }
@@ -219,8 +199,7 @@ storageReadRemoteOpen(THIS_VOID)
                 compressFilterP(compressTypeGz, (int)this->interface.compressLevel, .raw = true));
         }
 
-        ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ);
+        PackWrite *const param = protocolPackNew();
-        PackWrite *const param = protocolCommandParam(command);
 
         pckWriteStrP(param, this->interface.name);
         pckWriteBoolP(param, this->interface.ignoreMissing);
@@ -233,10 +212,9 @@ storageReadRemoteOpen(THIS_VOID)
 
         pckWritePackP(param, ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read))));
 
-        protocolClientCommandPut(this->client, command, false);
-
         // If the file exists
-        result = pckReadBoolP(protocolClientDataGet(this->client));
+        PackRead *const packRead = protocolClientSessionOpenP(this->session, .param = param);
+        result = pckReadBoolP(packRead);
 
         if (result)
         {
@@ -247,12 +225,9 @@ storageReadRemoteOpen(THIS_VOID)
             if (this->interface.compressible)
                 ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilterP(compressTypeGz, .raw = true));
 
-            // Set free callback to ensure the protocol is cleared on a short read
+            // Read the first block or eof
-            memContextCallbackSet(objMemContext(this), storageReadRemoteFreeResource, this);
+            storageReadRemoteInternal(this, packRead);
         }
-        // Else nothing to do
-        else
-            protocolClientDataEndGet(this->client);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -274,8 +249,7 @@ storageReadRemoteClose(THIS_VOID)
 
     ASSERT(this != NULL);
 
-    memContextCallbackClear(objMemContext(this));
+    protocolClientSessionCancel(this->session);
-    storageReadRemoteFreeResource(this);
 
     FUNCTION_LOG_RETURN_VOID();
 }
@@ -307,6 +281,7 @@ storageReadRemoteNew(
         {
             .storage = storage,
             .client = client,
+            .session = protocolClientSessionNewP(client, PROTOCOL_COMMAND_STORAGE_READ, .async = true),
 
             .interface = (StorageReadInterface)
             {

src/storage/remote/storage.c

@@ -136,18 +136,14 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, Storage
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO);
+        PackWrite *const commandParam = protocolPackNew();
-        PackWrite *const commandParam = protocolCommandParam(command);
 
         pckWriteStrP(commandParam, file);
         pckWriteU32P(commandParam, level);
         pckWriteBoolP(commandParam, param.followLink);
 
-        // Put command
-        protocolClientCommandPut(this->client, command, false);
-
         // Read info from protocol
-        PackRead *read = protocolClientDataGet(this->client);
+        PackRead *const read = protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_INFO, .param = commandParam);
 
         result.exists = pckReadBoolP(read);
 
@@ -162,8 +158,6 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, Storage
             }
             MEM_CONTEXT_PRIOR_END();
         }
-
-        protocolClientDataEndGet(this->client);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -190,14 +184,13 @@ storageRemoteLinkCreate(
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        ProtocolCommand *const command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_LINK_CREATE);
+        PackWrite *const commandParam = protocolPackNew();
-        PackWrite *const commandParam = protocolCommandParam(command);
 
         pckWriteStrP(commandParam, target);
         pckWriteStrP(commandParam, linkPath);
         pckWriteU32P(commandParam, param.linkType);
 
-        protocolClientExecute(this->client, command, false);
+        protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_LINK_CREATE, .param = commandParam);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -224,45 +217,39 @@ storageRemoteList(THIS_VOID, const String *const path, const StorageInfoLevel le
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_LIST);
+        PackWrite *const commandParam = protocolPackNew();
-        PackWrite *const commandParam = protocolCommandParam(command);
 
         pckWriteStrP(commandParam, path);
         pckWriteU32P(commandParam, level);
 
-        // Put command
-        protocolClientCommandPut(this->client, command, false);
-
         // Read list
         StorageRemoteInfoData parseData = {.memContext = memContextCurrent()};
-        result = storageLstNew(level);
+        PackRead *const read = protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_LIST, .param = commandParam);
 
-        MEM_CONTEXT_TEMP_RESET_BEGIN()
+        if (pckReadBoolP(read))
         {
-            PackRead *read = protocolClientDataGet(this->client);
+            result = storageLstNew(level);
-            pckReadNext(read);
 
-            while (pckReadType(read) == pckTypeStr)
+            MEM_CONTEXT_TEMP_RESET_BEGIN()
             {
-                StorageInfo info = {.exists = true, .level = level, .name = pckReadStrP(read)};
+                while (pckReadNext(read))
+                {
+                    pckReadObjBeginP(read);
 
-                storageRemoteInfoGet(&parseData, read, &info);
+                    StorageInfo info = {.exists = true, .level = level, .name = pckReadStrP(read)};
-                storageLstAdd(result, &info);
 
-                // Reset the memory context occasionally so we don't use too much memory or slow down processing
+                    storageRemoteInfoGet(&parseData, read, &info);
-                MEM_CONTEXT_TEMP_RESET(1000);
+                    storageLstAdd(result, &info);
+                    pckReadObjEndP(read);
 
-                read = protocolClientDataGet(this->client);
+                    // Reset the memory context occasionally so we don't use too much memory or slow down processing
-                pckReadNext(read);
+                    MEM_CONTEXT_TEMP_RESET(1000);
+                }
             }
+            MEM_CONTEXT_TEMP_END();
 
-            if (!pckReadBoolP(read))
+            storageLstMove(result, memContextPrior());
-                result = NULL;
         }
-        MEM_CONTEXT_TEMP_END();
-
-        protocolClientDataEndGet(this->client);
-        storageLstMove(result, memContextPrior());
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -349,15 +336,14 @@ storageRemotePathCreate(
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_CREATE);
+        PackWrite *const commandParam = protocolPackNew();
-        PackWrite *const commandParam = protocolCommandParam(command);
 
         pckWriteStrP(commandParam, path);
         pckWriteBoolP(commandParam, errorOnExists);
         pckWriteBoolP(commandParam, noParentCreate);
         pckWriteModeP(commandParam, mode);
 
-        protocolClientExecute(this->client, command, false);
+        protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_PATH_CREATE, .param = commandParam);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -384,13 +370,12 @@ storageRemotePathRemove(THIS_VOID, const String *path, bool recurse, StorageInte
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE);
+        PackWrite *const commandParam = protocolPackNew();
-        PackWrite *const commandParam = protocolCommandParam(command);
 
         pckWriteStrP(commandParam, path);
         pckWriteBoolP(commandParam, recurse);
 
-        result = pckReadBoolP(protocolClientExecute(this->client, command, true));
+        result = pckReadBoolP(protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE, .param = commandParam));
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -414,10 +399,11 @@ storageRemotePathSync(THIS_VOID, const String *path, StorageInterfacePathSyncPar
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_SYNC);
+        PackWrite *const commandParam = protocolPackNew();
-        pckWriteStrP(protocolCommandParam(command), path);
 
-        protocolClientExecute(this->client, command, false);
+        pckWriteStrP(commandParam, path);
+
+        protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_PATH_SYNC, .param = commandParam);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -441,13 +427,12 @@ storageRemoteRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam p
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_REMOVE);
+        PackWrite *const commandParam = protocolPackNew();
-        PackWrite *const commandParam = protocolCommandParam(command);
 
         pckWriteStrP(commandParam, file);
         pckWriteBoolP(commandParam, param.errorOnMissing);
 
-        protocolClientExecute(this->client, command, false);
+        protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_REMOVE, .param = commandParam);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -501,7 +486,7 @@ storageRemoteNew(
         MEM_CONTEXT_TEMP_BEGIN()
         {
             // Execute command and get result
-            PackRead *result = protocolClientExecute(this->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE), true);
+            PackRead *const result = protocolClientRequestP(this->client, PROTOCOL_COMMAND_STORAGE_FEATURE);
 
             // Get path in parent context
             MEM_CONTEXT_PRIOR_BEGIN()

src/storage/remote/write.c

@@ -22,6 +22,7 @@ typedef struct StorageWriteRemote
     StorageRemote *storage;                                         // Storage that created this object
     StorageWrite *write;                                            // Storage write interface
     ProtocolClient *client;                                         // Protocol client to make requests with
+    ProtocolClientSession *session;                                 // Protocol session for requests
 
 #ifdef DEBUG
     uint64_t protocolWriteBytes;                                    // How many bytes were written to the protocol layer?
@@ -36,30 +37,6 @@ Macros for function logging
 #define FUNCTION_LOG_STORAGE_WRITE_REMOTE_FORMAT(value, buffer, bufferSize)                                                        \
     objNameToLog(value, "StorageWriteRemote", buffer, bufferSize)
 
-/***********************************************************************************************************************************
-Close file on the remote
-***********************************************************************************************************************************/
-static void
-storageWriteRemoteFreeResource(THIS_VOID)
-{
-    THIS(StorageWriteRemote);
-
-    FUNCTION_LOG_BEGIN(logLevelTrace);
-        FUNCTION_LOG_PARAM(STORAGE_WRITE_REMOTE, this);
-    FUNCTION_LOG_END();
-
-    ASSERT(this != NULL);
-
-    PackWrite *const write = protocolPackNew();
-    protocolClientDataPut(this->client, pckWriteBoolP(write, false));
-    pckWriteFree(write);
-
-    protocolClientDataPut(this->client, NULL);
-    protocolClientDataEndGet(this->client);
-
-    FUNCTION_LOG_RETURN_VOID();
-}
-
 /***********************************************************************************************************************************
 Open the file
 ***********************************************************************************************************************************/
@@ -80,8 +57,7 @@ storageWriteRemoteOpen(THIS_VOID)
         if (this->interface.compressible)
             ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilterP(compressTypeGz));
 
-        ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE);
+        PackWrite *const param = protocolPackNew();
-        PackWrite *const param = protocolCommandParam(command);
 
         pckWriteStrP(param, this->interface.name);
         pckWriteModeP(param, this->interface.modeFile);
@@ -95,8 +71,7 @@ storageWriteRemoteOpen(THIS_VOID)
         pckWriteBoolP(param, this->interface.atomic);
         pckWritePackP(param, ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write))));
 
-        protocolClientCommandPut(this->client, command, true);
+        protocolClientSessionOpenP(this->session, .param = param);
-        protocolClientDataGet(this->client);
 
         // Clear filters since they will be run on the remote side
         ioFilterGroupClear(ioWriteFilterGroup(storageWriteIo(this->write)));
@@ -108,9 +83,6 @@ storageWriteRemoteOpen(THIS_VOID)
                 ioWriteFilterGroup(storageWriteIo(this->write)),
                 compressFilterP(compressTypeGz, (int)this->interface.compressLevel));
         }
-
-        // Set free callback to ensure remote file is freed
-        memContextCallbackSet(objMemContext(this), storageWriteRemoteFreeResource, this);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -121,7 +93,7 @@ storageWriteRemoteOpen(THIS_VOID)
 Write to the file
 ***********************************************************************************************************************************/
 static void
-storageWriteRemote(THIS_VOID, const Buffer *buffer)
+storageWriteRemote(THIS_VOID, const Buffer *const buffer)
 {
     THIS(StorageWriteRemote);
 
@@ -135,8 +107,13 @@ storageWriteRemote(THIS_VOID, const Buffer *buffer)
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        protocolClientDataPut(
+        if (protocolClientSessionQueued(this->session))
-            this->client, pckWriteBinP(pckWriteNewP(.size = ioBufferSize() + PROTOCOL_PACK_DEFAULT_SIZE), buffer));
+            protocolClientSessionResponse(this->session);
+
+        PackWrite *const param = pckWriteNewP(.size = PROTOCOL_PACK_DEFAULT_SIZE + bufUsed(buffer));
+        pckWriteBinP(param, buffer);
+
+        protocolClientSessionRequestAsyncP(this->session, .param = param);
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -166,15 +143,15 @@ storageWriteRemoteClose(THIS_VOID)
     {
         MEM_CONTEXT_TEMP_BEGIN()
         {
-            protocolClientDataPut(this->client, NULL);
+            if (protocolClientSessionQueued(this->session))
+                protocolClientSessionResponse(this->session);
+
             ioFilterGroupResultAllSet(
-                ioWriteFilterGroup(storageWriteIo(this->write)), pckReadPackP(protocolClientDataGet(this->client)));
+                ioWriteFilterGroup(storageWriteIo(this->write)), pckReadPackP(protocolClientSessionClose(this->session)));
-            protocolClientDataEndGet(this->client);
         }
         MEM_CONTEXT_TEMP_END();
 
         this->client = NULL;
-        memContextCallbackClear(objMemContext(this));
     }
 
     FUNCTION_LOG_RETURN_VOID();
@@ -215,6 +192,7 @@ storageWriteRemoteNew(
         {
             .storage = storage,
             .client = client,
+            .session = protocolClientSessionNewP(client, PROTOCOL_COMMAND_STORAGE_WRITE, .async = true),
 
             .interface = (StorageWriteInterface)
             {

test/code-count/file-type.yaml

@@ -2175,14 +2175,6 @@ src/protocol/client.h:
   class: core
   type: c/h
 
-src/protocol/command.c:
-  class: core
-  type: c
-
-src/protocol/command.h:
-  class: core
-  type: c/h
-
 src/protocol/helper.c:
   class: core
   type: c

test/define.yaml

@@ -453,7 +453,6 @@ unit:
           - postgres/interface/crc32
           - postgres/interface/page
           - protocol/client
-          - protocol/command
           - protocol/helper
           - protocol/server
           - storage/helper
@@ -481,7 +480,6 @@ unit:
 
         coverage:
           - protocol/client
-          - protocol/command
           - protocol/helper
           - protocol/parallel
           - protocol/parallelJob
@@ -562,6 +560,7 @@ unit:
           - storage/storage
 
         include:
+          - protocol/client
           - storage/read
           - storage/write
 

test/src/common/harnessDebug.h

@@ -52,6 +52,14 @@ C Debug Harness
     }                                                                                                                              \
     while (0)
 
+#define FUNCTION_HARNESS_RETURN_STRUCT(...)                                                                                        \
+    do                                                                                                                             \
+    {                                                                                                                              \
+        STACK_TRACE_POP(false);                                                                                                    \
+        return __VA_ARGS__;                                                                                                        \
+    }                                                                                                                              \
+    while (0)
+
 #define FUNCTION_HARNESS_RETURN_VOID()                                                                                             \
         STACK_TRACE_POP(false);
 

test/src/common/harnessProtocol.c

@@ -235,7 +235,7 @@ protocolRemoteExec(
             protocolServerProcess(server, NULL, hrnProtocolStatic.remoteHandlerList, hrnProtocolStatic.remoteHandlerListSize);
 
             // Put an end message here to sync with the client to ensure that coverage data is written before exiting
-            protocolServerDataEndPut(server);
+            protocolServerResponseP(server);
 
             // Exit when done
             exit(0);

test/src/module/command/backupTest.c

@@ -1823,7 +1823,7 @@ testRun(void)
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("report job error");
 
-        ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("key"), protocolCommandNew(strIdFromZ("x")));
+        ProtocolParallelJob *job = protocolParallelJobNew(VARSTRDEF("key"), strIdFromZ("x"), NULL);
         protocolParallelJobErrorSet(job, errorTypeCode(&AssertError), STRDEF("error message"));
 
         unsigned int currentPercentComplete = 0;
@@ -1837,7 +1837,7 @@ testRun(void)
         TEST_TITLE("report host/100% progress on noop result");
 
         // Create job that skips file
-        job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), protocolCommandNew(strIdFromZ("x")));
+        job = protocolParallelJobNew(VARSTRDEF("pg_data/test"), strIdFromZ("x"), NULL);
 
         PackWrite *const resultPack = protocolPackNew();
         pckWriteStrP(resultPack, STRDEF("pg_data/test"));

test/src/module/db/dbTest.c

@@ -151,7 +151,6 @@ testRun(void)
                     TRY_BEGIN()
                     {
                         TEST_RESULT_VOID(dbOpen(db), "open db");
-                        TEST_RESULT_UINT(db->remoteIdx, 0, "check remote idx");
                         TEST_RESULT_VOID(dbFree(db), "free db");
                         db = NULL;
                     }
@@ -170,7 +169,6 @@ testRun(void)
                     TRY_BEGIN()
                     {
                         TEST_RESULT_VOID(dbOpen(db), "open db");
-                        TEST_RESULT_UINT(db->remoteIdx, 1, "check idx");
                         TEST_RESULT_STR_Z(dbWalSwitch(db), "000000030000000200000003", "wal switch");
                         TEST_RESULT_UINT(dbDbTimeout(db), 777000, "check timeout");
                         TEST_RESULT_VOID(memContextCallbackClear(db->pub.memContext), "clear context so close is not called");

test/src/module/postgres/clientTest.c

@@ -497,8 +497,7 @@ testRun(void)
             {.function = HRN_PQ_FINISH},
             {.function = HRN_PQ_GETRESULT, .resultNull = true});
 #endif
-        TEST_RESULT_VOID(pgClientClose(client), "close client");
+        TEST_RESULT_VOID(pgClientFree(client), "free client");
-        TEST_RESULT_VOID(pgClientClose(client), "close client again");
     }
 
     FUNCTION_HARNESS_RETURN_VOID();

test/src/module/protocol/protocolTest.c

@@ -21,113 +21,150 @@ Test protocol server command handlers
 ***********************************************************************************************************************************/
 #define TEST_PROTOCOL_COMMAND_ASSERT                                STRID5("assert", 0x2922ce610)
 
-__attribute__((__noreturn__)) static void
+__attribute__((__noreturn__)) static ProtocolServerResult *
-testCommandAssertProtocol(PackRead *const param, ProtocolServer *const server)
+testCommandAssertProtocol(PackRead *const param)
 {
     FUNCTION_HARNESS_BEGIN();
         FUNCTION_HARNESS_PARAM(PACK_READ, param);
-        FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_HARNESS_END();
 
     ASSERT(param == NULL);
-    ASSERT(server != NULL);
 
     hrnErrorThrowP();
 
-    // No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
+    // No FUNCTION_HARNESS_RETURN() because the function does not return
 }
 
 #define TEST_PROTOCOL_COMMAND_ERROR                                 STRID5("error", 0x127ca450)
 
 static unsigned int testCommandErrorProtocolTotal = 0;
 
-__attribute__((__noreturn__)) static void
+__attribute__((__noreturn__)) static ProtocolServerResult *
-testCommandErrorProtocol(PackRead *const param, ProtocolServer *const server)
+testCommandErrorProtocol(PackRead *const param)
 {
     FUNCTION_HARNESS_BEGIN();
         FUNCTION_HARNESS_PARAM(PACK_READ, param);
-        FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_HARNESS_END();
 
     ASSERT(param == NULL);
-    ASSERT(server != NULL);
 
     testCommandErrorProtocolTotal++;
     hrnErrorThrowP(.errorType = &FormatError, .message = testCommandErrorProtocolTotal <= 2 ? NULL : "ERR_MESSAGE_RETRY");
 
-    // No FUNCTION_HARNESS_RETURN_VOID() because the function does not return
+    // No FUNCTION_HARNESS_RETURN() because the function does not return
 }
 
 #define TEST_PROTOCOL_COMMAND_SIMPLE                                STRID5("c-simple", 0x2b20d4cf630)
 
-static void
+static ProtocolServerResult *
-testCommandRequestSimpleProtocol(PackRead *const param, ProtocolServer *const server)
+testCommandRequestSimpleProtocol(PackRead *const param)
 {
     FUNCTION_HARNESS_BEGIN();
         FUNCTION_HARNESS_PARAM(PACK_READ, param);
-        FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_HARNESS_END();
 
-    ASSERT(param == NULL);
+    ProtocolServerResult *const result = param != NULL ? protocolServerResultNewP() : NULL;
-    ASSERT(server != NULL);
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        protocolServerDataPut(server, pckWriteStrP(protocolPackNew(), STRDEF("output")));
+        if (param != NULL)
-        protocolServerDataEndPut(server);
+            pckWriteStrP(protocolServerResultData(result), strNewFmt("output%u", pckReadU32P(param)));
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_HARNESS_RETURN_VOID();
+    FUNCTION_HARNESS_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
 #define TEST_PROTOCOL_COMMAND_COMPLEX                               STRID5("c-complex", 0x182b20d78f630)
+#define TEST_PROTOCOL_COMMAND_COMPLEX_CLOSE                         STRID5("c-complex-c", 0xf782b20d78f630)
 
-static void
+static bool testCommandRequestComplexOpenReturn = false;
-testCommandRequestComplexProtocol(PackRead *const param, ProtocolServer *const server)
+
+static ProtocolServerResult *
+testCommandRequestComplexOpenProtocol(PackRead *const param)
 {
     FUNCTION_HARNESS_BEGIN();
         FUNCTION_HARNESS_PARAM(PACK_READ, param);
-        FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_HARNESS_END();
 
     ASSERT(param != NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
         TEST_RESULT_UINT(pckReadU32P(param), 87, "param check");
         TEST_RESULT_STR_Z(pckReadStrP(param), "data", "param check");
 
-        TEST_RESULT_VOID(protocolServerDataPut(server, NULL), "sync");
+        pckWriteBoolP(protocolServerResultData(result), testCommandRequestComplexOpenReturn);
 
-        TEST_RESULT_BOOL(pckReadBoolP(protocolServerDataGet(server)), true, "data get");
+        if (testCommandRequestComplexOpenReturn)
-        TEST_RESULT_UINT(pckReadModeP(protocolServerDataGet(server)), 0644, "data get");
+            protocolServerResultSessionDataSet(result, strNewZ("DATA"));
-        TEST_RESULT_PTR(protocolServerDataGet(server), NULL, "data end get");
-
-        TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true)), "data put");
-        TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteI32P(protocolPackNew(), -1)), "data put");
-        TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_HARNESS_RETURN_VOID();
+    testCommandRequestComplexOpenReturn = true;
+
+    FUNCTION_HARNESS_RETURN(PROTOCOL_SERVER_RESULT, result);
+}
+
+static bool testCommandRequestComplexReturn = false;
+
+static ProtocolServerResult *
+testCommandRequestComplexProtocol(PackRead *const param, void *const data)
+{
+    FUNCTION_HARNESS_BEGIN();
+        FUNCTION_HARNESS_PARAM(PACK_READ, param);
+        FUNCTION_HARNESS_PARAM(STRING, data);
+    FUNCTION_HARNESS_END();
+
+    ASSERT(param == NULL);
+    ASSERT(data != NULL);
+    ASSERT(strEqZ(data, "DATA"));
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
+    pckWriteBoolP(protocolServerResultData(result), testCommandRequestComplexReturn);
+
+    if (!testCommandRequestComplexReturn)
+        protocolServerResultCloseSet(result);
+
+    testCommandRequestComplexReturn = true;
+
+    FUNCTION_HARNESS_RETURN_STRUCT(result);
+}
+
+static ProtocolServerResult *
+testCommandRequestComplexCloseProtocol(PackRead *const param, void *const data)
+{
+    FUNCTION_HARNESS_BEGIN();
+        FUNCTION_HARNESS_PARAM(PACK_READ, param);
+        FUNCTION_HARNESS_PARAM(STRING, data);
+    FUNCTION_HARNESS_END();
+
+    ASSERT(param == NULL);
+    ASSERT(data != NULL);
+    ASSERT(strEqZ(data, "DATA"));
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
+    pckWriteBoolP(protocolServerResultData(result), true);
+
+    FUNCTION_HARNESS_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
 #define TEST_PROTOCOL_COMMAND_RETRY                                 STRID5("retry", 0x19950b20)
 
 static unsigned int testCommandRetryTotal = 1;
 
-static void
+static ProtocolServerResult *
-testCommandRetryProtocol(PackRead *const param, ProtocolServer *const server)
+testCommandRetryProtocol(PackRead *const param)
 {
     FUNCTION_HARNESS_BEGIN();
         FUNCTION_HARNESS_PARAM(PACK_READ, param);
-        FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
     FUNCTION_HARNESS_END();
 
     ASSERT(param == NULL);
-    ASSERT(server != NULL);
+
+    ProtocolServerResult *const result = protocolServerResultNewP();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
@@ -137,20 +174,22 @@ testCommandRetryProtocol(PackRead *const param, ProtocolServer *const server)
             THROW(FormatError, "error-until-0");
         }
 
-        protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), true));
+        pckWriteBoolP(protocolServerResultData(result), true);
-        protocolServerDataEndPut(server);
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_HARNESS_RETURN_VOID();
+    FUNCTION_HARNESS_RETURN(PROTOCOL_SERVER_RESULT, result);
 }
 
 #define TEST_PROTOCOL_SERVER_HANDLER_LIST                                                                                          \
-    {.command = TEST_PROTOCOL_COMMAND_ASSERT, .handler = testCommandAssertProtocol},                                               \
+    {.command = TEST_PROTOCOL_COMMAND_ASSERT, .process = testCommandAssertProtocol},                                               \
-    {.command = TEST_PROTOCOL_COMMAND_ERROR, .handler = testCommandErrorProtocol},                                                 \
+    {.command = TEST_PROTOCOL_COMMAND_ERROR, .process = testCommandErrorProtocol},                                                 \
-    {.command = TEST_PROTOCOL_COMMAND_SIMPLE, .handler = testCommandRequestSimpleProtocol},                                        \
+    {.command = TEST_PROTOCOL_COMMAND_SIMPLE, .process = testCommandRequestSimpleProtocol},                                        \
-    {.command = TEST_PROTOCOL_COMMAND_COMPLEX, .handler = testCommandRequestComplexProtocol},                                      \
+    {.command = TEST_PROTOCOL_COMMAND_COMPLEX, .open = testCommandRequestComplexOpenProtocol,                                      \
-    {.command = TEST_PROTOCOL_COMMAND_RETRY, .handler = testCommandRetryProtocol},
+        .processSession = testCommandRequestComplexProtocol},                                                                      \
+    {.command = TEST_PROTOCOL_COMMAND_COMPLEX_CLOSE, .open = testCommandRequestComplexOpenProtocol,                                \
+        .processSession = testCommandRequestComplexProtocol, .close = testCommandRequestComplexCloseProtocol},                     \
+    {.command = TEST_PROTOCOL_COMMAND_RETRY, .process = testCommandRetryProtocol},
 
 /***********************************************************************************************************************************
 Test ParallelJobCallback
@@ -290,6 +329,7 @@ testRun(void)
                 .name = strNewZ("test"),
                 .state = protocolClientStateIdle,
                 .write = write,
+                .sessionList = lstNewP(sizeof(ProtocolClientSession)),
             };
 
             memContextCallbackSet(memContextCurrent(), protocolClientFreeResource, protocolHelperClient.client);
@@ -527,7 +567,7 @@ testRun(void)
 
                 TEST_ERROR(
                     protocolServerProcess(server, NULL, commandHandler, LENGTH_OF(commandHandler)), ProtocolError,
-                    "invalid command 'BOGUS' (0x38eacd271)");
+                    "invalid request 'BOGUS' (0x38eacd271)");
 
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("server restart and assert");
@@ -615,24 +655,24 @@ testRun(void)
                 TEST_TITLE("invalid command");
 
                 TEST_ERROR(
-                    protocolClientExecute(client, protocolCommandNew(strIdFromZ("BOGUS")), false), ProtocolError,
+                    protocolClientRequestP(client, strIdFromZ("BOGUS")), ProtocolError,
-                    "raised from test client: invalid command 'BOGUS' (0x38eacd271)");
+                    "raised from test client: invalid request 'BOGUS' (0x38eacd271)");
 
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("command throws assert");
 
                 TRY_BEGIN()
                 {
-                    protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ASSERT), false);
+                    protocolClientRequestP(client, TEST_PROTOCOL_COMMAND_ASSERT);
                     THROW(TestError, "error was expected");
                 }
                 CATCH_FATAL()
                 {
+                    TEST_RESULT_Z(errorMessage(), "raised from test client: ERR_MESSAGE", "check message");
                     TEST_RESULT_PTR(errorType(), &AssertError, "check type");
                     TEST_RESULT_Z(errorFileName(), TEST_PGB_PATH "/src/protocol/client.c", "check file");
                     TEST_RESULT_Z(errorFunctionName(), "protocolClientError", "check function");
                     TEST_RESULT_BOOL(errorFileLine() > 0, true, "check file line > 0");
-                    TEST_RESULT_Z(errorMessage(), "raised from test client: ERR_MESSAGE", "check message");
                     TEST_RESULT_Z(errorStackTrace(), "ERR_STACK_TRACE", "check stack trace");
                 }
                 TRY_END();
@@ -645,36 +685,141 @@ testRun(void)
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("simple command");
 
+                PackWrite *commandParam = protocolPackNew();
+                pckWriteU32P(commandParam, 99);
+
                 TEST_RESULT_STR_Z(
-                    pckReadStrP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_SIMPLE), true)), "output",
+                    pckReadStrP(protocolClientRequestP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .param = commandParam)), "output99",
                     "execute");
 
                 // -----------------------------------------------------------------------------------------------------------------
-                TEST_TITLE("complex command");
+                TEST_TITLE("simple command with out of order results");
 
-                // Put the command to the server
+                ProtocolClientSession *session1 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .async = true);
-                ProtocolCommand *command = NULL;
+                PackWrite *param1 = protocolPackNew();
-                TEST_ASSIGN(command, protocolCommandNew(TEST_PROTOCOL_COMMAND_COMPLEX), "command");
+                pckWriteU32P(param1, 1);
-                TEST_RESULT_VOID(pckWriteU32P(protocolCommandParam(command), 87), "param");
+                protocolClientSessionRequestAsyncP(session1, .param = param1);
-                TEST_RESULT_VOID(pckWriteStrP(protocolCommandParam(command), STRDEF("data")), "param");
+
-                TEST_RESULT_VOID(protocolClientCommandPut(client, command, true), "command put");
+                ProtocolClientSession *session2 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .async = true);
+                protocolClientSessionRequestAsyncP(session2);
+
+                ProtocolClientSession *session3 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE);
+                PackWrite *param3 = protocolPackNew();
+                pckWriteU32P(param3, 3);
+
+                TEST_RESULT_STR_Z(pckReadStrP(protocolClientSessionRequestP(session3, .param = param3)), "output3", "output 3");
+
+                ProtocolClientSession *session4 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .async = true);
+                PackWrite *param4 = protocolPackNew();
+                pckWriteU32P(param4, 4);
+                protocolClientSessionRequestAsyncP(session4, .param = param4);
+
+                ProtocolClientSession *session5 = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_SIMPLE, .async = true);
+                PackWrite *param5 = protocolPackNew();
+                pckWriteU32P(param5, 5);
+                protocolClientSessionRequestAsyncP(session5, .param = param5);
+
+                TEST_RESULT_VOID(protocolClientSessionCancel(session2), "cancel 2");
+                TEST_RESULT_STR_Z(pckReadStrP(protocolClientSessionResponse(session1)), "output1", "output 1");
+                TEST_RESULT_VOID(protocolClientSessionCancel(session4), "cancel 4");
+                TEST_RESULT_STR_Z(pckReadStrP(protocolClientSessionResponse(session5)), "output5", "output 5");
+
+                TEST_RESULT_VOID(protocolClientSessionFree(session1), "free 1");
+                TEST_RESULT_VOID(protocolClientSessionFree(session5), "free 5");
+
+                TEST_RESULT_UINT(lstSize(client->sessionList), 0, "session list is empty");
+
+                // -----------------------------------------------------------------------------------------------------------------
+                TEST_TITLE("invalid session");
+
+                TEST_ERROR(protocolClientSessionFindIdx(client, 999), ProtocolError, "unable to find protocol client session 999");
+
+                // -----------------------------------------------------------------------------------------------------------------
+                TEST_TITLE("open returns false");
+
+                PackWrite *commandOpenParam = protocolPackNew();
+                pckWriteU32P(commandOpenParam, 87);
+                pckWriteStrP(commandOpenParam, STRDEF("data"));
+
+                TEST_RESULT_BOOL(
+                    pckReadBoolP(
+                        protocolClientSessionOpenP(
+                            protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX), .param = commandOpenParam)),
+                    false, "open request");
+
+                // -----------------------------------------------------------------------------------------------------------------
+                TEST_TITLE("process returns false (no close needed)");
+
+                ProtocolClientSession *session = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX);
+                TEST_RESULT_BOOL(
+                    pckReadBoolP(protocolClientSessionOpenP(session, .param = commandOpenParam)), true, "open succeed");
+
+                uint64_t sessionIdOld = session->sessionId;
+                session->sessionId = 9999;
 
                 TEST_ERROR(
-                    protocolClientStateExpect(client, protocolClientStateIdle), ProtocolError,
+                    protocolClientSessionRequestP(session), ProtocolError,
-                    "client state is 'cmd-data-get' but expected 'idle'");
+                    "raised from test client: unable to find session id 9999 for request c-complex:prc");
 
-                // Read null data to indicate that the server has started the command and is read to receive data
+                session->sessionId = sessionIdOld;
-                TEST_RESULT_PTR(protocolClientDataGet(client), NULL, "command started and ready for data");
 
-                // Write data to the server
+                TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionRequestP(session)), false, "no more to process");
-                TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteBoolP(protocolPackNew(), true)), "data put");
-                TEST_RESULT_VOID(protocolClientDataPut(client, pckWriteModeP(protocolPackNew(), 0644)), "data put");
-                TEST_RESULT_VOID(protocolClientDataPut(client, NULL), "data end put");
 
-                // Get data from the server
+                // -----------------------------------------------------------------------------------------------------------------
-                TEST_RESULT_BOOL(pckReadBoolP(protocolClientDataGet(client)), true, "data get");
+                TEST_TITLE("error if state not idle");
-                TEST_RESULT_INT(pckReadI32P(protocolClientDataGet(client)), -1, "data get");
+
-                TEST_RESULT_VOID(protocolClientDataEndGet(client), "data end get");
+                client->state = protocolClientStateResponse;
+                session->pub.open = true;
+
+                TEST_ERROR(
+                    protocolClientSessionRequestP(session), ProtocolError,
+                    "client state is 'response' but expected 'idle'");
+
+                client->state = protocolClientStateIdle;
+
+                // -----------------------------------------------------------------------------------------------------------------
+                TEST_TITLE("process returns true");
+
+                session = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX);
+
+                TEST_RESULT_BOOL(
+                    pckReadBoolP(protocolClientSessionOpenP(session, .param = commandOpenParam)), true, "open succeed");
+
+                TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionRequestP(session)), true, "more to process");
+                TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionRequestP(session)), true, "more to process");
+
+                TEST_RESULT_PTR(protocolClientSessionClose(session), NULL, "close request");
+
+                // -----------------------------------------------------------------------------------------------------------------
+                TEST_TITLE("close handler");
+
+                session = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX_CLOSE, .async = true);
+
+                TEST_RESULT_BOOL(
+                    pckReadBoolP(protocolClientSessionOpenP(session, .param = commandOpenParam)), true, "open succeed");
+
+                TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionRequestP(session)), true, "more to process");
+                TEST_RESULT_BOOL(pckReadBoolP(protocolClientSessionClose(session)), true, "close request");
+
+                session->pub.open = true;
+                TEST_RESULT_VOID(protocolClientRequestInternal(session, protocolCommandTypeClose, NULL), "close after close");
+                TEST_ERROR_FMT(
+                    protocolClientResponseInternal(session), ProtocolError,
+                    "raised from test client: unable to find session id %" PRIu64 " for request c-complex-c:cls",
+                    session->sessionId);
+
+                TEST_RESULT_VOID(protocolClientRequestInternal(session, protocolCommandTypeCancel, NULL), "cancel after close");
+                TEST_RESULT_VOID(protocolClientResponseInternal(session), "cancel request succeeds");
+                session->pub.open = false;
+
+                // -----------------------------------------------------------------------------------------------------------------
+                TEST_TITLE("cancel handler");
+
+                session = protocolClientSessionNewP(client, TEST_PROTOCOL_COMMAND_COMPLEX);
+
+                TEST_RESULT_BOOL(
+                    pckReadBoolP(protocolClientSessionOpenP(session, .param = commandOpenParam)), true, "open succeed");
+                TEST_RESULT_VOID(protocolClientSessionFree(session), "free request");
 
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("free client");
@@ -692,15 +837,13 @@ testRun(void)
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("command with retry");
 
-                TEST_RESULT_BOOL(
+                TEST_RESULT_BOOL(pckReadBoolP(protocolClientRequestP(client, TEST_PROTOCOL_COMMAND_RETRY)), true, "execute");
-                    pckReadBoolP(protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_RETRY), true)), true,
-                    "execute");
 
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("command throws assert with retry messages");
 
                 TEST_ERROR(
-                    protocolClientExecute(client, protocolCommandNew(TEST_PROTOCOL_COMMAND_ERROR), false), FormatError,
+                    protocolClientRequestP(client, TEST_PROTOCOL_COMMAND_ERROR), FormatError,
                     "raised from test client: ERR_MESSAGE\n"
                     "[RETRY DETAIL OMITTED]");
 
@@ -843,7 +986,7 @@ testRun(void)
 
                 TEST_ASSIGN(server, protocolServer(tlsServer, socketSession), "server start");
                 TEST_RESULT_PTR_NE(server, NULL, "server is not null");
-                TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "server exit");
+                TEST_RESULT_UINT(protocolServerRequest(server).id, PROTOCOL_COMMAND_EXIT, "server exit");
 
                 // Repo server access denied (archive-get) invalid stanza
                 // -----------------------------------------------------------------------------------------------------------------
@@ -885,7 +1028,7 @@ testRun(void)
 
                 TEST_ASSIGN(server, protocolServer(tlsServer, socketSession), "server start");
                 TEST_RESULT_PTR_NE(server, NULL, "server is not null");
-                TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "server exit");
+                TEST_RESULT_UINT(protocolServerRequest(server).id, PROTOCOL_COMMAND_EXIT, "server exit");
             }
             HRN_FORK_PARENT_END();
         }
@@ -906,7 +1049,7 @@ testRun(void)
         {
             TEST_ASSIGN(
                 job,
-                protocolParallelJobNew(VARSTRDEF("test"), protocolCommandNew(strIdFromZ("c"))), "new job");
+                protocolParallelJobNew(VARSTRDEF("test"), strIdFromZ("c"), NULL), "new job");
             TEST_RESULT_PTR(protocolParallelJobMove(job, memContextPrior()), job, "move job");
             TEST_RESULT_PTR(protocolParallelJobMove(NULL, memContextPrior()), NULL, "move null job");
         }
@@ -938,16 +1081,15 @@ testRun(void)
                     "local server 1");
 
                 // Command with output
-                TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ("c-one"), "c-one command get");
+                TEST_RESULT_UINT(protocolServerRequest(server).id, strIdFromZ("c-one"), "c-one command get");
 
                 // Wait for notify from parent
                 HRN_FORK_CHILD_NOTIFY_GET();
 
-                TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 1)), "data end put");
+                TEST_RESULT_VOID(protocolServerResponseP(server, .data = pckWriteU32P(protocolPackNew(), 1)), "data end put");
-                TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
 
                 // Wait for exit
-                TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "noop command get");
+                TEST_RESULT_UINT(protocolServerRequest(server).id, PROTOCOL_COMMAND_EXIT, "noop command get");
             }
             HRN_FORK_CHILD_END();
 
@@ -961,20 +1103,19 @@ testRun(void)
                     "local server 2");
 
                 // Command with output
-                TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ("c2"), "c2 command get");
+                TEST_RESULT_UINT(protocolServerRequest(server).id, strIdFromZ("c2"), "c2 command get");
 
                 // Wait for notify from parent
                 HRN_FORK_CHILD_NOTIFY_GET();
 
-                TEST_RESULT_VOID(protocolServerDataPut(server, pckWriteU32P(protocolPackNew(), 2)), "data end put");
+                TEST_RESULT_VOID(protocolServerResponseP(server, .data = pckWriteU32P(protocolPackNew(), 2)), "data end put");
-                TEST_RESULT_VOID(protocolServerDataEndPut(server), "data end put");
 
                 // Command with error
-                TEST_RESULT_UINT(protocolServerCommandGet(server).id, strIdFromZ("c-three"), "c-three command get");
+                TEST_RESULT_UINT(protocolServerRequest(server).id, strIdFromZ("c-three"), "c-three command get");
                 TEST_RESULT_VOID(protocolServerError(server, 39, STRDEF("very serious error"), STRDEF("stack")), "error put");
 
                 // Wait for exit
-                TEST_RESULT_UINT(protocolServerCommandGet(server).id, PROTOCOL_COMMAND_EXIT, "wait for exit");
+                TEST_RESULT_UINT(protocolServerRequest(server).id, PROTOCOL_COMMAND_EXIT, "wait for exit");
             }
             HRN_FORK_CHILD_END();
 
@@ -1018,23 +1159,26 @@ testRun(void)
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_TITLE("add jobs");
 
-                ProtocolCommand *command = protocolCommandNew(strIdFromZ("c-one"));
+                StringId command = strIdFromZ("c-one");
-                pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
+                PackWrite *param = protocolPackNew();
-                pckWriteStrP(protocolCommandParam(command), STRDEF("param2"));
+                pckWriteStrP(param, STRDEF("param1"));
+                pckWriteStrP(param, STRDEF("param2"));
 
-                ProtocolParallelJob *job = protocolParallelJobNew(varNewStr(STRDEF("job1")), command);
+                ProtocolParallelJob *job = protocolParallelJobNew(varNewStr(STRDEF("job1")), command, param);
                 TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
 
-                command = protocolCommandNew(strIdFromZ("c2"));
+                command = strIdFromZ("c2");
-                pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
+                param = protocolPackNew();
+                pckWriteStrP(param, STRDEF("param1"));
 
-                job = protocolParallelJobNew(varNewStr(STRDEF("job2")), command);
+                job = protocolParallelJobNew(varNewStr(STRDEF("job2")), command, param);
                 TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
 
-                command = protocolCommandNew(strIdFromZ("c-three"));
+                command = strIdFromZ("c-three");
-                pckWriteStrP(protocolCommandParam(command), STRDEF("param1"));
+                param = protocolPackNew();
+                pckWriteStrP(param, STRDEF("param1"));
 
-                job = protocolParallelJobNew(varNewStr(STRDEF("job3")), command);
+                job = protocolParallelJobNew(varNewStr(STRDEF("job3")), command, param);
                 TEST_RESULT_VOID(lstAdd(data.jobList, &job), "add job");
 
                 // -----------------------------------------------------------------------------------------------------------------

test/src/module/storage/remoteTest.c

@@ -325,16 +325,30 @@ testRun(void)
         ioBufferSizeSet(bufferOld);
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_TITLE("read partial file then free");
+        TEST_TITLE("read partial file then free (interleaved with normal read)");
 
         buffer = bufNew(6);
+        Buffer *buffer2 = bufNew(7);
 
+        StorageRead *fileRead2 = NULL;
         TEST_ASSIGN(fileRead, storageNewReadP(storageRepo, STRDEF("test.txt")), "get file");
+        TEST_ASSIGN(fileRead2, storageNewReadP(storageRepo, STRDEF("test.txt"), .limit = VARUINT64(11)), "get file");
+
         TEST_RESULT_BOOL(ioReadOpen(storageReadIo(fileRead)), true, "open read");
+        TEST_RESULT_BOOL(ioReadOpen(storageReadIo(fileRead2)), true, "open read file 2");
+
         TEST_RESULT_UINT(ioRead(storageReadIo(fileRead), buffer), 6, "partial read");
         TEST_RESULT_STR_Z(strNewBuf(buffer), "BABABA", "check contents");
         TEST_RESULT_BOOL(ioReadEof(storageReadIo(fileRead)), false, "no eof");
+
+        TEST_RESULT_UINT(ioRead(storageReadIo(fileRead2), buffer2), 7, "partial read file 2");
+        TEST_RESULT_STR_Z(strNewBuf(buffer2), "BABABAB", "check contents");
+        bufUsedZero(buffer2);
+        TEST_RESULT_UINT(ioRead(storageReadIo(fileRead2), buffer2), 4, "partial read file 2");
+        TEST_RESULT_STR_Z(strNewBuf(buffer2), "ABAB", "check contents");
+
         TEST_RESULT_VOID(storageReadFree(fileRead), "free");
+        TEST_RESULT_VOID(ioReadClose(storageReadIo(fileRead2)), "close");
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("read file with compression");
@@ -452,17 +466,35 @@ testRun(void)
         ((StorageRemote *)storageDriver(storageRepoWrite))->compressLevel = 3;
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_TITLE("write file, free before close, make sure the .tmp file remains");
+        TEST_TITLE("write file, free before close, make sure the .tmp file remains (interleaved with normal write)");
 
+        StorageWrite *write3 = NULL;
         TEST_ASSIGN(write, storageNewWriteP(storageRepoWrite, STRDEF("test2.txt")), "new write file");
+        TEST_ASSIGN(write3, storageNewWriteP(storageRepoWrite, STRDEF("test3.txt")), "new write file");
 
         TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(write)), "open file");
+        TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(write3)), "open file 3");
+
         TEST_RESULT_VOID(ioWrite(storageWriteIo(write), contentBuf), "write bytes");
+        TEST_RESULT_VOID(ioWrite(storageWriteIo(write3), contentBuf), "write bytes to file 3");
 
         TEST_RESULT_VOID(storageWriteFree(write), "free file");
+        TEST_RESULT_VOID(ioWriteClose(storageWriteIo(write3)), "close file 3");
 
         TEST_RESULT_UINT(
             storageInfoP(storageTest, STRDEF("repo128/test2.txt.pgbackrest.tmp")).size, 16384, "file exists and is partial");
+        TEST_RESULT_BOOL(bufEq(storageGetP(storageNewReadP(storageRepo, STRDEF("test3.txt"))), contentBuf), true, "check file 3");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("write free and close before write");
+
+        TEST_ASSIGN(write, storageNewWriteP(storageRepoWrite, STRDEF("test2.txt")), "new write file");
+        TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(write)), "open file");
+        TEST_RESULT_VOID(storageWriteFree(write), "free file");
+
+        TEST_ASSIGN(write, storageNewWriteP(storageRepoWrite, STRDEF("test2.txt")), "new write file");
+        TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(write)), "open file");
+        TEST_RESULT_VOID(ioWriteClose(storageWriteIo(write)), "close file");
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("write the file again with protocol compression");
@@ -711,10 +743,23 @@ testRun(void)
     // sure coverage data has been written by the remote. We also need to make sure that the mem context callback is cleared so that
     // protocolClientFreeResource() will not be called and send another exit. protocolFree() is still required to free the client
     // objects.
-    memContextCallbackClear(objMemContext(protocolRemoteGet(protocolStorageTypeRepo, 0)));
+    ProtocolClient *client = protocolRemoteGet(protocolStorageTypeRepo, 0);
-    protocolClientExecute(protocolRemoteGet(protocolStorageTypeRepo, 0), protocolCommandNew(PROTOCOL_COMMAND_EXIT), false);
+
-    memContextCallbackClear(objMemContext(protocolRemoteGet(protocolStorageTypePg, 1)));
+    memContextCallbackClear(objMemContext(client));
-    protocolClientExecute(protocolRemoteGet(protocolStorageTypePg, 1), protocolCommandNew(PROTOCOL_COMMAND_EXIT), false);
+
+    for (unsigned int sessionIdx = 0; sessionIdx < lstSize(client->sessionList); sessionIdx++)
+        memContextCallbackClear(objMemContext(*(ProtocolClientSession **)lstGet(client->sessionList, sessionIdx)));
+
+    protocolClientRequestP(client, PROTOCOL_COMMAND_EXIT);
+
+    client = protocolRemoteGet(protocolStorageTypePg, 1);
+
+    for (unsigned int sessionIdx = 0; sessionIdx < lstSize(client->sessionList); sessionIdx++)
+        memContextCallbackClear(objMemContext(*(ProtocolClientSession **)lstGet(client->sessionList, sessionIdx)));
+
+    memContextCallbackClear(objMemContext(client));
+    protocolClientRequestP(client, PROTOCOL_COMMAND_EXIT);
+
     protocolFree();
 
     FUNCTION_HARNESS_RETURN_VOID();