Migrate remote command to C.

Prior to this the Perl remote was used to satisfy C requests. This worked fine but since the remote needed to be migrated to C anyway there was no reason to wait.

Add the ProtocolServer object and tweak ProtocolClient to work with it. It was also necessary to add a mechanism to get option values from the remote so that encryption settings could be read and used in the storage object.

Update the remote storage objects to comply with the protocol changes and add the storage protocol handler.

Ideally this commit would have been broken up into smaller chunks but there are cross-dependencies in the protocol layer and it didn't seem worth the extra effort.

doc/xml/release.xml

@@ -25,6 +25,10 @@
                 </release-bug-list>
 
                 <release-development-list>
+                    <release-item>
+                        <p>Migrate <cmd>remote</cmd> command to C.</p>
+                    </release-item>
+
                     <release-item>
                         <release-item-contributor-list>
                             <release-item-contributor id="stephen.frost"/>

src/Makefile

@@ -65,6 +65,7 @@ SRCS = \
 	command/info/info.c \
 	command/command.c \
 	command/control/control.c \
+	command/remote/remote.c \
 	common/debug.c \
 	common/encode.c \
 	common/encode/base64.c \
@@ -114,6 +115,7 @@ SRCS = \
 	config/exec.c \
 	config/load.c \
 	config/parse.c \
+	config/protocol.c \
 	crypto/cipherBlock.c \
 	crypto/hash.c \
 	crypto/crypto.c \
@@ -137,6 +139,7 @@ SRCS = \
 	postgres/interface/v100.c \
 	postgres/interface/v110.c \
 	postgres/pageChecksum.c \
+	protocol/server.c \
 	protocol/client.c \
 	protocol/helper.c \
 	storage/driver/posix/storage.c \
@@ -144,6 +147,7 @@ SRCS = \
 	storage/driver/posix/fileRead.c \
 	storage/driver/posix/fileWrite.c \
 	storage/driver/remote/fileRead.c \
+	storage/driver/remote/protocol.c \
 	storage/driver/remote/storage.c \
 	storage/driver/s3/fileRead.c \
 	storage/driver/s3/storage.c \
@@ -196,6 +200,9 @@ command/help/help.o: command/help/help.c common/assert.h common/debug.h common/e
 command/info/info.o: command/info/info.c command/archive/common.h command/info/info.h common/assert.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h crypto/crypto.h crypto/hash.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h perl/exec.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h
 	$(CC) $(CFLAGS) -c command/info/info.c -o command/info/info.o
 
+command/remote/remote.o: command/remote/remote.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/helper.h protocol/server.h storage/driver/remote/protocol.h
+	$(CC) $(CFLAGS) -c command/remote/remote.c -o command/remote/remote.o
+
 common/debug.o: common/debug.c common/assert.h common/debug.h common/error.auto.h common/error.h common/logLevel.h common/stackTrace.h common/type/convert.h
 	$(CC) $(CFLAGS) -c common/debug.c -o common/debug.o
 
@@ -343,6 +350,9 @@ config/load.o: config/load.c command/command.h common/assert.h common/debug.h co
 config/parse.o: config/parse.c common/assert.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/parse.auto.c config/parse.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
 	$(CC) $(CFLAGS) -c config/parse.c -o config/parse.o
 
+config/protocol.o: config/protocol.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/server.h
+	$(CC) $(CFLAGS) -c config/protocol.c -o config/protocol.o
+
 crypto/cipherBlock.o: crypto/cipherBlock.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h crypto/cipherBlock.h crypto/crypto.h
 	$(CC) $(CFLAGS) -c crypto/cipherBlock.c -o crypto/cipherBlock.o
 
@@ -367,7 +377,7 @@ info/infoManifest.o: info/infoManifest.c common/error.auto.h common/error.h comm
 info/infoPg.o: info/infoPg.c common/assert.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h crypto/crypto.h crypto/hash.h info/info.h info/infoPg.h postgres/interface.h postgres/version.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h
 	$(CC) $(CFLAGS) -c info/infoPg.c -o info/infoPg.o
 
-main.o: main.c command/archive/get/get.h command/archive/push/push.h command/command.h command/help/help.h command/info/info.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h version.h
+main.o: main.c command/archive/get/get.h command/archive/push/push.h command/command.h command/help/help.h command/info/info.h command/remote/remote.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h version.h
 	$(CC) $(CFLAGS) -c main.c -o main.o
 
 perl/config.o: perl/config.c common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h
@@ -418,9 +428,12 @@ postgres/pageChecksum.o: postgres/pageChecksum.c common/assert.h common/debug.h
 protocol/client.o: protocol/client.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h protocol/client.h version.h
 	$(CC) $(CFLAGS) -c protocol/client.c -o protocol/client.o
 
-protocol/helper.o: protocol/helper.c common/assert.h common/debug.h common/error.auto.h common/error.h common/exec.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h protocol/client.h protocol/helper.h
+protocol/helper.o: protocol/helper.c common/assert.h common/debug.h common/error.auto.h common/error.h common/exec.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h config/protocol.h crypto/crypto.h protocol/client.h protocol/helper.h protocol/server.h
 	$(CC) $(CFLAGS) -c protocol/helper.c -o protocol/helper.o
 
+protocol/server.o: protocol/server.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/server.h version.h
+	$(CC) $(CFLAGS) -c protocol/server.c -o protocol/server.o
+
 storage/driver/posix/common.o: storage/driver/posix/common.c common/assert.h common/debug.h common/error.auto.h common/error.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/string.h storage/driver/posix/common.h
 	$(CC) $(CFLAGS) -c storage/driver/posix/common.c -o storage/driver/posix/common.o
 
@@ -433,10 +446,13 @@ storage/driver/posix/fileWrite.o: storage/driver/posix/fileWrite.c common/assert
 storage/driver/posix/storage.o: storage/driver/posix/storage.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h storage/driver/posix/common.h storage/driver/posix/fileRead.h storage/driver/posix/fileWrite.h storage/driver/posix/storage.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
 	$(CC) $(CFLAGS) -c storage/driver/posix/storage.c -o storage/driver/posix/storage.o
 
-storage/driver/remote/fileRead.o: storage/driver/remote/fileRead.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h storage/driver/remote/fileRead.h storage/driver/remote/storage.h storage/fileRead.h storage/fileRead.intern.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
+storage/driver/remote/fileRead.o: storage/driver/remote/fileRead.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/server.h storage/driver/remote/fileRead.h storage/driver/remote/protocol.h storage/driver/remote/storage.h storage/fileRead.h storage/fileRead.intern.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
 	$(CC) $(CFLAGS) -c storage/driver/remote/fileRead.c -o storage/driver/remote/fileRead.o
 
-storage/driver/remote/storage.o: storage/driver/remote/storage.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/helper.h storage/driver/remote/fileRead.h storage/driver/remote/storage.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
+storage/driver/remote/protocol.o: storage/driver/remote/protocol.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/server.h storage/driver/remote/protocol.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h
+	$(CC) $(CFLAGS) -c storage/driver/remote/protocol.c -o storage/driver/remote/protocol.o
+
+storage/driver/remote/storage.o: storage/driver/remote/storage.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h protocol/client.h protocol/helper.h protocol/server.h storage/driver/remote/fileRead.h storage/driver/remote/protocol.h storage/driver/remote/storage.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h
 	$(CC) $(CFLAGS) -c storage/driver/remote/storage.c -o storage/driver/remote/storage.o
 
 storage/driver/s3/fileRead.o: storage/driver/s3/fileRead.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h storage/driver/s3/fileRead.h storage/driver/s3/storage.h storage/fileRead.h storage/fileRead.intern.h storage/fileWrite.h storage/info.h storage/storage.h storage/storage.intern.h

src/command/remote/remote.c

@@ -0,0 +1,38 @@
+/***********************************************************************************************************************************
+Remote Command
+***********************************************************************************************************************************/
+#include "common/debug.h"
+#include "common/io/handleRead.h"
+#include "common/io/handleWrite.h"
+#include "common/log.h"
+#include "config/config.h"
+#include "config/protocol.h"
+#include "protocol/helper.h"
+#include "protocol/server.h"
+#include "storage/driver/remote/protocol.h"
+
+/***********************************************************************************************************************************
+Remote command
+***********************************************************************************************************************************/
+void
+cmdRemote(int handleRead, int handleWrite)
+{
+    FUNCTION_LOG_VOID(logLevelDebug);
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        String *name = strNewFmt(PROTOCOL_SERVICE_REMOTE "-%d", cfgOptionInt(cfgOptProcess));
+        IoRead *read = ioHandleReadIo(ioHandleReadNew(name, handleRead, (TimeMSec)(cfgOptionDbl(cfgOptProtocolTimeout) * 1000)));
+        ioReadOpen(read);
+        IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(name, handleWrite));
+        ioWriteOpen(write);
+
+        ProtocolServer *server = protocolServerNew(name, PROTOCOL_SERVICE_REMOTE_STR, read, write);
+        protocolServerHandlerAdd(server, storageDriverRemoteProtocol);
+        protocolServerHandlerAdd(server, configProtocol);
+        protocolServerProcess(server);
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN_VOID();
+}

src/command/remote/remote.h

@@ -0,0 +1,12 @@
+/***********************************************************************************************************************************
+Remote Command
+***********************************************************************************************************************************/
+#ifndef COMMAND_REMOTE_REMOTE_H
+#define COMMAND_REMOTE_REMOTE_H
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+void cmdRemote(int handleRead, int handleWrite);
+
+#endif

src/config/protocol.c

@@ -0,0 +1,77 @@
+/***********************************************************************************************************************************
+Configuration Protocol Handler
+***********************************************************************************************************************************/
+#include "common/debug.h"
+#include "common/io/io.h"
+#include "common/log.h"
+#include "common/memContext.h"
+#include "config/config.h"
+#include "config/protocol.h"
+
+/***********************************************************************************************************************************
+Constants
+***********************************************************************************************************************************/
+STRING_EXTERN(PROTOCOL_COMMAND_CONFIG_OPTION_STR,                   PROTOCOL_COMMAND_CONFIG_OPTION);
+
+/***********************************************************************************************************************************
+Process config protocol requests
+***********************************************************************************************************************************/
+bool
+configProtocol(const String *command, const VariantList *paramList, ProtocolServer *server)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(STRING, command);
+        FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
+        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
+    FUNCTION_LOG_END();
+
+    ASSERT(command != NULL);
+
+    // Attempt to satisfy the request -- we may get requests that are meant for other handlers
+    bool found = true;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        if (strEq(command, PROTOCOL_COMMAND_CONFIG_OPTION_STR))
+        {
+            VariantList *optionList = varLstNew();
+
+            for (unsigned int optionIdx = 0; optionIdx < varLstSize(paramList); optionIdx++)
+                varLstAdd(optionList, varDup(cfgOption(cfgOptionId(strPtr(varStr(varLstGet(paramList, optionIdx)))))));
+
+            protocolServerResponse(server, varNewVarLst(optionList));
+        }
+        else
+            found = false;
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN(BOOL, found);
+}
+
+/***********************************************************************************************************************************
+Get option values from another process
+***********************************************************************************************************************************/
+VariantList *
+configProtocolOption(ProtocolClient *client, const VariantList *paramList)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_CLIENT, client);
+        FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
+    FUNCTION_LOG_END();
+
+    VariantList *result = NULL;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        KeyValue *command = kvPut(kvNew(), varNewStr(PROTOCOL_COMMAND_STR), varNewStr(PROTOCOL_COMMAND_CONFIG_OPTION_STR));
+        kvPut(command, varNewStr(PROTOCOL_PARAMETER_STR), varNewVarLst(paramList));
+
+        memContextSwitch(MEM_CONTEXT_OLD());
+        result = varVarLst(protocolClientExecute(client, command, true));
+        memContextSwitch(MEM_CONTEXT_TEMP());
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN(VARIANT_LIST, result);
+}

src/config/protocol.h

@@ -0,0 +1,24 @@
+/***********************************************************************************************************************************
+Configuration Protocol Handler
+***********************************************************************************************************************************/
+#ifndef CONFIG_PROTOCOL_H
+#define CONFIG_PROTOCOL_H
+
+#include "common/type/string.h"
+#include "common/type/variantList.h"
+#include "protocol/client.h"
+#include "protocol/server.h"
+
+/***********************************************************************************************************************************
+Constants
+***********************************************************************************************************************************/
+#define PROTOCOL_COMMAND_CONFIG_OPTION                              "configOption"
+    STRING_DECLARE(PROTOCOL_COMMAND_CONFIG_OPTION_STR);
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+bool configProtocol(const String *command, const VariantList *paramList, ProtocolServer *server);
+VariantList *configProtocolOption(ProtocolClient *client, const VariantList *paramList);
+
+#endif

src/main.c

@@ -3,12 +3,14 @@ Main
 ***********************************************************************************************************************************/
 #include <stdio.h>
 #include <stdlib.h>
+#include <unistd.h>
 
 #include "command/archive/get/get.h"
 #include "command/archive/push/push.h"
 #include "command/help/help.h"
 #include "command/info/info.h"
 #include "command/command.h"
+#include "command/remote/remote.h"
 #include "common/debug.h"
 #include "common/error.h"
 #include "common/exit.h"
@@ -60,6 +62,13 @@ main(int argListSize, const char *argList[])
             fflush(stdout);
         }
 
+        // Remote command.  Currently only implements a subset.
+        // -------------------------------------------------------------------------------------------------------------------------
+        else if (cfgCommand() == cfgCmdRemote && strEqZ(cfgOptionStr(cfgOptCommand), "info"))
+        {
+            cmdRemote(STDIN_FILENO, STDOUT_FILENO);
+        }
+
         // Archive get command
         // -------------------------------------------------------------------------------------------------------------------------
         else if (cfgCommand() == cfgCmdArchiveGet)

src/protocol/client.c

@@ -120,7 +120,7 @@ protocolClientNew(const String *name, const String *service, IoRead *read, IoWri
 /***********************************************************************************************************************************
 Read the command output
 ***********************************************************************************************************************************/
-const VariantList *
+const Variant *
 protocolClientReadOutput(ProtocolClient *this, bool outputRequired)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
@@ -130,7 +130,7 @@ protocolClientReadOutput(ProtocolClient *this, bool outputRequired)
 
     ASSERT(this != NULL);
 
-    const VariantList *result = NULL;
+    const Variant *result = NULL;
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
@@ -151,19 +151,15 @@ protocolClientReadOutput(ProtocolClient *this, bool outputRequired)
         }
 
         // Get output
-        const Variant *output = kvGet(responseKv, varNewStr(PROTOCOL_OUTPUT_STR));
+        result = kvGet(responseKv, varNewStr(PROTOCOL_OUTPUT_STR));
-
-        ASSERT(output != NULL);
-        ASSERT(varType(output) == varTypeVariantList);
 
         if (outputRequired)
         {
             // Just move the entire response kv since the output is the largest part if it
             kvMove(responseKv, MEM_CONTEXT_OLD());
-            result = varVarLst(output);
         }
         // Else if no output is required then there should not be any
-        else if (varLstSize(varVarLst(output)) != 0)
+        else if (result != NULL)
             THROW(AssertError, "no output required by command");
 
         // Reset the keep alive time
@@ -171,7 +167,7 @@ protocolClientReadOutput(ProtocolClient *this, bool outputRequired)
     }
     MEM_CONTEXT_TEMP_END();
 
-    FUNCTION_LOG_RETURN_CONST(VARIANT_LIST, result);
+    FUNCTION_LOG_RETURN_CONST(VARIANT, result);
 }
 
 /***********************************************************************************************************************************
@@ -201,7 +197,7 @@ protocolClientWriteCommand(ProtocolClient *this, const KeyValue *command)
 /***********************************************************************************************************************************
 Execute a protocol command and get the output
 ***********************************************************************************************************************************/
-const VariantList *
+const Variant *
 protocolClientExecute(ProtocolClient *this, const KeyValue *command, bool outputRequired)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
@@ -215,7 +211,7 @@ protocolClientExecute(ProtocolClient *this, const KeyValue *command, bool output
 
     protocolClientWriteCommand(this, command);
 
-    FUNCTION_LOG_RETURN_CONST(VARIANT_LIST, protocolClientReadOutput(this, outputRequired));
+    FUNCTION_LOG_RETURN_CONST(VARIANT, protocolClientReadOutput(this, outputRequired));
 }
 
 /***********************************************************************************************************************************

src/protocol/client.h

@@ -55,10 +55,10 @@ ProtocolClient *protocolClientNew(const String *name, const String *service, IoR
 /***********************************************************************************************************************************
 Functions
 ***********************************************************************************************************************************/
-const VariantList *protocolClientExecute(ProtocolClient *this, const KeyValue *command, bool outputRequired);
+const Variant *protocolClientExecute(ProtocolClient *this, const KeyValue *command, bool outputRequired);
 ProtocolClient *protocolClientMove(ProtocolClient *this, MemContext *parentNew);
 void protocolClientNoOp(ProtocolClient *this);
-const VariantList *protocolClientReadOutput(ProtocolClient *this, bool outputRequired);
+const Variant *protocolClientReadOutput(ProtocolClient *this, bool outputRequired);
 void protocolClientWriteCommand(ProtocolClient *this, const KeyValue *command);
 
 /***********************************************************************************************************************************

src/protocol/helper.c

@@ -4,8 +4,10 @@ Protocol Helper
 #include "common/debug.h"
 #include "common/exec.h"
 #include "common/memContext.h"
+#include "crypto/crypto.h"
 #include "config/config.h"
 #include "config/exec.h"
+#include "config/protocol.h"
 #include "protocol/helper.h"
 
 /***********************************************************************************************************************************
@@ -134,6 +136,23 @@ protocolGet(RemoteType remoteType, unsigned int remoteId)
                 strNewFmt("remote-%u protocol on '%s'", remoteId, strPtr(cfgOptionStr(cfgOptRepoHost))),
                 PROTOCOL_SERVICE_REMOTE_STR, execIoRead(protocolHelper.remoteExec), execIoWrite(protocolHelper.remoteExec));
 
+            // Get cipher options from the remote if none are locally configured
+            if (strEq(cfgOptionStr(cfgOptRepoCipherType), CIPHER_TYPE_NONE_STR))
+            {
+                // Options to query
+                VariantList *param = varLstNew();
+                varLstAdd(param, varNewStr(strNew(cfgOptionName(cfgOptRepoCipherType))));
+                varLstAdd(param, varNewStr(strNew(cfgOptionName(cfgOptRepoCipherPass))));
+
+                VariantList *optionList = configProtocolOption(protocolHelper.remote, param);
+
+                if (!strEq(varStr(varLstGet(optionList, 0)), CIPHER_TYPE_NONE_STR))
+                {
+                    cfgOptionSet(cfgOptRepoCipherType, cfgSourceConfig, varLstGet(optionList, 0));
+                    cfgOptionSet(cfgOptRepoCipherPass, cfgSourceConfig, varLstGet(optionList, 1));
+                }
+            }
+
             protocolClientMove(protocolHelper.remote, execMemContext(protocolHelper.remoteExec));
         }
         MEM_CONTEXT_END();

src/protocol/server.c

@@ -0,0 +1,259 @@
+/***********************************************************************************************************************************
+Protocol Server
+***********************************************************************************************************************************/
+#include "common/debug.h"
+#include "common/log.h"
+#include "common/memContext.h"
+#include "common/time.h"
+#include "common/type/json.h"
+#include "common/type/keyValue.h"
+#include "common/type/list.h"
+#include "protocol/client.h"
+#include "protocol/server.h"
+#include "version.h"
+
+/***********************************************************************************************************************************
+Object type
+***********************************************************************************************************************************/
+struct ProtocolServer
+{
+    MemContext *memContext;
+    const String *name;
+    IoRead *read;
+    IoWrite *write;
+
+    List *handlerList;
+};
+
+/***********************************************************************************************************************************
+Create object
+***********************************************************************************************************************************/
+ProtocolServer *
+protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(STRING, name);
+        FUNCTION_LOG_PARAM(STRING, service);
+        FUNCTION_LOG_PARAM(IO_READ, read);
+        FUNCTION_LOG_PARAM(IO_WRITE, write);
+    FUNCTION_LOG_END();
+
+    ASSERT(name != NULL);
+    ASSERT(read != NULL);
+    ASSERT(write != NULL);
+
+    ProtocolServer *this = NULL;
+
+    MEM_CONTEXT_NEW_BEGIN("ProtocolServer")
+    {
+        this = memNew(sizeof(ProtocolServer));
+        this->memContext = memContextCurrent();
+
+        this->name = strDup(name);
+        this->read = read;
+        this->write = write;
+
+        this->handlerList = lstNew(sizeof(ProtocolServerProcessHandler));
+
+        // Send the protocol greeting
+        MEM_CONTEXT_TEMP_BEGIN()
+        {
+            KeyValue *greetingKv = kvNew();
+            kvPut(greetingKv, varNewStr(PROTOCOL_GREETING_NAME_STR), varNewStr(strNew(PROJECT_NAME)));
+            kvPut(greetingKv, varNewStr(PROTOCOL_GREETING_SERVICE_STR), varNewStr(service));
+            kvPut(greetingKv, varNewStr(PROTOCOL_GREETING_VERSION_STR), varNewStr(strNew(PROJECT_VERSION)));
+
+            ioWriteLine(this->write, kvToJson(greetingKv, 0));
+            ioWriteFlush(this->write);
+        }
+        MEM_CONTEXT_TEMP_END();
+    }
+    MEM_CONTEXT_NEW_END();
+
+    FUNCTION_LOG_RETURN(PROTOCOL_SERVER, this);
+}
+
+/***********************************************************************************************************************************
+Add a new handler
+***********************************************************************************************************************************/
+void
+protocolServerHandlerAdd(ProtocolServer *this, ProtocolServerProcessHandler handler)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
+        FUNCTION_LOG_PARAM(FUNCTIONP, handler);
+    FUNCTION_LOG_END();
+
+    lstAdd(this->handlerList, &handler);
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Process requests
+***********************************************************************************************************************************/
+void
+protocolServerProcess(ProtocolServer *this)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
+    FUNCTION_LOG_END();
+
+    // Loop until exit command is received
+    bool exit = false;
+
+    do
+    {
+        TRY_BEGIN()
+        {
+            MEM_CONTEXT_TEMP_BEGIN()
+            {
+                // Read command
+                KeyValue *commandKv = varKv(jsonToVar(ioReadLine(this->read)));
+                String *command = varStr(kvGet(commandKv, varNewStr(PROTOCOL_COMMAND_STR)));
+                VariantList *paramList = varVarLst(kvGet(commandKv, varNewStr(PROTOCOL_PARAMETER_STR)));
+
+                // Process command
+                bool found = false;
+
+                for (unsigned int handlerIdx = 0; handlerIdx < lstSize(this->handlerList); handlerIdx++)
+                {
+                    // Get the next handler
+                    ProtocolServerProcessHandler handler = *(ProtocolServerProcessHandler *)lstGet(this->handlerList, handlerIdx);
+
+                    // Send the command to the handler
+                    found = handler(command, paramList, this);
+
+                    // If the handler processed the command then exit the handler loop
+                    if (found)
+                        break;
+                }
+
+                if (!found)
+                {
+                    if (strEq(command, PROTOCOL_COMMAND_NOOP_STR))
+                        protocolServerResponse(this, NULL);
+                    else if (strEq(command, PROTOCOL_COMMAND_EXIT_STR))
+                        exit = true;
+                    else
+                        THROW_FMT(ProtocolError, "invalid command '%s'", strPtr(command));
+                }
+            }
+            MEM_CONTEXT_TEMP_END();
+        }
+        // Asserts are thrown so a stack trace will be output to aid in debugging
+        CATCH(AssertError)
+        {
+            RETHROW();
+        }
+        CATCH_ANY()
+        {
+            KeyValue *error = kvNew();
+            kvPut(error, varNewStr(PROTOCOL_ERROR_STR), varNewInt(errorCode()));
+            kvPut(error, varNewStr(PROTOCOL_OUTPUT_STR), varNewStr(strNew(errorMessage())));
+
+            ioWriteLine(this->write, kvToJson(error, 0));
+            ioWriteFlush(this->write);
+        }
+        TRY_END();
+    }
+    while (!exit);
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Respond to request with output if provided
+***********************************************************************************************************************************/
+void
+protocolServerResponse(ProtocolServer *this, const Variant *output)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
+        FUNCTION_LOG_PARAM(VARIANT, output);
+    FUNCTION_LOG_END();
+
+    KeyValue *result = kvNew();
+
+    if (output != NULL)
+        kvAdd(result, varNewStr(PROTOCOL_OUTPUT_STR), output);
+
+    ioWriteLine(this->write, kvToJson(result, 0));
+    ioWriteFlush(this->write);
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Move the file object to a new context
+***********************************************************************************************************************************/
+ProtocolServer *
+protocolServerMove(ProtocolServer *this, MemContext *parentNew)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(PROTOCOL_SERVER, this);
+        FUNCTION_TEST_PARAM(MEM_CONTEXT, parentNew);
+    FUNCTION_TEST_END();
+
+    ASSERT(parentNew != NULL);
+
+    if (this != NULL)
+        memContextMove(this->memContext, parentNew);
+
+    FUNCTION_TEST_RETURN(this);
+}
+
+/***********************************************************************************************************************************
+Get read interface
+***********************************************************************************************************************************/
+IoRead *
+protocolServerIoRead(const ProtocolServer *this)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(PROTOCOL_SERVER, this);
+    FUNCTION_TEST_END();
+
+    ASSERT(this != NULL);
+
+    FUNCTION_TEST_RETURN(this->read);
+}
+
+/***********************************************************************************************************************************
+Get write interface
+***********************************************************************************************************************************/
+IoWrite *
+protocolServerIoWrite(const ProtocolServer *this)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(PROTOCOL_SERVER, this);
+    FUNCTION_TEST_END();
+
+    ASSERT(this != NULL);
+
+    FUNCTION_TEST_RETURN(this->write);
+}
+
+/***********************************************************************************************************************************
+Render as string for logging
+***********************************************************************************************************************************/
+String *
+protocolServerToLog(const ProtocolServer *this)
+{
+    return strNewFmt("{name: %s}", strPtr(this->name));
+}
+
+/***********************************************************************************************************************************
+Free the file
+***********************************************************************************************************************************/
+void
+protocolServerFree(ProtocolServer *this)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, this);
+    FUNCTION_LOG_END();
+
+    if (this != NULL)
+        memContextFree(this->memContext);
+
+    FUNCTION_LOG_RETURN_VOID();
+}

src/protocol/server.h

@@ -0,0 +1,54 @@
+/***********************************************************************************************************************************
+Protocol Server
+***********************************************************************************************************************************/
+#ifndef PROTOCOL_SERVER_H
+#define PROTOCOL_SERVER_H
+
+/***********************************************************************************************************************************
+Object type
+***********************************************************************************************************************************/
+typedef struct ProtocolServer ProtocolServer;
+
+#include "common/io/read.h"
+#include "common/io/write.h"
+
+/***********************************************************************************************************************************
+Protocol process handler type
+***********************************************************************************************************************************/
+typedef bool (*ProtocolServerProcessHandler)(const String *command, const VariantList *paramList, ProtocolServer *server);
+
+/***********************************************************************************************************************************
+Constructor
+***********************************************************************************************************************************/
+ProtocolServer *protocolServerNew(const String *name, const String *service, IoRead *read, IoWrite *write);
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+void protocolServerProcess(ProtocolServer *this);
+void protocolServerResponse(ProtocolServer *this, const Variant *output);
+void protocolServerHandlerAdd(ProtocolServer *this, ProtocolServerProcessHandler handler);
+ProtocolServer *protocolServerMove(ProtocolServer *this, MemContext *parentNew);
+
+/***********************************************************************************************************************************
+Getters
+***********************************************************************************************************************************/
+IoRead *protocolServerIoRead(const ProtocolServer *this);
+IoWrite *protocolServerIoWrite(const ProtocolServer *this);
+
+/***********************************************************************************************************************************
+Destructor
+***********************************************************************************************************************************/
+void protocolServerFree(ProtocolServer *this);
+
+/***********************************************************************************************************************************
+Macros for function logging
+***********************************************************************************************************************************/
+String *protocolServerToLog(const ProtocolServer *this);
+
+#define FUNCTION_LOG_PROTOCOL_SERVER_TYPE                                                                                          \
+    ProtocolServer *
+#define FUNCTION_LOG_PROTOCOL_SERVER_FORMAT(value, buffer, bufferSize)                                                             \
+    FUNCTION_LOG_STRING_OBJECT_FORMAT(value, protocolServerToLog, buffer, bufferSize)
+
+#endif

src/storage/driver/remote/fileRead.c

@@ -11,19 +11,13 @@ Remote Storage File Read Driver
 #include "common/regExp.h"
 #include "common/type/convert.h"
 #include "storage/driver/remote/fileRead.h"
+#include "storage/driver/remote/protocol.h"
 #include "storage/fileRead.intern.h"
 
 /***********************************************************************************************************************************
 Regular expressions
 ***********************************************************************************************************************************/
-#define BLOCK_HEADER                                                "BRBLOCK"
+STRING_STATIC(BLOCK_REG_EXP_STR,                                    PROTOCOL_BLOCK_HEADER "[0-9]+");
-STRING_STATIC(BLOCK_REG_EXP_STR,                                    BLOCK_HEADER "[0-9]+");
-
-/***********************************************************************************************************************************
-Command constants
-***********************************************************************************************************************************/
-STRING_STATIC(STORAGE_REMOTE_COMMAND_OPEN_READ_STR,                 "storageOpenRead");
-STRING_STATIC(STORAGE_REMOTE_COMMAND_OPEN_READ_IGNORE_MISSING_STR,  "bIgnoreMissing");
 
 /***********************************************************************************************************************************
 Object type
@@ -127,20 +121,16 @@ storageDriverRemoteFileReadOpen(StorageDriverRemoteFileRead *this)
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        // Add optional parameters
-        Variant *paramOpt = varNewKv();
-        kvPut(varKv(paramOpt), varNewStr(STORAGE_REMOTE_COMMAND_OPEN_READ_IGNORE_MISSING_STR), varNewBool(this->ignoreMissing));
-
         // Add parameters
         Variant *param = varNewVarLst(varLstNew());
         varLstAdd(varVarLst(param), varNewStr(this->name));
-        varLstAdd(varVarLst(param), paramOpt);
+        varLstAdd(varVarLst(param), varNewBool(this->ignoreMissing));
 
         // Construct command
-        KeyValue *command = kvPut(kvNew(), varNewStr(PROTOCOL_COMMAND_STR), varNewStr(STORAGE_REMOTE_COMMAND_OPEN_READ_STR));
+        KeyValue *command = kvPut(kvNew(), varNewStr(PROTOCOL_COMMAND_STR), varNewStr(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR));
         kvPut(command, varNewStr(PROTOCOL_PARAMETER_STR), param);
 
-        result = varIntForce(varLstGet(protocolClientExecute(this->client, command, true), 0));
+        result = varBool(protocolClientExecute(this->client, command, true));
     }
     MEM_CONTEXT_TEMP_END();
 
@@ -162,7 +152,7 @@ storageDriverRemoteFileReadBlockSize(const String *message)
     if (!regExpMatch(storageDriverRemoteFileReadLocal.blockRegExp, message))
         THROW_FMT(ProtocolError, "'%s' is not a valid block size message", strPtr(message));
 
-    FUNCTION_LOG_RETURN(SIZE, (size_t)cvtZToUInt64(strPtr(message) + sizeof(BLOCK_HEADER) - 1));
+    FUNCTION_LOG_RETURN(SIZE, (size_t)cvtZToUInt64(strPtr(message) + sizeof(PROTOCOL_BLOCK_HEADER) - 1));
 }
 
 /***********************************************************************************************************************************
@@ -195,16 +185,7 @@ storageDriverRemoteFileRead(StorageDriverRemoteFileRead *this, Buffer *buffer, b
                     this->remaining = storageDriverRemoteFileReadBlockSize(ioReadLine(protocolClientIoRead(this->client)));
 
                     if (this->remaining == 0)
-                    {
                         this->eof = true;
-
-                        // ??? Read line with filter data -- ignored for the time-being but will need to be implemented
-                        ioReadLine(protocolClientIoRead(this->client));
-
-                        // The last message sent can be ignored because it is always 1.  This is an aritifact of the protocl layer
-                        // in Perl which should be removed when converted to C.
-                        ioReadLine(protocolClientIoRead(this->client));
-                    }
                 }
                 MEM_CONTEXT_TEMP_END();
             }

src/storage/driver/remote/protocol.c

@@ -0,0 +1,91 @@
+/***********************************************************************************************************************************
+Remote Storage Protocol Handler
+***********************************************************************************************************************************/
+#include "common/debug.h"
+#include "common/io/io.h"
+#include "common/log.h"
+#include "common/memContext.h"
+#include "storage/driver/remote/protocol.h"
+#include "storage/helper.h"
+
+/***********************************************************************************************************************************
+Constants
+***********************************************************************************************************************************/
+STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_LIST_STR,                    PROTOCOL_COMMAND_STORAGE_LIST);
+STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR,               PROTOCOL_COMMAND_STORAGE_OPEN_READ);
+
+/***********************************************************************************************************************************
+Process storage protocol requests
+***********************************************************************************************************************************/
+bool
+storageDriverRemoteProtocol(const String *command, const VariantList *paramList, ProtocolServer *server)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(STRING, command);
+        FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
+        FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
+    FUNCTION_LOG_END();
+
+    ASSERT(command != NULL);
+
+    // Determine which storage should be used (??? for now this is only repo)
+    const Storage *storage = storageRepo();
+
+    // Attempt to satisfy the request -- we may get requests that are meant for other handlers
+    bool found = true;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        if (strEq(command, PROTOCOL_COMMAND_STORAGE_LIST_STR))
+        {
+            protocolServerResponse(
+                server,
+                varNewVarLst(
+                    varLstNewStrLst(
+                        storageListP(
+                            storage, varStr(varLstGet(paramList, 0)), .errorOnMissing = varBool(varLstGet(paramList, 1)),
+                            varStr(varLstGet(paramList, 2))))));
+        }
+        else if (strEq(command, PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR))
+        {
+            // Create the read object
+            IoRead *fileRead = storageFileReadIo(
+                storageNewReadP(storage, varStr(varLstGet(paramList, 0)), .ignoreMissing = varBool(varLstGet(paramList, 1))));
+
+            // Check if the file exists
+            bool exists = ioReadOpen(fileRead);
+            protocolServerResponse(server, varNewBool(exists));
+
+            // Transfer the file if it exists
+            if (exists)
+            {
+                Buffer *buffer = bufNew(ioBufferSize());
+
+                // Write file out to protocol layer
+                do
+                {
+                    ioRead(fileRead, buffer);
+
+                    if (bufUsed(buffer) > 0)
+                    {
+                        ioWriteLine(protocolServerIoWrite(server), strNewFmt(PROTOCOL_BLOCK_HEADER "%zu", bufUsed(buffer)));
+                        ioWrite(protocolServerIoWrite(server), buffer);
+                        ioWriteFlush(protocolServerIoWrite(server));
+
+                        bufUsedZero(buffer);
+                    }
+                }
+                while (!ioReadEof(fileRead));
+
+                // Write a zero block to show file is complete
+                ioWriteLine(protocolServerIoWrite(server), strNew(PROTOCOL_BLOCK_HEADER "0"));
+                ioWriteFlush(protocolServerIoWrite(server));
+            }
+        }
+        else
+            found = false;
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN(BOOL, found);
+}

src/storage/driver/remote/protocol.h

@@ -0,0 +1,26 @@
+/***********************************************************************************************************************************
+Remote Storage Protocol Handler
+***********************************************************************************************************************************/
+#ifndef STORAGE_DRIVER_REMOTE_PROTOCOL_H
+#define STORAGE_DRIVER_REMOTE_PROTOCOL_H
+
+#include "common/type/string.h"
+#include "common/type/variantList.h"
+#include "protocol/server.h"
+
+/***********************************************************************************************************************************
+Constants
+***********************************************************************************************************************************/
+#define PROTOCOL_BLOCK_HEADER                                       "BRBLOCK"
+
+#define PROTOCOL_COMMAND_STORAGE_LIST                               "storageList"
+    STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_LIST_STR);
+#define PROTOCOL_COMMAND_STORAGE_OPEN_READ                          "storageOpenRead"
+    STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR);
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+bool storageDriverRemoteProtocol(const String *command, const VariantList *paramList, ProtocolServer *server);
+
+#endif

src/storage/driver/remote/storage.c

@@ -7,6 +7,7 @@ Remote Storage Driver
 #include "protocol/client.h"
 #include "protocol/helper.h"
 #include "storage/driver/remote/fileRead.h"
+#include "storage/driver/remote/protocol.h"
 #include "storage/driver/remote/storage.h"
 
 /***********************************************************************************************************************************
@@ -14,13 +15,6 @@ Driver type constant string
 ***********************************************************************************************************************************/
 STRING_EXTERN(STORAGE_DRIVER_REMOTE_TYPE_STR,                           STORAGE_DRIVER_REMOTE_TYPE);
 
-/***********************************************************************************************************************************
-Command constants
-***********************************************************************************************************************************/
-STRING_STATIC(STORAGE_REMOTE_COMMAND_LIST_STR,                          "storageList");
-STRING_STATIC(STORAGE_REMOTE_COMMAND_LIST_EXPRESSION_STR,               "strExpression");
-STRING_STATIC(STORAGE_REMOTE_COMMAND_LIST_IGNORE_MISSING_STR,           "bIgnoreMissing");
-
 /***********************************************************************************************************************************
 Object type
 ***********************************************************************************************************************************/
@@ -138,21 +132,17 @@ storageDriverRemoteList(StorageDriverRemote *this, const String *path, bool erro
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        // Add optional parameters
-        Variant *paramOpt = varNewKv();
-        kvPut(varKv(paramOpt), varNewStr(STORAGE_REMOTE_COMMAND_LIST_IGNORE_MISSING_STR), varNewBool(!errorOnMissing));
-        kvPut(varKv(paramOpt), varNewStr(STORAGE_REMOTE_COMMAND_LIST_EXPRESSION_STR), varNewStr(expression));
-
         // Add parameters
         Variant *param = varNewVarLst(varLstNew());
         varLstAdd(varVarLst(param), varNewStr(path));
-        varLstAdd(varVarLst(param), paramOpt);
+        varLstAdd(varVarLst(param), varNewBool(errorOnMissing));
+        varLstAdd(varVarLst(param), varNewStr(expression));
 
         // Construct command
-        KeyValue *command = kvPut(kvNew(), varNewStr(PROTOCOL_COMMAND_STR), varNewStr(STORAGE_REMOTE_COMMAND_LIST_STR));
+        KeyValue *command = kvPut(kvNew(), varNewStr(PROTOCOL_COMMAND_STR), varNewStr(PROTOCOL_COMMAND_STORAGE_LIST_STR));
         kvPut(command, varNewStr(PROTOCOL_PARAMETER_STR), param);
 
-        result = strLstMove(strLstNewVarLst(protocolClientExecute(this->client, command, true)), MEM_CONTEXT_OLD());
+        result = strLstMove(strLstNewVarLst(varVarLst(protocolClientExecute(this->client, command, true))), MEM_CONTEXT_OLD());
     }
     MEM_CONTEXT_TEMP_END();
 

test/define.yaml

@@ -434,6 +434,13 @@ unit:
         coverage:
           config/exec: full
 
+      # ----------------------------------------------------------------------------------------------------------------------------
+      - name: protocol
+        total: 1
+
+        coverage:
+          config/protocol: full
+
   # ********************************************************************************************************************************
   - name: storage
 
@@ -532,11 +539,12 @@ unit:
 
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: remote
-        total: 1
+        total: 3
         perlReq: true
 
         coverage:
           storage/driver/remote/fileRead: full
+          storage/driver/remote/protocol: full
           storage/driver/remote/storage: full
           storage/helper: full
           storage/storage: full
@@ -571,12 +579,13 @@ unit:
 
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: protocol
-        total: 4
+        total: 5
         perlReq: true
 
         coverage:
           protocol/client: full
           protocol/helper: full
+          protocol/server: full
 
   # ********************************************************************************************************************************
   - name: info
@@ -635,6 +644,13 @@ unit:
         coverage:
           command/info/info: full
 
+      # ----------------------------------------------------------------------------------------------------------------------------
+      - name: remote
+        total: 1
+
+        coverage:
+          command/remote/remote: full
+
   # ********************************************************************************************************************************
   - name: archive
 

test/src/module/command/remoteTest.c

@@ -0,0 +1,73 @@
+/***********************************************************************************************************************************
+Test Remote Command
+***********************************************************************************************************************************/
+#include "common/io/handleRead.h"
+#include "common/io/handleWrite.h"
+#include "protocol/client.h"
+#include "protocol/server.h"
+
+#include "common/harnessConfig.h"
+#include "common/harnessFork.h"
+
+/***********************************************************************************************************************************
+Test Run
+***********************************************************************************************************************************/
+void
+testRun(void)
+{
+    FUNCTION_HARNESS_VOID();
+
+    // *****************************************************************************************************************************
+    if (testBegin("cmdRemote()"))
+    {
+        // Create pipes for testing.  Read/write is from the perspective of the client.
+        int pipeRead[2];
+        int pipeWrite[2];
+        THROW_ON_SYS_ERROR(pipe(pipeRead) == -1, KernelError, "unable to read test pipe");
+        THROW_ON_SYS_ERROR(pipe(pipeWrite) == -1, KernelError, "unable to write test pipe");
+
+        HARNESS_FORK_BEGIN()
+        {
+            HARNESS_FORK_CHILD()
+            {
+                close(pipeRead[0]);
+                close(pipeWrite[1]);
+
+                StringList *argList = strLstNew();
+                strLstAddZ(argList, "pgbackrest");
+                strLstAddZ(argList, "--stanza=test1");
+                strLstAddZ(argList, "--command=info");
+                strLstAddZ(argList, "--process=1");
+                strLstAddZ(argList, "--type=backup");
+                strLstAddZ(argList, "remote");
+                harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
+
+                cmdRemote(pipeWrite[0], pipeRead[1]);
+
+                close(pipeRead[1]);
+                close(pipeWrite[0]);
+            }
+
+            HARNESS_FORK_PARENT()
+            {
+                close(pipeRead[1]);
+                close(pipeWrite[0]);
+
+                IoRead *read = ioHandleReadIo(ioHandleReadNew(strNew("server read"), pipeRead[0], 2000));
+                ioReadOpen(read);
+                IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(strNew("server write"), pipeWrite[1]));
+                ioWriteOpen(write);
+
+                ProtocolClient *client = protocolClientNew(strNew("test"), PROTOCOL_SERVICE_REMOTE_STR, read, write);
+                protocolClientNoOp(client);
+                protocolClientFree(client);
+
+                close(pipeRead[0]);
+                close(pipeWrite[1]);
+            }
+        }
+        HARNESS_FORK_END();
+    }
+
+    FUNCTION_HARNESS_RESULT_VOID();
+}

test/src/module/config/protocolTest.c

@@ -0,0 +1,87 @@
+/***********************************************************************************************************************************
+Test Configuration Protocol
+***********************************************************************************************************************************/
+#include "common/io/handleRead.h"
+#include "common/io/handleWrite.h"
+#include "protocol/client.h"
+#include "protocol/server.h"
+
+#include "common/harnessConfig.h"
+#include "common/harnessFork.h"
+
+/***********************************************************************************************************************************
+Test run
+***********************************************************************************************************************************/
+void
+testRun(void)
+{
+    FUNCTION_HARNESS_VOID();
+
+    // *****************************************************************************************************************************
+    if (testBegin("configProtocol() and configProtocolOption()"))
+    {
+        // Create pipes for testing.  Read/write is from the perspective of the client.
+        int pipeRead[2];
+        int pipeWrite[2];
+        THROW_ON_SYS_ERROR(pipe(pipeRead) == -1, KernelError, "unable to read test pipe");
+        THROW_ON_SYS_ERROR(pipe(pipeWrite) == -1, KernelError, "unable to write test pipe");
+
+        HARNESS_FORK_BEGIN()
+        {
+            HARNESS_FORK_CHILD()
+            {
+                close(pipeRead[0]);
+                close(pipeWrite[1]);
+
+                IoRead *read = ioHandleReadIo(ioHandleReadNew(strNew("client read"), pipeWrite[0], 2000));
+                ioReadOpen(read);
+                IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(strNew("client write"), pipeRead[1]));
+                ioWriteOpen(write);
+
+                StringList *argList = strLstNew();
+                strLstAddZ(argList, "pgbackrest");
+                strLstAddZ(argList, "--stanza=test1");
+                strLstAddZ(argList, "--repo1-host=repo-host");
+                strLstAddZ(argList, "--repo1-host-user=repo-host-user");
+                strLstAddZ(argList, "archive-get");
+                harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
+
+                ProtocolServer *server = protocolServerNew(strNew("test"), strNew("config"), read, write);
+                protocolServerHandlerAdd(server, configProtocol);
+                protocolServerProcess(server);
+
+                close(pipeRead[1]);
+                close(pipeWrite[0]);
+            }
+
+            HARNESS_FORK_PARENT()
+            {
+                close(pipeRead[1]);
+                close(pipeWrite[0]);
+
+                IoRead *read = ioHandleReadIo(ioHandleReadNew(strNew("server read"), pipeRead[0], 2000));
+                ioReadOpen(read);
+                IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(strNew("server write"), pipeWrite[1]));
+                ioWriteOpen(write);
+
+                ProtocolClient *client = protocolClientNew(strNew("test"), strNew("config"), read, write);
+
+                VariantList *list = varLstNew();
+                varLstAdd(list, varNewStr(strNew("repo1-host")));
+                varLstAdd(list, varNewStr(strNew("repo1-host-user")));
+
+                TEST_RESULT_STR(
+                    strPtr(strLstJoin(strLstNewVarLst(configProtocolOption(client, list)), "|")), "repo-host|repo-host-user",
+                    "get options");
+
+                protocolClientFree(client);
+
+                close(pipeRead[0]);
+                close(pipeWrite[1]);
+            }
+        }
+        HARNESS_FORK_END();
+    }
+
+    FUNCTION_HARNESS_RESULT_VOID();
+}

test/src/module/protocol/protocolTest.c

@@ -3,10 +3,54 @@ Test Protocol
 ***********************************************************************************************************************************/
 #include "common/io/handleRead.h"
 #include "common/io/handleWrite.h"
+#include "storage/storage.h"
+#include "storage/driver/posix/storage.h"
+#include "version.h"
 
 #include "common/harnessConfig.h"
 #include "common/harnessFork.h"
 
+/***********************************************************************************************************************************
+Test protocol request handler
+***********************************************************************************************************************************/
+bool
+testServerProtocol(const String *command, const VariantList *paramList, ProtocolServer *server)
+{
+    FUNCTION_HARNESS_BEGIN();
+        FUNCTION_HARNESS_PARAM(STRING, command);
+        FUNCTION_HARNESS_PARAM(VARIANT_LIST, paramList);
+        FUNCTION_HARNESS_PARAM(PROTOCOL_SERVER, server);
+    FUNCTION_HARNESS_END();
+
+    ASSERT(command != NULL);
+
+    // Attempt to satisfy the request -- we may get requests that are meant for other handlers
+    bool found = true;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        if (strEq(command, strNew("assert")))
+        {
+            THROW(AssertError, "test assert");
+        }
+        else if (strEq(command, strNew("request-simple")))
+        {
+            protocolServerResponse(server, varNewBool(true));
+        }
+        else if (strEq(command, strNew("request-complex")))
+        {
+            protocolServerResponse(server, varNewBool(false));
+            ioWriteLine(protocolServerIoWrite(server), strNew("LINEOFTEXT"));
+            ioWriteFlush(protocolServerIoWrite(server));
+        }
+        else
+            found = false;
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_HARNESS_RESULT(BOOL, found);
+}
+
 /***********************************************************************************************************************************
 Test Run
 ***********************************************************************************************************************************/
@@ -15,6 +59,9 @@ testRun(void)
 {
     FUNCTION_HARNESS_VOID();
 
+    Storage *storageTest = storageDriverPosixInterface(
+        storageDriverPosixNew(strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL));
+
     // *****************************************************************************************************************************
     if (testBegin("repoIsLocal()"))
     {
@@ -119,7 +166,7 @@ testRun(void)
                 ioWriteFlush(write);
 
                 TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"cmd\":\"noop\"}", "noop");
-                ioWriteLine(write, strNew("{\"out\":[]}"));
+                ioWriteLine(write, strNew("{}"));
                 ioWriteFlush(write);
 
                 // Throw errors
@@ -204,7 +251,7 @@ testRun(void)
                 const VariantList *output = NULL;
                 KeyValue *command = kvPut(kvNew(), varNewStr(PROTOCOL_COMMAND_STR), varNewStr(strNew("test")));
 
-                TEST_ASSIGN(output, protocolClientExecute(client, command, true), "execute command with output");
+                TEST_ASSIGN(output, varVarLst(protocolClientExecute(client, command, true)), "execute command with output");
                 TEST_RESULT_UINT(varLstSize(output), 2, "check output size");
                 TEST_RESULT_STR(strPtr(varStr(varLstGet(output, 0))), "value1", "check value1");
                 TEST_RESULT_STR(strPtr(varStr(varLstGet(output, 1))), "value2", "check value2");
@@ -220,17 +267,119 @@ testRun(void)
         HARNESS_FORK_END();
     }
 
+    // *****************************************************************************************************************************
+    if (testBegin("ProtocolServer"))
+    {
+        // Create pipes for testing.  Read/write is from the perspective of the client.
+        int pipeRead[2];
+        int pipeWrite[2];
+        THROW_ON_SYS_ERROR(pipe(pipeRead) == -1, KernelError, "unable to read test pipe");
+        THROW_ON_SYS_ERROR(pipe(pipeWrite) == -1, KernelError, "unable to write test pipe");
+
+        HARNESS_FORK_BEGIN()
+        {
+            HARNESS_FORK_CHILD()
+            {
+                close(pipeRead[0]);
+                close(pipeWrite[1]);
+
+                IoRead *read = ioHandleReadIo(ioHandleReadNew(strNew("client read"), pipeWrite[0], 2000));
+                ioReadOpen(read);
+                IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(strNew("client write"), pipeRead[1]));
+                ioWriteOpen(write);
+
+                // Check greeting
+                TEST_RESULT_STR(
+                    strPtr(ioReadLine(read)), "{\"name\":\"pgBackRest\",\"service\":\"test\",\"version\":\"" PROJECT_VERSION "\"}",
+                    "check greeting");
+
+                // Noop
+                TEST_RESULT_VOID(ioWriteLine(write, strNew("{\"cmd\":\"noop\"}")), "write noop");
+                TEST_RESULT_VOID(ioWriteFlush(write), "flush noop");
+                TEST_RESULT_STR(strPtr(ioReadLine(read)), "{}", "noop result");
+
+                // Invalid command
+                TEST_RESULT_VOID(ioWriteLine(write, strNew("{\"cmd\":\"bogus\"}")), "write bogus");
+                TEST_RESULT_VOID(ioWriteFlush(write), "flush bogus");
+                TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"err\":39,\"out\":\"invalid command 'bogus'\"}", "bogus error");
+
+                // Simple request
+                TEST_RESULT_VOID(ioWriteLine(write, strNew("{\"cmd\":\"request-simple\"}")), "write simple request");
+                TEST_RESULT_VOID(ioWriteFlush(write), "flush simple request");
+                TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"out\":true}", "simple request result");
+
+                // Assert -- no response will come backup because the process loop will terminate
+                TEST_RESULT_VOID(ioWriteLine(write, strNew("{\"cmd\":\"assert\"}")), "write assert");
+                TEST_RESULT_VOID(ioWriteFlush(write), "flush simple request");
+
+                // Complex request -- after process loop has been restarted
+                TEST_RESULT_VOID(ioWriteLine(write, strNew("{\"cmd\":\"request-complex\"}")), "write complex request");
+                TEST_RESULT_VOID(ioWriteFlush(write), "flush complex request");
+                TEST_RESULT_STR(strPtr(ioReadLine(read)), "{\"out\":false}", "complex request result");
+                TEST_RESULT_STR(strPtr(ioReadLine(read)), "LINEOFTEXT", "complex request result");
+
+                // Exit
+                TEST_RESULT_VOID(ioWriteLine(write, strNew("{\"cmd\":\"exit\"}")), "write exit");
+                TEST_RESULT_VOID(ioWriteFlush(write), "flush exit");
+
+                close(pipeRead[1]);
+                close(pipeWrite[0]);
+            }
+
+            HARNESS_FORK_PARENT()
+            {
+                close(pipeRead[1]);
+                close(pipeWrite[0]);
+
+                IoRead *read = ioHandleReadIo(ioHandleReadNew(strNew("server read"), pipeRead[0], 2000));
+                ioReadOpen(read);
+                IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(strNew("server write"), pipeWrite[1]));
+                ioWriteOpen(write);
+
+                // Send greeting
+                ProtocolServer *server = NULL;
+
+                MEM_CONTEXT_TEMP_BEGIN()
+                {
+                    TEST_ASSIGN(
+                        server,
+                        protocolServerMove(
+                            protocolServerNew(strNew("test server"), strNew("test"), read, write), MEM_CONTEXT_OLD()),
+                        "create server");
+                    TEST_RESULT_VOID(protocolServerMove(NULL, MEM_CONTEXT_OLD()), "move null server");
+                }
+                MEM_CONTEXT_TEMP_END();
+
+                TEST_RESULT_PTR(protocolServerIoRead(server), server->read, "get read io");
+                TEST_RESULT_PTR(protocolServerIoWrite(server), server->write, "get write io");
+
+                TEST_RESULT_VOID(protocolServerHandlerAdd(server, testServerProtocol), "add handler");
+
+                TEST_ERROR(protocolServerProcess(server), AssertError, "test assert");
+                TEST_RESULT_VOID(protocolServerProcess(server), "run process loop again");
+
+                TEST_RESULT_VOID(protocolServerFree(server), "free server");
+                TEST_RESULT_VOID(protocolServerFree(NULL), "free null server");
+
+                close(pipeRead[0]);
+                close(pipeWrite[1]);
+            }
+        }
+        HARNESS_FORK_END();
+    }
+
     // *****************************************************************************************************************************
     if (testBegin("protocolGet()"))
     {
+        // Simple protocol start
+        // -------------------------------------------------------------------------------------------------------------------------
         StringList *argList = strLstNew();
         strLstAddZ(argList, "/usr/bin/pgbackrest");
         strLstAddZ(argList, "--stanza=db");
-        strLstAddZ(argList, "--db-timeout=9");
         strLstAddZ(argList, "--protocol-timeout=10");
         strLstAddZ(argList, "--repo1-host=localhost");
         strLstAdd(argList, strNewFmt("--repo1-path=%s", testPath()));
-        strLstAddZ(argList, "archive-push");
+        strLstAddZ(argList, "info");
         harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
 
         ProtocolClient *client = NULL;
@@ -239,6 +388,58 @@ testRun(void)
         TEST_RESULT_PTR(protocolGet(remoteTypeRepo, 1), client, "get cached protocol");
         TEST_RESULT_VOID(protocolFree(), "free protocol objects");
         TEST_RESULT_VOID(protocolFree(), "free protocol objects again");
+
+        // Start protocol with local encryption settings
+        // -------------------------------------------------------------------------------------------------------------------------
+        storagePut(
+            storageNewWriteNP(storageTest, strNew("pgbackrest.conf")),
+            bufNewStr(
+                strNew(
+                    "[global]\n"
+                    "repo1-cipher-type=aes-256-cbc\n"
+                    "repo1-cipher-pass=acbd\n")));
+
+        argList = strLstNew();
+        strLstAddZ(argList, "/usr/bin/pgbackrest");
+        strLstAddZ(argList, "--stanza=db");
+        strLstAddZ(argList, "--protocol-timeout=10");
+        strLstAdd(argList, strNewFmt("--config=%s/pgbackrest.conf", testPath()));
+        strLstAddZ(argList, "--repo1-host=localhost");
+        strLstAdd(argList, strNewFmt("--repo1-path=%s", testPath()));
+        strLstAddZ(argList, "info");
+        harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
+
+        TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptRepoCipherPass)), "acbd", "check cipher pass before");
+        TEST_ASSIGN(client, protocolGet(remoteTypeRepo, 1), "get protocol");
+        TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptRepoCipherPass)), "acbd", "check cipher pass after");
+
+        TEST_RESULT_VOID(protocolFree(), "free protocol objects");
+
+        // Start protocol with remote encryption settings
+        // -------------------------------------------------------------------------------------------------------------------------
+        storagePut(
+            storageNewWriteNP(storageTest, strNew("pgbackrest.conf")),
+            bufNewStr(
+                strNew(
+                    "[global]\n"
+                    "repo1-cipher-type=aes-256-cbc\n"
+                    "repo1-cipher-pass=dcba\n")));
+
+        argList = strLstNew();
+        strLstAddZ(argList, "/usr/bin/pgbackrest");
+        strLstAddZ(argList, "--stanza=db");
+        strLstAddZ(argList, "--protocol-timeout=10");
+        strLstAdd(argList, strNewFmt("--repo1-host-config=%s/pgbackrest.conf", testPath()));
+        strLstAddZ(argList, "--repo1-host=localhost");
+        strLstAdd(argList, strNewFmt("--repo1-path=%s", testPath()));
+        strLstAddZ(argList, "info");
+        harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
+
+        TEST_RESULT_PTR(cfgOptionStr(cfgOptRepoCipherPass), NULL, "check cipher pass before");
+        TEST_ASSIGN(client, protocolGet(remoteTypeRepo, 1), "get protocol");
+        TEST_RESULT_STR(strPtr(cfgOptionStr(cfgOptRepoCipherPass)), "dcba", "check cipher pass after");
+
+        TEST_RESULT_VOID(protocolFree(), "free protocol objects");
     }
 
     FUNCTION_HARNESS_RESULT_VOID();

test/src/module/storage/remoteTest.c

@@ -1,6 +1,9 @@
 /***********************************************************************************************************************************
 Test Remote Storage Driver
 ***********************************************************************************************************************************/
+#include "common/io/bufferRead.h"
+#include "common/io/bufferWrite.h"
+
 #include "common/harnessConfig.h"
 
 /***********************************************************************************************************************************
@@ -19,24 +22,32 @@ testRun(void)
     StringList *argList = strLstNew();
     strLstAddZ(argList, "/usr/bin/pgbackrest");
     strLstAddZ(argList, "--stanza=db");
-    strLstAddZ(argList, "--db-timeout=9");
     strLstAddZ(argList, "--protocol-timeout=10");
     strLstAddZ(argList, "--buffer-size=16384");
     strLstAddZ(argList, "--repo1-host=localhost");
     strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
-    strLstAddZ(argList, "archive-push");
+    strLstAddZ(argList, "info");
     harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
 
+    // Start a protocol server to test the remote protocol
+    Buffer *serverWrite = bufNew(8192);
+    IoWrite *serverWriteIo = ioBufferWriteIo(ioBufferWriteNew(serverWrite));
+    ioWriteOpen(serverWriteIo);
+
+    ProtocolServer *server = protocolServerNew(
+        strNew("test"), strNew("test"), ioBufferReadIo(ioBufferReadNew(bufNew(0))), serverWriteIo);
+
+    bufUsedSet(serverWrite, 0);
+
     // *****************************************************************************************************************************
-    if (testBegin("storageList(), storageNewRead()"))
+    if (testBegin("storageList()"))
     {
         Storage *storageRemote = NULL;
-
+        TEST_ASSIGN(storageRemote, storageRepoGet(strNew(STORAGE_TYPE_POSIX), false), "get remote repo storage");
-        // Create repo path since the remote will not start without it
         storagePathCreateNP(storageTest, strNew("repo"));
 
-        TEST_ASSIGN(storageRemote, storageRepoGet(strNew(STORAGE_TYPE_POSIX), false), "get remote repo storage");
         TEST_RESULT_STR(strPtr(strLstJoin(storageListNP(storageRemote, NULL), ",")), "" , "list empty path");
+        TEST_RESULT_PTR(storageListNP(storageRemote, strNew(BOGUS_STR)), NULL , "missing directory ignored");
 
         // -------------------------------------------------------------------------------------------------------------------------
         storagePathCreateNP(storageTest, strNew("repo/testy"));
@@ -44,10 +55,33 @@ testRun(void)
 
         storagePathCreateNP(storageTest, strNew("repo/testy2\""));
         TEST_RESULT_STR(
-            strPtr(strLstJoin(storageListNP(storageRemote, strNewFmt("%s/repo", testPath())), ",")), "testy,testy2\"" ,
+            strPtr(strLstJoin(strLstSort(storageListNP(storageRemote, strNewFmt("%s/repo", testPath())), sortOrderAsc), ",")),
-            "list 2 paths");
+            "testy,testy2\"" , "list 2 paths");
 
+        // Check protocol function directly
         // -------------------------------------------------------------------------------------------------------------------------
+        VariantList *paramList = varLstNew();
+        varLstAdd(paramList, NULL);
+        varLstAdd(paramList, varNewBool(false));
+        varLstAdd(paramList, varNewStr(strNew("^testy$")));
+
+        TEST_RESULT_BOOL(storageDriverRemoteProtocol(PROTOCOL_COMMAND_STORAGE_LIST_STR, paramList, server), true, "protocol list");
+        TEST_RESULT_STR(strPtr(strNewBuf(serverWrite)), "{\"out\":[\"testy\"]}\n", "check result");
+
+        bufUsedSet(serverWrite, 0);
+
+        // Check invalid protocol function
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_RESULT_BOOL(storageDriverRemoteProtocol(strNew(BOGUS_STR), paramList, server), false, "invalid function");
+    }
+
+    // *****************************************************************************************************************************
+    if (testBegin("storageNewRead()"))
+    {
+        Storage *storageRemote = NULL;
+        TEST_ASSIGN(storageRemote, storageRepoGet(strNew(STORAGE_TYPE_POSIX), false), "get remote repo storage");
+        storagePathCreateNP(storageTest, strNew("repo"));
+
         Buffer *contentBuf = bufNew(32768);
 
         for (unsigned int contentIdx = 0; contentIdx < bufSize(contentBuf); contentIdx++)
@@ -59,7 +93,8 @@ testRun(void)
             strPtr(strNewBuf(storageGetNP(storageNewReadNP(storageRemote, strNew("test.txt"))))), FileMissingError,
             strPtr(
                 strNewFmt(
-                    "raised from remote-1 protocol on 'localhost': unable to open '%s/repo/test.txt': No such file or directory",
+                    "raised from remote-1 protocol on 'localhost': unable to open '%s/repo/test.txt' for read:"
+                        " [2] No such file or directory",
                     testPath())));
 
         storagePutNP(storageNewWriteNP(storageTest, strNew("repo/test.txt")), contentBuf);
@@ -81,7 +116,48 @@ testRun(void)
         TEST_ERROR(
             storageDriverRemoteFileReadBlockSize(strNew("bogus")), ProtocolError, "'bogus' is not a valid block size message");
 
+        // Check protocol function directly (file missing)
         // -------------------------------------------------------------------------------------------------------------------------
+        VariantList *paramList = varLstNew();
+        varLstAdd(paramList, varNewStr(strNew("missing.txt")));
+        varLstAdd(paramList, varNewBool(true));
+
+        TEST_RESULT_BOOL(
+            storageDriverRemoteProtocol(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR, paramList, server), true,
+            "protocol open read (missing)");
+        TEST_RESULT_STR(strPtr(strNewBuf(serverWrite)), "{\"out\":false}\n", "check result");
+
+        bufUsedSet(serverWrite, 0);
+
+        // Check protocol function directly (file exists)
+        // -------------------------------------------------------------------------------------------------------------------------
+        storagePutNP(storageNewWriteNP(storageTest, strNew("repo/test.txt")), bufNewStr(strNew("TESTDATA")));
+        ioBufferSizeSet(4);
+
+        paramList = varLstNew();
+        varLstAdd(paramList, varNewStr(strNew("test.txt")));
+        varLstAdd(paramList, varNewBool(false));
+
+        TEST_RESULT_BOOL(
+            storageDriverRemoteProtocol(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR, paramList, server), true, "protocol open read");
+        TEST_RESULT_STR(
+            strPtr(strNewBuf(serverWrite)),
+            "{\"out\":true}\n"
+                "BRBLOCK4\n"
+                "TESTBRBLOCK4\n"
+                "DATABRBLOCK0\n",
+            "check result");
+
+        bufUsedSet(serverWrite, 0);
+        ioBufferSizeSet(8192);
+    }
+
+    // *****************************************************************************************************************************
+    if (testBegin("UNIMPLEMENTED"))
+    {
+        Storage *storageRemote = NULL;
+        TEST_ASSIGN(storageRemote, storageRepoGet(strNew(STORAGE_TYPE_POSIX), false), "get remote repo storage");
+
         storageRemote->write = true;
         TEST_ERROR(storageExistsNP(storageRemote, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");
         TEST_ERROR(storageInfoNP(storageRemote, strNew("file.txt")), AssertError, "NOT YET IMPLEMENTED");