Add performance/storage test.

The primary purpose of this test (currently) is to measure the performance of storageRemoteInfoList(), which is critical for building a manifest when the PostgreSQL host is remote.

The starting baseline of 1 million files is perhaps a bit aggressive but it seems very likely to blow up if there are performance regressions.

test/define.yaml

@@ -733,3 +733,10 @@ performance:
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: type
         total: 3
+
+      # ----------------------------------------------------------------------------------------------------------------------------
+      - name: storage
+        total: 1
+
+        include:
+          - storage/helper

test/src/module/performance/storageTest.c

@@ -0,0 +1,210 @@
+/***********************************************************************************************************************************
+Storage Performance
+
+Test the performance of various storage functions, in particular when implemented remotely.
+
+Generally speaking, the starting values should be high enough to "blow up" in terms of execution time if there are performance
+problems without taking very long if everything is running smoothly. These starting values can then be scaled up for profiling and
+stress testing as needed.  In general we hope to scale to 1000 without running out of memory on the test systems or taking an undue
+amount of time.  It should be noted that in this context scaling to 1000 is nowhere near to turning it up to 11.
+***********************************************************************************************************************************/
+#include "common/harnessConfig.h"
+#include "common/harnessFork.h"
+
+#include "common/io/handleRead.h"
+#include "common/io/handleWrite.h"
+#include "common/object.h"
+#include "protocol/client.h"
+#include "protocol/server.h"
+#include "storage/remote/protocol.h"
+#include "storage/storage.intern.h"
+
+/***********************************************************************************************************************************
+Dummy functions and interface for constructing test drivers
+***********************************************************************************************************************************/
+static bool
+storageTestDummyExists(THIS_VOID, const String *file, StorageInterfaceExistsParam param)
+{
+    (void)thisVoid; (void)file; (void)param; return false;
+}
+
+static StorageInfo
+storageTestDummyInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param)
+{
+    (void)thisVoid; (void)file; (void)param; return (StorageInfo){.exists = false};
+}
+
+static bool
+storageTestDummyInfoList(
+    THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param)
+{
+    (void)thisVoid; (void)path; (void)callback; (void)callbackData; (void)param; return false;
+}
+
+static StringList *
+storageTestDummyList(THIS_VOID, const String *path, StorageInterfaceListParam param)
+{
+    (void)thisVoid; (void)path; (void)param; return NULL;
+}
+
+static StorageRead *
+storageTestDummyNewRead(THIS_VOID, const String *file, bool ignoreMissing, StorageInterfaceNewReadParam param)
+{
+    (void)thisVoid; (void)file; (void)ignoreMissing; (void)param; return NULL;
+}
+
+static StorageWrite *
+storageTestDummyNewWrite(THIS_VOID, const String *file, StorageInterfaceNewWriteParam param)
+{
+    (void)thisVoid; (void)file; (void)param; return NULL;
+}
+
+static bool
+storageTestDummyPathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfacePathRemoveParam param)
+{
+    (void)thisVoid; (void)path; (void)recurse; (void)param; return false;
+}
+
+static void
+storageTestDummyRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam param)
+{
+    (void)thisVoid; (void)file; (void)param;
+}
+
+static const StorageInterface storageInterfaceTestDummy =
+{
+    .exists = storageTestDummyExists,
+    .info = storageTestDummyInfo,
+    .infoList = storageTestDummyInfoList,
+    .list = storageTestDummyList,
+    .newRead = storageTestDummyNewRead,
+    .newWrite = storageTestDummyNewWrite,
+    .pathRemove = storageTestDummyPathRemove,
+    .remove = storageTestDummyRemove,
+};
+
+/***********************************************************************************************************************************
+Dummy callback functions
+***********************************************************************************************************************************/
+static void
+storageTestDummyInfoListCallback(void *data, const StorageInfo *info)
+{
+    (void)data;
+    (void)info;
+
+    // Do some work in the mem context to blow up the total time if this is not efficient
+    memResize(memNew(16), 32);
+}
+
+/***********************************************************************************************************************************
+Driver to test storageInfoList
+***********************************************************************************************************************************/
+typedef struct
+{
+    STORAGE_COMMON_MEMBER;
+    uint64_t fileTotal;
+} StorageTestPerfInfoList;
+
+static bool
+storageTestPerfInfoList(
+    THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param)
+{
+    THIS(StorageTestPerfInfoList);
+    (void)path; (void)param;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        MEM_CONTEXT_TEMP_RESET_BEGIN()
+        {
+            for (uint64_t fileIdx = 0; fileIdx < this->fileTotal; fileIdx++)
+            {
+                callback(callbackData, &(StorageInfo){.exists = true});
+                MEM_CONTEXT_TEMP_RESET(1000);
+            }
+        }
+        MEM_CONTEXT_TEMP_END();
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    return this->fileTotal != 0;
+}
+
+/***********************************************************************************************************************************
+Test Run
+***********************************************************************************************************************************/
+void
+testRun(void)
+{
+    FUNCTION_HARNESS_VOID();
+
+    // *****************************************************************************************************************************
+    if (testBegin("storageInfoList()"))
+    {
+        // One million files represents a fairly large cluster
+        CHECK(testScale() <= 2000);
+        uint64_t fileTotal = (uint64_t)1000000 * testScale();
+
+        HARNESS_FORK_BEGIN()
+        {
+            HARNESS_FORK_CHILD_BEGIN(0, true)
+            {
+                // Create a basic configuration so the remote storage driver can determine the storage type
+                StringList *argList = strLstNew();
+                strLstAddZ(argList, "--" CFGOPT_STANZA "=test");
+                strLstAddZ(argList, "--" CFGOPT_PROCESS "=0");
+                strLstAddZ(argList, "--" CFGOPT_REMOTE_TYPE "=" PROTOCOL_REMOTE_TYPE_REPO);
+                harnessCfgLoadRole(cfgCmdArchivePush, cfgCmdRoleRemote, argList);
+
+                // Create a driver to test remote performance of storageInfoList() and inject it into storageRepo()
+                StorageTestPerfInfoList driver =
+                {
+                    .interface = storageInterfaceTestDummy,
+                    .fileTotal = fileTotal,
+                };
+
+                driver.interface.infoList = storageTestPerfInfoList;
+
+                storageHelper.storageRepo = storageNew(STRDEF("TEST"), STRDEF("/"), 0, 0, false, NULL, &driver, driver.interface);
+
+                // Setup handler for remote storage protocol
+                IoRead *read = ioHandleReadNew(strNew("storage server read"), HARNESS_FORK_CHILD_READ(), 60000);
+                ioReadOpen(read);
+                IoWrite *write = ioHandleWriteNew(strNew("storage server write"), HARNESS_FORK_CHILD_WRITE());
+                ioWriteOpen(write);
+
+                ProtocolServer *server = protocolServerNew(strNew("storage test server"), strNew("test"), read, write);
+                protocolServerHandlerAdd(server, storageRemoteProtocol);
+                protocolServerProcess(server);
+
+            }
+            HARNESS_FORK_CHILD_END();
+
+            HARNESS_FORK_PARENT_BEGIN()
+            {
+                // Create client
+                IoRead *read = ioHandleReadNew(strNew("storage client read"), HARNESS_FORK_PARENT_READ_PROCESS(0), 60000);
+                ioReadOpen(read);
+                IoWrite *write = ioHandleWriteNew(strNew("storage client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0));
+                ioWriteOpen(write);
+
+                ProtocolClient *client = protocolClientNew(strNew("storage test client"), strNew("test"), read, write);
+
+                // Create remote storage
+                Storage *storageRemote = storageRemoteNew(
+                    STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, false, NULL, client, 1);
+
+                // Storage info list
+                TEST_RESULT_VOID(
+                    storageInfoListP(storageRemote, NULL, storageTestDummyInfoListCallback, NULL),
+                    "list %" PRIu64 " remote files", fileTotal);
+
+                // Free client
+                protocolClientFree(client);
+            }
+            HARNESS_FORK_PARENT_END();
+        }
+        HARNESS_FORK_END();
+    }
+
+    FUNCTION_HARNESS_RESULT_VOID();
+}