Add manifestNewBuild() to performance/type test.

Use a test storage driver to allow manifestNewBuild() to be run against a test cluster at any scale without having to write files to disk.

Simplify the test by using the output of manifestNewBuild() to feed manifestSave() and manifestNewLoad().

Also add manifest size to the output.

test/src/module/performance/typeTest.c

@@ -6,16 +6,19 @@ up" in terms of execution time if there are performance problems without taking
 
 These starting values can then be scaled up for profiling and stress testing as needed.  In general we hope to scale to 1000 without
 running out of memory on the test systems or taking an undue amount of time.  It should be noted that in this context scaling to
-1000 is nowhere near to turning it up to 11.
+1000 is nowhere near turning it up to 11.
 ***********************************************************************************************************************************/
 #include "common/ini.h"
 #include "common/io/bufferRead.h"
 #include "common/io/bufferWrite.h"
 #include "common/time.h"
 #include "common/type/list.h"
+#include "common/type/object.h"
 #include "info/manifest.h"
+#include "postgres/version.h"
 
 #include "common/harnessInfo.h"
+#include "common/harnessStorage.h"
 
 /***********************************************************************************************************************************
 Test sort comparator
@@ -47,6 +50,96 @@ testIniLoadCountCallback(void *data, const String *section, const String *key, c
     (void)value;
 }
 
+/***********************************************************************************************************************************
+Driver to test manifestNewBuild(). Generates files for a valid-looking PostgreSQL cluster that can be scaled to any size.
+***********************************************************************************************************************************/
+typedef struct
+{
+    STORAGE_COMMON_MEMBER;
+    uint64_t fileTotal;
+} StorageTestManifestNewBuild;
+
+static StorageInfo
+storageTestManifestNewBuildInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param)
+{
+    (void)thisVoid; (void)level; (void)param;
+
+    StorageInfo result =
+    {
+        .level = storageInfoLevelDetail,
+        .exists = true,
+        .type = storageTypePath,
+        .mode = 0600,
+        .userId = 100,
+        .groupId = 100,
+        .user = STRDEF("test"),
+        .group = STRDEF("test"),
+    };
+
+    if (strEq(file, STRDEF("/pg")))
+    {
+        result.type = storageTypePath;
+    }
+    else
+        THROW_FMT(AssertError, "unhandled file info '%s'", strPtr(file));
+
+    return result;
+}
+
+static bool
+storageTestManifestNewBuildInfoList(
+    THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
+    StorageInterfaceInfoListParam param)
+{
+    THIS(StorageTestManifestNewBuild);
+    (void)path; (void)level; (void)param;
+
+    MEM_CONTEXT_TEMP_RESET_BEGIN()
+    {
+        StorageInfo result =
+        {
+            .level = storageInfoLevelDetail,
+            .exists = true,
+            .type = storageTypePath,
+            .mode = 0700,
+            .userId = 100,
+            .groupId = 100,
+            .user = STRDEF("test"),
+            .group = STRDEF("test"),
+        };
+
+        if (strEq(path, STRDEF("/pg")))
+        {
+            result.name = STRDEF("base");
+            callback(callbackData, &result);
+        }
+        else if (strEq(path, STRDEF("/pg/base")))
+        {
+            result.name = STRDEF("1000000000");
+            callback(callbackData, &result);
+        }
+        else if (strEq(path, STRDEF("/pg/base/1000000000")))
+        {
+            result.type = storageTypeFile;
+            result.size = 8192;
+            result.mode = 0600;
+            result.timeModified = 1595627966;
+
+            for (unsigned int fileIdx = 0; fileIdx < this->fileTotal; fileIdx++)
+            {
+                result.name = strNewFmt("%u", 1000000000 + fileIdx);
+                callback(callbackData, &result);
+                MEM_CONTEXT_TEMP_RESET(10000);
+            }
+        }
+        else
+            THROW_FMT(AssertError, "unhandled file list info '%s'", strPtr(path));
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    return true;
+}
+
 /***********************************************************************************************************************************
 Test Run
 ***********************************************************************************************************************************/
@@ -113,145 +206,76 @@ testRun(void)
         TEST_RESULT_INT(iniTotal, iniMax, "    check ini total");
     }
 
-    // Load/save a larger manifest to test performance and memory usage.  The default sizing is for a "typical" cluster but this can
+    // Build/load/save a larger manifest to test performance and memory usage. The default sizing is for a "typical" large cluster
-    // be scaled to test larger cluster sizes.
+    // but this can be scaled to test larger cluster sizes.
     // *****************************************************************************************************************************
-    if (testBegin("manifestNewLoad()/manifestSave()"))
+    if (testBegin("manifestNewBuild()/manifestNewLoad()/manifestSave()"))
     {
         CHECK(testScale() <= 1000000);
 
-        // Manifest with all features
+        // Create a storage driver to test manifest build with an arbitrary number of files
-        // -------------------------------------------------------------------------------------------------------------------------
+        StorageTestManifestNewBuild driver =
-        TEST_TITLE("generate manifest");
-
-        String *manifestStr = strNew
-        (
-            "[backup]\n"
-            "backup-label=\"20190818-084502F_20190820-084502D\"\n"
-            "backup-prior=\"20190818-084502F\"\n"
-            "backup-timestamp-copy-start=1566290707\n"
-            "backup-timestamp-start=1566290702\n"
-            "backup-timestamp-stop=1566290710\n"
-            "backup-type=\"diff\"\n"
-            "\n"
-            "[backup:db]\n"
-            "db-catalog-version=201809051\n"
-            "db-control-version=1100\n"
-            "db-id=2\n"
-            "db-system-id=6689162560678426440\n"
-            "db-version=\"11\"\n"
-            "\n"
-            "[backup:option]\n"
-            "option-archive-check=true\n"
-            "option-archive-copy=false\n"
-            "option-backup-standby=false\n"
-            "option-buffer-size=1048576\n"
-            "option-checksum-page=true\n"
-            "option-compress=true\n"
-            "option-compress-level=9\n"
-            "option-compress-level-network=3\n"
-            "option-delta=false\n"
-            "option-hardlink=false\n"
-            "option-online=false\n"
-            "option-process-max=2\n"
-            "\n"
-            "[backup:target]\n"
-            "pg_data={\"path\":\"/pg/base\",\"type\":\"path\"}\n");
-
-        for (unsigned int linkIdx = 0; linkIdx < 1; linkIdx++)
-            strCatFmt(manifestStr, "pg_data/pg_stat%u={\"path\":\"../pg_stat\",\"type\":\"link\"}\n", linkIdx);
-
-        strCatZ(
-            manifestStr,
-            "\n"
-            "[target:file]\n");
-
-        unsigned int fileTotal = 100000 * (unsigned int)testScale();
-
-        // Because of the way the filenames are formatted they will end up badly out of order.  We'll be depending on the sort after
-        // load the fix this.  Normally the files won't need sorting, but a collation issue could well cause problems for us without
-        // it.
-        for (unsigned int fileIdx = 0; fileIdx < fileTotal; fileIdx++)
         {
-            strCatFmt(
+            .interface = storageInterfaceTestDummy,
-                manifestStr,
+            .fileTotal = 100000 * (unsigned int)testScale(),
-                "pg_data/base/16384/%u={\"checksum\":\"184473f470864e067ee3a22e64b47b0a1c356f29\",\"size\":16384"
+        };
-                    ",\"timestamp\":1565282114}\n",
-                16384 + fileIdx);
-        }
 
-        strCatZ(
+        driver.interface.info = storageTestManifestNewBuildInfo;
-            manifestStr,
+        driver.interface.infoList = storageTestManifestNewBuildInfoList;
-            "\n"
-            "[target:file:default]\n"
-            "group=\"postgres\"\n"
-            "master=false\n"
-            "mode=\"0600\"\n"
-            "user=\"postgres\"\n"
-            "\n"
-            "[target:link]\n"
-            "pg_data/pg_stat={\"destination\":\"../pg_stat\"}\n"
-            "\n"
-            "[target:link:default]\n"
-            "group=\"postgres\"\n"
-            "user=\"postgres\"\n"
-            "\n"
-            "[target:path]\n"
-            "pg_data={}\n"
-            "pg_data/base={}\n"
-            "pg_data/base/1={}\n"
-            "pg_data/base/13124={}\n"
-            "pg_data/base/13125={}\n"
-            "pg_data/base/16391={}\n"
-            "pg_data/global={}\n"
-            "pg_data/pg_commit_ts={}\n"
-            "pg_data/pg_dynshmem={}\n"
-            "pg_data/pg_logical={}\n"
-            "pg_data/pg_logical/mappings={}\n"
-            "pg_data/pg_logical/snapshots={}\n"
-            "pg_data/pg_multixact={}\n"
-            "pg_data/pg_multixact/members={}\n"
-            "pg_data/pg_multixact/offsets={}\n"
-            "pg_data/pg_notify={}\n"
-            "pg_data/pg_replslot={}\n"
-            "pg_data/pg_serial={}\n"
-            "pg_data/pg_snapshots={}\n"
-            "pg_data/pg_stat={}\n"
-            "pg_data/pg_stat_tmp={}\n"
-            "pg_data/pg_subtrans={}\n"
-            "pg_data/pg_tblspc={}\n"
-            "pg_data/pg_twophase={}\n"
-            "pg_data/pg_wal={}\n"
-            "pg_data/pg_wal/archive_status={}\n"
-            "pg_data/pg_xact={}\n"
-            "\n"
-            "[target:path:default]\n"
-            "group=\"postgres\"\n"
-            "mode=\"0700\"\n"
-            "user=\"postgres\"\n"
-        );
 
-        const Buffer *contentLoad = harnessInfoChecksum(manifestStr);
+        Storage *storagePg = storageNew(STRDEF("TEST"), STRDEF("/pg"), 0, 0, false, NULL, &driver, driver.interface);
-
-        TEST_LOG_FMT("%s manifest generated with %u files", strPtr(strSizeFormat(bufUsed(contentLoad))), fileTotal);
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_TITLE("load manifest");
+        TEST_TITLE("build manifest");
 
+        MemContext *testContext = memContextNew("test");
+        memContextKeep();
+        Manifest *manifest = NULL;
         TimeMSec timeBegin = timeMSec();
-        Manifest *manifest = manifestNewLoad(ioBufferReadNew(contentLoad));
-        TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
 
-        TEST_RESULT_UINT(manifestFileTotal(manifest), fileTotal, "   check file total");
+        MEM_CONTEXT_BEGIN(testContext)
+        {
+            TEST_ASSIGN(
+                manifest, manifestNewBuild(storagePg, PG_VERSION_91, false, false, NULL, NULL), "build with %" PRIu64 " files",
+                driver.fileTotal);
+        }
+        MEM_CONTEXT_END();
+
+        TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
+        TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
+
+        TEST_RESULT_UINT(manifestFileTotal(manifest), driver.fileTotal, "   check file total");
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("save manifest");
 
         Buffer *contentSave = bufNew(0);
         timeBegin = timeMSec();
+
         manifestSave(manifest, ioBufferWriteNew(contentSave));
+
         TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
 
+        memContextFree(testContext);
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("load manifest");
+
+        testContext = memContextNew("test");
+        memContextKeep();
+        timeBegin = timeMSec();
+
+        MEM_CONTEXT_BEGIN(testContext)
+        {
+            manifest = manifestNewLoad(ioBufferReadNew(contentSave));
+        }
+        MEM_CONTEXT_END();
+
+        TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
+        TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
+
+        TEST_RESULT_UINT(manifestFileTotal(manifest), driver.fileTotal, "   check file total");
+
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("find all files");