More C test harness improvements and CI.

Build pgbackrest binary and auto-generated code automatically.

Remove --module option and allow modules to run by parameter. This is less verbose and multiple modules can be run at a time.

Allow filtering of modules. Multiple tests can be passed as parameters and if the module ends in / it will be used as a prefix filter. For example, common/ will run all the common modules.

If a test errors the remaining tests will still run but the test process will eventually exit with an error.

CI tests are included but unit tests remain on the development branch.

With these changes all unit tests run except those that specify the define (e.g. common/assert-off) or containerReq (e.g. protocol/protocol) keywords.

Building the C test harness has been simplified:

meson -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug test/build/none pgbackrest
ninja -C test/build/none test/src/test-pgbackrest

To run all modules:

test/build/none/test/src/test-pgbackrest test

Just the common/error module:

test/build/none/test/src/test-pgbackrest test common/error

All info modules:

test/build/none/test/src/test-pgbackrest test info/

.github/workflows/test.yml

@@ -61,6 +61,34 @@ jobs:
       - name: Run Test
         run: cd ${HOME?} && ${GITHUB_WORKSPACE?}/pgbackrest/test/ci.pl ${{matrix.param}} --param=build-max=2
 
+  # C test harness. Run inside a container so tests that require socket binding work correctly.
+  testc:
+    runs-on: ubuntu-22.04
+    container:
+      image: ubuntu:22.04
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+        with:
+          path: pgbackrest
+
+      - name: Install
+        run: |
+            apt-get update
+            DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y sudo zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config gcc ccache meson liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev tzdata
+            adduser --disabled-password --gecos \"\" runner
+
+      - name: Build
+        run: |
+          sudo -u runner cp -rp ${GITHUB_WORKSPACE?}/pgbackrest /home/runner
+          sudo -u runner meson -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug /home/runner/test/build/none /home/runner/pgbackrest
+          sudo -u runner ninja -C /home/runner/test/build/none test/src/test-pgbackrest
+
+      - name: Test
+        run: |
+          sudo -u runner /home/runner/test/build/none/test/src/test-pgbackrest --repo-path=/home/runner/pgbackrest --test-path=/home/runner/test --no-repo-copy test
+
   # Basic tests on other architectures using emulation. The emulation is so slow that running all the unit tests would be too
   # expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to
   # test on one big-endian architecture to be sure that checksums are correct.

test/src/build/config/config.yaml

@@ -10,7 +10,8 @@ command:
     log-level-default: DEBUG
     parameter-allowed: true
 
-  test: {}
+  test:
+    parameter-allowed: true
 
   noop:
     internal: true
@@ -44,16 +45,18 @@ option:
       - 512KiB
       - 1MiB
 
-  module:
-    type: string
-    command:
-      test: {}
-
   neutral-umask:
     type: boolean
     internal: true
     default: true
 
+  repo-copy:
+    type: boolean
+    default: true
+    negate: true
+    command:
+      test: {}
+
   repo-path:
     type: string
     default: pgbackrest

test/src/build/help/help.xml

@@ -77,16 +77,6 @@
                     <example>n</example>
                 </option>
 
-                <option id="module">
-                    <summary>Module to test.</summary>
-
-                    <text>
-                        <p>Unit test module to be tested.</p>
-                    </text>
-
-                    <example>common/error</example>
-                </option>
-
                 <option id="neutral-umask">
                     <summary>Use a neutral umask.</summary>
 
@@ -99,6 +89,16 @@
                     <example>n</example>
                 </option>
 
+                <option id="repo-copy">
+                    <summary>Make copy of code repository.</summary>
+
+                    <text>
+                        <p>Make a copy of the code repository for testing so changes to the code repository do not affect tests.</p>
+                    </text>
+
+                    <example>n</example>
+                </option>
+
                 <option id="repo-path">
                     <summary>Path to code repository.</summary>
 
@@ -106,7 +106,7 @@
                         <p>Path to the original code repository or a copy.</p>
                     </text>
 
-                    <example>n</example>
+                    <example>/path/to/pgbackrest</example>
                 </option>
 
                 <option id="scale">

test/src/command/help/meson.build

@@ -12,6 +12,6 @@ test_help_auto_c_inc = custom_target(
         build_code,
         'help',
         '@CURRENT_SOURCE_DIR@/../../..',
-        '@BUILD_ROOT@/test/src',
+        '@BUILD_ROOT@/test',
     ],
 )

test/src/command/test/build.c

@@ -30,7 +30,7 @@ Constants
 TestBuild *
 testBldNew(
     const String *const pathRepo, const String *const pathTest, const String *const vm, const unsigned int vmId,
-    const String *const moduleName, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime,
+    const TestDefModule *const module, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime,
     const String *const timeZone)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
@@ -38,7 +38,7 @@ testBldNew(
         FUNCTION_LOG_PARAM(STRING, pathTest);
         FUNCTION_LOG_PARAM(STRING, vm);
         FUNCTION_LOG_PARAM(UINT, vmId);
-        FUNCTION_LOG_PARAM(STRING, moduleName);
+        FUNCTION_LOG_PARAM_P(VOID, module);
         FUNCTION_LOG_PARAM(UINT, test);
         FUNCTION_LOG_PARAM(UINT64, scale);
         FUNCTION_LOG_PARAM(ENUM, logLevel);
@@ -49,7 +49,7 @@ testBldNew(
     ASSERT(pathRepo != NULL);
     ASSERT(pathTest != NULL);
     ASSERT(vm != NULL);
-    ASSERT(moduleName != NULL);
+    ASSERT(module != NULL);
     ASSERT(scale != 0);
 
     TestBuild *this = NULL;
@@ -67,7 +67,7 @@ testBldNew(
                 .pathTest = strDup(pathTest),
                 .vm = strDup(vm),
                 .vmId = vmId,
-                .moduleName = strDup(moduleName),
+                .module = module,
                 .test = test,
                 .scale = scale,
                 .logLevel = logLevel,
@@ -78,10 +78,6 @@ testBldNew(
 
         this->pub.storageRepo = storagePosixNewP(testBldPathRepo(this));
         this->pub.storageTest = storagePosixNewP(testBldPathTest(this));
-
-        // Find the module to test
-        this->pub.module = lstFind(testDefParse(testBldStorageRepo(this)).moduleList, &this->pub.moduleName);
-        CHECK(AssertError, this->pub.module != NULL, "unable to find module");
     }
     OBJ_NEW_END();
 
@@ -451,8 +447,7 @@ testBldUnit(TestBuild *const this)
 
         // Path to the project exe when it exists
         const String *const pathProjectExe = storagePathP(
-            testBldStorageTest(this),
-            strNewFmt("bin/%s%s/" PROJECT_BIN, strZ(testBldVm(this)), strEqZ(testBldVm(this), "none") ? "/src" : ""));
+            testBldStorageTest(this), strNewFmt("build/%s/src/" PROJECT_BIN, strZ(testBldVm(this))));
         strReplace(testC, STRDEF("{[C_TEST_PROJECT_EXE]}"), pathProjectExe);
 
         // Path to source -- used to construct __FILENAME__ tests
@@ -490,7 +485,7 @@ testBldUnit(TestBuild *const this)
         // Include test file
         strReplace(
             testC, STRDEF("{[C_TEST_INCLUDE]}"),
-            strNewFmt("#include \"%s/test/src/module/%sTest.c\"", strZ(pathRepo), strZ(bldEnum(NULL, testBldModuleName(this)))));
+            strNewFmt("#include \"%s/test/src/module/%sTest.c\"", strZ(pathRepo), strZ(bldEnum(NULL, module->name))));
 
         // Test list
         String *const testList = strNew();

test/src/command/test/build.h

@@ -19,7 +19,7 @@ typedef struct TestBuild TestBuild;
 Constructors
 ***********************************************************************************************************************************/
 TestBuild *testBldNew(
-    const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const String *moduleName,
+    const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const TestDefModule *module,
     unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone);
 
 /***********************************************************************************************************************************
@@ -33,7 +33,6 @@ typedef struct TestBuildPub
     const Storage *storageTest;                                     // Test storage
     const String *vm;                                               // Vm to run the test on
     unsigned int vmId;                                              // Vm id (0-based) to run the test on
-    const String *moduleName;                                       // Module to run
     const TestDefModule *module;                                    // Module definition
     unsigned int test;                                              // Specific test to run (0 if all)
     LogLevel logLevel;                                              // Test log level
@@ -93,13 +92,6 @@ testBldModule(const TestBuild *const this)
     return THIS_PUB(TestBuild)->module;
 }
 
-// Module to run
-__attribute__((always_inline)) static inline const String *
-testBldModuleName(const TestBuild *const this)
-{
-    return THIS_PUB(TestBuild)->moduleName;
-}
-
 // Specific test to run
 __attribute__((always_inline)) static inline unsigned int
 testBldTest(const TestBuild *const this)

test/src/command/test/test.c

@@ -11,67 +11,235 @@ Test Command
 #include "config/config.h"
 #include "storage/posix/storage.h"
 
+/**********************************************************************************************************************************/
+static const String *cmdTestExecLog;
+
+static void
+cmdTestExec(const String *const command)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(STRING, command);
+    FUNCTION_LOG_END();
+
+    ASSERT(cmdTestExecLog != NULL);
+    ASSERT(command != NULL);
+
+    LOG_DETAIL_FMT("exec: %s", strZ(command));
+
+    if (system(zNewFmt("%s > %s 2>&1", strZ(command), strZ(cmdTestExecLog))) != 0)
+    {
+        const Buffer *const buffer = storageGetP(
+            storageNewReadP(storagePosixNewP(FSLASH_STR), cmdTestExecLog, .ignoreMissing = true));
+
+        THROW_FMT(
+            ExecuteError, "unable to execute: %s > %s 2>&1:%s", strZ(command), strZ(cmdTestExecLog),
+            buffer == NULL || bufEmpty(buffer) ? " no log output" : zNewFmt("\n%s", strZ(strNewBuf(buffer))));
+    }
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/**********************************************************************************************************************************/
+static void
+cmdTestPathCreate(const Storage *const storage, const String *const path)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(STORAGE, storage);
+        FUNCTION_LOG_PARAM(STRING, path);
+    FUNCTION_LOG_END();
+
+    ASSERT(storage != NULL);
+
+    TRY_BEGIN()
+    {
+        storagePathRemoveP(storage, path, .recurse = true);
+    }
+    CATCH_ANY()
+    {
+        // Reset permissions
+        cmdTestExec(strNewFmt("chmod -R 777 %s", strZ(storagePathP(storage, path))));
+
+        // Try to remove again
+        storagePathRemoveP(storage, path, .recurse = true);
+    }
+    TRY_END();
+
+    storagePathCreateP(storage, path, .mode = 0770);
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
 /**********************************************************************************************************************************/
 void
 cmdTest(
     const String *const pathRepo, const String *const pathTest, const String *const vm, const unsigned int vmId,
-    const String *const moduleName, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime,
-    const String *const timeZone)
+    const StringList *moduleFilterList, const unsigned int test, const uint64_t scale, const LogLevel logLevel,
+    const bool logTime, const String *const timeZone, bool repoCopy)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(STRING, pathRepo);
         FUNCTION_LOG_PARAM(STRING, pathTest);
         FUNCTION_LOG_PARAM(STRING, vm);
         FUNCTION_LOG_PARAM(UINT, vmId);
-        FUNCTION_LOG_PARAM(STRING, moduleName);
+        FUNCTION_LOG_PARAM(STRING_LIST, moduleFilterList);
         FUNCTION_LOG_PARAM(UINT, test);
         FUNCTION_LOG_PARAM(UINT64, scale);
         FUNCTION_LOG_PARAM(ENUM, logLevel);
         FUNCTION_LOG_PARAM(BOOL, logTime);
         FUNCTION_LOG_PARAM(STRING, timeZone);
+        FUNCTION_LOG_PARAM(BOOL, repoCopy);
     FUNCTION_LOG_END();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        // Build unit test
-        TestBuild *const testBld = testBldNew(pathRepo, pathTest, vm, vmId, moduleName, test, scale, logLevel, logTime, timeZone);
-        testBldUnit(testBld);
+        // Create the data path
+        const Storage *const storageHrnId = storagePosixNewP(strNewFmt("%s/data-%u", strZ(pathTest), vmId), .write = true);
+        cmdTestExecLog = storagePathP(storageHrnId, STRDEF("exec.log"));
 
-        // Remove and recreate the test path
-        const Storage *const storageTestId = storagePosixNewP(
-            strNewFmt("%s/test-%u", strZ(testBldPathTest(testBld)), testBldVmId(testBld)), .write = true);
-        const char *const permReset = zNewFmt("chmod -R 777 %s", strZ(storagePathP(storageTestId, NULL)));
+        cmdTestPathCreate(storageHrnId, NULL);
 
-        if (system(permReset) != 0)
-            THROW_FMT(ExecuteError, "unable to execute: %s", permReset);
+        // Copy the source repository if requested (otherwise defaults to source code repository)
+        const String *pathRepoCopy = pathRepo;
 
-        storagePathRemoveP(storageTestId, NULL, .recurse = true);
-        storagePathCreateP(storageTestId, NULL, .mode = 0770);
-
-        // Meson setup
-        const String *const pathUnit = strNewFmt("%s/unit-%u", strZ(testBldPathTest(testBld)), testBldVmId(testBld));
-        const String *const pathUnitBuild = strNewFmt("%s/build", strZ(pathUnit));
-
-        if (!storageExistsP(testBldStorageTest(testBld), strNewFmt("%s/build.ninja", strZ(pathUnitBuild))))
+        if (repoCopy)
         {
-            const char *const mesonSetup = zNewFmt(
-                "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug %s %s", strZ(pathUnitBuild), strZ(pathUnit));
+            pathRepoCopy = strNewFmt("%s/repo", strZ(pathTest));
+            const Storage *const storageRepoCopy = storagePosixNewP(pathRepoCopy, .write = true);
 
-            if (system(mesonSetup) != 0)
-                THROW_FMT(ExecuteError, "unable to execute: %s", mesonSetup);
+            LOG_DETAIL_FMT("sync repo to %s", strZ(pathRepoCopy));
+            storagePathCreateP(storageRepoCopy, NULL, .mode = 0770);
+
+            cmdTestExec(
+                strNewFmt(
+                    "git -C %s ls-files -c --others --exclude-standard | rsync -LtW --files-from=- %s/ %s", strZ(pathRepo),
+                    strZ(pathRepo), strZ(pathRepoCopy)));
         }
 
-        // Ninja build
-        const char *const ninjaBuild = zNewFmt("ninja -C %s", strZ(pathUnitBuild));
+        // Build code (??? better to do this only when it is needed)
+        cmdTestExec(strNewFmt("%s/build/none/src/build-code postgres %s/extra", strZ(pathTest), strZ(pathRepoCopy)));
 
-        if (system(ninjaBuild) != 0)
-            THROW_FMT(ExecuteError, "unable to execute: %s", ninjaBuild);
+        // Build test list
+        const TestDef testDef = testDefParse(storagePosixNewP(pathRepoCopy));
+        StringList *const moduleList = strLstNew();
+        bool binRequired = false;
 
-        // Unit test
-        const char *const unitTest = zNewFmt("%s/test-unit", strZ(pathUnitBuild));
+        if (strLstEmpty(moduleFilterList))
+        {
+            StringList *const moduleFilterListEmpty = strLstNew();
+            strLstAddZ(moduleFilterListEmpty, "");
 
-        if (system(unitTest) != 0)
-            THROW_FMT(ExecuteError, "unable to execute: %s", unitTest);
+            moduleFilterList = moduleFilterListEmpty;
+        }
+
+        for (unsigned int moduleFilterIdx = 0; moduleFilterIdx < strLstSize(moduleFilterList); moduleFilterIdx++)
+        {
+            const String *const moduleFilter = strLstGet(moduleFilterList, moduleFilterIdx);
+
+            if (strEmpty(moduleFilter) || strEndsWithZ(moduleFilter, "/"))
+            {
+                bool found = false;
+
+                for (unsigned int moduleIdx = 0; moduleIdx < lstSize(testDef.moduleList); moduleIdx++)
+                {
+                    const TestDefModule *const module = lstGet(testDef.moduleList, moduleIdx);
+
+                    // ??? These test types don't run yet
+                    if (module->flag != NULL || module->containerRequired)
+                        continue;
+
+                    if (strEmpty(moduleFilter) || strBeginsWith(module->name, moduleFilter))
+                    {
+                        strLstAddIfMissing(moduleList, module->name);
+                        found = true;
+
+                        if (module->binRequired)
+                            binRequired = true;
+                    }
+                }
+
+                if (!found)
+                    THROW_FMT(ParamInvalidError, "'%s' prefix does not match any tests", strZ(moduleFilter));
+            }
+            else
+            {
+                const TestDefModule *const module = lstFind(testDef.moduleList, &moduleFilter);
+
+                if (module == NULL)
+                    THROW_FMT(ParamInvalidError, "'%s' is not a valid test", strZ(moduleFilter));
+
+                strLstAddIfMissing(moduleList, module->name);
+            }
+        }
+
+        // Build pgbackrest exe
+        if (binRequired)
+        {
+            LOG_INFO("build pgbackrest");
+            cmdTestExec(strNewFmt("ninja -C %s/build/none src/pgbackrest", strZ(pathTest)));
+        }
+
+        // Process test list
+        unsigned int errorTotal = 0;
+
+        for (unsigned int moduleIdx = 0; moduleIdx < strLstSize(moduleList); moduleIdx++)
+        {
+            const String *const moduleName = strLstGet(moduleList, moduleIdx);
+            const TestDefModule *const module = lstFind(testDef.moduleList, &moduleName);
+            CHECK(AssertError, module != NULL, "unable to find module");
+
+            TRY_BEGIN()
+            {
+                // Build unit test
+                const TimeMSec buildTimeBegin = timeMSec();
+                TestBuild *const testBld = testBldNew(
+                    pathRepoCopy, pathTest, vm, vmId, module, test, scale, logLevel, logTime, timeZone);
+                testBldUnit(testBld);
+
+                // Create the test path
+                const Storage *const storageTestId = storagePosixNewP(
+                    strNewFmt("%s/test-%u", strZ(testBldPathTest(testBld)), testBldVmId(testBld)), .write = true);
+
+                cmdTestPathCreate(storageTestId, NULL);
+
+                // Meson setup
+                const String *const pathUnit = strNewFmt("%s/unit-%u", strZ(pathTest), vmId);
+                const String *const pathUnitBuild = strNewFmt("%s/build", strZ(pathUnit));
+
+                if (!storageExistsP(testBldStorageTest(testBld), strNewFmt("%s/build.ninja", strZ(pathUnitBuild))))
+                {
+                    LOG_DETAIL("meson setup");
+                    cmdTestExec(
+                        strNewFmt(
+                            "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug %s %s", strZ(pathUnitBuild),
+                            strZ(pathUnit)));
+                }
+
+                // Ninja build
+                cmdTestExec(strNewFmt("ninja -C %s", strZ(pathUnitBuild)));
+                const TimeMSec buildTimeEnd = timeMSec();
+
+                // Unit test
+                const TimeMSec runTimeBegin = timeMSec();
+                cmdTestExec(strNewFmt("%s/test-unit", strZ(pathUnitBuild)));
+                const TimeMSec runTimeEnd = timeMSec();
+
+                LOG_INFO_FMT(
+                    "test unit %s (bld=%.3fs, run=%.3fs)", strZ(moduleName),
+                    (double)(buildTimeEnd - buildTimeBegin) / (double)MSEC_PER_SEC,
+                    (double)(runTimeEnd - runTimeBegin) / (double)MSEC_PER_SEC);
+            }
+            CATCH_ANY()
+            {
+                LOG_INFO_FMT("test unit %s", strZ(moduleName));
+                LOG_ERROR(errorCode(), errorMessage());
+                errorTotal++;
+            }
+            TRY_END();
+        }
+
+        // Return error
+        if (errorTotal > 0)
+            THROW_FMT(CommandError, "%u test failure(s)", errorTotal);
     }
     MEM_CONTEXT_TEMP_END();
 

test/src/command/test/test.h

@@ -7,13 +7,13 @@ Perform a test.
 #define TEST_COMMAND_TEST_TEST_H
 
 #include "common/logLevel.h"
-#include "common/type/string.h"
+#include "common/type/stringList.h"
 
 /***********************************************************************************************************************************
 Functions
 ***********************************************************************************************************************************/
 void cmdTest(
-    const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const String *moduleName,
-    unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone);
+    const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const StringList *moduleFilterList,
+    unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, bool repoCopy);
 
 #endif

test/src/main.c

@@ -72,10 +72,9 @@ main(int argListSize, const char *argList[])
                 {
                     cmdTest(
                         cfgOptionStr(cfgOptRepoPath), cfgOptionStr(cfgOptTestPath), cfgOptionStr(cfgOptVm),
-                        cfgOptionUInt(cfgOptVmId), cfgOptionStr(cfgOptModule),
-                        cfgOptionTest(cfgOptTest) ? cfgOptionUInt(cfgOptTest) : 0, cfgOptionUInt64(cfgOptScale),
-                        logLevelEnum(cfgOptionStrId(cfgOptLogLevelTest)), cfgOptionBool(cfgOptLogTimestamp),
-                        cfgOptionStrNull(cfgOptTz));
+                        cfgOptionUInt(cfgOptVmId), cfgCommandParam(), cfgOptionTest(cfgOptTest) ? cfgOptionUInt(cfgOptTest) : 0,
+                        cfgOptionUInt64(cfgOptScale), logLevelEnum(cfgOptionStrId(cfgOptLogLevelTest)),
+                        cfgOptionBool(cfgOptLogTimestamp), cfgOptionStrNull(cfgOptTz), cfgOptionBool(cfgOptRepoCopy));
 
                     break;
                 }