Integrate C test harness with Perl test harness.

The C test harness is used for unit tests from the Perl harness where possible. Currently, unit tests can be run in the C harness when --no-coverage is specified and --profile is not specified.

C harness tests work on meson 0.45.

The C harness runs with valgrind by default. Valgrind can be disabled with --no-valgrind.

Also rebuild containers to add meson and update the documentation so that meson builds will work (even though we don't do them yet).

.github/workflows/test.yml

@@ -65,7 +65,7 @@ jobs:
   testc:
     runs-on: ubuntu-22.04
     container:
-      image: ubuntu:22.04
+      image: ubuntu:18.04
 
     steps:
       - name: Checkout Code
@@ -76,18 +76,18 @@ jobs:
       - name: Install
         run: |
             apt-get update
-            DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y sudo zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config gcc ccache meson liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev tzdata
+            DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y sudo zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config gcc ccache meson liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev tzdata valgrind
             adduser --disabled-password --gecos \"\" runner
 
       - name: Build
         run: |
-          sudo -u runner cp -rp ${GITHUB_WORKSPACE?}/pgbackrest /home/runner
+          sudo -iu runner cp -rp ${GITHUB_WORKSPACE?}/pgbackrest /home/runner
-          sudo -u runner meson -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug /home/runner/test/build/none /home/runner/pgbackrest
+          sudo -iu runner meson -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug /home/runner/test/build/none /home/runner/pgbackrest
-          sudo -u runner ninja -C /home/runner/test/build/none test/src/test-pgbackrest
+          sudo -iu runner ninja -C /home/runner/test/build/none test/src/test-pgbackrest
 
       - name: Test
         run: |
-          sudo -u runner /home/runner/test/build/none/test/src/test-pgbackrest --repo-path=/home/runner/pgbackrest --test-path=/home/runner/test --no-repo-copy test
+          sudo -iu runner /home/runner/test/build/none/test/src/test-pgbackrest --repo-path=/home/runner/pgbackrest --test-path=/home/runner/test --no-repo-copy test
 
   # Basic tests on other architectures using emulation. The emulation is so slow that running all the unit tests would be too
   # expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to

doc/xml/user-guide.xml

@@ -396,8 +396,9 @@
         # Set locale
         RUN echo en_US.UTF-8 UTF-8 > /etc/locale.conf
 
-        # Add path to PostgreSQL
+        # Add path to PostgreSQL and package
         ENV PATH=/usr/pgsql-{[pg-version]}/bin:$PATH
+        ENV PKG_CONFIG_PATH=/usr/pgsql-{[pg-version]}/lib/pkgconfig:$PKG_CONFIG_PATH
 
         CMD ["/usr/sbin/init"]
     </host-define>

test/container.yaml

@@ -12,13 +12,10 @@
 # - docker login -u pgbackrest
 # - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
 # **********************************************************************************************************************************
-20220606A:
+20220726A:
   x86_64:
-    f36: 876069489d2b58e505cbd46366076447fd56aa5a
+    f36: 099b329ca7988b05f2cb8ef759e146ea9faab108
-    u22: 592b61a9db5f0418b027318548f9af1480ada7e1
+    rh7: 6072f05804b369681efad5cebe01704cb9d2a81a
-
+    u18: 7cf94d338a8d399f15eb551977a00a7db8e5b879
-20220519A:
+    u20: f5b22e94c4bac5589e92977a3edd52507f8fa150
-  x86_64:
+    u22: a1a0e9f6f96a75d4212c347a746d137306056606
-    u18: 36f27a31ed9af97fcbfaba09794c719c225f885d
-    u20: 17c74ed3fd3d76119f672740d77caf873fc57bac
-    rh7: 31d4a6c10534e69bc0251e11d86ee9b00971d823

test/lib/pgBackRestTest/Common/ContainerTest.pm

@@ -385,7 +385,7 @@ sub containerBuild
                 "    yum -y install openssh-server openssh-clients wget sudo valgrind git \\\n" .
                 "        perl perl-Digest-SHA perl-DBD-Pg perl-YAML-LibYAML openssl \\\n" .
                 "        gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel perl-ExtUtils-Embed rpm-build \\\n" .
-                "        libyaml-devel zlib-devel libxml2-devel lz4-devel lz4 bzip2-devel bzip2 perl-JSON-PP";
+                "        libyaml-devel zlib-devel libxml2-devel lz4-devel lz4 bzip2-devel bzip2 perl-JSON-PP ccache meson";
         }
         else
         {
@@ -396,7 +396,7 @@ sub containerBuild
                 "        libdbd-pg-perl libhtml-parser-perl libssl-dev libperl-dev \\\n" .
                 "        libyaml-libyaml-perl tzdata devscripts lintian libxml-checker-perl txt2man debhelper \\\n" .
                 "        libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev pkg-config \\\n" .
-                "        libbz2-dev bzip2 libyaml-dev libjson-pp-perl liblz4-dev liblz4-tool gnupg lsb-release";
+                "        libbz2-dev bzip2 libyaml-dev libjson-pp-perl liblz4-dev liblz4-tool gnupg lsb-release ccache meson";
 
             # This package is required to build valgrind on 32-bit
             if ($oVm->{$strOS}{&VM_ARCH} eq VM_ARCH_I386)
@@ -556,7 +556,8 @@ sub containerBuild
         if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL)
         {
             $strScript .=
-                "\n\nENV PATH=/usr/pgsql-" . @{$oOS->{&VM_DB}}[-1] . "/bin:\$PATH\n";
+                "\n\nENV PATH=/usr/pgsql-" . @{$oOS->{&VM_DB}}[-1] . "/bin:\$PATH\n" .
+                "ENV PKG_CONFIG_PATH=/usr/pgsql-" . @{$oOS->{&VM_DB}}[-1] . "/lib/pkgconfig:\$PKG_CONFIG_PATH\n";
         }
 
         #---------------------------------------------------------------------------------------------------------------------------

test/lib/pgBackRestTest/Common/JobTest.pm

@@ -114,8 +114,14 @@ sub new
     # Set try to 0
     $self->{iTry} = 0;
 
-    # Setup the path where gcc coverage will be performed
+    # Use the new C test harness?
-    $self->{strGCovPath} = "$self->{strTestPath}/gcov-$self->{oTest}->{&TEST_VM}-$self->{iVmIdx}";
+    $self->{bTestC} =
+        $self->{oTest}->{&TEST_C} && !$self->{bCoverageUnit} && !$self->{bProfile} &&
+        $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE;
+
+    # Setup the path where unit test will be built
+    $self->{strUnitPath} =
+        "$self->{strTestPath}/" . ($self->{bTestC} ? 'unit' : 'gcov') . "-$self->{iVmIdx}/$self->{oTest}->{&TEST_VM}";
     $self->{strDataPath} = "$self->{strTestPath}/data-$self->{iVmIdx}";
     $self->{strRepoPath} = "$self->{strTestPath}/repo";
 
@@ -180,13 +186,10 @@ sub run
             # Create host test directory
             $self->{oStorageTest}->pathCreate($strHostTestPath, {strMode => '0770'});
 
-            # Create gcov directory
+            # Create unit directory
-            my $bGCovExists = true;
+            if ($self->{oTest}->{&TEST_C} && !$self->{oStorageTest}->pathExists($self->{strUnitPath}))
-
-            if ($self->{oTest}->{&TEST_C} && !$self->{oStorageTest}->pathExists($self->{strGCovPath}))
             {
-                $self->{oStorageTest}->pathCreate($self->{strGCovPath}, {strMode => '0770'});
+                $self->{oStorageTest}->pathCreate($self->{strUnitPath}, {strMode => '0770', bCreateParent => true});
-                $bGCovExists = false;
             }
 
             # Create data directory
@@ -195,20 +198,31 @@ sub run
                 $self->{oStorageTest}->pathCreate($self->{strDataPath}, {strMode => '0770'});
             }
 
+            # Create ccache directory
+            my $strCCachePath = "$self->{strTestPath}/ccache-$self->{iVmIdx}/$self->{oTest}->{&TEST_VM}";
+
+            if ($self->{oTest}->{&TEST_C} && !$self->{oStorageTest}->pathExists($strCCachePath))
+            {
+                $self->{oStorageTest}->pathCreate($strCCachePath, {strMode => '0770', bCreateParent => true});
+            }
+
             if ($self->{oTest}->{&TEST_CONTAINER})
             {
                 if ($self->{oTest}->{&TEST_VM} ne VM_NONE)
                 {
                     my $strBinPath = $self->{strTestPath} . '/bin/' . $self->{oTest}->{&TEST_VM} . '/' . PROJECT_EXE;
+                    my $strBuildPath = $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM};
 
                     executeTest(
                         'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" .
                         " -v ${strHostTestPath}:${strVmTestPath}" .
-                        ($self->{oTest}->{&TEST_C} ? " -v $self->{strGCovPath}:$self->{strGCovPath}" : '') .
+                        ($self->{oTest}->{&TEST_C} ? " -v $self->{strUnitPath}:$self->{strUnitPath}" : '') .
                         ($self->{oTest}->{&TEST_C} ? " -v $self->{strDataPath}:$self->{strDataPath}" : '') .
                         " -v $self->{strBackRestBase}:$self->{strBackRestBase}" .
-                        " -v $self->{strRepoPath}:$self->{strRepoPath}:ro" .
+                        " -v $self->{strRepoPath}:$self->{strRepoPath}" .
                         ($self->{oTest}->{&TEST_BIN_REQ} ? " -v ${strBinPath}:${strBinPath}:ro" : '') .
+                        ($self->{bTestC} ? " -v ${strBuildPath}:${strBuildPath}:ro" : '') .
+                        ($self->{bTestC} ? " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" : '') .
                         ' ' . containerRepo() . ':' . $self->{oTest}->{&TEST_VM} . '-test',
                         {bSuppressStdErr => true});
                 }
@@ -227,27 +241,52 @@ sub run
         {
             my $strCommand = undef;                                 # Command to run test
 
-            # If testing C code
+            # If testing with C harness
-            if ($self->{oTest}->{&TEST_C})
+            if ($self->{bTestC})
+            {
+                # Create command
+                # ------------------------------------------------------------------------------------------------------------------
+                # Build filename for valgrind suppressions
+                my $strValgrindSuppress = $self->{strRepoPath} . '/test/src/valgrind.suppress.' . $self->{oTest}->{&TEST_VM};
+
+                $strCommand =
+                    ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '" : '') .
+                    " \\\n" .
+                    $self->{strTestPath} . '/build/' . $self->{oTest}->{&TEST_VM} . '/test/src/test-pgbackrest --no-run' .
+                        ' --no-repo-copy --repo-path=' . $self->{strTestPath} . '/repo' . '  --test-path=' . $self->{strTestPath} .
+                        ' --vm=' . $self->{oTest}->{&TEST_VM} . ' --vm-id=' . $self->{iVmIdx} . ' test ' .
+                        $self->{oTest}->{&TEST_MODULE} . '/' . $self->{oTest}->{&TEST_NAME} . " && \\\n" .
+                    # Allow stderr to be copied to stderr and stdout
+                    "exec 3>&1 && \\\n" .
+                    # Test with valgrind when requested
+                    ($self->{bValgrindUnit} && $self->{oTest}->{&TEST_TYPE} ne TESTDEF_PERFORMANCE ?
+                        'valgrind -q --gen-suppressions=all' .
+                        ($self->{oStorageTest}->exists($strValgrindSuppress) ? " --suppressions=${strValgrindSuppress}" : '') .
+                        " --exit-on-first-error=yes --leak-check=full --leak-resolution=high --error-exitcode=25" . ' ' : '') .
+                        $self->{strUnitPath} . '/build/test-unit 2>&1 1>&3 | tee /dev/stderr' .
+                    ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "'" : '');
+            }
+            # Else still testing with Perl harness
+            elsif ($self->{oTest}->{&TEST_C})
             {
                 my $strRepoCopyPath = $self->{strTestPath} . '/repo';           # Path to repo copy
                 my $strRepoCopySrcPath = $strRepoCopyPath . '/src';             # Path to repo copy src
                 my $strRepoCopyTestSrcPath = $strRepoCopyPath . '/test/src';    # Path to repo copy test src
-                my $strShimSrcPath = $self->{strGCovPath} . '/src';             # Path to shim src
+                my $strShimSrcPath = $self->{strUnitPath} . '/src';             # Path to shim src
-                my $strShimTestSrcPath = $self->{strGCovPath} . '/test/src';    # Path to shim test src
+                my $strShimTestSrcPath = $self->{strUnitPath} . '/test/src';    # Path to shim test src
 
                 my $bCleanAll = false;                              # Do all object files need to be cleaned?
                 my $bConfigure = false;                             # Does configure need to be run?
 
                 # If the build.processing file exists then wipe the path to start clean
                 # ------------------------------------------------------------------------------------------------------------------
-                my $strBuildProcessingFile = $self->{strGCovPath} . "/build.processing";
+                my $strBuildProcessingFile = $self->{strUnitPath} . "/build.processing";
 
                 # If the file exists then processing terminated before test.bin was run in the last test and the path might be in a
                 # bad state.
                 if ($self->{oStorageTest}->exists($strBuildProcessingFile))
                 {
-                    executeTest("find $self->{strGCovPath} -mindepth 1 -print0 | xargs -0 rm -rf");
+                    executeTest("find $self->{strUnitPath} -mindepth 1 -print0 | xargs -0 rm -rf");
                 }
 
                 # Write build.processing to track processing of this test
@@ -263,7 +302,7 @@ sub run
                     "LIBS_CONFIG = \@LIBS\@ \@LIBS_BUILD\@\n";
 
                 # If Makefile.in has changed then configure needs to be run and all files cleaned
-                if (buildPutDiffers($self->{oStorageTest}, $self->{strGCovPath} . "/Makefile.in", $strMakefileIn))
+                if (buildPutDiffers($self->{oStorageTest}, $self->{strUnitPath} . "/Makefile.in", $strMakefileIn))
                 {
                     $bConfigure = true;
                     $bCleanAll = true;
@@ -334,7 +373,7 @@ sub run
                     "vpath \%.c ${strShimSrcPath}:${strShimTestSrcPath}:${strRepoCopySrcPath}:${strRepoCopyTestSrcPath}\n";
 
                 # If Makefile.param has changed then clean all files
-                if (buildPutDiffers($self->{oStorageTest}, $self->{strGCovPath} . "/Makefile.param", $strMakefileParam))
+                if (buildPutDiffers($self->{oStorageTest}, $self->{strUnitPath} . "/Makefile.param", $strMakefileParam))
                 {
                     $bCleanAll = true;
                 }
@@ -543,7 +582,7 @@ sub run
                     "DEP_FILES = \$(call rwildcard,.build,*.dep)\n" .
                     "include \$(DEP_FILES)\n";
 
-                buildPutDiffers($self->{oStorageTest}, $self->{strGCovPath} . "/Makefile", $strMakefile);
+                buildPutDiffers($self->{oStorageTest}, $self->{strUnitPath} . "/Makefile", $strMakefile);
 
                 # Create test.c
                 # ------------------------------------------------------------------------------------------------------------------
@@ -673,12 +712,12 @@ sub run
 
                 # Save test.c and make sure it gets a new timestamp
                 # ------------------------------------------------------------------------------------------------------------------
-                my $strTestCFile = "$self->{strGCovPath}/test.c";
+                my $strTestCFile = "$self->{strUnitPath}/test.c";
 
-                if (buildPutDiffers($self->{oStorageTest}, "$self->{strGCovPath}/test.c", $strTestC))
+                if (buildPutDiffers($self->{oStorageTest}, "$self->{strUnitPath}/test.c", $strTestC))
                 {
                     # Get timestamp for test.bin if it existss
-                    my $oTestBinInfo = $self->{oStorageTest}->info("$self->{strGCovPath}/test.bin", {bIgnoreMissing => true});
+                    my $oTestBinInfo = $self->{oStorageTest}->info("$self->{strUnitPath}/test.bin", {bIgnoreMissing => true});
                     my $iTestBinOriginalTime = defined($oTestBinInfo) ? $oTestBinInfo->mtime : 0;
 
                     # Get timestamp for test.c
@@ -709,7 +748,7 @@ sub run
                 $strCommand =
                     ($self->{oTest}->{&TEST_VM} ne VM_NONE ? "docker exec -i -u ${\TEST_USER} ${strImage} bash -l -c '" : '') .
                     " \\\n" .
-                    "cd $self->{strGCovPath} && \\\n" .
+                    "cd $self->{strUnitPath} && \\\n" .
                     # Clean build
                     ($bCleanAll ? "rm -rf .build && \\\n" : '') .
                     # Remove coverage data
@@ -811,12 +850,12 @@ sub end
         {
             executeTest(
                 ($self->{oTest}->{&TEST_VM} ne VM_NONE ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') .
-                    "gprof $self->{strGCovPath}/test.bin $self->{strGCovPath}/gmon.out > $self->{strGCovPath}/gprof.txt");
+                    "gprof $self->{strUnitPath}/test.bin $self->{strUnitPath}/gmon.out > $self->{strUnitPath}/gprof.txt");
 
             $self->{oStorageTest}->pathCreate(
                 "$self->{strBackRestBase}/test/result/profile", {strMode => '0750', bIgnoreExists => true, bCreateParent => true});
             $self->{oStorageTest}->copy(
-                "$self->{strGCovPath}/gprof.txt", "$self->{strBackRestBase}/test/result/profile/gprof.txt");
+                "$self->{strUnitPath}/gprof.txt", "$self->{strBackRestBase}/test/result/profile/gprof.txt");
         }
 
         # If C code generate coverage info
@@ -826,7 +865,7 @@ sub end
                 $self->{oStorageTest}, $self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME},
                 $self->{oTest}->{&TEST_VM} ne VM_NONE, $self->{bCoverageSummary},
                 $self->{oTest}->{&TEST_VM} eq VM_NONE ? undef : $strImage, $self->{strTestPath}, "$self->{strTestPath}/temp",
-                $self->{strGCovPath}, $self->{strBackRestBase} . '/test/result');
+                $self->{strUnitPath}, $self->{strBackRestBase} . '/test/result');
         }
 
         # Record elapsed time

test/src/build/config/config.yaml

@@ -63,6 +63,13 @@ option:
     command:
       test: {}
 
+  run:
+    type: boolean
+    default: true
+    negate: true
+    command:
+      test: {}
+
   scale:
     type: integer
     internal: true
@@ -92,6 +99,13 @@ option:
     command:
       test: {}
 
+  valgrind:
+    type: boolean
+    default: true
+    negate: true
+    command:
+      test: {}
+
   vm:
     type: string
     internal: true

test/src/build/help/help.xml

@@ -109,6 +109,16 @@
                     <example>/path/to/pgbackrest</example>
                 </option>
 
+                <option id="run">
+                    <summary>Run the test?</summary>
+
+                    <text>
+                        <p>When negated the test will only be built.</p>
+                    </text>
+
+                    <example>n</example>
+                </option>
+
                 <option id="scale">
                     <summary>Scale performance test.</summary>
 
@@ -149,6 +159,16 @@
                     <example>America/New_York</example>
                 </option>
 
+                <option id="valgrind">
+                    <summary>Run test with valgrind.</summary>
+
+                    <text>
+                        <p>Valgrind helps find various memory issues.</p>
+                    </text>
+
+                    <example>n</example>
+                </option>
+
                 <option id="vm">
                     <summary>VM to run test on.</summary>
 

test/src/command/test/build.c

@@ -190,6 +190,60 @@ testBldWrite(const Storage *const storage, StringList *const fileList, const cha
     FUNCTION_LOG_RETURN_VOID();
 }
 
+/***********************************************************************************************************************************
+Generate a relative path from the compare path to the base path
+
+??? This function has not been hardened for edge cases, e.g. paths are equal. Probably this should he moved to the storage module.
+***********************************************************************************************************************************/
+static String *
+cmdBldPathRelative(const String *const base, const String *const compare)
+{
+    FUNCTION_LOG_BEGIN(logLevelDebug);
+        FUNCTION_LOG_PARAM(STRING, base);
+        FUNCTION_LOG_PARAM(STRING, compare);
+    FUNCTION_LOG_END();
+
+    ASSERT(base != NULL);
+    ASSERT(compare != NULL);
+
+    String *const result = strNew();
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        const StringList *const baseList = strLstNewSplitZ(base, "/");
+        const StringList *const compareList = strLstNewSplitZ(compare, "/");
+        unsigned int compareIdx = 0;
+
+        // Find the part of the paths that is the same
+        while (
+            compareIdx < strLstSize(baseList) && compareIdx < strLstSize(compareList) &&
+            strEq(strLstGet(baseList, compareIdx), strLstGet(compareList, compareIdx)))
+        {
+            compareIdx++;
+        }
+
+        // Generate ../ part of relative path
+        bool first = true;
+
+        for (unsigned int dotIdx = compareIdx; dotIdx < strLstSize(baseList); dotIdx++)
+        {
+            if (!first)
+                strCatChr(result, '/');
+            else
+                first = false;
+
+            strCatZ(result, "..");
+        }
+
+        // Add remaining path
+        for (unsigned int pathIdx = compareIdx; pathIdx < strLstSize(compareList); pathIdx++)
+            strCatFmt(result, "/%s", strZ(strLstGet(compareList, pathIdx)));
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN(STRING, result);
+}
+
 /**********************************************************************************************************************************/
 void
 testBldUnit(TestBuild *const this)
@@ -205,13 +259,14 @@ testBldUnit(TestBuild *const this)
     MEM_CONTEXT_TEMP_BEGIN()
     {
         const Storage *const storageUnit = storagePosixNewP(
-            strNewFmt("%s/unit-%u", strZ(testBldPathTest(this)), testBldVmId(this)), .write = true);
+            strNewFmt("%s/unit-%u/%s", strZ(testBldPathTest(this)), testBldVmId(this), strZ(testBldVm(this))), .write = true);
         const Storage *const storageTestId = storagePosixNewP(
             strNewFmt("%s/test-%u", strZ(testBldPathTest(this)), testBldVmId(this)), .write = true);
         StringList *const storageUnitList = strLstNew();
         const TestDefModule *const module = testBldModule(this);
-        const String *const pathRepo = testBldPathRepo(this);
         const String *const pathUnit = storagePathP(storageUnit, NULL);
+        const String *const pathRepo = testBldPathRepo(this);
+        const String *const pathRepoRel = cmdBldPathRelative(pathUnit, pathRepo);
 
         // Build shim modules
         // -------------------------------------------------------------------------------------------------------------------------
@@ -258,11 +313,11 @@ testBldUnit(TestBuild *const this)
 
                 strReplace(harnessC, STRDEF("{[SHIM_MODULE]}"), includeReplace);
                 testBldWrite(storageUnit, storageUnitList, strZ(harnessFile), BUFSTR(harnessC));
-                strLstAddFmt(harnessList, "%s/%s", strZ(pathUnit), strZ(harnessFile));
+                strLstAdd(harnessList, harnessFile);
             }
             // Else harness can be referenced directly from the repo path
             else
-                strLstAdd(harnessList, harnessPath);
+                strLstAddFmt(harnessList, "%s/%s", strZ(pathRepoRel), strZ(harnessFile));
         }
 
         // Copy meson_options.txt
@@ -315,8 +370,7 @@ testBldUnit(TestBuild *const this)
             MESON_COMMENT_BLOCK "\n"
             "# Unit test\n"
             MESON_COMMENT_BLOCK "\n"
-            "executable(\n"
+            "src_unit = files(\n");
-            "    'test-unit',\n");
 
         for (unsigned int dependIdx = 0; dependIdx < strLstSize(module->dependList); dependIdx++)
         {
@@ -325,7 +379,7 @@ testBldUnit(TestBuild *const this)
             if (strLstExists(harnessIncludeList, depend))
                 continue;
 
-            strCatFmt(mesonBuild, "    '%s/src/%s.c',\n", strZ(pathRepo), strZ(depend));
+            strCatFmt(mesonBuild, "    '%s/src/%s.c',\n", strZ(pathRepoRel), strZ(depend));
         }
 
         // Add harnesses
@@ -334,13 +388,16 @@ testBldUnit(TestBuild *const this)
             const TestDefHarness *const harness = lstGet(module->harnessList, harnessIdx);
 
             // Add harness depends
-            const String *const harnessDependPath = strNewFmt(
+            const String *const harnessDependPath = strNewFmt("test/src/common/%s", strZ(bldEnum("harness", harness->name)));
-                "%s/test/src/common/%s", strZ(pathRepo), strZ(bldEnum("harness", harness->name)));
             StorageIterator *const storageItr = storageNewItrP(
                 testBldStorageRepo(this), harnessDependPath, .expression = STRDEF("\\.c$"), .sortOrder = sortOrderAsc);
 
             while (storageItrMore(storageItr))
-                strCatFmt(mesonBuild, "    '%s/%s',\n", strZ(harnessDependPath), strZ(storageItrNext(storageItr).name));
+            {
+                strCatFmt(
+                    mesonBuild, "    '%s/%s/%s',\n", strZ(pathRepoRel), strZ(harnessDependPath),
+                    strZ(storageItrNext(storageItr).name));
+            }
 
             // Add harness if no includes are in module or coverage includes
             unsigned int includeIdx = 0;
@@ -363,6 +420,11 @@ testBldUnit(TestBuild *const this)
             mesonBuild,
             "    '%s/test/src/common/harnessTest.c',\n"
             "    'test.c',\n"
+            ")\n"
+            "\n"
+            "executable(\n"
+            "    'test-unit',\n"
+            "    sources: src_unit,\n"
             "    include_directories:\n"
             "        include_directories(\n"
             "            '.',\n"
@@ -380,7 +442,7 @@ testBldUnit(TestBuild *const this)
             "        lib_zstd,\n"
             "    ],\n"
             ")\n",
-            strZ(pathRepo), strZ(pathRepo), strZ(pathRepo));
+            strZ(pathRepoRel), strZ(pathRepoRel), strZ(pathRepoRel));
 
         testBldWrite(storageUnit, storageUnitList, "meson.build", BUFSTR(mesonBuild));
 
@@ -394,9 +456,9 @@ testBldUnit(TestBuild *const this)
 
         if (module->coverageList != NULL)
         {
-        for (unsigned int coverageIdx = 0; coverageIdx < lstSize(module->coverageList); coverageIdx++)
+            for (unsigned int coverageIdx = 0; coverageIdx < lstSize(module->coverageList); coverageIdx++)
-        {
+            {
-            const TestDefCoverage *const coverage = lstGet(module->coverageList, coverageIdx);
+                const TestDefCoverage *const coverage = lstGet(module->coverageList, coverageIdx);
 
                 if (coverage->coverable && !coverage->include)
                     strLstAdd(testIncludeFileList, coverage->name);
@@ -405,8 +467,8 @@ testBldUnit(TestBuild *const this)
 
         if (module->includeList != NULL)
         {
-        for (unsigned int includeIdx = 0; includeIdx < strLstSize(module->includeList); includeIdx++)
+            for (unsigned int includeIdx = 0; includeIdx < strLstSize(module->includeList); includeIdx++)
-            strLstAdd(testIncludeFileList, strLstGet(module->includeList, includeIdx));
+                strLstAdd(testIncludeFileList, strLstGet(module->includeList, includeIdx));
         }
 
         String *const testIncludeFile = strNew();
@@ -430,7 +492,7 @@ testBldUnit(TestBuild *const this)
             if (harnessIdx != lstSize(module->harnessList))
                 strCatFmt(testIncludeFile, "#include \"%s\"", strZ(strLstGet(harnessList, harnessIdx)));
             else
-                strCatFmt(testIncludeFile, "#include \"%s/src/%s.c\"", strZ(pathRepo), strZ(include));
+                strCatFmt(testIncludeFile, "#include \"%s/src/%s.c\"", strZ(pathRepoRel), strZ(include));
         }
 
         strReplace(testC, STRDEF("{[C_INCLUDE]}"), testIncludeFile);
@@ -447,11 +509,14 @@ testBldUnit(TestBuild *const this)
 
         // Path to the project exe when it exists
         const String *const pathProjectExe = storagePathP(
-            testBldStorageTest(this), strNewFmt("build/%s/src/" PROJECT_BIN, strZ(testBldVm(this))));
+            testBldStorageTest(this),
+            strNewFmt(
+                "%s/%s%s/" PROJECT_BIN, strEqZ(testBldVm(this), "none") ? "build" : "bin", strZ(testBldVm(this)),
+                strEqZ(testBldVm(this), "none") ? "/src" : ""));
         strReplace(testC, STRDEF("{[C_TEST_PROJECT_EXE]}"), pathProjectExe);
 
         // Path to source -- used to construct __FILENAME__ tests
-        strReplace(testC, STRDEF("{[C_TEST_PGB_PATH]}"), pathRepo);
+        strReplace(testC, STRDEF("{[C_TEST_PGB_PATH]}"), strNewFmt("../%s", strZ(pathRepoRel)));
 
         // Test log level
         strReplace(
@@ -485,7 +550,7 @@ testBldUnit(TestBuild *const this)
         // Include test file
         strReplace(
             testC, STRDEF("{[C_TEST_INCLUDE]}"),
-            strNewFmt("#include \"%s/test/src/module/%sTest.c\"", strZ(pathRepo), strZ(bldEnum(NULL, module->name))));
+            strNewFmt("#include \"%s/test/src/module/%sTest.c\"", strZ(pathRepoRel), strZ(bldEnum(NULL, module->name))));
 
         // Test list
         String *const testList = strNew();

test/src/command/test/test.c

@@ -74,7 +74,7 @@ void
 cmdTest(
     const String *const pathRepo, const String *const pathTest, const String *const vm, const unsigned int vmId,
     const StringList *moduleFilterList, const unsigned int test, const uint64_t scale, const LogLevel logLevel,
-    const bool logTime, const String *const timeZone, bool repoCopy)
+    const bool logTime, const String *const timeZone, const bool repoCopy, const bool valgrind, const bool run)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(STRING, pathRepo);
@@ -88,15 +88,21 @@ cmdTest(
         FUNCTION_LOG_PARAM(BOOL, logTime);
         FUNCTION_LOG_PARAM(STRING, timeZone);
         FUNCTION_LOG_PARAM(BOOL, repoCopy);
+        FUNCTION_LOG_PARAM(BOOL, valgrind);
+        FUNCTION_LOG_PARAM(BOOL, run);
     FUNCTION_LOG_END();
 
     MEM_CONTEXT_TEMP_BEGIN()
     {
-        // Create the data path
+        // Log file name
-        const Storage *const storageHrnId = storagePosixNewP(strNewFmt("%s/data-%u", strZ(pathTest), vmId), .write = true);
+        cmdTestExecLog = strNewFmt("%s/exec-%u.log", strZ(pathTest), vmId);
-        cmdTestExecLog = storagePathP(storageHrnId, STRDEF("exec.log"));
 
-        cmdTestPathCreate(storageHrnId, NULL);
+        // Create data path
+        if (run)
+        {
+            const Storage *const storageHrnId = storagePosixNewP(strNewFmt("%s/data-%u", strZ(pathTest), vmId), .write = true);
+            cmdTestPathCreate(storageHrnId, NULL);
+        }
 
         // Copy the source repository if requested (otherwise defaults to source code repository)
         const String *pathRepoCopy = pathRepo;
@@ -116,7 +122,8 @@ cmdTest(
         }
 
         // Build code (??? better to do this only when it is needed)
-        cmdTestExec(strNewFmt("%s/build/none/src/build-code postgres %s/extra", strZ(pathTest), strZ(pathRepoCopy)));
+        if (run)
+            cmdTestExec(strNewFmt("%s/build/%s/src/build-code postgres %s/extra", strZ(pathTest), strZ(vm), strZ(pathRepoCopy)));
 
         // Build test list
         const TestDef testDef = testDefParse(storagePosixNewP(pathRepoCopy));
@@ -143,8 +150,8 @@ cmdTest(
                 {
                     const TestDefModule *const module = lstGet(testDef.moduleList, moduleIdx);
 
-                    // ??? These test types don't run yet
+                    // ??? Container tests don not run yet
-                    if (module->flag != NULL || module->containerRequired)
+                    if (module->containerRequired)
                         continue;
 
                     if (strEmpty(moduleFilter) || strBeginsWith(module->name, moduleFilter))
@@ -172,7 +179,7 @@ cmdTest(
         }
 
         // Build pgbackrest exe
-        if (binRequired)
+        if (run && binRequired)
         {
             LOG_INFO("build pgbackrest");
             cmdTestExec(strNewFmt("ninja -C %s/build/none src/pgbackrest", strZ(pathTest)));
@@ -180,6 +187,8 @@ cmdTest(
 
         // Process test list
         unsigned int errorTotal = 0;
+        bool buildRetry = false;
+        const char *buildTypeLast = NULL;
 
         for (unsigned int moduleIdx = 0; moduleIdx < strLstSize(moduleList); moduleIdx++)
         {
@@ -190,54 +199,109 @@ cmdTest(
             TRY_BEGIN()
             {
                 // Build unit test
-                const TimeMSec buildTimeBegin = timeMSec();
+                const String *const pathUnit = strNewFmt("%s/unit-%u/%s", strZ(pathTest), vmId, strZ(vm));
-                TestBuild *const testBld = testBldNew(
-                    pathRepoCopy, pathTest, vm, vmId, module, test, scale, logLevel, logTime, timeZone);
-                testBldUnit(testBld);
-
-                // Create the test path
-                const Storage *const storageTestId = storagePosixNewP(
-                    strNewFmt("%s/test-%u", strZ(testBldPathTest(testBld)), testBldVmId(testBld)), .write = true);
-
-                cmdTestPathCreate(storageTestId, NULL);
-
-                // Meson setup
-                const String *const pathUnit = strNewFmt("%s/unit-%u", strZ(pathTest), vmId);
                 const String *const pathUnitBuild = strNewFmt("%s/build", strZ(pathUnit));
+                const TimeMSec buildTimeBegin = timeMSec();
+                TimeMSec buildTimeEnd;
+                TestBuild *testBld;
 
-                if (!storageExistsP(testBldStorageTest(testBld), strNewFmt("%s/build.ninja", strZ(pathUnitBuild))))
+                TRY_BEGIN()
                 {
-                    LOG_DETAIL("meson setup");
+                    // Build unit
-                    cmdTestExec(
+                    testBld = testBldNew(
-                        strNewFmt(
+                        pathRepoCopy, pathTest, vm, vmId, module, test, scale, logLevel, logTime, timeZone);
-                            "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug %s %s", strZ(pathUnitBuild),
+                    testBldUnit(testBld);
-                            strZ(pathUnit)));
+
+                    // Meson setup
+                    const char *buildType = "debug";
+
+                    if (module->flag != NULL)
+                    {
+                        ASSERT(strEqZ(module->flag, "-DNDEBUG"));
+                        buildType = "release";
+                    }
+
+                    if (!storageExistsP(testBldStorageTest(testBld), strNewFmt("%s/build.ninja", strZ(pathUnitBuild))))
+                    {
+                        LOG_DETAIL("meson setup");
+                        cmdTestExec(
+                            strNewFmt(
+                                "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=%s %s %s", buildType,
+                                strZ(pathUnitBuild), strZ(pathUnit)));
+
+                        buildTypeLast = buildType;
+                    }
+
+                    // Reconfigure as needed
+                    if (buildType != buildTypeLast)
+                    {
+                        LOG_DETAIL("meson configure");
+                        cmdTestExec(strNewFmt("meson configure -Dbuildtype=%s %s", buildType, strZ(pathUnitBuild)));
+                        buildTypeLast = buildType;
+                    }
+
+                    // Ninja build
+                    cmdTestExec(strNewFmt("ninja -C %s", strZ(pathUnitBuild)));
+                    buildTimeEnd = timeMSec();
+
+                    buildRetry = false;
                 }
+                CATCH_ANY()
+                {
+                    // If this is the first build failure then clean the build path a retry
+                    if (buildRetry == false)
+                    {
+                        buildRetry = true;
+                        moduleIdx--;
 
-                // Ninja build
+                        LOG_WARN_FMT("build failed for unit %s -- will retry: %s", strZ(moduleName), errorMessage());
-                cmdTestExec(strNewFmt("ninja -C %s", strZ(pathUnitBuild)));
+                        cmdTestPathCreate(storagePosixNewP(pathTest, .write = true), pathUnit);
-                const TimeMSec buildTimeEnd = timeMSec();
+                    }
+                    // Else error
+                    else
+                    {
+                        buildRetry = false;
+                        RETHROW();
+                    }
+                }
+                TRY_END();
 
-                // Unit test
+                // Skip test if build needs to be retried
-                const TimeMSec runTimeBegin = timeMSec();
+                if (run && !buildRetry)
-                cmdTestExec(strNewFmt("%s/test-unit", strZ(pathUnitBuild)));
+                {
-                const TimeMSec runTimeEnd = timeMSec();
+                    // Create test path
+                    const Storage *const storageTestId = storagePosixNewP(
+                        strNewFmt("%s/test-%u", strZ(testBldPathTest(testBld)), testBldVmId(testBld)), .write = true);
 
-                LOG_INFO_FMT(
+                    cmdTestPathCreate(storageTestId, NULL);
-                    "test unit %s (bld=%.3fs, run=%.3fs)", strZ(moduleName),
+
-                    (double)(buildTimeEnd - buildTimeBegin) / (double)MSEC_PER_SEC,
+                    // Unit test
-                    (double)(runTimeEnd - runTimeBegin) / (double)MSEC_PER_SEC);
+                    const TimeMSec runTimeBegin = timeMSec();
+                    String *const command = strNew();
+
+                    if (valgrind)
+                        strCatZ(command, "valgrind -q ");
+
+                    strCatFmt(command, "%s/test-unit", strZ(pathUnitBuild));
+                    cmdTestExec(command);
+
+                    const TimeMSec runTimeEnd = timeMSec();
+
+                    LOG_INFO_FMT(
+                        "test unit %s (bld=%.3fs, run=%.3fs)", strZ(moduleName),
+                        (double)(buildTimeEnd - buildTimeBegin) / (double)MSEC_PER_SEC,
+                        (double)(runTimeEnd - runTimeBegin) / (double)MSEC_PER_SEC);
+                }
             }
             CATCH_ANY()
             {
-                LOG_INFO_FMT("test unit %s", strZ(moduleName));
+                LOG_ERROR_FMT(errorCode(), "test unit %s failed: %s", strZ(moduleName), errorMessage());
-                LOG_ERROR(errorCode(), errorMessage());
                 errorTotal++;
             }
             TRY_END();
         }
 
-        // Return error
+        // Report errors
         if (errorTotal > 0)
             THROW_FMT(CommandError, "%u test failure(s)", errorTotal);
     }

test/src/command/test/test.h

@@ -14,6 +14,7 @@ Functions
 ***********************************************************************************************************************************/
 void cmdTest(
     const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const StringList *moduleFilterList,
-    unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, bool repoCopy);
+    unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, bool repoCopy, bool valgrind,
+    bool run);
 
 #endif

test/src/main.c

@@ -74,7 +74,8 @@ main(int argListSize, const char *argList[])
                         cfgOptionStr(cfgOptRepoPath), cfgOptionStr(cfgOptTestPath), cfgOptionStr(cfgOptVm),
                         cfgOptionUInt(cfgOptVmId), cfgCommandParam(), cfgOptionTest(cfgOptTest) ? cfgOptionUInt(cfgOptTest) : 0,
                         cfgOptionUInt64(cfgOptScale), logLevelEnum(cfgOptionStrId(cfgOptLogLevelTest)),
-                        cfgOptionBool(cfgOptLogTimestamp), cfgOptionStrNull(cfgOptTz), cfgOptionBool(cfgOptRepoCopy));
+                        cfgOptionBool(cfgOptLogTimestamp), cfgOptionStrNull(cfgOptTz), cfgOptionBool(cfgOptRepoCopy),
+                        cfgOptionBool(cfgOptValgrind), cfgOptionBool(cfgOptRun));
 
                     break;
                 }

test/test.pl

@@ -141,7 +141,7 @@ my $bVersion = false;
 my $bHelp = false;
 my $bQuiet = false;
 my $strPgVersion = 'minimal';
-my $strVm;
+my $strVm = VM_NONE;
 my $bVmBuild = false;
 my $bVmForce = false;
 my $bBuildOnly = false;
@@ -298,6 +298,9 @@ eval
         confess "Only one --test can be provided when --run is specified";
     }
 
+    # Check vm
+    vmValid($strVm);
+
     # Set test path if not explicitly set
     if (!defined($strTestPath))
     {
@@ -319,17 +322,6 @@ eval
         }
     }
 
-    # If VM is not defined then set it to all
-    if (!defined($strVm))
-    {
-        $strVm = VM_ALL;
-    }
-    # Else make sure vm is valid
-    elsif ($strVm ne VM_ALL)
-    {
-        vmValid($strVm);
-    }
-
     # Get the base backrest path
     my $strBackRestBase = dirname(dirname(abs_path($0)));
     my $strVagrantPath = "${strBackRestBase}/test/.vagrant";
@@ -393,6 +385,49 @@ eval
         #---------------------------------------------------------------------------------------------------------------------------
         executeTest("make -C ${strBackRestBase}/src -f Makefile.in clean-all");
 
+        # Clean up
+        #---------------------------------------------------------------------------------------------------------------------------
+        my $iTestFail = 0;
+        my $iTestRetry = 0;
+        my $oyProcess = [];
+        my $strCodePath = "${strBackRestBase}/test/result/coverage/raw";
+
+        if (!$bDryRun || $bVmOut)
+        {
+            &log(INFO, "cleanup old data" . ($strVm ne VM_NONE ? " and containers" : ''));
+
+            if ($strVm ne VM_NONE)
+            {
+                containerRemove('test-([0-9]+|build)');
+            }
+
+            for (my $iVmIdx = 0; $iVmIdx < 8; $iVmIdx++)
+            {
+                push(@{$oyProcess}, undef);
+            }
+
+            executeTest(
+                "chmod 700 -R ${strTestPath}/test-* 2>&1 || true && rm -rf ${strTestPath}/temp ${strTestPath}/test-*" .
+                    " ${strTestPath}/data-*");
+            $oStorageTest->pathCreate("${strTestPath}/temp", {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
+
+            # Remove old lcov dirs -- do it this way so the dirs stay open in finder/explorer, etc.
+            executeTest("rm -rf ${strBackRestBase}/test/result/coverage/lcov/*");
+
+            # Overwrite the C coverage report so it will load but not show old coverage
+            $oStorageTest->pathCreate(
+                "${strBackRestBase}/test/result/coverage", {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
+            $oStorageBackRest->put(
+                "${strBackRestBase}/test/result/coverage/coverage.html", "<center>[ Generating New Report ]</center>");
+
+            # Copy C code for coverage tests
+            if (vmCoverageC($strVm) && !$bDryRun)
+            {
+                executeTest("rm -rf ${strBackRestBase}/test/result/coverage/raw/*");
+                $oStorageTest->pathCreate("${strCodePath}", {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
+            }
+        }
+
         # Auto-generate configure files unless --min-gen specified
         #---------------------------------------------------------------------------------------------------------------------------
         if (!$bMinGen)
@@ -491,28 +526,8 @@ eval
             &log(INFO, "    autogenerated configure script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes'));
         }
 
-        # Make a copy of the repo to track which files have been changed
-        #---------------------------------------------------------------------------------------------------------------------------
-        my $strRepoCachePath = "${strTestPath}/repo";
-
-        # Create the repo path -- this should hopefully prevent obvious rsync errors below
-        $oStorageTest->pathCreate($strRepoCachePath, {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
-
-        # Copy the repo
-        executeTest(
-            "git -C ${strBackRestBase} ls-files -c --others --exclude-standard |" .
-                " rsync -rLtW --delete --files-from=- --exclude=test/result" .
-                # This option is not supported on MacOS. The eventual plan is to remove the need for it.
-                (trim(`uname`) ne 'Darwin' ? ' --ignore-missing-args' : '') .
-                " ${strBackRestBase}/ ${strRepoCachePath}");
-
-        # Auto-generate code files (if --min-gen specified then do minimum required)
-        #---------------------------------------------------------------------------------------------------------------------------
-        my $strBuildPath = "${strTestPath}/build";
-
-        &log(INFO, (!-e $strBuildPath ? 'clean ' : '') . 'autogenerate code');
-
         # Auto-generate version for root meson.build script
+        #---------------------------------------------------------------------------------------------------------------------------
         my $strMesonBuildOld = ${$oStorageTest->get("${strBackRestBase}/meson.build")};
         my $strMesonBuildNew;
 
@@ -528,24 +543,71 @@ eval
 
         buildPutDiffers($oStorageBackRest, "${strBackRestBase}/meson.build", $strMesonBuildNew);
 
-        # Setup build if it does not exist
+        # Start build container if vm is not none
-        if (!-e $strBuildPath)
+        #---------------------------------------------------------------------------------------------------------------------------
+        if ($strVm ne VM_NONE)
         {
-            executeTest("meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath} ${strBackRestBase}");
+            my $strCCachePath = "${strTestPath}/ccache-0/${strVm}";
+
+            if (!$oStorageTest->pathExists($strCCachePath))
+            {
+                $oStorageTest->pathCreate($strCCachePath, {strMode => '0770', bCreateParent => true});
+            }
+
+            executeTest(
+                "docker run -itd -h test-build --name=test-build" .
+                    " -v ${strBackRestBase}:${strBackRestBase} -v ${strTestPath}:${strTestPath}" .
+                    " -v ${strCCachePath}:/home/${\TEST_USER}/.ccache" . ' ' . containerRepo() . ":${strVm}-test",
+                {bSuppressStdErr => true});
+        }
+
+        # Create path for repo cache
+        #---------------------------------------------------------------------------------------------------------------------------
+        my $strRepoCachePath = "${strTestPath}/repo";
+
+        # Create the repo path -- this should hopefully prevent obvious rsync errors below
+        $oStorageTest->pathCreate($strRepoCachePath, {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
+
+        # Auto-generate code files (if --min-gen specified then do minimum required)
+        #---------------------------------------------------------------------------------------------------------------------------
+        my $strBuildPath = "${strTestPath}/build/${strVm}";
+        my $strBuildNinja = "${strBuildPath}/build.ninja";
+
+        &log(INFO, (!-e $strBuildNinja ? 'clean ' : '') . 'autogenerate code');
+
+        # Setup build if it does not exist
+        my $strGenerateCommand =
+            "ninja -C ${strBuildPath} src/build-code test/src/test-pgbackrest" .
+            ($bMinGen ? '' : " && \\\n${strBuildPath}/src/build-code config ${strBackRestBase}/src") .
+            ($bMinGen ? '' : " && \\\n${strBuildPath}/src/build-code error ${strBackRestBase}/src") .
+            " && \\\n${strBuildPath}/src/build-code postgres ${strBackRestBase}/src ${strRepoCachePath}";
+
+        if (!-e $strBuildNinja)
+        {
+            $strGenerateCommand =
+                "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath} ${strBackRestBase} && \\\n" .
+                $strGenerateCommand;
         }
 
         # Build code
         executeTest(
-            "ninja -C ${strBuildPath} src/build-code" .
+            ($strVm ne VM_NONE ? "docker exec -i -u ${\TEST_USER} test-build bash -l -c ' \\\n" : '') .
-            ($bMinGen ? '' : " && ${strBuildPath}/src/build-code config ${strBackRestBase}/src") .
+            $strGenerateCommand . ($strVm ne VM_NONE ? "'" : ''));
-            ($bMinGen ? '' : " && ${strBuildPath}/src/build-code error ${strBackRestBase}/src") .
-            " && cd $strRepoCachePath/src && ${strBuildPath}/src/build-code postgres");
 
         if ($bGenOnly)
         {
             exit 0;
         }
 
+        # Make a copy of the repo to track which files have been changed
+        #---------------------------------------------------------------------------------------------------------------------------
+        executeTest(
+            "git -C ${strBackRestBase} ls-files -c --others --exclude-standard |" .
+                " rsync -rLtW --delete --files-from=- --exclude=test/result" .
+                # This option is not supported on MacOS. The eventual plan is to remove the need for it.
+                (trim(`uname`) ne 'Darwin' ? ' --ignore-missing-args' : '') .
+                " ${strBackRestBase}/ ${strRepoCachePath}");
+
         # Generate code counts
         #---------------------------------------------------------------------------------------------------------------------------
         if ($bCodeCount)
@@ -556,49 +618,6 @@ eval
             exit 0;
         }
 
-        # Clean up
-        #---------------------------------------------------------------------------------------------------------------------------
-        my $iTestFail = 0;
-        my $iTestRetry = 0;
-        my $oyProcess = [];
-        my $strCodePath = "${strBackRestBase}/test/result/coverage/raw";
-
-        if (!$bDryRun || $bVmOut)
-        {
-            &log(INFO, "cleanup old data" . ($strVm ne VM_NONE ? " and containers" : ''));
-
-            if ($strVm ne VM_NONE)
-            {
-                containerRemove('test-([0-9]+|build)');
-            }
-
-            for (my $iVmIdx = 0; $iVmIdx < 8; $iVmIdx++)
-            {
-                push(@{$oyProcess}, undef);
-            }
-
-            executeTest(
-                "chmod 700 -R ${strTestPath}/test-* 2>&1 || true && rm -rf ${strTestPath}/temp ${strTestPath}/test-*" .
-                    " ${strTestPath}/data-*");
-            $oStorageTest->pathCreate("${strTestPath}/temp", {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
-
-            # Remove old lcov dirs -- do it this way so the dirs stay open in finder/explorer, etc.
-            executeTest("rm -rf ${strBackRestBase}/test/result/coverage/lcov/*");
-
-            # Overwrite the C coverage report so it will load but not show old coverage
-            $oStorageTest->pathCreate(
-                "${strBackRestBase}/test/result/coverage", {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
-            $oStorageBackRest->put(
-                "${strBackRestBase}/test/result/coverage/coverage.html", "<center>[ Generating New Report ]</center>");
-
-            # Copy C code for coverage tests
-            if (vmCoverageC($strVm) && !$bDryRun)
-            {
-                executeTest("rm -rf ${strBackRestBase}/test/result/coverage/raw/*");
-                $oStorageTest->pathCreate("${strCodePath}", {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
-            }
-        }
-
         # Determine which tests to run
         #---------------------------------------------------------------------------------------------------------------------------
         my $oyTestRun;
@@ -657,7 +676,6 @@ eval
         {
             my $oVm = vmGet();
             my $lTimestampLast;
-            my $strBinPath = "${strTestPath}/bin";
             my $rhBinBuild = {};
 
             # Build the binary
@@ -679,25 +697,32 @@ eval
 
                 foreach my $strBuildVM (@stryBuildVm)
                 {
-                    my $strBuildPath = "${strBinPath}/${strBuildVM}";
-
                     if ($strBuildVM eq VM_NONE)
                     {
-                        &log(INFO, "    " . (!-e $strBuildPath ? 'clean ' : '') . "bin build for ${strBuildVM} (${strBuildPath})");
+                        my $strBuildPath = "${strTestPath}/build/${strBuildVM}";
+                        my $strBuildNinja = "${strBuildPath}/build.ninja";
+
+                        &log(INFO, "    " . (!-e $strBuildNinja ? 'clean ' : '') . "bin build for ${strBuildVM} (${strBuildPath})");
 
                         # Setup build if it does not exist
-                        if (!-e $strBuildPath)
+                        my $strBuildCommand = "ninja -C ${strBuildPath} src/pgbackrest";
+
+                        if (!-e $strBuildNinja)
                         {
-                            executeTest(
+                            $strBuildCommand =
                                 "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath}" .
-                                    " ${strBackRestBase}");
+                                    " ${strBackRestBase} && \\\n" .
+                                $strBuildCommand;
                         }
 
                         # Build code
-                        executeTest("ninja -C ${strBuildPath}");
+                        executeTest($strBuildCommand);
                     }
                     else
                     {
+                        my $strBinPath = "${strTestPath}/bin";
+                        my $strBuildPath = "${strBinPath}/${strBuildVM}";
+
                         &log(INFO, "    bin build for ${strBuildVM} (${strBuildPath})");
 
                         my $bRebuild = false;
@@ -714,12 +739,6 @@ eval
 
                         my $bBuildOptionsDiffer = buildPutDiffers($oStorageBackRest, $strBuildFlagFile, $strBuildFlags);
 
-                        executeTest(
-                            "docker run -itd -h test-build --name=test-build" .
-                                " -v ${strBackRestBase}:${strBackRestBase} -v ${strTestPath}:${strTestPath} " .
-                                containerRepo() . ":${strBuildVM}-test",
-                            {bSuppressStdErr => true});
-
                         if ($bBuildOptionsDiffer ||
                             !-e "${strBuildPath}/Makefile" ||
                             stat("${strBackRestBase}/src/Makefile.in")->mtime > stat("${strBuildPath}/Makefile")->mtime ||
@@ -744,12 +763,17 @@ eval
                                 " --directory ${strBuildPath} CFLAGS_EXTRA='${strCFlags}'" .
                                 ($strLdFlags ne '' ? " LDFLAGS_EXTRA='${strLdFlags}'" : ''),
                             {bShowOutputAsync => $bLogDetail});
-
-                        executeTest("docker rm -f test-build");
                     }
                 }
             }
 
+            # Shut down the build vm
+            #-----------------------------------------------------------------------------------------------------------------------
+            if ($strVm ne VM_NONE)
+            {
+                executeTest("docker rm -f test-build");
+            }
+
             # Build the package
             #-----------------------------------------------------------------------------------------------------------------------
             if ($bBuildPackage && $strVm ne VM_NONE)
@@ -1097,7 +1121,7 @@ eval
         $strBackRestBase,                                           # Base backrest directory
         $strTestPath,                                               # Path where the tests will run
         dirname($strTestPath) . "/bin/${strVm}/" . PROJECT_EXE,     # Path to the pgbackrest binary
-        dirname($strTestPath) . "/bin/" . VM_NONE . '/src/' . PROJECT_EXE,  # Path to the pgbackrest storage helper
+        dirname($strTestPath) . "/build/" . VM_NONE . '/src/' . PROJECT_EXE,  # Path to the pgbackrest storage helper
         $strPgVersion ne 'minimal' ? $strPgSqlBin: undef,           # Pg bin path
         $strPgVersion ne 'minimal' ? $strPgVersion: undef,          # Pg version
         $stryModule[0], $stryModuleTest[0], \@iyModuleTestRun,      # Module info