Test harness improvements.

* Allow more than one test to provide coverage for the same module.
* Add option to disable valgrind.
* Add option to disabled coverage.
* Add option to disable debug build.
* Add option to disable compiler optimization.
* Add --dev-test mode.

doc/xml/release.xml

@@ -33,6 +33,10 @@
 
             <release-test-list>
                 <release-development-list>
+                    <release-item>
+                        <p>Test harness improvements.  Allow more than one test to provide coverage for the same module.  Add option to disable valgrind.  Add option to disabled coverage.  Add option to disable debug build.  Add option to disable compiler optimization.  Add <id>--dev-test</id> mode.</p>
+                    </release-item>
+
                     <release-item>
                         <p>Update SSL error message test on CentOS 7.</p>
                     </release-item>

test/.gitignore

@@ -1,6 +1,6 @@
-test
 .vagrant
 nytprof*
 scratch.txt
 coverage*
 ubuntu-xenial-16.04-cloudimg-console.log
+profile

test/lib/pgBackRestTest/Common/JobTest.pm

@@ -66,6 +66,11 @@ sub new
         $self->{bShowOutputAsync},
         $self->{bNoCleanup},
         $self->{iRetry},
+        $self->{bValgrindUnit},
+        $self->{bCoverageUnit},
+        $self->{bOptimize},
+        $self->{bProfile},
+        $self->{bDebug},
     ) =
         logDebugParam
         (
@@ -86,6 +91,11 @@ sub new
             {name => 'bShowOutputAsync'},
             {name => 'bNoCleanup'},
             {name => 'iRetry'},
+            {name => 'bValgrindUnit'},
+            {name => 'bCoverageUnit'},
+            {name => 'bOptimize'},
+            {name => 'bProfile'},
+            {name => 'bDebug'},
         );
 
     # Set try to 0
@@ -179,16 +189,16 @@ sub run
                 # If testing C code copy source files to the test directory
                 if ($self->{oTest}->{&TEST_C})
                 {
-                    if (!$bGCovExists)
+                    # If any of the build flags have changed then we'll need to rebuild from scratch
-                    {
+                    my $bFlagsChanged =
-                        executeTest("cp -r $self->{strBackRestBase}/src/*  $self->{strGCovPath}");
+                        defined($rhBuildFlags->{$self->{iVmIdx}}) != defined($self->{oTest}->{&TEST_CDEF}) ||
-                    }
-
-                    if (!$bGCovExists || defined($rhBuildFlags->{$self->{iVmIdx}}) != defined($self->{oTest}->{&TEST_CDEF}) ||
                         defined($rhBuildFlags->{$self->{iVmIdx}}) &&
-                        $rhBuildFlags->{$self->{iVmIdx}} ne $self->{oTest}->{&TEST_CDEF})
+                        $rhBuildFlags->{$self->{iVmIdx}} ne $self->{oTest}->{&TEST_CDEF};
+
+                    if (!$bGCovExists || $bFlagsChanged)
                     {
-                        executeTest("cp -r $self->{strBackRestBase}/test/src/*  $self->{strGCovPath}");
+                        executeTest("rsync -rt --delete $self->{strBackRestBase}/src/ $self->{strGCovPath}");
+                        executeTest("rsync -rt $self->{strBackRestBase}/test/src/ $self->{strGCovPath}");
                     }
 
                     $rhBuildFlags->{$self->{iVmIdx}} = $self->{oTest}->{&TEST_CDEF};
@@ -246,8 +256,6 @@ sub run
                 ($self->{bNoCleanup} ? " --no-cleanup" : '');
         }
 
-        &log(DETAIL, $strCommand);
-
         if (!$self->{bDryRun} || $self->{bVmOut})
         {
             # If testing C code
@@ -341,7 +349,7 @@ sub run
                 # Build the Makefile
                 my $strMakefile =
                     "CC=gcc\n" .
-                    "CFLAGS=-I. -std=c99 -fPIC -g \\\n" .
+                    "CFLAGS=-I. -std=c99 -fPIC -g" . ($self->{bProfile} ? " -pg" : '') . "\\\n" .
                     "       -Werror -Wfatal-errors -Wall -Wextra -Wwrite-strings -Wno-clobbered -Wswitch-enum -Wconversion \\\n" .
                     ($self->{oTest}->{&TEST_VM} eq VM_U16 ? "       -Wformat-signedness \\\n" : '') .
                     # This warning appears to be broken on U12 even though the functionality is fine
@@ -351,24 +359,26 @@ sub run
                         "       -Wpedantic \\\n" : '') .
                     "       -Wformat=2 -Wformat-nonliteral \\\n" .
                     "       `perl -MExtUtils::Embed -e ccopts`\n" .
-                    "LDFLAGS=-lcrypto" . (vmCoverage($self->{oTest}->{&TEST_VM}) ? " -lgcov" : '') .
+                    "LDFLAGS=-lcrypto" . (vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .
                         " `perl -MExtUtils::Embed -e ldopts`\n" .
-                    'TESTFLAGS=' . ($self->{oTest}->{&TEST_DEBUG_UNIT_SUPPRESS} ? '' : "-DDEBUG_UNIT ") .
+                    'TESTFLAGS=' . ($self->{oTest}->{&TEST_DEBUG_UNIT_SUPPRESS} ? '' : "-DDEBUG_UNIT") .
-                        ($self->{oTest}->{&TEST_CDEF} ? "$self->{oTest}->{&TEST_CDEF}" : '') .
+                        ($self->{oTest}->{&TEST_CDEF} ? " $self->{oTest}->{&TEST_CDEF}" : '') .
+                        ($self->{bDebug} ? '' : " -DNDEBUG") .
                     "\n" .
                     "\nSRCS=" . join(' ', @stryCFile) . "\n" .
                     "OBJS=\$(SRCS:.c=.o)\n" .
                     "\n" .
                     "test: \$(OBJS) test.o\n" .
-                    "\t\$(CC) -o test \$(OBJS) test.o \$(LDFLAGS)\n" .
+                    "\t\$(CC) -o test \$(OBJS) test.o"  . ($self->{bProfile} ? " -pg" : '') . " \$(LDFLAGS)\n" .
                     "\n" .
                     "test.o: test.c\n" .
-	                "\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) " .
+	                "\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) -O0" .
-                        (vmCoverage($self->{oTest}->{&TEST_VM}) ? '-fprofile-arcs -ftest-coverage -O0' : '-O2') .
+                        (vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ?
+                            ' -fprofile-arcs -ftest-coverage' : '') .
                         " -c test.c\n" .
                     "\n" .
                     ".c.o:\n" .
-	                "\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) -O2 -c \$< -o \$@\n";
+	                "\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) " . ($self->{bOptimize} ? '-O2' : '-O0') . " -c \$< -o \$@\n";
 
                 $self->{oStorageTest}->put($self->{strGCovPath} . "/Makefile", $strMakefile);
             }
@@ -426,8 +436,20 @@ sub end
             syswrite(*STDOUT, "\n");
         }
 
-        # If C library generate coverage info
+        # If C code generate profile info
-        if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && vmCoverage($self->{oTest}->{&TEST_VM}))
+        if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && $self->{bProfile})
+        {
+            executeTest(
+                'docker exec -i -u ' . TEST_USER . " ${strImage} " .
+                "gprof --flat-profile $self->{strGCovPath}/test $self->{strGCovPath}/gmon.out > $self->{strGCovPath}/gprof.flat.txt");
+
+            $self->{oStorageTest}->pathCreate("$self->{strBackRestBase}/test/profile", {strMode => '0750', bIgnoreExists => true});
+            $self->{oStorageTest}->copy(
+                "$self->{strGCovPath}/gprof.flat.txt", "$self->{strBackRestBase}/test/profile/gprof.flat.txt");
+        }
+
+        # If C code generate coverage info
+        if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit})
         {
             # Generate a list of files to cover
             my $hTestCoverage =
@@ -447,6 +469,7 @@ sub end
             # Generate coverage reports for the modules
             my $strLCovExe = "lcov --config-file=$self->{strBackRestBase}/test/src/lcov.conf";
             my $strLCovOut = $self->{strGCovPath} . '/test.lcov';
+            my $strLCovOutTmp = $self->{strGCovPath} . '/test.tmp.lcov';
 
             executeTest(
                 'docker exec -i -u ' . TEST_USER . " ${strImage} " .
@@ -472,10 +495,18 @@ sub end
 
                 executeTest(
                     'docker exec -i -u ' . TEST_USER . " ${strImage} " .
-                    "${strLCovExe} --extract=${strLCovOut} */${strModuleName}.c --o=${strLCovFile}");
+                    "${strLCovExe} --extract=${strLCovOut} */${strModuleName}.c --o=${strLCovOutTmp}");
+
+                # Combine with prior run if there was one
+                if ($self->{oStorageTest}->exists($strLCovFile))
+                {
+                    executeTest(
+                        'docker exec -i -u ' . TEST_USER . " ${strImage} " .
+                        "${strLCovExe} --add-tracefile=${strLCovOutTmp} --add-tracefile=${strLCovFile} --o=${strLCovOutTmp}");
+                }
 
                 # Update source file
-                my $strCoverage = ${$self->{oStorageTest}->get($strLCovFile)};
+                my $strCoverage = ${$self->{oStorageTest}->get($strLCovOutTmp)};
 
                 if (defined($strCoverage))
                 {
@@ -506,10 +537,9 @@ sub end
                         # Fix source file name
                         $strCoverage =~ s/^SF\:.*$/SF:$strModulePath\.c/mg;
 
-                        # Save coverage file
                         $self->{oStorageTest}->put($strLCovFile, $strCoverage);
 
-                        if ($self->{oStorageTest}->exists(${strLCovTotal}))
+                        if ($self->{oStorageTest}->exists($strLCovTotal))
                         {
                             executeTest(
                                 'docker exec -i -u ' . TEST_USER . " ${strImage} " .

test/lib/pgBackRestTest/Common/VmTest.pm

@@ -25,6 +25,8 @@ use constant VM_DB                                                  => 'db';
     push @EXPORT, qw(VM_DB);
 use constant VM_DB_TEST                                             => 'db-test';
     push @EXPORT, qw(VM_DB_TEST);
+use constant VMDEF_DEBUG_INTEGRATION                                => 'debug-integration';
+    push @EXPORT, qw(VMDEF_DEBUG_INTEGRATION);
 use constant VM_CONTROL_MASTER                                      => 'control-master';
     push @EXPORT, qw(VM_CONTROL_MASTER);
 use constant VM_DEPRECATED                                          => 'deprecated';
@@ -41,6 +43,8 @@ use constant VMDEF_PGSQL_BIN                                        => 'pgsql-bi
     push @EXPORT, qw(VMDEF_PGSQL_BIN);
 use constant VMDEF_PERL_ARCH_PATH                                   => 'perl-arch-path';
     push @EXPORT, qw(VMDEF_PERL_ARCH_PATH);
+use constant VMDEF_WITH_BACKTRACE                                   => 'with-backtrace';
+    push @EXPORT, qw(VMDEF_WITH_BACKTRACE);
 
 ####################################################################################################################################
 # Valid OS base List
@@ -142,6 +146,8 @@ my $oyVm =
         &VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin',
         &VMDEF_PERL_ARCH_PATH => '/usr/local/lib64/perl5',
 
+        &VMDEF_DEBUG_INTEGRATION => false,
+
         &VM_DB =>
         [
             PG_VERSION_96,
@@ -230,6 +236,8 @@ my $oyVm =
         &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
         &VMDEF_PERL_ARCH_PATH => '/usr/local/lib/x86_64-linux-gnu/perl/5.22.1',
 
+        &VMDEF_WITH_BACKTRACE => true,
+
         &VM_DB =>
         [
             PG_VERSION_94,
@@ -345,4 +353,28 @@ sub vmArchBits
 
 push @EXPORT, qw(vmArchBits);
 
+####################################################################################################################################
+# Does the VM support libbacktrace?
+####################################################################################################################################
+sub vmWithBackTrace
+{
+    my $strVm = shift;
+
+    return ($oyVm->{$strVm}{&VMDEF_WITH_BACKTRACE} ? true : false);
+}
+
+push @EXPORT, qw(vmWithBackTrace);
+
+####################################################################################################################################
+# Will integration tests be run in debug mode?
+####################################################################################################################################
+sub vmDebugIntegration
+{
+    my $strVm = shift;
+
+    return (defined($oyVm->{$strVm}{&VMDEF_DEBUG_INTEGRATION}) ? $oyVm->{$strVm}{&VMDEF_DEBUG_INTEGRATION} : true);
+}
+
+push @EXPORT, qw(vmDebugIntegration);
+
 1;

test/test.pl

@@ -86,8 +86,14 @@ test.pl [options]
    --smart              perform libc/package builds only when source timestamps have changed
    --no-package         do not build packages
    --no-ci-config       don't overwrite the current continuous integration config
-   --dev                --no-lint --smart --no-package
+   --dev                --no-lint --smart --no-package --no-optimize
-   --expect             --no-lint --smart --no-package --vm=co7 --db=9.6 --log-force
+   --dev-test           --no-lint --no-package
+   --expect             --no-lint --no-package --vm=co7 --db=9.6 --log-force
+   --no-valgrind        don't run valgrind on C unit tests (saves time)
+   --no-coverage        don't run coverage on C unit tests (saves time)
+   --no-optimize        don't do compile optimization for C (saves compile time)
+   --profile            generate profile info
+   --no-debug           don't generate a debug build
 
  Configuration Options:
    --psql-bin           path to the psql executables (e.g. /usr/lib/postgresql/9.3/bin/)
@@ -133,6 +139,7 @@ my $bVmForce = false;
 my $bNoLint = false;
 my $bBuildOnly = false;
 my $bCoverageOnly = false;
+my $bNoCoverage = false;
 my $bCOnly = false;
 my $bGenOnly = false;
 my $bCodeCount = false;
@@ -140,7 +147,12 @@ my $bSmart = false;
 my $bNoPackage = false;
 my $bNoCiConfig = false;
 my $bDev = false;
+my $bDevTest = false;
+my $bProfile = false;
 my $bExpect = false;
+my $bNoValgrind = false;
+my $bNoOptimize = false;
+my $bNoDebug = false;
 my $iRetry = 0;
 
 GetOptions ('q|quiet' => \$bQuiet,
@@ -168,12 +180,18 @@ GetOptions ('q|quiet' => \$bQuiet,
             'no-package' => \$bNoPackage,
             'no-ci-config' => \$bNoCiConfig,
             'coverage-only' => \$bCoverageOnly,
+            'no-coverage' => \$bNoCoverage,
             'c-only' => \$bCOnly,
             'gen-only' => \$bGenOnly,
             'code-count' => \$bCodeCount,
             'smart' => \$bSmart,
             'dev' => \$bDev,
+            'dev-test' => \$bDevTest,
+            'profile' => \$bProfile,
             'expect' => \$bExpect,
+            'no-valgrind' => \$bNoValgrind,
+            'no-optimize' => \$bNoOptimize,
+            'no-debug', => \$bNoDebug,
             'retry=s' => \$iRetry)
     or pod2usage(2);
 
@@ -206,13 +224,34 @@ eval
     }
 
     ################################################################################################################################
-    # Update options for --dev
+    # Update options for --dev and --dev-fast and --dev-test
     ################################################################################################################################
+    if ($bDev && $bDevTest)
+    {
+        confess "cannot combine --dev and --dev-test";
+    }
+
     if ($bDev)
     {
         $bNoLint = true;
         $bSmart = true;
         $bNoPackage = true;
+        $bNoOptimize = true;
+    }
+
+    if ($bDevTest)
+    {
+        $bNoPackage = true;
+        $bNoLint = true;
+    }
+
+    ################################################################################################################################
+    # Update options for --profile
+    ################################################################################################################################
+    if ($bProfile)
+    {
+        $bNoValgrind = true;
+        $bNoCoverage = true;
     }
 
     ################################################################################################################################
@@ -588,10 +627,12 @@ eval
                         &log(INFO, "    clang static analyzer ${strBuildVM} (${strBuildPath})");
                     }
 
+                    my $strCDebug = vmDebugIntegration($strVm) ? 'CDEBUG=' : '';
+
                     executeTest(
                         'docker exec -i test-build' .
                         (vmCoverage($strVm) && !$bNoLint ? ' scan-build-5.0' : '') .
-                        " make --silent --directory ${strBuildPath} CEXTRA=-g CDEBUG=",
+                        " make --silent --directory ${strBuildPath} CEXTRA=-g ${strCDebug}",
                         {bShowOutputAsync => $bLogDetail});
 
                     executeTest(
@@ -1019,7 +1060,8 @@ eval
                 {
                     my $oJob = new pgBackRestTest::Common::JobTest(
                         $oStorageTest, $strBackRestBase, $strTestPath, $strCoveragePath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut,
-                        $iVmIdx, $iVmMax, $iTestIdx, $iTestMax, $strLogLevel, $bLogForce, $bShowOutputAsync, $bNoCleanup, $iRetry);
+                        $iVmIdx, $iVmMax, $iTestIdx, $iTestMax, $strLogLevel, $bLogForce, $bShowOutputAsync, $bNoCleanup, $iRetry,
+                        !$bNoValgrind, !$bNoCoverage, !$bNoOptimize, $bProfile, !$bNoDebug);
                     $iTestIdx++;
 
                     if ($oJob->run())
@@ -1037,7 +1079,7 @@ eval
         #---------------------------------------------------------------------------------------------------------------------------
         my $iUncoveredCodeModuleTotal = 0;
 
-        if (vmCoverage($strVm) && !$bDryRun)
+        if (vmCoverage($strVm) && !$bNoCoverage && !$bDryRun)
         {
             &log(INFO, 'writing coverage report');
             executeTest("cp -rp ${strCoveragePath} ${strCoveragePath}_temp");