Test harness improvements.

* Allow more than one test to provide coverage for the same module. * Add option to disable valgrind. * Add option to disabled coverage. * Add option to disable debug build. * Add option to disable compiler optimization. * Add --dev-test mode.
2025-09-16 09:06:18 +02:00 · 2018-05-18 06:45:14 -04:00
parent bce3d0fe1c
commit abb9651f4c
5 changed files with 139 additions and 31 deletions
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -33,6 +33,10 @@

            <release-test-list>
                <release-development-list>
+                    <release-item>
+                        <p>Test harness improvements.  Allow more than one test to provide coverage for the same module.  Add option to disable valgrind.  Add option to disabled coverage.  Add option to disable debug build.  Add option to disable compiler optimization.  Add <id>--dev-test</id> mode.</p>
+                    </release-item>
+
                    <release-item>
                        <p>Update SSL error message test on CentOS 7.</p>
                    </release-item>
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -1,6 +1,6 @@
-test
 .vagrant
 nytprof*
 scratch.txt
 coverage*
 ubuntu-xenial-16.04-cloudimg-console.log
+profile
--- a/test/lib/pgBackRestTest/Common/JobTest.pm
+++ b/test/lib/pgBackRestTest/Common/JobTest.pm
@@ -66,6 +66,11 @@ sub new
        $self->{bShowOutputAsync},
        $self->{bNoCleanup},
        $self->{iRetry},
+        $self->{bValgrindUnit},
+        $self->{bCoverageUnit},
+        $self->{bOptimize},
+        $self->{bProfile},
+        $self->{bDebug},
    ) =
        logDebugParam
        (
@@ -86,6 +91,11 @@ sub new
            {name => 'bShowOutputAsync'},
            {name => 'bNoCleanup'},
            {name => 'iRetry'},
+            {name => 'bValgrindUnit'},
+            {name => 'bCoverageUnit'},
+            {name => 'bOptimize'},
+            {name => 'bProfile'},
+            {name => 'bDebug'},
        );

    # Set try to 0
@@ -179,16 +189,16 @@ sub run
                # If testing C code copy source files to the test directory
                if ($self->{oTest}->{&TEST_C})
                {
-                    if (!$bGCovExists)
-                    {
-                        executeTest("cp -r $self->{strBackRestBase}/src/*  $self->{strGCovPath}");
-                    }
-
-                    if (!$bGCovExists || defined($rhBuildFlags->{$self->{iVmIdx}}) != defined($self->{oTest}->{&TEST_CDEF}) ||
+                    # If any of the build flags have changed then we'll need to rebuild from scratch
+                    my $bFlagsChanged =
+                        defined($rhBuildFlags->{$self->{iVmIdx}}) != defined($self->{oTest}->{&TEST_CDEF}) ||
                        defined($rhBuildFlags->{$self->{iVmIdx}}) &&
-                        $rhBuildFlags->{$self->{iVmIdx}} ne $self->{oTest}->{&TEST_CDEF})
+                        $rhBuildFlags->{$self->{iVmIdx}} ne $self->{oTest}->{&TEST_CDEF};
+
+                    if (!$bGCovExists || $bFlagsChanged)
                    {
-                        executeTest("cp -r $self->{strBackRestBase}/test/src/*  $self->{strGCovPath}");
+                        executeTest("rsync -rt --delete $self->{strBackRestBase}/src/ $self->{strGCovPath}");
+                        executeTest("rsync -rt $self->{strBackRestBase}/test/src/ $self->{strGCovPath}");
                    }

                    $rhBuildFlags->{$self->{iVmIdx}} = $self->{oTest}->{&TEST_CDEF};
@@ -246,8 +256,6 @@ sub run
                ($self->{bNoCleanup} ? " --no-cleanup" : '');
        }

-        &log(DETAIL, $strCommand);
-
        if (!$self->{bDryRun} || $self->{bVmOut})
        {
            # If testing C code
@@ -341,7 +349,7 @@ sub run
                # Build the Makefile
                my $strMakefile =
                    "CC=gcc\n" .
-                    "CFLAGS=-I. -std=c99 -fPIC -g \\\n" .
+                    "CFLAGS=-I. -std=c99 -fPIC -g" . ($self->{bProfile} ? " -pg" : '') . "\\\n" .
                    "       -Werror -Wfatal-errors -Wall -Wextra -Wwrite-strings -Wno-clobbered -Wswitch-enum -Wconversion \\\n" .
                    ($self->{oTest}->{&TEST_VM} eq VM_U16 ? "       -Wformat-signedness \\\n" : '') .
                    # This warning appears to be broken on U12 even though the functionality is fine
@@ -351,24 +359,26 @@ sub run
                        "       -Wpedantic \\\n" : '') .
                    "       -Wformat=2 -Wformat-nonliteral \\\n" .
                    "       `perl -MExtUtils::Embed -e ccopts`\n" .
-                    "LDFLAGS=-lcrypto" . (vmCoverage($self->{oTest}->{&TEST_VM}) ? " -lgcov" : '') .
+                    "LDFLAGS=-lcrypto" . (vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .
                        " `perl -MExtUtils::Embed -e ldopts`\n" .
-                    'TESTFLAGS=' . ($self->{oTest}->{&TEST_DEBUG_UNIT_SUPPRESS} ? '' : "-DDEBUG_UNIT ") .
-                        ($self->{oTest}->{&TEST_CDEF} ? "$self->{oTest}->{&TEST_CDEF}" : '') .
+                    'TESTFLAGS=' . ($self->{oTest}->{&TEST_DEBUG_UNIT_SUPPRESS} ? '' : "-DDEBUG_UNIT") .
+                        ($self->{oTest}->{&TEST_CDEF} ? " $self->{oTest}->{&TEST_CDEF}" : '') .
+                        ($self->{bDebug} ? '' : " -DNDEBUG") .
                    "\n" .
                    "\nSRCS=" . join(' ', @stryCFile) . "\n" .
                    "OBJS=\$(SRCS:.c=.o)\n" .
                    "\n" .
                    "test: \$(OBJS) test.o\n" .
-                    "\t\$(CC) -o test \$(OBJS) test.o \$(LDFLAGS)\n" .
+                    "\t\$(CC) -o test \$(OBJS) test.o"  . ($self->{bProfile} ? " -pg" : '') . " \$(LDFLAGS)\n" .
                    "\n" .
                    "test.o: test.c\n" .
-	                "\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) " .
-                        (vmCoverage($self->{oTest}->{&TEST_VM}) ? '-fprofile-arcs -ftest-coverage -O0' : '-O2') .
+	                "\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) -O0" .
+                        (vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ?
+                            ' -fprofile-arcs -ftest-coverage' : '') .
                        " -c test.c\n" .
                    "\n" .
                    ".c.o:\n" .
-	                "\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) -O2 -c \$< -o \$@\n";
+	                "\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) " . ($self->{bOptimize} ? '-O2' : '-O0') . " -c \$< -o \$@\n";

                $self->{oStorageTest}->put($self->{strGCovPath} . "/Makefile", $strMakefile);
            }
@@ -426,8 +436,20 @@ sub end
            syswrite(*STDOUT, "\n");
        }

-        # If C library generate coverage info
-        if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && vmCoverage($self->{oTest}->{&TEST_VM}))
+        # If C code generate profile info
+        if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && $self->{bProfile})
+        {
+            executeTest(
+                'docker exec -i -u ' . TEST_USER . " ${strImage} " .
+                "gprof --flat-profile $self->{strGCovPath}/test $self->{strGCovPath}/gmon.out > $self->{strGCovPath}/gprof.flat.txt");
+
+            $self->{oStorageTest}->pathCreate("$self->{strBackRestBase}/test/profile", {strMode => '0750', bIgnoreExists => true});
+            $self->{oStorageTest}->copy(
+                "$self->{strGCovPath}/gprof.flat.txt", "$self->{strBackRestBase}/test/profile/gprof.flat.txt");
+        }
+
+        # If C code generate coverage info
+        if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit})
        {
            # Generate a list of files to cover
            my $hTestCoverage =
@@ -447,6 +469,7 @@ sub end
            # Generate coverage reports for the modules
            my $strLCovExe = "lcov --config-file=$self->{strBackRestBase}/test/src/lcov.conf";
            my $strLCovOut = $self->{strGCovPath} . '/test.lcov';
+            my $strLCovOutTmp = $self->{strGCovPath} . '/test.tmp.lcov';

            executeTest(
                'docker exec -i -u ' . TEST_USER . " ${strImage} " .
@@ -472,10 +495,18 @@ sub end

                executeTest(
                    'docker exec -i -u ' . TEST_USER . " ${strImage} " .
-                    "${strLCovExe} --extract=${strLCovOut} */${strModuleName}.c --o=${strLCovFile}");
+                    "${strLCovExe} --extract=${strLCovOut} */${strModuleName}.c --o=${strLCovOutTmp}");
+
+                # Combine with prior run if there was one
+                if ($self->{oStorageTest}->exists($strLCovFile))
+                {
+                    executeTest(
+                        'docker exec -i -u ' . TEST_USER . " ${strImage} " .
+                        "${strLCovExe} --add-tracefile=${strLCovOutTmp} --add-tracefile=${strLCovFile} --o=${strLCovOutTmp}");
+                }

                # Update source file
-                my $strCoverage = ${$self->{oStorageTest}->get($strLCovFile)};
+                my $strCoverage = ${$self->{oStorageTest}->get($strLCovOutTmp)};

                if (defined($strCoverage))
                {
@@ -506,10 +537,9 @@ sub end
                        # Fix source file name
                        $strCoverage =~ s/^SF\:.*$/SF:$strModulePath\.c/mg;

-                        # Save coverage file
                        $self->{oStorageTest}->put($strLCovFile, $strCoverage);

-                        if ($self->{oStorageTest}->exists(${strLCovTotal}))
+                        if ($self->{oStorageTest}->exists($strLCovTotal))
                        {
                            executeTest(
                                'docker exec -i -u ' . TEST_USER . " ${strImage} " .
--- a/test/lib/pgBackRestTest/Common/VmTest.pm
+++ b/test/lib/pgBackRestTest/Common/VmTest.pm
@@ -25,6 +25,8 @@ use constant VM_DB                                                  => 'db';
    push @EXPORT, qw(VM_DB);
 use constant VM_DB_TEST                                             => 'db-test';
    push @EXPORT, qw(VM_DB_TEST);
+use constant VMDEF_DEBUG_INTEGRATION                                => 'debug-integration';
+    push @EXPORT, qw(VMDEF_DEBUG_INTEGRATION);
 use constant VM_CONTROL_MASTER                                      => 'control-master';
    push @EXPORT, qw(VM_CONTROL_MASTER);
 use constant VM_DEPRECATED                                          => 'deprecated';
@@ -41,6 +43,8 @@ use constant VMDEF_PGSQL_BIN                                        => 'pgsql-bi
    push @EXPORT, qw(VMDEF_PGSQL_BIN);
 use constant VMDEF_PERL_ARCH_PATH                                   => 'perl-arch-path';
    push @EXPORT, qw(VMDEF_PERL_ARCH_PATH);
+use constant VMDEF_WITH_BACKTRACE                                   => 'with-backtrace';
+    push @EXPORT, qw(VMDEF_WITH_BACKTRACE);

 ####################################################################################################################################
 # Valid OS base List
@@ -142,6 +146,8 @@ my $oyVm =
        &VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin',
        &VMDEF_PERL_ARCH_PATH => '/usr/local/lib64/perl5',

+        &VMDEF_DEBUG_INTEGRATION => false,
+
        &VM_DB =>
        [
            PG_VERSION_96,
@@ -230,6 +236,8 @@ my $oyVm =
        &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
        &VMDEF_PERL_ARCH_PATH => '/usr/local/lib/x86_64-linux-gnu/perl/5.22.1',

+        &VMDEF_WITH_BACKTRACE => true,
+
        &VM_DB =>
        [
            PG_VERSION_94,
@@ -345,4 +353,28 @@ sub vmArchBits

 push @EXPORT, qw(vmArchBits);

+####################################################################################################################################
+# Does the VM support libbacktrace?
+####################################################################################################################################
+sub vmWithBackTrace
+{
+    my $strVm = shift;
+
+    return ($oyVm->{$strVm}{&VMDEF_WITH_BACKTRACE} ? true : false);
+}
+
+push @EXPORT, qw(vmWithBackTrace);
+
+####################################################################################################################################
+# Will integration tests be run in debug mode?
+####################################################################################################################################
+sub vmDebugIntegration
+{
+    my $strVm = shift;
+
+    return (defined($oyVm->{$strVm}{&VMDEF_DEBUG_INTEGRATION}) ? $oyVm->{$strVm}{&VMDEF_DEBUG_INTEGRATION} : true);
+}
+
+push @EXPORT, qw(vmDebugIntegration);
+
 1;
--- a/test/test.pl
+++ b/test/test.pl
@@ -86,8 +86,14 @@ test.pl [options]
   --smart              perform libc/package builds only when source timestamps have changed
   --no-package         do not build packages
   --no-ci-config       don't overwrite the current continuous integration config
-   --dev                --no-lint --smart --no-package
-   --expect             --no-lint --smart --no-package --vm=co7 --db=9.6 --log-force
+   --dev                --no-lint --smart --no-package --no-optimize
+   --dev-test           --no-lint --no-package
+   --expect             --no-lint --no-package --vm=co7 --db=9.6 --log-force
+   --no-valgrind        don't run valgrind on C unit tests (saves time)
+   --no-coverage        don't run coverage on C unit tests (saves time)
+   --no-optimize        don't do compile optimization for C (saves compile time)
+   --profile            generate profile info
+   --no-debug           don't generate a debug build

 Configuration Options:
   --psql-bin           path to the psql executables (e.g. /usr/lib/postgresql/9.3/bin/)
@@ -133,6 +139,7 @@ my $bVmForce = false;
 my $bNoLint = false;
 my $bBuildOnly = false;
 my $bCoverageOnly = false;
+my $bNoCoverage = false;
 my $bCOnly = false;
 my $bGenOnly = false;
 my $bCodeCount = false;
@@ -140,7 +147,12 @@ my $bSmart = false;
 my $bNoPackage = false;
 my $bNoCiConfig = false;
 my $bDev = false;
+my $bDevTest = false;
+my $bProfile = false;
 my $bExpect = false;
+my $bNoValgrind = false;
+my $bNoOptimize = false;
+my $bNoDebug = false;
 my $iRetry = 0;

 GetOptions ('q|quiet' => \$bQuiet,
@@ -168,12 +180,18 @@ GetOptions ('q|quiet' => \$bQuiet,
            'no-package' => \$bNoPackage,
            'no-ci-config' => \$bNoCiConfig,
            'coverage-only' => \$bCoverageOnly,
+            'no-coverage' => \$bNoCoverage,
            'c-only' => \$bCOnly,
            'gen-only' => \$bGenOnly,
            'code-count' => \$bCodeCount,
            'smart' => \$bSmart,
            'dev' => \$bDev,
+            'dev-test' => \$bDevTest,
+            'profile' => \$bProfile,
            'expect' => \$bExpect,
+            'no-valgrind' => \$bNoValgrind,
+            'no-optimize' => \$bNoOptimize,
+            'no-debug', => \$bNoDebug,
            'retry=s' => \$iRetry)
    or pod2usage(2);

@@ -206,13 +224,34 @@ eval
    }

    ################################################################################################################################
-    # Update options for --dev
+    # Update options for --dev and --dev-fast and --dev-test
    ################################################################################################################################
+    if ($bDev && $bDevTest)
+    {
+        confess "cannot combine --dev and --dev-test";
+    }
+
    if ($bDev)
    {
        $bNoLint = true;
        $bSmart = true;
        $bNoPackage = true;
+        $bNoOptimize = true;
+    }
+
+    if ($bDevTest)
+    {
+        $bNoPackage = true;
+        $bNoLint = true;
+    }
+
+    ################################################################################################################################
+    # Update options for --profile
+    ################################################################################################################################
+    if ($bProfile)
+    {
+        $bNoValgrind = true;
+        $bNoCoverage = true;
    }

    ################################################################################################################################
@@ -588,10 +627,12 @@ eval
                        &log(INFO, "    clang static analyzer ${strBuildVM} (${strBuildPath})");
                    }

+                    my $strCDebug = vmDebugIntegration($strVm) ? 'CDEBUG=' : '';
+
                    executeTest(
                        'docker exec -i test-build' .
                        (vmCoverage($strVm) && !$bNoLint ? ' scan-build-5.0' : '') .
-                        " make --silent --directory ${strBuildPath} CEXTRA=-g CDEBUG=",
+                        " make --silent --directory ${strBuildPath} CEXTRA=-g ${strCDebug}",
                        {bShowOutputAsync => $bLogDetail});

                    executeTest(
@@ -1019,7 +1060,8 @@ eval
                {
                    my $oJob = new pgBackRestTest::Common::JobTest(
                        $oStorageTest, $strBackRestBase, $strTestPath, $strCoveragePath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut,
-                        $iVmIdx, $iVmMax, $iTestIdx, $iTestMax, $strLogLevel, $bLogForce, $bShowOutputAsync, $bNoCleanup, $iRetry);
+                        $iVmIdx, $iVmMax, $iTestIdx, $iTestMax, $strLogLevel, $bLogForce, $bShowOutputAsync, $bNoCleanup, $iRetry,
+                        !$bNoValgrind, !$bNoCoverage, !$bNoOptimize, $bProfile, !$bNoDebug);
                    $iTestIdx++;

                    if ($oJob->run())
@@ -1037,7 +1079,7 @@ eval
        #---------------------------------------------------------------------------------------------------------------------------
        my $iUncoveredCodeModuleTotal = 0;

-        if (vmCoverage($strVm) && !$bDryRun)
+        if (vmCoverage($strVm) && !$bNoCoverage && !$bDryRun)
        {
            &log(INFO, 'writing coverage report');
            executeTest("cp -rp ${strCoveragePath} ${strCoveragePath}_temp");