From f0ed89f21fe2ae6278e64395f4191519807e529b Mon Sep 17 00:00:00 2001 From: David Steele Date: Sat, 15 Sep 2018 13:27:06 -0400 Subject: [PATCH] Allow C or Perl coverage to run on more than one VM. C or Perl coverage tests can now be run on any VM provided a recent enough version of Devel::Cover or lcov is available. For now, leave u18 as the only VM to run coverage tests due to some issues with older versions of lcov. --- doc/xml/release.xml | 4 + .../pgBackRestTest/Common/ContainerTest.pm | 21 ++- test/lib/pgBackRestTest/Common/JobTest.pm | 11 +- test/lib/pgBackRestTest/Common/RunTest.pm | 2 +- test/lib/pgBackRestTest/Common/VmTest.pm | 65 ++++++-- test/test.pl | 142 +++++++++--------- 6 files changed, 153 insertions(+), 92 deletions(-) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 16420e877..afcecf776 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -105,6 +105,10 @@

Make Valgrind return an error even when a non-fatal issue is detected. Update some minor issues discovered in the tests as a result.

+ +

Allow C or Perl coverage to run on more than one VM.

+ +

Don't perform valgrind when requested.

 diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index 513058059..8fe293ba3 100755 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -421,6 +421,11 @@ sub containerBuild { $strScript .= ' perl-JSON-PP'; } + + if (vmCoverageC($strOS)) + { + $strScript .= ' lcov'; + } } else { @@ -439,9 +444,15 @@ sub containerBuild { $strScript .= ' libperl5.14'; } - elsif ($strOS eq VM_U18) + + if (vmLintC($strOS)) { - $strScript .= ' clang-6.0 clang-tools-6.0 lcov'; + $strScript .= ' clang-6.0 clang-tools-6.0'; + } + + if (vmCoverageC($strOS)) + { + $strScript .= ' lcov'; } } @@ -580,7 +591,7 @@ sub containerBuild $strCopy = undef; my $strPkgDevelCover = packageDevelCover($oVm->{$strOS}{&VM_ARCH}); - my $bPkgDevelCoverBuild = vmCoverage($strOS) && !$oStorageDocker->exists("test/package/${strOS}-${strPkgDevelCover}"); + my $bPkgDevelCoverBuild = vmCoveragePerl($strOS) && !$oStorageDocker->exists("test/package/${strOS}-${strPkgDevelCover}"); $strScript = sectionHeader() . "# Create test user\n" . @@ -599,7 +610,7 @@ sub containerBuild { $strScript .= sectionHeader() . "# Install Devel::Cover package source & build\n" . - " git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git" . + " git clone https://salsa.debian.org/perl-team/modules/packages/libdevel-cover-perl.git" . " /root/libdevel-cover-perl && \\\n" . " cd /root/libdevel-cover-perl && \\\n" . " git checkout debian/" . LIB_COVER_VERSION . " && \\\n" . @@ -669,7 +680,7 @@ sub containerBuild $strImageParent = containerRepo() . ":${strOS}-base"; $strImage = "${strOS}-test"; - if (vmCoverage($strOS)) + if (vmCoveragePerl($strOS)) { $oStorageDocker->copy( "test/package/${strOS}-${strPkgDevelCover}", "test/.vagrant/docker/${strOS}-${strPkgDevelCover}"); diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 7658d0c6d..e1f9ec510 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -247,7 +247,7 @@ sub run $strCommand = ($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . testRunExe( - vmCoverage($self->{oTest}->{&TEST_VM}), undef, abs_path($0), dirname($self->{strCoveragePath}), + vmCoverageC($self->{oTest}->{&TEST_VM}), undef, abs_path($0), dirname($self->{strCoveragePath}), $self->{strBackRestBase}, $self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME}) . " --test-path=${strVmTestPath}" . " --vm=$self->{oTest}->{&TEST_VM}" . @@ -375,7 +375,7 @@ sub run # " -Wpedantic \\\n" : '') . " -Wformat=2 -Wformat-nonliteral -Wstrict-prototypes -Wpointer-arith -Wvla \\\n" . " `perl -MExtUtils::Embed -e ccopts`\n" . - "LDFLAGS=-lcrypto -lz" . (vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') . + "LDFLAGS=-lcrypto -lz" . (vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') . (vmWithBackTrace($self->{oTest}->{&TEST_VM}) && $self->{bBackTrace} ? ' -lbacktrace' : '') . " `perl -MExtUtils::Embed -e ldopts`\n" . 'TESTFLAGS=' . ($self->{oTest}->{&TEST_DEBUG_UNIT_SUPPRESS} ? '' : "-DDEBUG_UNIT") . @@ -391,7 +391,7 @@ sub run "\n" . "test.o: test.c\n" . "\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) -O0" . - (vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? + (vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? ' -fprofile-arcs -ftest-coverage' : '') . " -c test.c\n" . "\n" . @@ -467,7 +467,7 @@ sub end } # If C code generate coverage info - if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit}) + if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit}) { # Generate a list of files to cover my $hTestCoverage = @@ -512,7 +512,6 @@ sub end my $strLCovTotal = $self->{strBackRestBase} . "/test/.vagrant/code/all.lcov"; executeTest( - 'docker exec -i -u ' . TEST_USER . " ${strImage} " . "${strLCovExe} --extract=${strLCovOut} */${strModuleName}.c --o=${strLCovOutTmp}"); # Combine with prior run if there was one @@ -523,7 +522,6 @@ sub end $self->{oStorageTest}->put($strLCovOutTmp, $strCoverage); executeTest( - 'docker exec -i -u ' . TEST_USER . " ${strImage} " . "${strLCovExe} --add-tracefile=${strLCovOutTmp} --add-tracefile=${strLCovFile} --o=${strLCovOutTmp}"); } @@ -564,7 +562,6 @@ sub end if ($self->{oStorageTest}->exists($strLCovTotal)) { executeTest( - 'docker exec -i -u ' . TEST_USER . " ${strImage} " . "${strLCovExe} --add-tracefile=${strLCovFile} --add-tracefile=${strLCovTotal} --o=${strLCovTotal}"); } else diff --git a/test/lib/pgBackRestTest/Common/RunTest.pm b/test/lib/pgBackRestTest/Common/RunTest.pm index 20c40b3bf..0394fd8ec 100644 --- a/test/lib/pgBackRestTest/Common/RunTest.pm +++ b/test/lib/pgBackRestTest/Common/RunTest.pm @@ -606,7 +606,7 @@ sub archBits {return vmArchBits(shift->{strVm})} sub backrestExe {return shift->{strBackRestExe}} sub backrestUser {return shift->{strBackRestUser}} sub basePath {return shift->{strBasePath}} -sub coverage {vmCoverage(shift->{strVm})} +sub coverage {vmCoveragePerl(shift->{strVm})} sub dataPath {return shift->basePath() . '/test/data'} sub doCleanup {return shift->{bCleanup}} sub doExpect {return shift->{bExpect}} diff --git a/test/lib/pgBackRestTest/Common/VmTest.pm b/test/lib/pgBackRestTest/Common/VmTest.pm index 67d44b449..b290ab25b 100644 --- a/test/lib/pgBackRestTest/Common/VmTest.pm +++ b/test/lib/pgBackRestTest/Common/VmTest.pm @@ -29,10 +29,16 @@ use constant VMDEF_DEBUG_INTEGRATION => 'debug-in push @EXPORT, qw(VMDEF_DEBUG_INTEGRATION); use constant VM_CONTROL_MASTER => 'control-master'; push @EXPORT, qw(VM_CONTROL_MASTER); +# Will coverage testing be run for C? +use constant VMDEF_COVERAGE_C => 'coverage-c'; +# Will coverage testing be run for Perl? +use constant VMDEF_COVERAGE_PERL => 'coverage-perl'; use constant VM_DEPRECATED => 'deprecated'; push @EXPORT, qw(VM_DEPRECATED); use constant VM_IMAGE => 'image'; push @EXPORT, qw(VM_IMAGE); +# Will static code analysis be run for C? +use constant VMDEF_LINT_C => 'lint-c'; use constant VM_OS => 'os'; push @EXPORT, qw(VM_OS); use constant VM_OS_BASE => 'os-base'; @@ -103,9 +109,6 @@ use constant VM_EXPECT => VM_CO7; use constant VM_HOST_DEFAULT => VM_U18; push @EXPORT, qw(VM_HOST_DEFAULT); -# Defines the VM that will do coverage testing -use constant VM_COVERAGE => VM_U18; - # Lists valid VMs use constant VM_LIST => (VM_U18, VM_CO6, VM_CO7, VM_U12); push @EXPORT, qw(VM_LIST); @@ -334,6 +337,9 @@ my $oyVm = &VM_OS_REPO => 'bionic', &VM_IMAGE => 'ubuntu:18.04', &VM_ARCH => VM_ARCH_AMD64, + &VMDEF_COVERAGE_C => true, + &VMDEF_COVERAGE_PERL => true, + &VMDEF_LINT_C => true, &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin', &VMDEF_PERL_ARCH_PATH => '/usr/local/lib/x86_64-linux-gnu/perl/5.26.1', @@ -374,13 +380,19 @@ foreach my $strVm (sort(keys(%{$oyVm}))) foreach my $strPgVersion (versionSupport()) { my $strVmPgVersionRun; - my $strVmCoverage; + my $bVmCoveragePerl = false; + my $bVmCoverageC = false; foreach my $strVm (VM_LIST) { - if ($strVm eq VM_COVERAGE) + if (vmCoverageC($strVm)) { - $strVmCoverage = $strVm; + $bVmCoverageC = true; + } + + if (vmCoveragePerl($strVm)) + { + $bVmCoveragePerl = true; } foreach my $strVmPgVersion (@{$oyVm->{$strVm}{&VM_DB_TEST}}) @@ -399,9 +411,14 @@ foreach my $strPgVersion (versionSupport()) my $strErrorSuffix = 'is not configured to run on a default vm'; - if (!defined($strVmCoverage)) + if (!$bVmCoverageC) { - confess &log(ASSERT, 'vm designated for coverage testing (' . VM_COVERAGE . ") ${strErrorSuffix}"); + confess &log(ASSERT, "C coverage ${strErrorSuffix}"); + } + + if (!$bVmCoveragePerl) + { + confess &log(ASSERT, "Perl coverage ${strErrorSuffix}"); } if (!defined($strVmPgVersionRun)) @@ -434,16 +451,40 @@ sub vmBaseTest push @EXPORT, qw(vmBaseTest); #################################################################################################################################### -# vmCoverage +# vmCoverageC #################################################################################################################################### -sub vmCoverage +sub vmCoverageC { my $strVm = shift; - return ($strVm eq VM_COVERAGE ? true : false) + return $oyVm->{$strVm}{&VMDEF_COVERAGE_C} ? true : false; } -push @EXPORT, qw(vmCoverage); +push @EXPORT, qw(vmCoverageC); + +#################################################################################################################################### +# vmCoveragePerl +#################################################################################################################################### +sub vmCoveragePerl +{ + my $strVm = shift; + + return $oyVm->{$strVm}{&VMDEF_COVERAGE_PERL} ? true : false; +} + +push @EXPORT, qw(vmCoveragePerl); + +#################################################################################################################################### +# vmLintC +#################################################################################################################################### +sub vmLintC +{ + my $strVm = shift; + + return $oyVm->{$strVm}{&VMDEF_LINT_C} ? true : false; +} + +push @EXPORT, qw(vmLintC); #################################################################################################################################### # Get vm architecture bits diff --git a/test/test.pl b/test/test.pl index b6be942d3..b1329d0bb 100755 --- a/test/test.pl +++ b/test/test.pl @@ -321,7 +321,7 @@ eval { confess &log(ERROR, "select a single Debian-based VM for coverage testing"); } - elsif (!vmCoverage($strVm)) + elsif (!vmCoveragePerl($strVm)) { confess &log(ERROR, "only Debian-based VMs can be used for coverage testing"); } @@ -587,7 +587,7 @@ eval executeTest("rm -rf ${strBackRestBase}/test/coverage/perl/*"); # Copy C code for coverage tests - if (vmCoverage($strVm) && !$bDryRun) + if (vmCoverageC($strVm) && !$bDryRun) { $oStorageTest->pathCreate("${strCodePath}/test", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); @@ -721,7 +721,7 @@ eval " --include=" . join('/*** --include=', @stryBinSrcPath) . '/*** --exclude=*' . " ${strBackRestBase}/ ${strBinPath}/${strBuildVM}"); - if (vmCoverage($strVm) && !$bNoLint) + if (vmLintC($strVm) && !$bNoLint) { &log(INFO, " clang static analyzer ${strBuildVM} (${strBuildPath})"); } @@ -734,7 +734,7 @@ eval executeTest( 'docker exec -i test-build' . - (vmCoverage($strVm) && !$bNoLint ? ' scan-build-6.0' : '') . + (vmLintC($strVm) && !$bNoLint ? ' scan-build-6.0' : '') . " make --silent --directory ${strBuildPath} CEXTRA=${strCExtra} LDEXTRA=${strLdExtra} ${strCDebug}", {bShowOutputAsync => $bLogDetail}); @@ -1158,83 +1158,86 @@ eval #--------------------------------------------------------------------------------------------------------------------------- my $iUncoveredCodeModuleTotal = 0; - if (vmCoverage($strVm) && !$bNoCoverage && !$bDryRun) + if ((vmCoverageC($strVm) || vmCoveragePerl($strVm)) && !$bNoCoverage && !$bDryRun && $iTestFail == 0) { - &log(INFO, 'writing coverage report'); - executeTest("cp -rp ${strCoveragePath} ${strCoveragePath}_temp"); - executeTest( - "cd ${strCoveragePath}_temp && " . - LIB_COVER_EXE . " -report json -outputdir ${strBackRestBase}/test/coverage/perl ${strCoveragePath}_temp", - {bSuppressStdErr => true}); - executeTest("sudo rm -rf ${strCoveragePath}_temp"); - executeTest("sudo cp -rp ${strCoveragePath} ${strCoveragePath}_temp"); - executeTest( - "cd ${strCoveragePath}_temp && " . - LIB_COVER_EXE . " -outputdir ${strBackRestBase}/test/coverage/perl ${strCoveragePath}_temp", - {bSuppressStdErr => true}); - executeTest("sudo rm -rf ${strCoveragePath}_temp"); - # Determine which modules were covered (only check coverage if all tests were successful) #----------------------------------------------------------------------------------------------------------------------- - if ($iTestFail == 0) + my $hModuleTest; # Everything that was run + + # Build a hash of all modules, tests, and runs that were executed + foreach my $hTestRun (@{$oyTestRun}) { - my $hModuleTest; # Everything that was run + # Get coverage for the module + my $strModule = $hTestRun->{&TEST_MODULE}; + my $hModule = testDefModule($strModule); - # Build a hash of all modules, tests, and runs that were executed - foreach my $hTestRun (@{$oyTestRun}) + # Get coverage for the test + my $strTest = $hTestRun->{&TEST_NAME}; + my $hTest = testDefModuleTest($strModule, $strTest); + + # If no tests are listed it means all of them were run + if (@{$hTestRun->{&TEST_RUN}} == 0) { - # Get coverage for the module - my $strModule = $hTestRun->{&TEST_MODULE}; - my $hModule = testDefModule($strModule); + $hModuleTest->{$strModule}{$strTest} = true; + } + } - # Get coverage for the test - my $strTest = $hTestRun->{&TEST_NAME}; - my $hTest = testDefModuleTest($strModule, $strTest); + # Now compare against code modules that should have full coverage + my $hCoverageList = testDefCoverageList(); + my $hCoverageType = testDefCoverageType(); + my $hCoverageActual; - # If no tests are listed it means all of them were run - if (@{$hTestRun->{&TEST_RUN}} == 0) + foreach my $strCodeModule (sort(keys(%{$hCoverageList}))) + { + if (@{$hCoverageList->{$strCodeModule}} > 0) + { + my $iCoverageTotal = 0; + + foreach my $hTest (@{$hCoverageList->{$strCodeModule}}) { - $hModuleTest->{$strModule}{$strTest} = true; + if (!defined($hModuleTest->{$hTest->{strModule}}{$hTest->{strTest}})) + { + next; + } + + $iCoverageTotal++; + } + + if (@{$hCoverageList->{$strCodeModule}} == $iCoverageTotal) + { + $hCoverageActual->{testRunName($strCodeModule, false)} = $hCoverageType->{$strCodeModule}; } } + } + + if (keys(%{$hCoverageActual}) == 0) + { + &log(INFO, 'no code modules had all tests run required for coverage'); + } + + # Generate Perl coverage report + #----------------------------------------------------------------------------------------------------------------------- + if (vmCoveragePerl($strVm)) + { + &log(INFO, 'writing Perl coverage report'); + + executeTest("cp -rp ${strCoveragePath} ${strCoveragePath}_temp"); + executeTest( + "cd ${strCoveragePath}_temp && " . + LIB_COVER_EXE . " -report json -outputdir ${strBackRestBase}/test/coverage/perl ${strCoveragePath}_temp", + {bSuppressStdErr => true}); + executeTest("sudo rm -rf ${strCoveragePath}_temp"); + executeTest("sudo cp -rp ${strCoveragePath} ${strCoveragePath}_temp"); + executeTest( + "cd ${strCoveragePath}_temp && " . + LIB_COVER_EXE . " -outputdir ${strBackRestBase}/test/coverage/perl ${strCoveragePath}_temp", + {bSuppressStdErr => true}); + executeTest("sudo rm -rf ${strCoveragePath}_temp"); # Load the results of coverage testing from JSON my $oJSON = JSON::PP->new()->allow_nonref(); my $hCoverageResult = $oJSON->decode(${$oStorageBackRest->get('test/coverage/perl/cover.json')}); - # Now compare against code modules that should have full coverage - my $hCoverageList = testDefCoverageList(); - my $hCoverageType = testDefCoverageType(); - my $hCoverageActual; - - foreach my $strCodeModule (sort(keys(%{$hCoverageList}))) - { - if (@{$hCoverageList->{$strCodeModule}} > 0) - { - my $iCoverageTotal = 0; - - foreach my $hTest (@{$hCoverageList->{$strCodeModule}}) - { - if (!defined($hModuleTest->{$hTest->{strModule}}{$hTest->{strTest}})) - { - next; - } - - $iCoverageTotal++; - } - - if (@{$hCoverageList->{$strCodeModule}} == $iCoverageTotal) - { - $hCoverageActual->{testRunName($strCodeModule, false)} = $hCoverageType->{$strCodeModule}; - } - } - } - - if (keys(%{$hCoverageActual}) == 0) - { - &log(INFO, 'no code modules had all tests run required for coverage'); - } - foreach my $strCodeModule (sort(keys(%{$hCoverageActual}))) { # If the first char of the module is lower case then it's a c module @@ -1308,9 +1311,14 @@ eval } } } + } + + # Generate C coverage report + #--------------------------------------------------------------------------------------------------------------------------- + if (vmCoverageC($strVm)) + { + &log(INFO, 'writing C coverage report'); - # Generate C coverage with lcov - #--------------------------------------------------------------------------------------------------------------------------- my $strLCovFile = "${strBackRestBase}/test/.vagrant/code/all.lcov"; if ($oStorageBackRest->exists($strLCovFile))