db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-04-17 11:46:39 +02:00
Code Issues Releases Activity

Use lcov for C unit test coverage reporting.

Browse Source 
Switch from Devel::Cover because it would not report on branch coverage for reports converted from gcov.

Branch coverage is not complete, so for the time being errors will only be generated when statement coverage is not complete. Coverage of unit tests is not displayed in the report unless they are incomplete for either statement or branch coverage.
This commit is contained in:
David Steele 2018-03-19 23:33:28 -04:00
parent 2a3d6ecde8
commit 07f38f584a
11 changed files with 167 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -19,7 +19,7 @@ env:
- PGB_CI="doc"
before_install:
- sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl libyaml-perl python-pip
- sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl libyaml-perl python-pip lcov
- |
# Install & Configure AWS CLI
pip install --upgrade --user awscli

10
doc/xml/release.xml
View File

@ -12,6 +12,16 @@
</intro>
<release-list>
<release date="XXXX-XX-XX" version="2.02dev" title="UNDER DEVELOPMENT">
<release-test-list>
<release-feature-list>
<release-item>
<p>Use <proper>lcov</proper> for C unit test coverage reporting. Switch from <proper>Devel::Cover</proper> because it would not report on branch coverage for reports converted from <proper>gcov</proper>. Branch coverage is not complete, so for the time being errors will only be generated when statement coverage is not complete. Coverage of unit tests is not displayed in the report unless they are incomplete for either statement or branch coverage.</p>
</release-item>
</release-feature-list>
</release-test-list>
</release>
<release date="2018-03-19" version="2.01" title="Minor Bug Fixes and Improvements">
<release-core-list>
<release-bug-list>

4
src/common/regExp.c
View File

@ -68,8 +68,8 @@ regExpMatch(RegExp *this, const String *string)
int result = regexec(&this->regExp, strPtr(string), 0, NULL, 0);
// Check for an error
if (result != 0 && result != REG_NOMATCH)
regExpError(result); // {uncoverable - no error condition known}
if (result != 0 && result != REG_NOMATCH) // {uncoverable - no error condition known}
regExpError(result); // {+uncoverable}
return result == 0;
}

2
test/Vagrantfile vendored
View File

@ -52,7 +52,7 @@ Vagrant.configure(2) do |config|
#---------------------------------------------------------------------------------------------------------------------------
echo 'Install Build Tools' && date
apt-get install -y devscripts build-essential lintian git txt2man debhelper libssl-dev
apt-get install -y devscripts build-essential lintian git txt2man debhelper libssl-dev lcov
#---------------------------------------------------------------------------------------------------------------------------
echo 'Install AWS CLI' && date

2
test/lib/pgBackRestTest/Common/CiTest.pm
View File

@ -101,7 +101,7 @@ sub process
"before_install:\n" .
" - sudo apt-get -qq update && sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl" .
" libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man" .
" devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl libyaml-perl python-pip\n" .
" devscripts libjson-perl libio-socket-ssl-perl libxml-libxml-perl libyaml-perl python-pip lcov\n" .
" - |\n" .
" # Install & Configure AWS CLI\n" .
" pip install --upgrade --user awscli\n" .

2
test/lib/pgBackRestTest/Common/ContainerTest.pm
View File

@ -361,7 +361,7 @@ sub containerBuild
}
elsif ($strOS eq VM_U16)
{
$strScript .= ' clang-5.0';
$strScript .= ' clang-5.0 lcov';
}
}

114
test/lib/pgBackRestTest/Common/JobTest.pm
View File

@ -445,73 +445,91 @@ sub end
}
# Generate coverage reports for the modules
executeTest(
'docker exec -i -u ' . TEST_USER . " ${strImage} bash -l -c '" .
"cd $self->{strGCovPath} && " .
"gcov test.c'", {bSuppressStdErr => true});
my $strLCovExe = "lcov --config-file=$self->{strBackRestBase}/test/src/lcov.conf";
my $strLCovOut = $self->{strGCovPath} . '/test.lcov';
# Mark uncoverable statements as successful
executeTest(
'docker exec -i -u ' . TEST_USER . " ${strImage} " .
"${strLCovExe} --capture --directory=$self->{strGCovPath} --o=${strLCovOut}");
# Generate coverage report for each module
foreach my $strModule (sort(keys(%{$hTestCoverage})))
{
# File that contains coverage info for the module
my $strCoverageFile = "$self->{strGCovPath}/" . testRunName(basename($strModule), false) . ".c.gcov";
my $strModuleName = testRunName($strModule, false);
my $strModuleOutName = $strModuleName;
my $bTest = false;
# If marked as no code then error if there is code
if ($hTestCoverage->{$strModule} eq TESTDEF_COVERAGE_NOCODE)
if ($strModuleOutName =~ /^module/mg)
{
if ($self->{oStorageTest}->exists($strCoverageFile))
$strModuleOutName =~ s/^module/test/mg;
$bTest = true;
}
# Generate lcov reports
my $strModulePath = $self->{strBackRestBase} . "/test/.vagrant/code/${strModuleOutName}";
my $strLCovFile = "${strModulePath}.lcov";
my $strLCovTotal = $self->{strBackRestBase} . "/test/.vagrant/code/all.lcov";
executeTest(
'docker exec -i -u ' . TEST_USER . " ${strImage} " .
"${strLCovExe} --extract=${strLCovOut} */${strModuleName}.c --o=${strLCovFile}");
# Update source file
my $strCoverage = ${$self->{oStorageTest}->get($strLCovFile)};
if (defined($strCoverage))
{
if ($hTestCoverage->{$strModule} eq TESTDEF_COVERAGE_NOCODE)
{
confess &log(ERROR, "module '${strModule}' is marked 'no code' but has code");
}
next;
}
# Get coverage info
my $iTotalLines = (split(':', ($strCoverage =~ m/^LF:.*$/mg)[0]))[1] + 0;
my $iCoveredLines = (split(':', ($strCoverage =~ m/^LH:.*$/mg)[0]))[1] + 0;
# Load the coverage file
my $strCoverage = ${$self->{oStorageTest}->get($strCoverageFile)};
my $iTotalBranches = 0;
my $iCoveredBranches = 0;
# Go line by line and update uncovered statements
my $strUpdatedCoverage;
my $bFirst = true;
foreach my $strLine (split("\n", trim($strCoverage)))
{
# Rewrite source line to the original source path
if ($bFirst)
if ($strCoverage =~ /^BRF\:$/mg && $strCoverage =~ /^BRH\:$/mg)
{
$strLine =
" -: 0:Source:" .
$self->{strBackRestBase} . ($strModule =~ /Test$/ ? '/test' : '') . "/src/" .
testRunName(${strModule}, false) . ".c";
$bFirst = false;
$iTotalBranches = (split(':', ($strCoverage =~ m/^BRF:.*$/mg)[0]))[1] + 0;
$iCoveredBranches = (split(':', ($strCoverage =~ m/^BRH:.*$/mg)[0]))[1] + 0;
}
# Else if the statement is marked uncoverable
elsif ($strLine =~ /\/\/ \{(uncoverable - [^\}]+|\+uncoverable|uncovered - [^\}]+|\+uncovered)\}\s*$/)
# Report coverage if this is not a test or if the test does not have complete coverage
if (!$bTest || $iTotalLines != $iCoveredLines || $iTotalBranches != $iCoveredBranches)
{
# Error if the statement is marked uncoverable but is in fact covered
if ($strLine !~ /^ \#\#\#\#\#/)
# Fix source file name
$strCoverage =~ s/^SF\:.*$/SF:$strModulePath\.c/mg;
# Save coverage file
$self->{oStorageTest}->put($strLCovFile, $strCoverage);
if ($self->{oStorageTest}->exists(${strLCovTotal}))
{
&log(ERROR, "line in ${strModule}.c is marked as uncoverable but is covered:\n${strLine}");
executeTest(
'docker exec -i -u ' . TEST_USER . " ${strImage} " .
"${strLCovExe} --add-tracefile=${strLCovFile} --add-tracefile=${strLCovTotal} --o=${strLCovTotal}");
}
else
{
$self->{oStorageTest}->copy($strLCovFile, $strLCovTotal)
}
# Mark the statement as covered with 0 executions
$strLine =~ s/^ \#\#\#\#\#/^ 0/;
}
# Add to updated file
$strUpdatedCoverage .= "${strLine}\n";
else
{
$self->{oStorageTest}->remove($strLCovFile);
}
}
else
{
if ($hTestCoverage->{$strModule} ne TESTDEF_COVERAGE_NOCODE)
{
confess &log(ERROR, "module '${strModule}' is marked 'code' but has no code");
}
}
# Store the updated file
$self->{oStorageTest}->put($strCoverageFile, $strUpdatedCoverage);
}
executeTest(
'docker exec -i -u ' . TEST_USER . " ${strImage} bash -l -c '" .
"cd $self->{strGCovPath} && " .
# Only generate coverage files for VMs that support coverage
(vmCoverage($self->{oTest}->{&TEST_VM}) ?
"gcov2perl -db ../cover_db " . join(' ', @stryCoveredModule) : '') . "'");
}
# Record elapsed time

3
test/src/lcov.conf Normal file
View File

@ -0,0 +1,3 @@
lcov_branch_coverage=1
lcov_excl_br_line=[A-Z_]+\(|assert\(|testBegin\(
lcov_excl_line=\{\+*uncovered|\{\+*uncoverable

15
test/src/module/common/memContextTest.c
View File

@ -58,8 +58,7 @@ testRun()
int expectedTotal = 0;
for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++)
if (buffer2[charIdx] == 0)
expectedTotal++;
expectedTotal += buffer2[charIdx] == 0;
TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all bytes are 0");
@ -71,16 +70,14 @@ testRun()
expectedTotal = 0;
for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++)
if (buffer2[charIdx] == 0xC7)
expectedTotal++;
expectedTotal += buffer2[charIdx] == 0xC7;
TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all old bytes are filled");
expectedTotal = 0;
for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++)
if ((buffer2 + sizeof(size_t))[charIdx] == 0)
expectedTotal++;
expectedTotal += (buffer2 + sizeof(size_t))[charIdx] == 0;
TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all new bytes are 0");
memFreeInternal(buffer2);
@ -178,8 +175,7 @@ testRun()
int expectedTotal = 0;
for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++)
if (buffer[charIdx] == 0)
expectedTotal++;
expectedTotal += buffer[charIdx] == 0;
TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all bytes are 0");
@ -200,8 +196,7 @@ testRun()
int expectedTotal = 0;
for (unsigned int charIdx = 0; charIdx < sizeof(size_t); charIdx++)
if (buffer[charIdx] == 0xFE)
expectedTotal++;
expectedTotal += buffer[charIdx] == 0xFE;
TEST_RESULT_INT(expectedTotal, sizeof(size_t), "all bytes are 0xFE in original portion");

6
test/src/module/common/typeBufferTest.c
View File

@ -48,8 +48,7 @@ testRun()
unsigned int sameTotal = 0;
for (unsigned int bufferIdx = 0; bufferIdx < 256; bufferIdx++)
if (bufferPtr[bufferIdx] == bufferIdx)
sameTotal++;
sameTotal += bufferPtr[bufferIdx] == bufferIdx;
TEST_RESULT_INT(sameTotal, 256, "original bytes match");
@ -61,8 +60,7 @@ testRun()
sameTotal = 0;
for (unsigned int bufferIdx = 0; bufferIdx < bufSize(buffer); bufferIdx++)
if (bufferPtr[bufferIdx] == bufferIdx)
sameTotal++;
sameTotal += bufferPtr[bufferIdx] == bufferIdx;
TEST_RESULT_INT(sameTotal, 128, "original bytes match");

110
test/test.pl
View File

@ -433,6 +433,7 @@ eval
my $iTestRetry = 0;
my $oyProcess = [];
my $strCoveragePath = "${strTestPath}/cover_db";
my $strCodePath = "${strBackRestBase}/test/.vagrant/code";
if (!$bDryRun || $bVmOut)
{
@ -444,12 +445,24 @@ eval
push(@{$oyProcess}, undef);
}
executeTest("sudo umount ${strTestPath}", {bSuppressError => true});
executeTest("sudo rm -rf ${strTestPath}/*");
$oStorageTest->pathCreate($strTestPath, {strMode => '0770', bIgnoreExists => true});
executeTest("sudo mount -t tmpfs -o size=2560M test ${strTestPath}");
$oStorageTest->pathCreate($strCoveragePath, {strMode => '0770', bIgnoreMissing => true, bCreateParent => true});
# Remove old coverage dirs -- do it this way so the dirs stay open in finder/explorer, etc.
executeTest("rm -rf ${strBackRestBase}/test/coverage/c/*");
executeTest("rm -rf ${strBackRestBase}/test/coverage/perl/*");
# Copy C code for coverage tests
if (vmCoverage($strVm) && !$bDryRun)
{
$oStorageTest->pathCreate("${strCodePath}/test", {strMode => '0770', bIgnoreExists => true, bCreateParent => true});
executeTest("rsync -rt --delete --exclude=test ${strBackRestBase}/src/ ${strCodePath}");
executeTest("rsync -rt --delete ${strBackRestBase}/test/src/module/ ${strCodePath}/test");
}
}
# Determine which tests to run
@ -991,17 +1004,16 @@ eval
if (vmCoverage($strVm) && !$bDryRun)
{
&log(INFO, 'writing coverage report');
executeTest("rm -rf ${strBackRestBase}/test/coverage");
executeTest("cp -rp ${strCoveragePath} ${strCoveragePath}_temp");
executeTest(
"cd ${strCoveragePath}_temp && " .
LIB_COVER_EXE . " -report json -outputdir ${strBackRestBase}/test/coverage ${strCoveragePath}_temp",
LIB_COVER_EXE . " -report json -outputdir ${strBackRestBase}/test/coverage/perl ${strCoveragePath}_temp",
{bSuppressStdErr => true});
executeTest("sudo rm -rf ${strCoveragePath}_temp");
executeTest("sudo cp -rp ${strCoveragePath} ${strCoveragePath}_temp");
executeTest(
"cd ${strCoveragePath}_temp && " .
LIB_COVER_EXE . " -outputdir ${strBackRestBase}/test/coverage ${strCoveragePath}_temp",
LIB_COVER_EXE . " -outputdir ${strBackRestBase}/test/coverage/perl ${strCoveragePath}_temp",
{bSuppressStdErr => true});
executeTest("sudo rm -rf ${strCoveragePath}_temp");
@ -1031,7 +1043,7 @@ eval
# Load the results of coverage testing from JSON
my $oJSON = JSON::PP->new()->allow_nonref();
my $hCoverageResult = $oJSON->decode(${$oStorageBackRest->get('test/coverage/cover.json')});
my $hCoverageResult = $oJSON->decode(${$oStorageBackRest->get('test/coverage/perl/cover.json')});
# Now compare against code modules that should have full coverage
my $hCoverageList = testDefCoverageList();
@ -1074,27 +1086,15 @@ eval
foreach my $strCodeModule (sort(keys(%{$hCoverageActual})))
{
my $strCodeModulePath = "${strBackRestBase}/";
# If the first char of the module is lower case when this is a c module
# If the first char of the module is lower case then it's a c module
if (substr($strCodeModule, 0, 1) eq lc(substr($strCodeModule, 0, 1)))
{
# If it ends with Test then it is a test modile
if ($strCodeModule =~ /Test$/)
{
$strCodeModulePath .= "test/src/${strCodeModule}.c";
}
else
{
$strCodeModulePath .= "src/${strCodeModule}.c";
}
}
# Else a Perl module
else
{
$strCodeModulePath .= "lib/" . BACKREST_NAME . "/${strCodeModule}.pm"
next;
}
# Create code module path -- where the file is located on disk
my $strCodeModulePath = "${strBackRestBase}/lib/" . BACKREST_NAME . "/${strCodeModule}.pm";
# Get summary results
my $hCoverageResultAll = $hCoverageResult->{'summary'}{$strCodeModulePath}{total};
@ -1111,7 +1111,7 @@ eval
# Error if it really does have coverage
if ($hCoverageResultAll)
{
confess &log(ERROR, "module ${strCodeModule} is marked 'no code' but has code");
confess &log(ERROR, "perl module ${strCodeModule} is marked 'no code' but has code");
}
# Skip to next module
@ -1134,18 +1134,15 @@ eval
if ($iUncoveredLines != 0)
{
&log(ERROR, "code module ${strCodeModule} is not fully covered");
&log(ERROR, "perl module ${strCodeModule} is not fully covered");
$iUncoveredCodeModuleTotal++;
if ($strCodeModulePath !~ /\.c$/)
{
&log(ERROR, ('-' x 80));
executeTest(
"/usr/bin/cover -report text ${strCoveragePath} --select ${strBackRestBase}/lib/" .
BACKREST_NAME . "/${strCodeModule}.pm",
{bShowOutputAsync => true});
&log(ERROR, ('-' x 80));
}
&log(ERROR, ('-' x 80));
executeTest(
"/usr/bin/cover -report text ${strCoveragePath} --select ${strBackRestBase}/lib/" .
BACKREST_NAME . "/${strCodeModule}.pm",
{bShowOutputAsync => true});
&log(ERROR, ('-' x 80));
}
}
# Else test how much partial coverage there was
@ -1156,7 +1153,7 @@ eval
if ($iCoveragePercent == 100)
{
# This has been changed to a warning for now so archive/async tests will pass
&log(WARN, "code module ${strCodeModule} has 100% coverage but is not marked fully covered");
&log(WARN, "perl module ${strCodeModule} has 100% coverage but is not marked fully covered");
# $iUncoveredCodeModuleTotal++;
}
else
@ -1170,7 +1167,50 @@ eval
# If any modules had partial coverage then display them
if (defined($strPartialCoverage))
{
&log(INFO, "module (expected) partial coverage: ${strPartialCoverage}");
&log(INFO, "perl module (expected) partial coverage: ${strPartialCoverage}");
}
# Generate C coverage with lcov
#---------------------------------------------------------------------------------------------------------------------------
my $strLCovFile = "${strBackRestBase}/test/.vagrant/code/all.lcov";
if ($oStorageBackRest->exists($strLCovFile))
{
executeTest(
"genhtml ${strLCovFile} --branch-coverage --show-details --output-directory" .
" ${strBackRestBase}/test/coverage/c");
foreach my $strCodeModule (sort(keys(%{$hCoverageActual})))
{
# If the first char of the module is upper case then it's a Perl module
if (substr($strCodeModule, 0, 1) eq uc(substr($strCodeModule, 0, 1)))
{
next;
}
my $strCoverageFile = $strCodeModule;
$strCoverageFile =~ s/^module/test/mg;
$strCoverageFile = "${strBackRestBase}/test/.vagrant/code/${strCoverageFile}.lcov";
my $strCoverage = $oStorageBackRest->get(
$oStorageBackRest->openRead($strCoverageFile, {bIgnoreMissing => true}));
if (defined($strCoverage) && defined($$strCoverage))
{
my $iTotalLines = (split(':', ($$strCoverage =~ m/^LF:.*$/mg)[0]))[1] + 0;
my $iCoveredLines = (split(':', ($$strCoverage =~ m/^LH:.*$/mg)[0]))[1] + 0;
if ($iCoveredLines != $iTotalLines)
{
&log(ERROR, "c module ${strCodeModule} is not fully covered ($iCoveredLines/$iTotalLines)");
$iUncoveredCodeModuleTotal++;
}
}
}
}
else
{
executeTest("rm -rf ${strBackRestBase}/test/coverage/c");
}
}
}