From 5c1d4bcd0d41590e722f2d184cf64a62b740625c Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 15 May 2019 13:04:56 -0400 Subject: [PATCH] Automate coverage summary report generation. This report replaces the lcov report that was generated manually for each release. The lcov report was overly verbose just to say that we have virtually 100% coverage. --- doc/RELEASE.md | 80 +------ doc/lib/BackRestDoc/Html/DocHtmlPage.pm | 2 +- doc/manifest.xml | 2 + doc/release.pl | 15 +- doc/resource/html/default.css | 17 ++ doc/xml/auto/metric-coverage-report.auto.xml | 209 ++++++++++++++++++ doc/xml/metric.xml | 30 +++ doc/xml/release.xml | 8 + .../lib/pgBackRestTest/Common/CoverageTest.pm | 173 +++++++++++++++ test/lib/pgBackRestTest/Common/JobTest.pm | 18 +- test/src/lcov.conf | 24 -- test/test.pl | 40 +++- test/travis.pl | 2 +- 13 files changed, 505 insertions(+), 115 deletions(-) create mode 100644 doc/xml/auto/metric-coverage-report.auto.xml create mode 100644 doc/xml/metric.xml delete mode 100644 test/src/lcov.conf diff --git a/doc/RELEASE.md b/doc/RELEASE.md index fc34431a0..365999909 100644 --- a/doc/RELEASE.md +++ b/doc/RELEASE.md @@ -1,81 +1,3 @@ # Release Build Instructions -## Generate Coverage Report - -These instructions are temporary until a fully automated report is implemented. - -- In `test/src/lcov.conf` remove: -``` - # Specify the regular expression of lines to exclude - lcov_excl_line=lcov_excl_line=\{\+{0,1}uncovered[^_]|\{\+{0,1}uncoverable[^_] - - # Coverage rate limits - genhtml_hi_limit = 100 - genhtml_med_limit = 90 -``` - -And change `uncover(ed|able)_branch` to `uncoverable_branch`. - -- In `test/lib/pgBackRestTest/Common/JobTest.pm` modify: -``` -if (!$bTest || $iTotalLines != $iCoveredLines || $iTotalBranches != $iCoveredBranches) -``` -to: -``` -if (!$bTest) -``` - -- Run: -``` -/backrest/test/test.pl --dev-test --vm=u18 --c-only -``` - -- Copy coverage report: -``` -cd /doc/site -rm -rf coverage -cp -r ../../test/coverage/c coverage -``` - -- In `doc/site/coverage` replace: -``` - LCOV - all.lcov -``` -with: -``` - pgBackRest vX.XX C Code Coverage -``` - -- In `doc/site/coverage` replace: -``` - LCOV - code coverage report -``` -with: -``` - pgBackRest vX.XX C Code Coverage -``` - -- In `doc/site/coverage` replace: -``` - LCOV - all.lcov - -``` -with: -``` - <title>pgBackRest vX.XX C Code Coverage - -``` - -- In `doc/site/coverage` replace: -``` - <td class="headerValue">all.lcov</td> -``` -with: -``` - <td class="headerValue">all C unit</td> -``` - -- Switch to prior dir and copy coverage: -``` -cd prior/X.XX -rm -rf coverage -cp -r ../../coverage . -``` +## Run `doc/release.pl` diff --git a/doc/lib/BackRestDoc/Html/DocHtmlPage.pm b/doc/lib/BackRestDoc/Html/DocHtmlPage.pm index 7642b54dd..39d7cbffc 100644 --- a/doc/lib/BackRestDoc/Html/DocHtmlPage.pm +++ b/doc/lib/BackRestDoc/Html/DocHtmlPage.pm @@ -433,7 +433,7 @@ sub sectionProcess $oHeaderRowElement->addNew( HTML_TH, - "table-header-${strAlign}" . ($bFill ? ",table-header-fill" : ""), + "table-header-${strAlign}" . ($bFill ? " table-header-fill" : ""), {strContent => $self->processText($oColumn->textGet())}); } } diff --git a/doc/manifest.xml b/doc/manifest.xml index ba13095ae..d8f58548d 100644 --- a/doc/manifest.xml +++ b/doc/manifest.xml @@ -84,6 +84,7 @@ <source key="user-guide"/> <source key="reference" type="custom"/> <source key="release" type="custom"/> + <source key="metric"/> <source key="coding"/> <source key="documentation"/> <source key="test"/> @@ -96,6 +97,7 @@ <render-source key="release" menu="Releases"/> <render-source key="configuration" source="reference" menu="Configuration"/> <render-source key="command" source="reference" menu="Commands"/> + <render-source key="metric" menu="Metrics"/> </render> <render type="pdf" file="{[pdf-file]}"> diff --git a/doc/release.pl b/doc/release.pl index 844b0522c..9f8e2314f 100755 --- a/doc/release.pl +++ b/doc/release.pl @@ -62,6 +62,7 @@ release.pl [options] Release Options: --build Build the cache before release (should be included in the release commit) --deploy Deploy documentation to website (can be done as docs are updated) + --no-coverage Don't generate the coverage report =cut #################################################################################################################################### @@ -73,13 +74,15 @@ my $bQuiet = false; my $strLogLevel = 'info'; my $bBuild = false; my $bDeploy = false; +my $bNoCoverage = false; GetOptions ('help' => \$bHelp, 'version' => \$bVersion, 'quiet' => \$bQuiet, 'log-level=s' => \$strLogLevel, 'build' => \$bBuild, - 'deploy' => \$bDeploy) + 'deploy' => \$bDeploy, + 'no-coverage' => \$bNoCoverage) or pod2usage(2); #################################################################################################################################### @@ -119,6 +122,7 @@ eval my $strDocPath = dirname(abs_path($0)); my $strDocHtml = "${strDocPath}/output/html"; my $strDocExe = "${strDocPath}/doc.pl"; + my $strTestExe = dirname($strDocPath) . "/test/test.pl"; my $oStorageDoc = new pgBackRest::Storage::Local( $strDocPath, new pgBackRest::Storage::Posix::Driver({bFileSync => false, bPathSync => false})); @@ -135,6 +139,15 @@ eval # Remove all docker containers to get consistent IP address assignments executeTest('docker rm -f $(docker ps -a -q)', {bSuppressError => true}); + # Generate coverage summmary + if (!$bNoCoverage) + { + &log(INFO, "Generate Coverage Summary"); + executeTest( + "${strTestExe} --no-lint --no-package --no-valgrind --no-optimize --vm-max=3 --coverage-summary", + {bShowOutputAsync => true}); + } + # Generate deployment docs for RHEL/Centos 7 &log(INFO, "Generate RHEL/CentOS 7 documentation"); diff --git a/doc/resource/html/default.css b/doc/resource/html/default.css index c3ec2fae8..feb0799cc 100644 --- a/doc/resource/html/default.css +++ b/doc/resource/html/default.css @@ -304,6 +304,7 @@ Table Elements .table { margin-top: .5em; + border-spacing: .5em 0; } .table-caption @@ -318,6 +319,14 @@ Table Elements { } +.table-header-left, +.table-header-center, +.table-header-right +{ + white-space: nowrap; + border-bottom: 1px black solid; +} + .table-header-left { text-align: left; @@ -342,8 +351,16 @@ Table Elements { } +.table-data-left, +.table-data-center, +.table-data-right +{ + white-space: nowrap; +} + .table-data-left { + padding-top: 0px; text-align: left; } diff --git a/doc/xml/auto/metric-coverage-report.auto.xml b/doc/xml/auto/metric-coverage-report.auto.xml new file mode 100644 index 000000000..c78f4b659 --- /dev/null +++ b/doc/xml/auto/metric-coverage-report.auto.xml @@ -0,0 +1,209 @@ +<table-row> + <table-cell>command</table-cell> + <table-cell>3/3 (100.0%)</table-cell> + <table-cell>46/46 (100.0%)</table-cell> + <table-cell>82/82 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>command/archive</table-cell> + <table-cell>10/10 (100.0%)</table-cell> + <table-cell>56/56 (100.0%)</table-cell> + <table-cell>176/176 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>command/archive/get</table-cell> + <table-cell>6/6 (100.0%)</table-cell> + <table-cell>86/86 (100.0%)</table-cell> + <table-cell>212/212 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>command/archive/push</table-cell> + <table-cell>9/9 (100.0%)</table-cell> + <table-cell>86/86 (100.0%)</table-cell> + <table-cell>263/263 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>command/backup</table-cell> + <table-cell>1/1 (100.0%)</table-cell> + <table-cell>14/14 (100.0%)</table-cell> + <table-cell>23/23 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>command/control</table-cell> + <table-cell>2/2 (100.0%)</table-cell> + <table-cell>8/8 (100.0%)</table-cell> + <table-cell>17/17 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>command/help</table-cell> + <table-cell>4/4 (100.0%)</table-cell> + <table-cell>110/110 (100.0%)</table-cell> + <table-cell>163/163 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>command/info</table-cell> + <table-cell>7/7 (100.0%)</table-cell> + <table-cell>104/104 (100.0%)</table-cell> + <table-cell>259/259 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>command/local</table-cell> + <table-cell>1/1 (100.0%)</table-cell> + <table-cell>---</table-cell> + <table-cell>15/15 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>command/remote</table-cell> + <table-cell>1/1 (100.0%)</table-cell> + <table-cell>6/6 (100.0%)</table-cell> + <table-cell>27/27 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>common</table-cell> + <table-cell>129/129 (100.0%)</table-cell> + <table-cell>444/444 (100.0%)</table-cell> + <table-cell>1375/1375 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>common/compress/gzip</table-cell> + <table-cell>14/14 (100.0%)</table-cell> + <table-cell>22/22 (100.0%)</table-cell> + <table-cell>158/158 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>common/crypto</table-cell> + <table-cell>23/23 (100.0%)</table-cell> + <table-cell>68/68 (100.0%)</table-cell> + <table-cell>310/310 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>common/encode</table-cell> + <table-cell>5/5 (100.0%)</table-cell> + <table-cell>32/32 (100.0%)</table-cell> + <table-cell>81/81 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>common/io</table-cell> + <table-cell>45/45 (100.0%)</table-cell> + <table-cell>108/108 (100.0%)</table-cell> + <table-cell>498/498 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>common/io/filter</table-cell> + <table-cell>32/32 (100.0%)</table-cell> + <table-cell>80/80 (100.0%)</table-cell> + <table-cell>329/329 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>common/io/http</table-cell> + <table-cell>29/29 (100.0%)</table-cell> + <table-cell>146/146 (100.0%)</table-cell> + <table-cell>388/388 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>common/io/tls</table-cell> + <table-cell>13/13 (100.0%)</table-cell> + <table-cell>74/74 (100.0%)</table-cell> + <table-cell>228/228 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>common/type</table-cell> + <table-cell>227/227 (100.0%)</table-cell> + <table-cell>490/490 (100.0%)</table-cell> + <table-cell>2693/2693 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>config</table-cell> + <table-cell>97/97 (100.0%)</table-cell> + <table-cell>536/536 (100.0%)</table-cell> + <table-cell>1379/1379 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>info</table-cell> + <table-cell>33/33 (100.0%)</table-cell> + <table-cell>94/94 (100.0%)</table-cell> + <table-cell>484/484 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>perl</table-cell> + <table-cell>10/10 (100.0%)</table-cell> + <table-cell>26/26 (100.0%)</table-cell> + <table-cell>138/138 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>postgres</table-cell> + <table-cell>14/14 (100.0%)</table-cell> + <table-cell>64/64 (100.0%)</table-cell> + <table-cell>184/184 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>protocol</table-cell> + <table-cell>59/59 (100.0%)</table-cell> + <table-cell>136/136 (100.0%)</table-cell> + <table-cell>712/712 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>storage</table-cell> + <table-cell>59/59 (100.0%)</table-cell> + <table-cell>138/138 (100.0%)</table-cell> + <table-cell>668/668 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>storage/cifs</table-cell> + <table-cell>1/1 (100.0%)</table-cell> + <table-cell>---</table-cell> + <table-cell>9/9 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>storage/posix</table-cell> + <table-cell>31/31 (100.0%)</table-cell> + <table-cell>181/182 (99.45%)</table-cell> + <table-cell>536/536 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>storage/remote</table-cell> + <table-cell>22/22 (100.0%)</table-cell> + <table-cell>46/46 (100.0%)</table-cell> + <table-cell>387/387 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>storage/s3</table-cell> + <table-cell>23/23 (100.0%)</table-cell> + <table-cell>96/96 (100.0%)</table-cell> + <table-cell>487/487 (100.0%)</table-cell> +</table-row> + +<table-row> + <table-cell>TOTAL</table-cell> + <table-cell>910/910 (100.0%)</table-cell> + <table-cell>3297/3298 (99.97%)</table-cell> + <table-cell>12281/12281 (100.0%)</table-cell> +</table-row> \ No newline at end of file diff --git a/doc/xml/metric.xml b/doc/xml/metric.xml new file mode 100644 index 000000000..b83020acb --- /dev/null +++ b/doc/xml/metric.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE doc SYSTEM "doc.dtd" [<!ENTITY metricCoverageReport SYSTEM "auto/metric-coverage-report.auto.xml">]> +<doc title="{[project]}" subtitle="Metrics"> + <description>!!!</description> + + <section id="code-coverage"> + <title>Code Coverage + +

aims to have complete function/branch/line coverage for the core C code in /src.

+ +

Function/line coverage is complete with no exceptions.

+ +

Branch coverage excludes branches inside macros and assert() calls. Macros have their own unit tests so they do not need to be tested everywhere they appear. Asserts are not expected to have complete branch coverage since they test cases that should always be true.

+ + + + Directory + Functions + Branches + Lines + + + + &metricCoverageReport; + +
+ +

The C unit test modules in /test/src/module also have complete function/line coverage but are not included in the report.

+ + diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 971d1f2e7..b0dc4692d 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -185,6 +185,14 @@ + + + +

Automate coverage summary report generation.

+ + + + diff --git a/test/lib/pgBackRestTest/Common/CoverageTest.pm b/test/lib/pgBackRestTest/Common/CoverageTest.pm index daba117e0..d5d829cad 100644 --- a/test/lib/pgBackRestTest/Common/CoverageTest.pm +++ b/test/lib/pgBackRestTest/Common/CoverageTest.pm @@ -13,6 +13,7 @@ use English '-no_match_vars'; use Exporter qw(import); our @EXPORT = qw(); +use File::Basename qw(dirname); use pgBackRest::Common::Log; use pgBackRest::Common::String; @@ -21,6 +22,50 @@ use pgBackRest::Version; use BackRestDoc::Html::DocHtmlBuilder; use BackRestDoc::Html::DocHtmlElement; +#################################################################################################################################### +# Generate an lcov configuration file +#################################################################################################################################### +sub coverageLCovConfigGenerate +{ + my $oStorage = shift; + my $strOutFile = shift; + my $bCoverageSummary = shift; + + my $strBranchFilter = + 'OBJECT_DEFINE_[A-Z0-9_]+\(|\s{4}[A-Z][A-Z0-9_]+\([^\?]*\)|\s{4}(ASSERT|assert|switch\s)\(|\{\+{0,1}' . + ($bCoverageSummary ? 'uncoverable_branch' : 'uncover(ed|able)_branch'); + my $strLineFilter = '\{\+{0,1}uncover' . ($bCoverageSummary ? 'able' : '(ed|able)') . '[^_]'; + + my $strConfig = + "# LCOV Settings\n" . + "\n" . + "# Specify if branch coverage data should be collected and processed\n" . + "lcov_branch_coverage=1\n" . + "\n" . + "# Specify the regular expression of lines to exclude from branch coverage\n" . + "#\n" . + '# OBJECT_DEFINE_[A-Z0-9_]+\( - exclude object definitions' . "\n" . + '# \s{4}[A-Z][A-Z0-9_]+\([^\?]*\) - exclude macros that do not take a conditional parameter and are not themselves a parameter' . "\n" . + '# ASSERT/(|assert\( - exclude asserts since it usually not possible to trigger both branches' . "\n" . + '# switch \( - lcov requires default: to show complete coverage but --Wswitch-enum enforces all enum values be present' . "\n" . + "lcov_excl_br_line=${strBranchFilter}\n" . + "\n" . + "# Specify the regular expression of lines to exclude\n" . + "lcov_excl_line=${strLineFilter}\n" . + "\n" . + "# Coverage rate limits\n" . + "genhtml_hi_limit = 100\n" . + "genhtml_med_limit = 90\n" . + "\n" . + "# Width of line coverage field in source code view\n" . + "genhtml_line_field_width = 9\n"; + + # Write configuration file + $oStorage->put($strOutFile, $strConfig); +} + +push @EXPORT, qw(coverageLCovConfigGenerate); + #################################################################################################################################### # Generate a C coverage report #################################################################################################################################### @@ -480,4 +525,132 @@ sub coverageGenerate push @EXPORT, qw(coverageGenerate); +#################################################################################################################################### +# Generate a C coverage summary for the documentation +#################################################################################################################################### +sub coverageDocSummaryGenerateValue +{ + my $iHit = shift; + my $iFound = shift; + + if (!defined($iFound) || !defined($iHit) || $iFound == 0) + { + return "---"; + } + + my $fPercent = $iHit * 100 / $iFound; + my $strPercent; + + if ($fPercent == 100) + { + $strPercent = '100.0'; + } + elsif ($fPercent > 99.99) + { + $strPercent = '99.99'; + } + else + { + $strPercent = sprintf("%.2f", $fPercent); + } + + return "${iHit}/${iFound} (${strPercent}%)"; +} + +sub coverageDocSummaryGenerate +{ + my $oStorage = shift; + my $strCoveragePath = shift; + my $strOutFile = shift; + + # Track coverage summary + my $rhSummary; + + # Find all lcov files in the coverage path + my $rhManifest = $oStorage->manifest($strCoveragePath); + + foreach my $strFileCov (sort(keys(%{$rhManifest}))) + { + next if $strFileCov =~ /^test\//; + + if ($strFileCov =~ /\.lcov$/) + { + my $strCoverage = ${$oStorage->get("${strCoveragePath}/${strFileCov}")}; + my $strModule = dirname($strFileCov); + + foreach my $strLine (split("\n", $strCoverage)) + { + # Get Line Coverage + if ($strLine =~ /^LF\:/) + { + $rhSummary->{$strModule}{line}{found} += substr($strLine, 3) + 0; + $rhSummary->{zzztotal}{line}{found} += substr($strLine, 3) + 0; + } + + if ($strLine =~ /^LH\:/) + { + $rhSummary->{$strModule}{line}{hit} += substr($strLine, 3) + 0; + $rhSummary->{zzztotal}{line}{hit} += substr($strLine, 3) + 0; + } + + # Get Function Coverage + if ($strLine =~ /^FNF\:/) + { + $rhSummary->{$strModule}{function}{found} += substr($strLine, 4) + 0; + $rhSummary->{zzztotal}{function}{found} += substr($strLine, 4) + 0; + } + + if ($strLine =~ /^FNH\:/) + { + $rhSummary->{$strModule}{function}{hit} += substr($strLine, 4) + 0; + $rhSummary->{zzztotal}{function}{hit} += substr($strLine, 4) + 0; + } + + # Get Branch Coverage + if ($strLine =~ /^BRF\:/) + { + $rhSummary->{$strModule}{branch}{found} += substr($strLine, 4) + 0; + $rhSummary->{zzztotal}{branch}{found} += substr($strLine, 4) + 0; + } + + if ($strLine =~ /^BRH\:/) + { + $rhSummary->{$strModule}{branch}{hit} += substr($strLine, 4) + 0; + $rhSummary->{zzztotal}{branch}{hit} += substr($strLine, 4) + 0; + } + } + } + } + + # use Data::Dumper;confess Dumper($rhSummary); + + my $strSummary; + + foreach my $strModule (sort(keys(%{$rhSummary}))) + { + my $rhModuleData = $rhSummary->{$strModule}; + + $strSummary .= + (defined($strSummary) ? "\n\n" : '') . + "\n" . + " " . ($strModule eq 'zzztotal' ? 'TOTAL' : $strModule) . "\n" . + " " . + coverageDocSummaryGenerateValue($rhModuleData->{function}{hit}, $rhModuleData->{function}{found}) . + "\n" . + " " . + coverageDocSummaryGenerateValue($rhModuleData->{branch}{hit}, $rhModuleData->{branch}{found}) . + "\n" . + " " . + coverageDocSummaryGenerateValue($rhModuleData->{line}{hit}, $rhModuleData->{line}{found}) . + "\n" . + ""; + } + + + # Write coverage report + $oStorage->put($strOutFile, $strSummary); +} + +push @EXPORT, qw(coverageDocSummaryGenerate); + 1; diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 95793facb..e2176e035 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -26,6 +26,7 @@ use pgBackRest::Version; use pgBackRestTest::Common::BuildTest; use pgBackRestTest::Common::ContainerTest; +use pgBackRestTest::Common::CoverageTest; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::ListTest; @@ -70,6 +71,7 @@ sub new $self->{iRetry}, $self->{bValgrindUnit}, $self->{bCoverageUnit}, + $self->{bCoverageSummary}, $self->{bOptimize}, $self->{bBackTrace}, $self->{bProfile}, @@ -99,6 +101,7 @@ sub new {name => 'iRetry'}, {name => 'bValgrindUnit'}, {name => 'bCoverageUnit'}, + {name => 'bCoverageSummary'}, {name => 'bOptimize'}, {name => 'bBackTrace'}, {name => 'bProfile'}, @@ -578,13 +581,16 @@ sub end "module/$self->{oTest}->{&TEST_MODULE}/" . testRunName($self->{oTest}->{&TEST_NAME}, false) . 'Test'); # Generate coverage reports for the modules - my $strLCovExe = "lcov --config-file=$self->{strGCovPath}/test/lcov.conf"; + my $strLCovConf = $self->{strBackRestBase} . '/test/.vagrant/code/lcov.conf'; + coverageLCovConfigGenerate($self->{oStorageTest}, $strLCovConf, $self->{bCoverageSummary}); + + my $strLCovExeBase = "lcov --config-file=${strLCovConf}"; my $strLCovOut = $self->{strGCovPath} . '/test.lcov'; my $strLCovOutTmp = $self->{strGCovPath} . '/test.tmp.lcov'; executeTest( 'docker exec -i -u ' . TEST_USER . " ${strImage} " . - "${strLCovExe} --capture --directory=$self->{strGCovPath} --o=${strLCovOut}"); + "${strLCovExeBase} --capture --directory=$self->{strGCovPath} --o=${strLCovOut}"); # Generate coverage report for each module foreach my $strModule (@stryCoveredModule) @@ -599,6 +605,14 @@ sub end $bTest = true; } + # Disable branch coverage for test files + my $strLCovExe = $strLCovExeBase; + + if ($bTest) + { + $strLCovExe .= ' --rc lcov_branch_coverage=0'; + } + # Generate lcov reports my $strModulePath = $self->{strBackRestBase} . "/test/.vagrant/code/${strModuleOutName}"; my $strLCovFile = "${strModulePath}.lcov"; diff --git a/test/src/lcov.conf b/test/src/lcov.conf deleted file mode 100644 index c03301b0b..000000000 --- a/test/src/lcov.conf +++ /dev/null @@ -1,24 +0,0 @@ -# LCOV Settings - -# Specify if branch coverage data should be collected and processed -lcov_branch_coverage=1 - -# Specify the regular expression of lines to exclude from branch coverage -# -# OBJECT_DEFINE_[A-Z0-9_]+\( - exclude object definitions -# \s{4}[A-Z][A-Z0-9_]+\([^\?]*\) - exclude macros that do not take a conditional parameter and are not themselves a parameter -# \s{4}(TEST_|HARNESS_)[A-Z0-9_]+\( - exclude macros used in unit tests -# ASSERT/(|assert\( - exclude asserts since it usually not possible to trigger both branches -# (testBegin\( - exclude because it generally returns true, and is not needed for coverage -# switch \( - lcov requires default: to show complete coverage but --Wswitch-enum enforces all enum values be present -lcov_excl_br_line=OBJECT_DEFINE_[A-Z0-9_]+\(|\s{4}[A-Z][A-Z0-9_]+\([^\?]*\)|\s{4}(TEST_|HARNESS_)[A-Z0-9_]+\(|\s{4}(ASSERT|assert|switch\s)\(|\(testBegin\(|\{\+{0,1}uncover(ed|able)_branch - -# Specify the regular expression of lines to exclude -lcov_excl_line=\{\+{0,1}uncovered[^_]|\{\+{0,1}uncoverable[^_] - -# Coverage rate limits -genhtml_hi_limit = 100 -genhtml_med_limit = 90 - -# Width of line coverage field in source code view -genhtml_line_field_width = 9 diff --git a/test/test.pl b/test/test.pl index 56d18eebf..2a9f3d537 100755 --- a/test/test.pl +++ b/test/test.pl @@ -99,6 +99,9 @@ test.pl [options] --no-debug don't generate a debug build --debug-test-trace test stack trace for low-level functions (slow, esp w/valgrind, may cause timeouts) + Report Options: + --coverage-summary generate a coverage summary report for the documentation + Configuration Options: --psql-bin path to the psql executables (e.g. /usr/lib/postgresql/9.3/bin/) --test-path path where tests are executed (defaults to ./test) @@ -146,6 +149,7 @@ my $bNoLint = false; my $bBuildOnly = false; my $iBuildMax = 4; my $bCoverageOnly = false; +my $bCoverageSummary = false; my $bNoCoverage = false; my $bCOnly = false; my $bGenOnly = false; @@ -165,6 +169,8 @@ my $bNoDebug = false; my $bDebugTestTrace = false; my $iRetry = 0; +my @cmdOptions = @ARGV; + GetOptions ('q|quiet' => \$bQuiet, 'version' => \$bVersion, 'help' => \$bHelp, @@ -192,6 +198,7 @@ GetOptions ('q|quiet' => \$bQuiet, 'no-package' => \$bNoPackage, 'no-ci-config' => \$bNoCiConfig, 'coverage-only' => \$bCoverageOnly, + 'coverage-summary' => \$bCoverageSummary, 'no-coverage' => \$bNoCoverage, 'c-only' => \$bCOnly, 'gen-only' => \$bGenOnly, @@ -238,6 +245,15 @@ eval pod2usage(); } + ################################################################################################################################ + # Update options for --coverage-summary + ################################################################################################################################ + if ($bCoverageSummary) + { + $bCoverageOnly = true; + $bCOnly = true; + } + ################################################################################################################################ # Update options for --dev and --dev-fast and --dev-test ################################################################################################################################ @@ -1311,8 +1327,8 @@ eval my $oJob = new pgBackRestTest::Common::JobTest( $oStorageTest, $strBackRestBase, $strTestPath, $strCoveragePath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $iVmIdx, $iVmMax, $iTestIdx, $iTestMax, $strLogLevel, $strLogLevelTest, $bLogForce, $bShowOutputAsync, - $bNoCleanup, $iRetry, !$bNoValgrind, !$bNoCoverage, !$bNoOptimize, $bBackTrace, $bProfile, !$bNoDebug, - $bDebugTestTrace, $iBuildMax / $iVmMax < 1 ? 1 : int($iBuildMax / $iVmMax)); + $bNoCleanup, $iRetry, !$bNoValgrind, !$bNoCoverage, $bCoverageSummary, !$bNoOptimize, $bBackTrace, + $bProfile, !$bNoDebug, $bDebugTestTrace, $iBuildMax / $iVmMax < 1 ? 1 : int($iBuildMax / $iVmMax)); $iTestIdx++; if ($oJob->run()) @@ -1496,7 +1512,7 @@ eval if ($oStorageBackRest->exists($strLCovFile)) { executeTest( - "genhtml ${strLCovFile} --config-file=${strBackRestBase}/test/src/lcov.conf" . + "genhtml ${strLCovFile} --config-file=${strBackRestBase}/test/.vagrant/code/lcov.conf" . " --prefix=${strBackRestBase}/test/.vagrant/code" . " --output-directory=${strBackRestBase}/test/coverage/c"); @@ -1553,9 +1569,19 @@ eval } } - $oStorageBackRest->remove("${strBackRestBase}/test/.vagrant/code/all.lcov", {bIgnoreMissing => true}); - coverageGenerate( - $oStorageBackRest, "${strBackRestBase}/test/.vagrant/code", "${strBackRestBase}/test/coverage/c-coverage.html"); + $oStorageBackRest->remove("${strBackRestBase}/test/.vagrant/code/all.lcov", {bIgnoreMissing => true}); + coverageGenerate( + $oStorageBackRest, "${strBackRestBase}/test/.vagrant/code", + "${strBackRestBase}/test/coverage/c-coverage.html"); + + if ($bCoverageSummary) + { + &log(INFO, 'writing C coverage summary report'); + + coverageDocSummaryGenerate( + $oStorageBackRest, "${strBackRestBase}/test/.vagrant/code", + "${strBackRestBase}/doc/xml/auto/metric-coverage-report.auto.xml"); + } } else { @@ -1572,7 +1598,7 @@ eval " WITH ${iTestFail} FAILURE(S)") . ($iTestRetry == 0 ? '' : ", ${iTestRetry} RETRY(IES)") . ' (' . (time() - $lStartTime) . 's)'); - exit 1 if ($iTestFail > 0 || $iUncoveredCodeModuleTotal > 0); + exit 1 if ($iTestFail > 0 || ($iUncoveredCodeModuleTotal > 0 && !$bCoverageSummary)); exit 0; } diff --git a/test/travis.pl b/test/travis.pl index 7ac97c1bc..c64ab6907 100755 --- a/test/travis.pl +++ b/test/travis.pl @@ -119,7 +119,7 @@ eval processEnd(); processBegin('release documentation doc'); - executeTest("${strReleaseExe} --build", {bShowOutputAsync => true}); + executeTest("${strReleaseExe} --build --no-coverage", {bShowOutputAsync => true}); processEnd(); }