From 10dfbd90b5c3af6ebb6c651760d2d62ced738b84 Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 12 Oct 2017 12:55:48 -0400 Subject: [PATCH] Add C unit test infrastructure. --- doc/xml/release.xml | 6 + libc/Makefile.PL | 4 +- libc/t/sanity.t | 2 +- .../pgBackRestTest/Common/ContainerTest.pm | 2 +- test/lib/pgBackRestTest/Common/DefineTest.pm | 38 ++- test/lib/pgBackRestTest/Common/JobTest.pm | 291 +++++++++++++++--- test/lib/pgBackRestTest/Common/ListTest.pm | 11 + test/lib/pgBackRestTest/Common/RunTest.pm | 9 +- test/src/common/harnessTest.c | 66 ++++ test/src/common/harnessTest.h | 9 + test/src/module/common/typeTest.c | 20 ++ test/src/test.c | 43 +++ test/test.pl | 71 ++++- 13 files changed, 507 insertions(+), 65 deletions(-) create mode 100644 test/src/common/harnessTest.c create mode 100644 test/src/common/harnessTest.h create mode 100644 test/src/module/common/typeTest.c create mode 100644 test/src/test.c diff --git a/doc/xml/release.xml b/doc/xml/release.xml index a352b7eff..efafc3364 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -32,6 +32,12 @@ + + +

Add C unit test infrastructure.

+ + +

Warnings in C builds treated as errors.

diff --git a/libc/Makefile.PL b/libc/Makefile.PL index 7b6ca3792..7f2fb352b 100644 --- a/libc/Makefile.PL +++ b/libc/Makefile.PL @@ -312,12 +312,12 @@ use ExtUtils::MakeMaker; # Create C files array my @stryCFile = qw(LibC.c); -foreach my $strFile (sort(keys(%{$oStorage->manifest('')}))) +foreach my $strFile (sort(keys(%{$oStorage->manifest('src')}))) { # Skip all files except .c files (including .auto.c) next if $strFile !~ /(? 'coverage # Should expect log tests be run use constant TESTDEF_EXPECT => 'expect'; push @EXPORT, qw(TESTDEF_EXPECT); +# Is this a C test (instead of Perl)? +use constant TESTDEF_C => 'c'; + push @EXPORT, qw(TESTDEF_C); +# Is the C library required? This only applies to unit tests, the C library is always supplied for integration tests. +use constant TESTDEF_CLIB => 'clib'; + push @EXPORT, qw(TESTDEF_CLIB); # Determines if each run in a test will be run in a new container use constant TESTDEF_INDIVIDUAL => 'individual'; push @EXPORT, qw(TESTDEF_INDIVIDUAL); @@ -51,11 +57,14 @@ use constant TESTDEF_VM => 'vm'; push @EXPORT, qw(TESTDEF_VM); # The test provides full coverage for the module -use constant TESTDEF_COVERAGE_FULL => true; +use constant TESTDEF_COVERAGE_FULL => 'full'; push @EXPORT, qw(TESTDEF_COVERAGE_FULL); # The test provides partial coverage for the module -use constant TESTDEF_COVERAGE_PARTIAL => false; +use constant TESTDEF_COVERAGE_PARTIAL => 'partial'; push @EXPORT, qw(TESTDEF_COVERAGE_PARTIAL); +# The module does not have any code so does not have any coverage. An error will be thrown if the module has code in the future. +use constant TESTDEF_COVERAGE_NOCODE => 'nocode'; + push @EXPORT, qw(TESTDEF_COVERAGE_NOCODE); ################################################################################################################################ # Code modules @@ -88,6 +97,16 @@ my $oTestDef = &TESTDEF_TEST => [ + { + &TESTDEF_NAME => 'type', + &TESTDEF_TOTAL => 2, + &TESTDEF_C => true, + + &TESTDEF_COVERAGE => + { + 'common/type' => TESTDEF_COVERAGE_NOCODE, + }, + }, { &TESTDEF_NAME => 'http-client', &TESTDEF_TOTAL => 2, @@ -252,6 +271,7 @@ my $oTestDef = { &TESTDEF_NAME => 'local', &TESTDEF_TOTAL => 9, + &TESTDEF_CLIB => true, &TESTDEF_COVERAGE => { @@ -494,7 +514,7 @@ foreach my $hModule (@{$oTestDef->{&TESTDEF_MODULE}}) # Resolve variables that can be set in the module or the test foreach my $strVar ( - TESTDEF_CONTAINER, TESTDEF_EXPECT, TESTDEF_DB, TESTDEF_INDIVIDUAL, TESTDEF_VM) + TESTDEF_C, TESTDEF_CLIB, TESTDEF_CONTAINER, TESTDEF_EXPECT, TESTDEF_DB, TESTDEF_INDIVIDUAL, TESTDEF_VM) { $hTestDefHash->{$strModule}{$strTest}{$strVar} = coalesce( $hModuleTest->{$strVar}, $hModule->{$strVar}, $strVar eq TESTDEF_VM ? undef : false); @@ -503,6 +523,14 @@ foreach my $hModule (@{$oTestDef->{&TESTDEF_MODULE}}) # Set test count $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_TOTAL} = $hModuleTest->{&TESTDEF_TOTAL}; + # If this is a C test then add the test module to coverage + if ($hModuleTest->{&TESTDEF_C}) + { + my $strTestFile = "module/${strModule}/${strTest}Test"; + + $hModuleTest->{&TESTDEF_COVERAGE}{$strTestFile} = TESTDEF_COVERAGE_FULL; + } + # Concatenate coverage for modules and tests foreach my $hCoverage ($hModule->{&TESTDEF_COVERAGE}, $hModuleTest->{&TESTDEF_COVERAGE}) { @@ -521,9 +549,9 @@ foreach my $hModule (@{$oTestDef->{&TESTDEF_MODULE}}) { $hCoverageType->{$strCodeModule} = $hCoverage->{$strCodeModule}; } - elsif ($hCoverageType->{$strCodeModule} != $hCoverage->{$strCodeModule}) + elsif ($hCoverageType->{$strCodeModule} ne $hCoverage->{$strCodeModule}) { - confess &log(ASSERT, "cannot mix full/partial coverage for ${strCodeModule}"); + confess &log(ASSERT, "cannot mix coverage types for ${strCodeModule}"); } # Add to coverage list diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 899e896c1..d0954027c 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -14,7 +14,7 @@ use English '-no_match_vars'; use Cwd qw(abs_path); use Exporter qw(import); our @EXPORT = qw(); -use File::Basename qw(dirname); +use File::Basename qw(dirname basename); use POSIX qw(ceil); use Time::HiRes qw(gettimeofday); @@ -24,6 +24,7 @@ use pgBackRest::Common::Log; use pgBackRest::Common::String; use pgBackRestTest::Common::ContainerTest; +use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::ListTest; use pgBackRestTest::Common::RunTest; @@ -84,6 +85,9 @@ sub new # Set try to 0 $self->{iTry} = 0; + # Setup the path where gcc coverage will be performed + $self->{strGCovPath} = "$self->{strTestPath}/gcov-$self->{iVmIdx}"; + # Return from function and log return values if any return logDebugReturn ( @@ -102,6 +106,9 @@ sub run # Assign function parameters, defaults, and log debug info (my $strOperation) = logDebugParam (__PACKAGE__ . '->run', \@_,); + # Start the test timer + my $fTestStartTime = gettimeofday(); + # Was the job run? my $bRun = false; @@ -115,14 +122,15 @@ sub run $self->{strLogLevel} = lc(DEBUG); } - my $strTest = sprintf('P%0' . length($self->{iVmMax}) . 'd-T%0' . length($self->{iTestMax}) . 'd/%0' . - length($self->{iTestMax}) . "d - ", $self->{iVmIdx} + 1, $self->{iTestIdx} + 1, $self->{iTestMax}) . - 'vm=' . $self->{oTest}->{&TEST_VM} . - ', module=' . $self->{oTest}->{&TEST_MODULE} . - ', test=' . $self->{oTest}->{&TEST_NAME} . - (defined($self->{oTest}->{&TEST_RUN}) ? ', run=' . join(',', @{$self->{oTest}->{&TEST_RUN}}) : '') . - (defined($self->{oTest}->{&TEST_DB}) ? ', db=' . $self->{oTest}->{&TEST_DB} : '') . - ($self->{iTry} > 1 ? ' (retry ' . ($self->{iTry} - 1) . ')' : ''); + my $strTest = sprintf( + 'P%0' . length($self->{iVmMax}) . 'd-T%0' . length($self->{iTestMax}) . 'd/%0' . + length($self->{iTestMax}) . "d - ", $self->{iVmIdx} + 1, $self->{iTestIdx} + 1, $self->{iTestMax}) . + 'vm=' . $self->{oTest}->{&TEST_VM} . + ', module=' . $self->{oTest}->{&TEST_MODULE} . + ', test=' . $self->{oTest}->{&TEST_NAME} . + (defined($self->{oTest}->{&TEST_RUN}) ? ', run=' . join(',', sort(@{$self->{oTest}->{&TEST_RUN}})) : '') . + (defined($self->{oTest}->{&TEST_DB}) ? ', db=' . $self->{oTest}->{&TEST_DB} : '') . + ($self->{iTry} > 1 ? ' (retry ' . ($self->{iTry} - 1) . ')' : ''); my $strImage = 'test-' . $self->{iVmIdx}; my $strDbVersion = (defined($self->{oTest}->{&TEST_DB}) ? $self->{oTest}->{&TEST_DB} : PG_VERSION_94); @@ -141,31 +149,38 @@ sub run # Create host test directory $self->{oStorageTest}->pathCreate($strHostTestPath, {strMode => '0770'}); + # Create gcov directory + $self->{oStorageTest}->pathCreate($self->{strGCovPath}, {strMode => '0770'}); + if ($self->{oTest}->{&TEST_CONTAINER}) { executeTest( 'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" . " -v $self->{strCoveragePath}:$self->{strCoveragePath} " . " -v ${strHostTestPath}:${strVmTestPath}" . + " -v $self->{strGCovPath}:$self->{strGCovPath}" . " -v $self->{strBackRestBase}:$self->{strBackRestBase} " . containerRepo() . ':' . $self->{oTest}->{&TEST_VM} . "-test", {bSuppressStdErr => true}); # Install Perl C Library - # my $oVm = vmGet(); - # my $strOS = $self->{oTest}->{&TEST_VM}; - # my $strBuildPath = $self->{strBackRestBase} . "/test/.vagrant/libc/$strOS/libc/"; - # my $strPerlAutoPath = $$oVm{$strOS}{&VMDEF_PERL_ARCH_PATH} . '/auto/pgBackRest/LibC'; - # my $strPerlModulePath = $$oVm{$strOS}{&VMDEF_PERL_ARCH_PATH} . '/pgBackRest'; - # - # executeTest( - # "docker exec -i -u root ${strImage} bash -c '" . - # "mkdir -p -m 755 ${strPerlAutoPath} && " . - # "cp ${strBuildPath}/blib/arch/auto/pgBackRest/LibC/LibC.so ${strPerlAutoPath} && " . - # "cp ${strBuildPath}/blib/lib/auto/pgBackRest/LibC/autosplit.ix ${strPerlAutoPath} && " . - # "mkdir -p -m 755 ${strPerlModulePath} && " . - # "cp ${strBuildPath}/blib/lib/pgBackRest/LibC.pm ${strPerlModulePath}'"); + if ($self->{oTest}->{&TEST_CLIB}) + { + my $oVm = vmGet(); + my $strOS = $self->{oTest}->{&TEST_VM}; + my $strBuildPath = $self->{strBackRestBase} . "/test/.vagrant/libc/$strOS/libc/"; + my $strPerlAutoPath = $$oVm{$strOS}{&VMDEF_PERL_ARCH_PATH} . '/auto/pgBackRest/LibC'; + my $strPerlModulePath = $$oVm{$strOS}{&VMDEF_PERL_ARCH_PATH} . '/pgBackRest'; + + executeTest( + "docker exec -i -u root ${strImage} bash -c '" . + "mkdir -p -m 755 ${strPerlAutoPath} && " . + "cp ${strBuildPath}/blib/arch/auto/pgBackRest/LibC/LibC.so ${strPerlAutoPath} && " . + "cp ${strBuildPath}/blib/lib/auto/pgBackRest/LibC/autosplit.ix ${strPerlAutoPath} && " . + "mkdir -p -m 755 ${strPerlModulePath} && " . + "cp ${strBuildPath}/blib/lib/pgBackRest/LibC.pm ${strPerlModulePath}'"); + } } } @@ -178,32 +193,39 @@ sub run } # Create command - my $strCommand = - ($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . - (vmCoverage($self->{oTest}->{&TEST_VM}) ? testRunExe( - abs_path($0), dirname($self->{strCoveragePath}), $self->{strBackRestBase}, $self->{oTest}->{&TEST_MODULE}, - $self->{oTest}->{&TEST_NAME}, defined($self->{oTest}->{&TEST_RUN}) ? $self->{oTest}->{&TEST_RUN} : 'all') : - abs_path($0)) . - " --test-path=${strVmTestPath}" . - " --vm=$self->{oTest}->{&TEST_VM}" . - " --vm-id=$self->{iVmIdx}" . - " --module=" . $self->{oTest}->{&TEST_MODULE} . - ' --test=' . $self->{oTest}->{&TEST_NAME} . - $strCommandRunParam . - (defined($self->{oTest}->{&TEST_DB}) ? ' --db-version=' . $self->{oTest}->{&TEST_DB} : '') . - ($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') . - ' --pgsql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} . - ($self->{bLogForce} ? ' --log-force' : '') . - ($self->{bDryRun} ? ' --dry-run' : '') . - ($self->{bDryRun} ? ' --vm-out' : '') . - ($self->{bNoCleanup} ? " --no-cleanup" : ''); + my $strCommand; + + if ($self->{oTest}->{&TEST_C}) + { + $strCommand = 'docker exec -i -u ' . TEST_USER . " ${strImage} $self->{strGCovPath}/test"; + } + else + { + $strCommand = + ($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . + (vmCoverage($self->{oTest}->{&TEST_VM}) ? testRunExe( + abs_path($0), dirname($self->{strCoveragePath}), $self->{strBackRestBase}, $self->{oTest}->{&TEST_MODULE}, + $self->{oTest}->{&TEST_NAME}, defined($self->{oTest}->{&TEST_RUN}) ? $self->{oTest}->{&TEST_RUN} : 'all') : + abs_path($0)) . + " --test-path=${strVmTestPath}" . + " --vm=$self->{oTest}->{&TEST_VM}" . + " --vm-id=$self->{iVmIdx}" . + " --module=" . $self->{oTest}->{&TEST_MODULE} . + ' --test=' . $self->{oTest}->{&TEST_NAME} . + $strCommandRunParam . + (defined($self->{oTest}->{&TEST_DB}) ? ' --db-version=' . $self->{oTest}->{&TEST_DB} : '') . + ($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') . + ' --pgsql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} . + ($self->{bLogForce} ? ' --log-force' : '') . + ($self->{bDryRun} ? ' --dry-run' : '') . + ($self->{bDryRun} ? ' --vm-out' : '') . + ($self->{bNoCleanup} ? " --no-cleanup" : ''); + } &log(DETAIL, $strCommand); if (!$self->{bDryRun} || $self->{bVmOut}) { - my $fTestStartTime = gettimeofday(); - # Set permissions on the Docker test directory. This can be removed once users/groups are sync'd between # Docker and the host VM. if ($self->{oTest}->{&TEST_CONTAINER}) @@ -211,9 +233,104 @@ sub run executeTest("docker exec ${strImage} chown " . TEST_USER . ':' . TEST_GROUP . " -R ${strVmTestPath}"); } + if ($self->{oTest}->{&TEST_C}) + { + my $strCSrcPath = "$self->{strBackRestBase}/src"; + my $hTest = (testDefModuleTest($self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME})); + my $hTestCoverage = $hTest->{&TESTDEF_COVERAGE}; + + my @stryCFile; + + foreach my $strFile (sort(keys(%{$self->{oStorageTest}->manifest($strCSrcPath)}))) + { + # Skip all files except .c files (including .auto.c) + next if $strFile !~ /(?{substr($strFile, 0, length($strFile) - 2)})) + { + push(@stryCFile, "${strCSrcPath}/${strFile}"); + } + } + + # Generate list of C files to include for testing + my $strTestFile = + "module/$self->{oTest}->{&TEST_MODULE}/" . testRunName($self->{oTest}->{&TEST_NAME}, false) . 'Test.c'; + my $strCInclude; + + foreach my $strFile (sort(keys(%{$hTestCoverage}))) + { + # Don't include the test file as it is already included below + next if $strFile =~ /Test$/; + + # Don't any auto files as they are included in their companion C files + next if $strFile =~ /auto$/; + + # Include the C file if it exists + my $strCIncludeFile = "${strFile}.c"; + + # If the C file does not exist use the header file instead + if (!$self->{oStorageTest}->exists("${strCSrcPath}/${strCIncludeFile}")) + { + # Error if code was expected + if ($hTestCoverage->{$strFile} ne TESTDEF_COVERAGE_NOCODE) + { + confess &log(ERROR, "unable to file source file '${strCIncludeFile}'"); + } + + $strCIncludeFile = "${strFile}.h"; + } + + $strCInclude .= (defined($strCInclude) ? "\n" : '') . "#include \"${strCIncludeFile}\""; + } + + # Update C test file with test module + my $strTestC = ${$self->{oStorageTest}->get("$self->{strBackRestBase}/test/src/test.c")}; + $strTestC =~ s/\{\[C\_INCLUDE\]\}/$strCInclude/g; + $strTestC =~ s/\{\[C\_TEST\_INCLUDE\]\}/\#include \"$strTestFile\"/g; + + # Initialize tests + my $strTestInit; + + for (my $iTestIdx = 1; $iTestIdx <= $hTest->{&TESTDEF_TOTAL}; $iTestIdx++) + { + my $bSelected = false; + + # use Data::Dumper; confess "RUN: " . Dumper($self->{oTest}->{&TEST_RUN}); + + if (!defined($self->{oTest}->{&TEST_RUN}) || @{$self->{oTest}->{&TEST_RUN}} == 0 || + grep(/^$iTestIdx$/, @{$self->{oTest}->{&TEST_RUN}})) + { + $bSelected = true; + } + + $strTestInit .= + (defined($strTestInit) ? "\n " : '') . + sprintf("testAdd(%3d, %8s);" , $iTestIdx, ($bSelected ? 'true' : 'false')); + } + + $strTestC =~ s/\{\[C\_TEST\_LIST\]\}/$strTestInit/g; + + # Save C test file + $self->{oStorageTest}->put("$self->{strGCovPath}/test.c", $strTestC); + + my $strGccCommand = + 'gcc -std=c99 -fprofile-arcs -ftest-coverage -fPIC -O0 ' . + "-I/$self->{strBackRestBase}/src -I/$self->{strBackRestBase}/test/src test.c " . + "/$self->{strBackRestBase}/test/src/common/harnessTest.c " . + join(' ', @stryCFile) . ' -o test'; + + executeTest( + 'docker exec -i -u ' . TEST_USER . " ${strImage} bash -l -c '" . + "cd $self->{strGCovPath} && " . + "${strGccCommand}'"); + } + my $oExec = new pgBackRestTest::Common::ExecuteTest( $strCommand, - {bSuppressError => true, bShowOutputAsync => $self->{bShowOutputAsync}}); + {bSuppressError => !$self->{oTest}->{&TEST_C}, bShowOutputAsync => $self->{bShowOutputAsync}}); $oExec->begin(); @@ -261,13 +378,96 @@ sub end if (defined($iExitStatus)) { + my $strImage = 'test-' . $self->{iVmIdx}; + if ($self->{bShowOutputAsync}) { syswrite(*STDOUT, "\n"); } + # If C library generate coverage info + if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C}) + { + # Generate a list of files to cover + my $hTestCoverage = + (testDefModuleTest($self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME}))->{&TESTDEF_COVERAGE}; + + my @stryCoveredModule; + + foreach my $strModule (sort(keys(%{$hTestCoverage}))) + { + # Skip modules that have no code + next if ($hTestCoverage->{$strModule} eq TESTDEF_COVERAGE_NOCODE); + + push (@stryCoveredModule, testRunName(basename($strModule), false) . ".c.gcov"); + } + + # Generate coverage reports for the modules + executeTest( + 'docker exec -i -u ' . TEST_USER . " ${strImage} bash -l -c '" . + "cd $self->{strGCovPath} && " . + "gcov test.c'", {bSuppressStdErr => true}); + + # Mark uncoverable statements as successful + foreach my $strModule (sort(keys(%{$hTestCoverage}))) + { + # File that contains coverage info for the module + my $strCoverageFile = "$self->{strGCovPath}/" . testRunName(basename($strModule), false) . ".c.gcov"; + + # If marked as no code then error if there is code + if ($hTestCoverage->{$strModule} eq TESTDEF_COVERAGE_NOCODE) + { + if ($self->{oStorageTest}->exists($strCoverageFile)) + { + confess &log(ERROR, "module '${strModule}' is marked 'no code' but has code"); + } + + next; + } + + # Load the coverage file + my $strCoverage = ${$self->{oStorageTest}->get($strCoverageFile)}; + + # Go line by line and update uncovered statements + my $strUpdatedCoverage; + + foreach my $strLine (split("\n", trim($strCoverage))) + { + # If the statement is marked uncoverable + if ($strLine =~ /\/\/ \{(uncoverable - [^\}]+|\+uncoverable|uncovered - [^\}]+|\+uncovered)\}\s*$/) + { + # Error if the statement is marked uncoverable but is in fact covered + if ($strLine !~ /^ \#\#\#\#\#/) + { + &log(ERROR, "line in ${strModule}.c is marked as uncoverable but is covered:\n${strLine}"); + } + + # Mark the statement as covered with 0 executions + $strLine =~ s/^ \#\#\#\#\#/^ 0/; + } + + # Add to updated file + $strUpdatedCoverage .= "${strLine}\n"; + } + + # Store the updated file + $self->{oStorageTest}->put($strCoverageFile, $strUpdatedCoverage); + } + + executeTest( + 'docker exec -i -u ' . TEST_USER . " ${strImage} bash -l -c '" . + "cd $self->{strGCovPath} && " . + # Only generate coverage files for VMs that support coverage + (vmCoverage($self->{oTest}->{&TEST_VM}) ? + "gcov2perl -db ../cover_db " . join(' ', @stryCoveredModule) . " && " : '') . + # If these aren't deleted then cover automagically finds them and includes them in the report + "rm *.gcda *.c.gcov'"); + } + + # Record elapsed time my $fTestElapsedTime = ceil((gettimeofday() - $self->{oProcess}{start_time}) * 100) / 100; + # Output error if ($iExitStatus != 0) { &log(ERROR, "${strTestDone} (err${iExitStatus}-${fTestElapsedTime}s)" . @@ -275,6 +475,7 @@ sub end ":\n\n" . trim($oExecDone->{strOutLog}) . "\n" : ''), undef, undef, 4); $bFail = true; } + # Output success else { &log(INFO, "${strTestDone} (${fTestElapsedTime}s)". @@ -284,11 +485,11 @@ sub end if (!$self->{bNoCleanup}) { - my $strImage = 'test-' . $self->{iVmIdx}; my $strHostTestPath = "$self->{strTestPath}/${strImage}"; containerRemove("test-$self->{iVmIdx}"); executeTest("sudo rm -rf ${strHostTestPath}"); + executeTest("sudo rm -rf $self->{strGCovPath}"); } $bDone = true; diff --git a/test/lib/pgBackRestTest/Common/ListTest.pm b/test/lib/pgBackRestTest/Common/ListTest.pm index 32edf5ddc..74b4dd165 100644 --- a/test/lib/pgBackRestTest/Common/ListTest.pm +++ b/test/lib/pgBackRestTest/Common/ListTest.pm @@ -14,6 +14,7 @@ use Exporter qw(import); our @EXPORT = qw(); use pgBackRest::Common::Log; +use pgBackRest::Common::String; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::VmTest; @@ -23,6 +24,10 @@ use pgBackRestTest::Common::VmTest; ################################################################################################################################ use constant TEST_DB => 'db'; push @EXPORT, qw(TEST_DB); +use constant TEST_C => 'c'; + push @EXPORT, qw(TEST_C); +use constant TEST_CLIB => 'clib'; + push @EXPORT, qw(TEST_CLIB); use constant TEST_CONTAINER => 'container'; push @EXPORT, qw(TEST_CONTAINER); use constant TEST_MODULE => 'module'; @@ -49,6 +54,7 @@ sub testListGet my $iyModuleTestRun = shift; my $strDbVersion = shift; my $bCoverageOnly = shift; + my $bCOnly = shift; my $oyVm = vmGet(); my $oyTestRun = []; @@ -94,6 +100,9 @@ sub testListGet # Skip this test if it can't run on this VM next if (defined($hTest->{&TESTDEF_VM}) && grep(/^$strTestOS$/i, @{$hTest->{&TESTDEF_VM}}) == 0); + # Skip this test if only C tests are requested and this is not a C test + next if ($bCOnly && !$hTest->{&TESTDEF_C}); + for (my $iDbVersionIdx = $iDbVersionMax; $iDbVersionIdx >= $iDbVersionMin; $iDbVersionIdx--) { if ($iDbVersionIdx == -1 || $strDbVersion eq 'all' || $strDbVersion eq 'minimal' || @@ -134,6 +143,8 @@ sub testListGet my $oTestRun = { &TEST_VM => $strTestOS, + &TEST_C => coalesce($hTest->{&TESTDEF_C}, $hModule->{&TESTDEF_C}, false), + &TEST_CLIB => coalesce($hTest->{&TESTDEF_CLIB}, $hModule->{&TESTDEF_CLIB}, false), &TEST_CONTAINER => defined($hTest->{&TESTDEF_CONTAINER}) ? $hTest->{&TESTDEF_CONTAINER} : $hModule->{&TESTDEF_CONTAINER}, &TEST_PGSQL_BIN => $strPgSqlBin, diff --git a/test/lib/pgBackRestTest/Common/RunTest.pm b/test/lib/pgBackRestTest/Common/RunTest.pm index c1852cf3e..87eb389e3 100644 --- a/test/lib/pgBackRestTest/Common/RunTest.pm +++ b/test/lib/pgBackRestTest/Common/RunTest.pm @@ -454,18 +454,25 @@ sub testException sub testRunName { my $strName = shift; + my $bInitCapFirst = shift; + + $bInitCapFirst = defined($bInitCapFirst) ? $bInitCapFirst : true; + my $bFirst = true; my @stryName = split('\-', $strName); $strName = undef; foreach my $strPart (@stryName) { - $strName .= ucfirst($strPart); + $strName .= ($bFirst && $bInitCapFirst) || !$bFirst ? ucfirst($strPart) : $strPart; + $bFirst = false; } return $strName; } +push @EXPORT, qw(testRunName); + #################################################################################################################################### # testRun #################################################################################################################################### diff --git a/test/src/common/harnessTest.c b/test/src/common/harnessTest.c new file mode 100644 index 000000000..4c7751e1b --- /dev/null +++ b/test/src/common/harnessTest.c @@ -0,0 +1,66 @@ +/*********************************************************************************************************************************** +C Test Harness +***********************************************************************************************************************************/ +#include +#include +#include +#include + +#include "common/harnessTest.h" + +#define TEST_LIST_SIZE 64 + +typedef struct TestData +{ + bool selected; +} TestData; + +static TestData testList[TEST_LIST_SIZE]; + +static int testRun = 0; +static int testTotal = 0; + +/*********************************************************************************************************************************** +testAdd - add a new test +***********************************************************************************************************************************/ +void +testAdd(int run, bool selected) +{ + testList[testTotal].selected = selected; + testTotal++; +} + +/*********************************************************************************************************************************** +testBegin - should this test run? +***********************************************************************************************************************************/ +bool +testBegin(const char *name) +{ + testRun++; + + if (testList[testRun - 1].selected) + { + if (testRun != 1) + { + printf("\n"); + } + + printf("run %03d - %s\n", testRun, name); + fflush(stdout); + } +} + +/*********************************************************************************************************************************** +testComplete - make sure all expected tests ran +***********************************************************************************************************************************/ +void +testComplete() +{ + // Check that all tests ran + if (testRun != testTotal) + { + fprintf(stderr, "ERROR: expected %d tests but %d were run\n", testTotal, testRun); + fflush(stderr); + exit(255); + } +} diff --git a/test/src/common/harnessTest.h b/test/src/common/harnessTest.h new file mode 100644 index 000000000..cbeee54e8 --- /dev/null +++ b/test/src/common/harnessTest.h @@ -0,0 +1,9 @@ +/*********************************************************************************************************************************** +C Test Harness +***********************************************************************************************************************************/ +#include "common/type.h" + +// Functions +void testAdd(int run, bool selected); +bool testBegin(const char *name); +void testComplete(); diff --git a/test/src/module/common/typeTest.c b/test/src/module/common/typeTest.c new file mode 100644 index 000000000..b2214ad92 --- /dev/null +++ b/test/src/module/common/typeTest.c @@ -0,0 +1,20 @@ +/*********************************************************************************************************************************** +Test Types +***********************************************************************************************************************************/ +void testRun() +{ + // ----------------------------------------------------------------------------------------------------------------------------- + if (testBegin("test int size")) + { + // Ensure that int is at least 4 bytes + assert(sizeof(int) >= 4); + } + + // ----------------------------------------------------------------------------------------------------------------------------- + if (testBegin("test boolean values")) + { + // Ensure false and true are defined + assert(true); + assert(!false); + } +} diff --git a/test/src/test.c b/test/src/test.c new file mode 100644 index 000000000..c2a9c3b0b --- /dev/null +++ b/test/src/test.c @@ -0,0 +1,43 @@ +/*********************************************************************************************************************************** +C Test Wrapper + +This wrapper runs the the C unit tests. +***********************************************************************************************************************************/ +#include +#include +#include + +#include "common/harnessTest.h" + +/*********************************************************************************************************************************** +C files to be tested + +The files are included directly so the test can see and manipulate variables and functions in the module without the need to extern. +If a .c file does not exist for a module then the header file will be included instead. +***********************************************************************************************************************************/ +{[C_INCLUDE]} + +/*********************************************************************************************************************************** +The test code is included directly so it can freely interact with the included C files +***********************************************************************************************************************************/ +{[C_TEST_INCLUDE]} + +/*********************************************************************************************************************************** +main - run the tests +***********************************************************************************************************************************/ +int +main(void) +{ + // Initialize tests + // run, selected + {[C_TEST_LIST]} + + // Run the tests + testRun(); + + // End test run an make sure all tests ran + testComplete(); + + printf("\nTESTS COMPLETED SUCCESSFULLY (DESPITE ANY ERROR MESSAGES YOU SAW)\n"); + fflush(stdout); +} diff --git a/test/test.pl b/test/test.pl index f1e19befc..3ed64bbae 100755 --- a/test/test.pl +++ b/test/test.pl @@ -72,6 +72,7 @@ test.pl [options] --no-lint disable static source code analysis --build-only compile the C library / packages and run tests only --coverage-only only run coverage tests (as a subset of selected tests) + --c-only only run C tests --smart perform libc/package builds only when source timestamps have changed --no-package do not build packages --no-ci-config don't overwrite the current continuous integration config @@ -122,6 +123,7 @@ my $bVmForce = false; my $bNoLint = false; my $bBuildOnly = false; my $bCoverageOnly = false; +my $bCOnly = false; my $bSmart = false; my $bNoPackage = false; my $bNoCiConfig = false; @@ -154,6 +156,7 @@ GetOptions ('q|quiet' => \$bQuiet, 'no-package' => \$bNoPackage, 'no-ci-config' => \$bNoCiConfig, 'coverage-only' => \$bCoverageOnly, + 'c-only' => \$bCOnly, 'smart' => \$bSmart, 'dev' => \$bDev, 'expect' => \$bExpect, @@ -639,7 +642,7 @@ eval # Determine which tests to run #--------------------------------------------------------------------------------------------------------------------------- my $oyTestRun = testListGet( - $strVm, \@stryModule, \@stryModuleTest, \@iyModuleTestRun, $strDbVersion, $bCoverageOnly); + $strVm, \@stryModule, \@stryModuleTest, \@iyModuleTestRun, $strDbVersion, $bCoverageOnly, $bCOnly); if (@{$oyTestRun} == 0) { @@ -801,7 +804,7 @@ eval if (@{$hCoverageList->{$strCodeModule}} == $iCoverageTotal) { - $hCoverageActual->{$strCodeModule} = $hCoverageType->{$strCodeModule}; + $hCoverageActual->{testRunName($strCodeModule, false)} = $hCoverageType->{$strCodeModule}; } } } @@ -815,12 +818,53 @@ eval &log(INFO, 'no code modules had all tests run required for coverage'); } + my $strPartialCoverage; + foreach my $strCodeModule (sort(keys(%{$hCoverageActual}))) { + my $strCodeModulePath = "${strBackRestBase}/"; + + # If the first char of the module is lower case when this is a c module + if (substr($strCodeModule, 0, 1) eq lc(substr($strCodeModule, 0, 1))) + { + # If it ends with Test then it is a test modile + if ($strCodeModule =~ /Test$/) + { + $strCodeModulePath .= "test/src/${strCodeModule}.c"; + } + else + { + $strCodeModulePath .= "src/${strCodeModule}.c"; + } + } + # Else a Perl module + else + { + $strCodeModulePath .= "lib/" . BACKREST_NAME . "/${strCodeModule}.pm" + } + # Get summary results (??? Need to fix this for coverage testing on bin/pgbackrest since .pm is required) - my $hCoverageResultAll = - $hCoverageResult->{'summary'} - {"${strBackRestBase}/lib/" . BACKREST_NAME . "/${strCodeModule}.pm"}{total}; + my $hCoverageResultAll = $hCoverageResult->{'summary'}{$strCodeModulePath}{total}; + + # Try an extra / if the module is not found + if (!defined($hCoverageResultAll)) + { + $strCodeModulePath = "/${strCodeModulePath}"; + $hCoverageResultAll = $hCoverageResult->{'summary'}{$strCodeModulePath}{total}; + } + + # If module is marked as having no code + if ($hCoverageActual->{$strCodeModule} eq TESTDEF_COVERAGE_NOCODE) + { + # Error if it really does have coverage + if ($hCoverageResultAll) + { + confess &log(ERROR, "module ${strCodeModule} is marked 'no code' but has code"); + } + + # Skip to next module + next; + } if (!defined($hCoverageResultAll)) { @@ -830,9 +874,9 @@ eval # Check that all code has been covered my $iCoverageTotal = $hCoverageResultAll->{total}; my $iCoverageUncoverable = coalesce($hCoverageResultAll->{uncoverable}, 0); - my $iCoverageCovered = $hCoverageResultAll->{covered}; + my $iCoverageCovered = coalesce($hCoverageResultAll->{covered}, 0); - if ($hCoverageActual->{$strCodeModule} == TESTDEF_COVERAGE_FULL) + if ($hCoverageActual->{$strCodeModule} eq TESTDEF_COVERAGE_FULL) { my $iUncoveredLines = $iCoverageTotal - $iCoverageCovered - $iCoverageUncoverable; @@ -842,8 +886,8 @@ eval $iUncoveredCodeModuleTotal++; } } - # Else test how much partial coverage where was - else + # Else test how much partial coverage there was + elsif ($hCoverageActual->{$strCodeModule} eq TESTDEF_COVERAGE_PARTIAL) { my $iCoveragePercent = int(($iCoverageCovered + $iCoverageUncoverable) * 100 / $iCoverageTotal); @@ -854,10 +898,17 @@ eval } else { - &log(INFO, "code module ${strCodeModule} has (expected) partial coverage of ${iCoveragePercent}%"); + $strPartialCoverage .= + (defined($strPartialCoverage) ? ', ' : '') . "${strCodeModule} (${iCoveragePercent}%)"; } } } + + # If any modules had partial coverage then display them + if (defined($strPartialCoverage)) + { + &log(INFO, "module (expected) partial coverage: ${strPartialCoverage}"); + } } }