diff --git a/.travis.yml b/.travis.yml index d57081b64..1b4883a26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,31 +13,35 @@ services: - docker env: - - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=stanza --module=archive --module=backup --module=expire --module=info" + - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=info --module=stanza --module=archive --module=backup --module=expire" - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=full --test=synthetic --no-lint --no-package" - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=9.1" PGB_TEST_PARAM="--module=full --test=real --db=9.1 --process-max=2 --no-lint --no-package" - PGB_TEST_VM="co6" PGB_BUILD_PARAM="--db=9.0" PGB_TEST_PARAM="--module=full --test=real --db=9.0 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="u16" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=stanza --module=archive --module=backup --module=expire --module=info --no-lint" + - PGB_TEST_VM="u16" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=info --module=stanza --module=archive --module=backup --module=expire --no-lint" - PGB_TEST_VM="u16" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=full --test=synthetic --no-lint --no-package" - PGB_TEST_VM="u16" PGB_BUILD_PARAM="--db=9.4" PGB_TEST_PARAM="--module=full --test=real --db=9.4 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=stanza --module=archive --module=backup --module=expire --module=info --no-lint" + - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=info --module=stanza --module=archive --module=backup --module=expire --no-lint" - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=full --test=synthetic --no-lint --no-package" - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=9.3" PGB_TEST_PARAM="--module=full --test=real --db=9.3 --process-max=2 --no-lint --no-package" - PGB_TEST_VM="d8" PGB_BUILD_PARAM="--db=8.4" PGB_TEST_PARAM="--module=full --test=real --db=8.4 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=stanza --module=archive --module=backup --module=expire --module=info --no-lint" + - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=info --module=stanza --module=archive --module=backup --module=expire --no-lint" - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=full --test=synthetic --no-lint --no-package" - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=9.6" PGB_TEST_PARAM="--module=full --test=real --db=9.6 --process-max=2 --no-lint --no-package" - PGB_TEST_VM="co7" PGB_BUILD_PARAM="--db=9.5" PGB_TEST_PARAM="--module=full --test=real --db=9.5 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="u14" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=stanza --module=archive --module=backup --module=expire --module=info --no-lint" + - PGB_TEST_VM="u14" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=info --module=stanza --module=archive --module=backup --module=expire --no-lint" - PGB_TEST_VM="u14" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=full --test=synthetic --no-lint --no-package" - PGB_TEST_VM="u14" PGB_BUILD_PARAM="--db=9.2" PGB_TEST_PARAM="--module=full --test=real --db=9.2 --process-max=2 --no-lint --no-package" - - PGB_TEST_VM="u12" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=stanza --module=archive --module=backup --module=expire --module=info --no-lint" + - PGB_TEST_VM="u12" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=help --module=config --module=file --module=info --module=stanza --module=archive --module=backup --module=expire --no-lint" - PGB_TEST_VM="u12" PGB_BUILD_PARAM="--db=none" PGB_TEST_PARAM="--module=full --test=synthetic --no-lint --no-package" - PGB_TEST_VM="u12" PGB_BUILD_PARAM="--db=8.3" PGB_TEST_PARAM="--module=full --test=real --db=8.3 --process-max=2 --no-lint --no-package" before_install: - sudo apt-get -qq update - - sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libdevel-cover-perl + - sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl + - git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl + - cd ~/libdevel-cover-perl && git checkout debian/1.23-2 && debuild -i -us -uc -b + - sudo dpkg -i ~/libdevel-cover-perl_1.23-2_amd64.deb + - /usr/bin/cover -v install: - sudo adduser --ingroup=${USER?} --disabled-password --gecos "" backrest diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 992cd3fa8..cadeac32c 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -212,6 +212,10 @@ + +

Coverage testing always enabled on Debian-based containers.

+ +

Require description in every call to testResult().

 diff --git a/lib/pgBackRest/Common/String.pm b/lib/pgBackRest/Common/String.pm index aa9572748..aa07087e2 100644 --- a/lib/pgBackRest/Common/String.pm +++ b/lib/pgBackRest/Common/String.pm @@ -32,6 +32,24 @@ sub trim push @EXPORT, qw(trim); +#################################################################################################################################### +# coalesce +# +# Return the first non-null parameter. +#################################################################################################################################### +sub coalesce +{ + foreach my $strParam (@_) + { + if (defined($strParam)) + { + return $strParam; + } + } +} + +push @EXPORT, qw(coalesce); + #################################################################################################################################### # commonPrefix # diff --git a/test/Vagrantfile b/test/Vagrantfile index b5f1bf4c4..2df2dba8e 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -13,6 +13,7 @@ Vagrant.configure(2) do |config| # Provision the VM config.vm.provision "shell", inline: <<-SHELL + #--------------------------------------------------------------------------------------------------------------------------- echo 'Build Begin' && date # Suppress "dpkg-reconfigure: unable to re-open stdin: No file or directory" warning @@ -22,18 +23,32 @@ Vagrant.configure(2) do |config| sed -i 's/^127\.0\.0\.1\t.*/127\.0\.0\.1\tlocalhost pgbackrest-test/' /etc/hosts hostnamectl set-hostname pgbackrest-test - # Update Apt + #--------------------------------------------------------------------------------------------------------------------------- echo 'Update Apt' && date apt-get update - # Install build tools - apt-get install -y devscripts build-essential lintian git txt2man debhelper - - # Synchronize date + #--------------------------------------------------------------------------------------------------------------------------- + echo 'Synchronize Date' && date apt-get install -y ntpdate ntpdate pool.ntp.org - # Install Docker + #--------------------------------------------------------------------------------------------------------------------------- + echo 'Install Perl Modules' && date + apt-get install -y libdbd-pg-perl libxml-checker-perl libperl-critic-perl libdevel-nytprof-perl + + #--------------------------------------------------------------------------------------------------------------------------- + echo 'Install Build Tools' && date + apt-get install -y devscripts build-essential lintian git txt2man debhelper + + #--------------------------------------------------------------------------------------------------------------------------- + echo 'Build Devel::Cover' && date + apt-get install -y libpod-coverage-perl libtest-differences-perl libhtml-parser-perl libtemplate-perl + git clone --branch debian/1.23-2 \ + https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git /root/libdevel-cover-perl + cd /root/libdevel-cover-perl && debuild --no-lintian -i -us -uc -b > /dev/null + dpkg -i /root/libdevel-cover-perl_1.23-2_amd64.deb + + #--------------------------------------------------------------------------------------------------------------------------- echo 'Install Docker' && date apt-get install -y apt-transport-https ca-certificates apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D @@ -44,31 +59,28 @@ Vagrant.configure(2) do |config| service docker start sudo usermod -aG docker ubuntu - # Install Perl modules - echo 'Install Perl Modules' && date - apt-get install -y libdbd-pg-perl libxml-checker-perl libperl-critic-perl libdevel-nytprof-perl libdevel-cover-perl - - # Install utilities - echo 'Install Utilities' && date + #--------------------------------------------------------------------------------------------------------------------------- + echo 'Install Dev Utilities' && date apt-get install -y vim htop - # Install TeX Live + #--------------------------------------------------------------------------------------------------------------------------- echo 'Install TeX Live' && date apt-get install -y --no-install-recommends texlive-latex-base texlive-latex-extra texlive-fonts-recommended apt-get install -y texlive-font-utils - # Create backrest user and postgres group + #--------------------------------------------------------------------------------------------------------------------------- echo 'Create Postgres Group & pgBackRest User' && date groupadd -g5000 postgres adduser --uid=5001 --ingroup=ubuntu --disabled-password --gecos "" backrest - # Build VM images + #--------------------------------------------------------------------------------------------------------------------------- echo 'Build VM Images' && date rm -rf /backrest/test/.vagrant/docker/* rm -rf /backrest/test/.vagrant/libc/* rm -rf /backrest/test/.vagrant/package/* sudo su - ubuntu -c '/backrest/test/test.pl --vm-build' + #--------------------------------------------------------------------------------------------------------------------------- echo 'Build End' && date SHELL diff --git a/test/lib/pgBackRestTest/Common/CiTest.pm b/test/lib/pgBackRestTest/Common/CiTest.pm index 8d32c0f93..f98da87ae 100644 --- a/test/lib/pgBackRestTest/Common/CiTest.pm +++ b/test/lib/pgBackRestTest/Common/CiTest.pm @@ -94,23 +94,20 @@ sub process # Iterate each OS foreach my $strVm (VM_LIST) { - my $hTestDef = testDefGet(); my $hVm = vmGet(); my @stryModule; - my $hFullModule = undef; + my $strFullModule = undef; # Get all modules but full to break up the tests - foreach my $hModule (@{$hTestDef->{&TESTDEF_MODULE}}) + foreach my $strModule (testDefModuleList()) { - my $strModule = $hModule->{&TESTDEF_MODULE_NAME}; - if ($strModule ne 'full') { push(@stryModule, $strModule); } else { - $hFullModule = $hModule; + $strFullModule = $strModule; } } @@ -124,20 +121,18 @@ sub process $bFirst = false; # Now generate full tests - my $hRealTest = undef; + my $strRealTest = undef; - if (!defined($hFullModule)) + if (!defined($strFullModule)) { - confess "the full module is not defined, has the name changed?"; + confess "${strFullModule} module not found, has the name changed?"; } - foreach my $hTest (@{$hFullModule->{&TESTDEF_TEST}}) + foreach my $strTest (testDefModuleTestList($strFullModule)) { - my $strTest = $hTest->{&TESTDEF_TEST_NAME}; - if ($strTest eq 'real') { - $hRealTest = $hTest; + $strRealTest = $strTest; foreach my $strDbVersion (sort {$b cmp $a} @{$hVm->{$strVm}{&VM_DB_MINIMAL}}) { @@ -155,9 +150,9 @@ sub process } } - if (!defined($hRealTest)) + if (!defined($strRealTest)) { - confess 'real test not found in full module, has the name changed?'; + confess "${strRealTest} test not found in ${strFullModule} module, has the name changed?"; } } @@ -166,7 +161,12 @@ sub process "\n" . "before_install:\n" . " - sudo apt-get -qq update\n" . - " - sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libdevel-cover-perl\n" . + " - sudo apt-get install libxml-checker-perl libdbd-pg-perl libperl-critic-perl libtemplate-perl libpod-coverage-perl" . + " libtest-differences-perl libhtml-parser-perl lintian debhelper txt2man devscripts libjson-perl\n" . + " - git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl\n" . + ' - cd ~/libdevel-cover-perl && git checkout debian/' . LIB_COVER_VERSION . " && debuild -i -us -uc -b\n" . + ' - sudo dpkg -i ~/' . LIB_COVER_PACKAGE . "\n" . + ' - ' . LIB_COVER_EXE . " -v\n" . "\n" . "install:\n" . " - sudo adduser --ingroup=\${USER?} --disabled-password --gecos \"\" " . BACKREST_USER . "\n" . diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index b85d42f9e..2bde2e25d 100755 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -46,6 +46,16 @@ use constant BACKREST_USER => 'backrest push @EXPORT, qw(BACKREST_USER); use constant BACKREST_USER_ID => getpwnam(BACKREST_USER) . ''; +#################################################################################################################################### +# Package constants +#################################################################################################################################### +use constant LIB_COVER_VERSION => '1.23-2'; + push @EXPORT, qw(LIB_COVER_VERSION); +use constant LIB_COVER_PACKAGE => 'libdevel-cover-perl_' . LIB_COVER_VERSION . '_amd64.deb'; + push @EXPORT, qw(LIB_COVER_PACKAGE); +use constant LIB_COVER_EXE => '/usr/bin/cover'; + push @EXPORT, qw(LIB_COVER_EXE); + #################################################################################################################################### # Container repo #################################################################################################################################### @@ -257,6 +267,27 @@ sub sudoSetup return $strScript; } +#################################################################################################################################### +# Devel::Cover setup +#################################################################################################################################### +sub coverSetup +{ + my $strOS = shift; + + my $strScript = ''; + my $oVm = vmGet(); + + if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) + { + $strScript .= + "\n\n# Install Devel::Cover\n" . + "COPY ${strOS}-" . LIB_COVER_PACKAGE . ' /tmp/' . LIB_COVER_PACKAGE . "\n" . + 'RUN dpkg -i /tmp/' . LIB_COVER_PACKAGE; + } + + return $strScript; +} + #################################################################################################################################### # Install Perl packages #################################################################################################################################### @@ -269,27 +300,30 @@ sub perlInstall if ($strOS eq VM_CO6) { - return $strImage . + $strImage .= 'RUN yum install -y perl perl-Time-HiRes perl-parent perl-JSON perl-Digest-SHA perl-DBD-Pg'; } elsif ($strOS eq VM_CO7) { - return $strImage . + $strImage .= 'RUN yum install -y perl perl-JSON-PP perl-Digest-SHA perl-DBD-Pg'; } elsif ($strOS eq VM_U12 || $strOS eq VM_U14) { - return $strImage . - 'RUN apt-get install -y libdbd-pg-perl libdbi-perl libnet-daemon-perl libplrpc-perl'; + $strImage .= + 'RUN apt-get install -y libdbd-pg-perl libdbi-perl libnet-daemon-perl libplrpc-perl libhtml-parser-perl'; } elsif ($strOS eq VM_U16 || $strOS eq VM_D8) { - return $strImage . - 'RUN apt-get install -y libdbd-pg-perl libdbi-perl' . - ($strOS eq VM_U16 ? ' libdevel-cover-perl libtest-pod-coverage-perl' : ''); + $strImage .= + 'RUN apt-get install -y libdbd-pg-perl libdbi-perl libhtml-parser-perl'; + } + else + { + confess &log(ERROR, "unable to install perl for os '${strOS}'"); } - confess &log(ERROR, "unable to install perl for os '${strOS}'"); + return $strImage; } #################################################################################################################################### @@ -487,8 +521,12 @@ sub containerBuild { $strScript = "# Install package build tools and package source\n" . - "RUN apt-get install -y devscripts build-essential lintian git libxml-checker-perl txt2man debhelper && \\\n" . - " git clone https://anonscm.debian.org/git/pkg-postgresql/pgbackrest.git /root/package-src\n"; + "RUN apt-get install -y devscripts build-essential lintian git libxml-checker-perl txt2man debhelper" . + " libpod-coverage-perl libppi-html-perl libtemplate-perl libtest-differences-perl && \\\n" . + " git clone https://anonscm.debian.org/git/pkg-postgresql/pgbackrest.git /root/package-src && \\\n" . + " git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git" . + " /root/libdevel-cover-perl && \\\n" . + ' cd /root/libdevel-cover-perl && git checkout debian/' . LIB_COVER_VERSION . ' && debuild -i -us -uc -b'; } else { @@ -500,6 +538,20 @@ sub containerBuild # Write the image containerWrite($strTempPath, $strOS, 'Build', $strImageParent, $strImage, $strScript, $bVmForce, false); + # Copy Devel::Cover to host so it can be installed in other containers + if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) + { + executeTest('docker rm -f test-build', {bSuppressError => true}); + executeTest( + "docker run -itd -h test-build --name=test-build" . + " -v ${strTempPath}:${strTempPath} " . containerRepo() . ":${strOS}-build", + {bSuppressStdErr => true}); + executeTest( + "docker exec -i test-build " . + "bash -c 'cp /root/" . LIB_COVER_PACKAGE . " ${strTempPath}/${strOS}-" . LIB_COVER_PACKAGE . "'"); + executeTest('docker rm -f test-build'); + } + # Db image ########################################################################################################################### my @stryDbBuild; @@ -588,6 +640,9 @@ sub containerBuild # Install SSH key $strScript = sshSetup($strOS, TEST_USER, TEST_GROUP, $$oVm{$strOS}{&VM_CONTROL_MASTER}); + # Setup Devel::Cover + $strScript .= coverSetup($strOS); + # Write the image containerWrite($strTempPath, $strOS, "${strTitle} Test", $strImageParent, $strImage, $strScript, $bVmForce, true, true); } @@ -600,6 +655,9 @@ sub containerBuild # Install SSH key $strScript = sshSetup($strOS, TEST_USER, TEST_GROUP, $$oVm{$strOS}{&VM_CONTROL_MASTER}); + # Setup Devel::Cover + $strScript .= coverSetup($strOS); + # Write the image containerWrite($strTempPath, $strOS, 'Db Synthetic Test', $strImageParent, $strImage, $strScript, $bVmForce, true, true); @@ -627,6 +685,9 @@ sub containerBuild # Setup sudo $strScript .= "\n\n" . sudoSetup($strOS, TEST_GROUP); + # Setup Devel::Cover + $strScript .= coverSetup($strOS); + # Write the image containerWrite($strTempPath, $strOS, 'Loop Test', $strImageParent, $strImage, $strScript, $bVmForce, true, true); @@ -680,6 +741,9 @@ sub containerBuild "\n\n# Make " . TEST_USER . " home dir readable\n" . 'RUN chmod g+r,g+x /home/' . TEST_USER; + # Setup Devel::Cover + $strScript .= coverSetup($strOS); + # Write the image containerWrite($strTempPath, $strOS, "${strTitle} Test", $strImageParent, $strImage, $strScript, $bVmForce, true, true); } diff --git a/test/lib/pgBackRestTest/Common/DefineTest.pm b/test/lib/pgBackRestTest/Common/DefineTest.pm index 43cc11ddb..65ea2ac55 100644 --- a/test/lib/pgBackRestTest/Common/DefineTest.pm +++ b/test/lib/pgBackRestTest/Common/DefineTest.pm @@ -14,41 +14,50 @@ use Exporter qw(import); our @EXPORT = qw(); use pgBackRest::Common::Log; +use pgBackRest::Common::String; ################################################################################################################################ # Test definition constants ################################################################################################################################ use constant TESTDEF_MODULE => 'module'; push @EXPORT, qw(TESTDEF_MODULE); -use constant TESTDEF_MODULE_NAME => 'name'; - push @EXPORT, qw(TESTDEF_MODULE_NAME); - -use constant TESTDEF_EXPECT => 'expect'; - push @EXPORT, qw(TESTDEF_EXPECT); +use constant TESTDEF_NAME => 'name'; + push @EXPORT, qw(TESTDEF_NAME); use constant TESTDEF_TEST => 'test'; push @EXPORT, qw(TESTDEF_TEST); -use constant TESTDEF_TEST_ALL => 'all'; - push @EXPORT, qw(TESTDEF_TEST_ALL); -use constant TESTDEF_TEST_COVERAGE => 'coverage'; - push @EXPORT, qw(TESTDEF_TEST_COVERAGE); -use constant TESTDEF_TEST_INDIVIDUAL => 'individual'; - push @EXPORT, qw(TESTDEF_TEST_INDIVIDUAL); -use constant TESTDEF_TEST_NAME => 'name'; - push @EXPORT, qw(TESTDEF_TEST_NAME); -use constant TESTDEF_TEST_TOTAL => 'total'; - push @EXPORT, qw(TESTDEF_TEST_TOTAL); -use constant TESTDEF_TEST_CONTAINER => 'container'; - push @EXPORT, qw(TESTDEF_TEST_CONTAINER); -use constant TESTDEF_TEST_PROCESS => 'process'; - push @EXPORT, qw(TESTDEF_TEST_PROCESS); -use constant TESTDEF_TEST_DB => 'db'; - push @EXPORT, qw(TESTDEF_TEST_DB); +# Determines if the test will be run against multiple db versions +use constant TESTDEF_DB => 'db'; + push @EXPORT, qw(TESTDEF_DB); +# Determines if the test will be run in a container or will create containers itself +use constant TESTDEF_CONTAINER => 'container'; + push @EXPORT, qw(TESTDEF_CONTAINER); +# Determines coverage for the test +use constant TESTDEF_COVERAGE => 'coverage'; + push @EXPORT, qw(TESTDEF_COVERAGE); +# Should expect log tests be run +use constant TESTDEF_EXPECT => 'expect'; + push @EXPORT, qw(TESTDEF_EXPECT); +# Determines if each run in a test will be run in a new container +use constant TESTDEF_INDIVIDUAL => 'individual'; + push @EXPORT, qw(TESTDEF_INDIVIDUAL); +# Determines if the test will be run with multiple processes +use constant TESTDEF_PROCESS => 'process'; + push @EXPORT, qw(TESTDEF_PROCESS); +# Total runs in the test +use constant TESTDEF_TOTAL => 'total'; + push @EXPORT, qw(TESTDEF_TOTAL); + +# The test provides full coverage for the module use constant TESTDEF_COVERAGE_FULL => true; push @EXPORT, qw(TESTDEF_COVERAGE_FULL); +# The test provides partial coverage for the module use constant TESTDEF_COVERAGE_PARTIAL => false; push @EXPORT, qw(TESTDEF_COVERAGE_PARTIAL); +################################################################################################################################ +# Code modules +################################################################################################################################ use constant TESTDEF_MODULE_FILE => 'File'; push @EXPORT, qw(TESTDEF_MODULE_FILE); use constant TESTDEF_MODULE_FILE_COMMON => TESTDEF_MODULE_FILE . 'Common'; @@ -86,218 +95,211 @@ my $oTestDef = [ # Help tests { - &TESTDEF_MODULE_NAME => 'help', - &TESTDEF_TEST_CONTAINER => true, + &TESTDEF_NAME => 'help', + &TESTDEF_CONTAINER => true, &TESTDEF_EXPECT => true, &TESTDEF_TEST => [ { - &TESTDEF_TEST_NAME => 'help', - &TESTDEF_TEST_TOTAL => 1, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'help', + &TESTDEF_TOTAL => 1, } ] }, # Config tests { - &TESTDEF_MODULE_NAME => 'config', - &TESTDEF_TEST_CONTAINER => true, + &TESTDEF_NAME => 'config', + &TESTDEF_CONTAINER => true, &TESTDEF_TEST => [ { - &TESTDEF_TEST_NAME => 'unit', - &TESTDEF_TEST_TOTAL => 7, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'unit', + &TESTDEF_TOTAL => 7, }, { - &TESTDEF_TEST_NAME => 'option', - &TESTDEF_TEST_TOTAL => 34, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'option', + &TESTDEF_TOTAL => 34, }, { - &TESTDEF_TEST_NAME => 'config', - &TESTDEF_TEST_TOTAL => 25, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'config', + &TESTDEF_TOTAL => 25, } ] }, # File tests { - &TESTDEF_MODULE_NAME => 'file', - &TESTDEF_TEST_CONTAINER => true, + &TESTDEF_NAME => 'file', + &TESTDEF_CONTAINER => true, - &TESTDEF_TEST_COVERAGE => + &TESTDEF_COVERAGE => { - &TESTDEF_MODULE_FILE => TESTDEF_COVERAGE_FULL, - &TESTDEF_MODULE_FILE_COMMON => TESTDEF_COVERAGE_FULL, + &TESTDEF_MODULE_FILE => TESTDEF_COVERAGE_PARTIAL, + &TESTDEF_MODULE_FILE_COMMON => TESTDEF_COVERAGE_PARTIAL, }, &TESTDEF_TEST => [ { - &TESTDEF_TEST_NAME => 'unit', - &TESTDEF_TEST_TOTAL => 1, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'unit', + &TESTDEF_TOTAL => 1, }, { - &TESTDEF_TEST_NAME => 'owner', - &TESTDEF_TEST_TOTAL => 8, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'owner', + &TESTDEF_TOTAL => 8, }, { - &TESTDEF_TEST_NAME => 'path-create', - &TESTDEF_TEST_TOTAL => 8, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'path-create', + &TESTDEF_TOTAL => 8, }, { - &TESTDEF_TEST_NAME => 'move', - &TESTDEF_TEST_TOTAL => 24, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'move', + &TESTDEF_TOTAL => 24, }, { - &TESTDEF_TEST_NAME => 'compress', - &TESTDEF_TEST_TOTAL => 4, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'compress', + &TESTDEF_TOTAL => 4, }, { - &TESTDEF_TEST_NAME => 'wait', - &TESTDEF_TEST_TOTAL => 2, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'wait', + &TESTDEF_TOTAL => 2, }, { - &TESTDEF_TEST_NAME => 'link', - &TESTDEF_TEST_TOTAL => 1, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'link', + &TESTDEF_TOTAL => 1, }, { - &TESTDEF_TEST_NAME => 'stat', - &TESTDEF_TEST_TOTAL => 1, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'stat', + &TESTDEF_TOTAL => 1, }, { - &TESTDEF_TEST_NAME => 'manifest', - &TESTDEF_TEST_TOTAL => 5, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'manifest', + &TESTDEF_TOTAL => 5, }, { - &TESTDEF_TEST_NAME => 'list', - &TESTDEF_TEST_TOTAL => 72, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'list', + &TESTDEF_TOTAL => 72, }, { - &TESTDEF_TEST_NAME => 'remove', - &TESTDEF_TEST_TOTAL => 32, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'remove', + &TESTDEF_TOTAL => 32, }, { - &TESTDEF_TEST_NAME => 'hash', - &TESTDEF_TEST_TOTAL => 16, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'hash', + &TESTDEF_TOTAL => 16, }, { - &TESTDEF_TEST_NAME => 'exists', - &TESTDEF_TEST_TOTAL => 6, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'exists', + &TESTDEF_TOTAL => 6, }, { - &TESTDEF_TEST_NAME => 'copy', - &TESTDEF_TEST_TOTAL => 144, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'copy', + &TESTDEF_TOTAL => 144, } ] }, + # Info tests + { + &TESTDEF_NAME => 'info', + &TESTDEF_CONTAINER => true, + + &TESTDEF_TEST => + [ + { + &TESTDEF_NAME => 'unit', + &TESTDEF_TOTAL => 1, + + &TESTDEF_COVERAGE => + { + &TESTDEF_MODULE_INFO => TESTDEF_COVERAGE_PARTIAL, + }, + }, + ] + }, # Stanza tests { - &TESTDEF_MODULE_NAME => 'stanza', - &TESTDEF_TEST_CONTAINER => false, - &TESTDEF_EXPECT => true, + &TESTDEF_NAME => 'stanza', - &TESTDEF_TEST_COVERAGE => + &TESTDEF_COVERAGE => { - &TESTDEF_MODULE_STANZA => TESTDEF_COVERAGE_FULL, + &TESTDEF_MODULE_STANZA => TESTDEF_COVERAGE_PARTIAL, }, &TESTDEF_TEST => [ { - &TESTDEF_TEST_NAME => 'unit', - &TESTDEF_EXPECT => false, - &TESTDEF_TEST_TOTAL => 2, + &TESTDEF_NAME => 'unit', + &TESTDEF_TOTAL => 2, + &TESTDEF_CONTAINER => true, }, { - &TESTDEF_TEST_NAME => 'create', - &TESTDEF_TEST_TOTAL => 2 + &TESTDEF_NAME => 'create', + &TESTDEF_TOTAL => 2, + &TESTDEF_EXPECT => true, }, { - &TESTDEF_TEST_NAME => 'upgrade', - &TESTDEF_TEST_TOTAL => 2 + &TESTDEF_NAME => 'upgrade', + &TESTDEF_TOTAL => 2, + &TESTDEF_EXPECT => true, }, ] }, # Archive tests { - &TESTDEF_MODULE_NAME => 'archive', - &TESTDEF_TEST_CONTAINER => false, - &TESTDEF_EXPECT => true, + &TESTDEF_NAME => 'archive', &TESTDEF_TEST => [ { - &TESTDEF_TEST_NAME => 'unit', - &TESTDEF_TEST_TOTAL => 4, - &TESTDEF_TEST_CONTAINER => true, - &TESTDEF_TEST_INDIVIDUAL => false, - &TESTDEF_EXPECT => false, + &TESTDEF_NAME => 'unit', + &TESTDEF_TOTAL => 4, + &TESTDEF_CONTAINER => true, - &TESTDEF_TEST_COVERAGE => + &TESTDEF_COVERAGE => { - &TESTDEF_TEST_ALL => - { - &TESTDEF_MODULE_ARCHIVE_COMMON => TESTDEF_COVERAGE_PARTIAL, - } + &TESTDEF_MODULE_ARCHIVE_COMMON => TESTDEF_COVERAGE_PARTIAL, }, }, { - &TESTDEF_TEST_NAME => 'push-unit', - &TESTDEF_TEST_TOTAL => 7, - &TESTDEF_TEST_CONTAINER => true, - &TESTDEF_TEST_INDIVIDUAL => false, - &TESTDEF_EXPECT => false, + &TESTDEF_NAME => 'push-unit', + &TESTDEF_TOTAL => 7, + &TESTDEF_CONTAINER => true, - &TESTDEF_TEST_COVERAGE => + &TESTDEF_COVERAGE => { - &TESTDEF_TEST_ALL => - { - &TESTDEF_MODULE_ARCHIVE_PUSH => TESTDEF_COVERAGE_FULL, - &TESTDEF_MODULE_ARCHIVE_PUSH_ASYNC => TESTDEF_COVERAGE_FULL, - &TESTDEF_MODULE_ARCHIVE_PUSH_FILE => TESTDEF_COVERAGE_PARTIAL, - } + &TESTDEF_MODULE_ARCHIVE_PUSH => TESTDEF_COVERAGE_FULL, + &TESTDEF_MODULE_ARCHIVE_PUSH_ASYNC => TESTDEF_COVERAGE_FULL, + &TESTDEF_MODULE_ARCHIVE_PUSH_FILE => TESTDEF_COVERAGE_FULL, }, }, { - &TESTDEF_TEST_NAME => 'push', - &TESTDEF_TEST_TOTAL => 8, - &TESTDEF_TEST_PROCESS => true, + &TESTDEF_NAME => 'push', + &TESTDEF_TOTAL => 8, + &TESTDEF_PROCESS => true, + &TESTDEF_INDIVIDUAL => true, + &TESTDEF_EXPECT => true, }, { - &TESTDEF_TEST_NAME => 'stop', - &TESTDEF_TEST_TOTAL => 6 + &TESTDEF_NAME => 'stop', + &TESTDEF_TOTAL => 6, + &TESTDEF_INDIVIDUAL => true, + &TESTDEF_EXPECT => true, }, { - &TESTDEF_TEST_NAME => 'get', - &TESTDEF_TEST_TOTAL => 8 + &TESTDEF_NAME => 'get', + &TESTDEF_TOTAL => 8, + &TESTDEF_INDIVIDUAL => true, + &TESTDEF_EXPECT => true, }, ] }, # Backup tests { - &TESTDEF_MODULE_NAME => 'backup', - &TESTDEF_TEST_CONTAINER => false, - &TESTDEF_EXPECT => false, + &TESTDEF_NAME => 'backup', + &TESTDEF_CONTAINER => true, - &TESTDEF_TEST_COVERAGE => + &TESTDEF_COVERAGE => { &TESTDEF_MODULE_BACKUP_COMMON => TESTDEF_COVERAGE_FULL, }, @@ -305,79 +307,51 @@ my $oTestDef = &TESTDEF_TEST => [ { - &TESTDEF_TEST_NAME => 'unit', - &TESTDEF_TEST_TOTAL => 3, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'unit', + &TESTDEF_TOTAL => 3, }, { - &TESTDEF_TEST_NAME => 'info-unit', - &TESTDEF_TEST_TOTAL => 1, - &TESTDEF_TEST_INDIVIDUAL => false, + &TESTDEF_NAME => 'info-unit', + &TESTDEF_TOTAL => 1, }, ] }, # Expire tests { - &TESTDEF_MODULE_NAME => 'expire', - &TESTDEF_TEST_CONTAINER => false, + &TESTDEF_NAME => 'expire', &TESTDEF_EXPECT => true, + &TESTDEF_INDIVIDUAL => true, - &TESTDEF_TEST_COVERAGE => + &TESTDEF_COVERAGE => { - &TESTDEF_MODULE_EXPIRE => TESTDEF_COVERAGE_FULL, + &TESTDEF_MODULE_EXPIRE => TESTDEF_COVERAGE_PARTIAL, }, &TESTDEF_TEST => [ { - &TESTDEF_TEST_NAME => 'expire', - &TESTDEF_TEST_TOTAL => 2, - }, - ] - }, - # Info tests - { - &TESTDEF_MODULE_NAME => 'info', - &TESTDEF_TEST_CONTAINER => false, - &TESTDEF_EXPECT => true, - - &TESTDEF_TEST => - [ - { - &TESTDEF_TEST_NAME => 'unit', - &TESTDEF_TEST_CONTAINER => true, - &TESTDEF_TEST_INDIVIDUAL => false, - &TESTDEF_EXPECT => false, - &TESTDEF_TEST_TOTAL => 1, - - &TESTDEF_TEST_COVERAGE => - { - &TESTDEF_TEST_ALL => - { - &TESTDEF_MODULE_INFO => TESTDEF_COVERAGE_FULL, - } - }, + &TESTDEF_NAME => 'expire', + &TESTDEF_TOTAL => 2, }, ] }, # Full tests { - &TESTDEF_MODULE_NAME => 'full', - &TESTDEF_TEST_CONTAINER => false, + &TESTDEF_NAME => 'full', &TESTDEF_EXPECT => true, + &TESTDEF_INDIVIDUAL => true, + &TESTDEF_PROCESS => true, &TESTDEF_TEST => [ { - &TESTDEF_TEST_NAME => 'synthetic', - &TESTDEF_TEST_TOTAL => 8, - &TESTDEF_TEST_PROCESS => true + &TESTDEF_NAME => 'synthetic', + &TESTDEF_TOTAL => 8, }, { - &TESTDEF_TEST_NAME => 'real', - &TESTDEF_TEST_TOTAL => 11, - &TESTDEF_TEST_PROCESS => true, - &TESTDEF_TEST_DB => true + &TESTDEF_NAME => 'real', + &TESTDEF_TOTAL => 11, + &TESTDEF_DB => true, } ] }, @@ -385,55 +359,154 @@ my $oTestDef = }; #################################################################################################################################### -# testDefGet +# Process normalized data into a more queryable form #################################################################################################################################### -sub testDefGet +my $hTestDefHash; # An easier way to query hash version of the above +my @stryModule; # Ordered list of modules +my $hModuleTest; # Ordered list of tests for each module +my $hCoverageType; # Coverage type for each code module (full/partial) +my $hCoverageList; # Tests required for full code module coverage (if type full) + +# Iterate each module +foreach my $hModule (@{$oTestDef->{&TESTDEF_MODULE}}) { - return $oTestDef; -} + # Push the module onto the ordered list + my $strModule = $hModule->{&TESTDEF_NAME}; + push(@stryModule, $strModule); -push @EXPORT, qw(testDefGet); - -#################################################################################################################################### -# testDefModuleGet -#################################################################################################################################### -sub testDefModuleGet -{ - my $strModule = shift; - - # Find the module - foreach my $hModule (@{$oTestDef->{&TESTDEF_MODULE}}) - { - if ($hModule->{&TESTDEF_MODULE_NAME} eq $strModule) - { - return $hModule; - } - } - - confess &log(ASSERT, "unable to find module ${strModule}"); -} - -push @EXPORT, qw(testDefModuleGet); - -#################################################################################################################################### -# testDefModuleTestGet -#################################################################################################################################### -sub testDefModuleTestGet -{ - my $hModule = shift; - my $strModuleTest = shift; + # Iterate each test + my @stryModuleTest; foreach my $hModuleTest (@{$hModule->{&TESTDEF_TEST}}) { - if ($hModuleTest->{&TESTDEF_TEST_NAME} eq $strModuleTest) + # Push the test on the order list + my $strTest = $hModuleTest->{&TESTDEF_NAME}; + push(@stryModuleTest, $strTest); + + # Resolve variables that can be set in the module or the test + foreach my $strVar ( + TESTDEF_CONTAINER, TESTDEF_EXPECT, TESTDEF_PROCESS, TESTDEF_DB, TESTDEF_INDIVIDUAL) { - return $hModuleTest; + $hTestDefHash->{$strModule}{$strTest}{$strVar} = coalesce( + $hModuleTest->{$strVar}, coalesce($hModule->{$strVar}, false)); + } + + # Set test count + $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_TOTAL} = $hModuleTest->{&TESTDEF_TOTAL}; + + # Concatenate coverage for modules and tests + foreach my $hCoverage ($hModule->{&TESTDEF_COVERAGE}, $hModuleTest->{&TESTDEF_COVERAGE}) + { + foreach my $strCodeModule (sort(keys(%{$hCoverage}))) + { + if (defined($hTestDefHash->{$strModule}{$strTest}{&TESTDEF_COVERAGE}{$strCodeModule})) + { + confess &log(ASSERT, + "${strCodeModule} is defined for coverage in both module ${strModule} and test ${strTest}"); + } + + $hTestDefHash->{$strModule}{$strTest}{&TESTDEF_COVERAGE}{$strCodeModule} = $hCoverage->{$strCodeModule}; + + # Build coverage type hash and make sure coverage type does not change + if (!defined($hCoverageType->{$strCodeModule})) + { + $hCoverageType->{$strCodeModule} = $hCoverage->{$strCodeModule}; + } + elsif ($hCoverageType->{$strCodeModule} != $hCoverage->{$strCodeModule}) + { + confess &log(ASSERT, "cannot mix full/partial coverage for ${strCodeModule}"); + } + + # Add to coverage list + push(@{$hCoverageList->{$strCodeModule}}, {strModule=> $strModule, strTest => $strTest}); + } } } - confess &log(ASSERT, "unable to find module test ${strModuleTest}"); + $hModuleTest->{$strModule} = \@stryModuleTest; } -push @EXPORT, qw(testDefModuleTestGet); +#################################################################################################################################### +# testDefModuleList +#################################################################################################################################### +sub testDefModuleList +{ + return @stryModule; +} + +push @EXPORT, qw(testDefModuleList); + +#################################################################################################################################### +# testDefModule +#################################################################################################################################### +sub testDefModule +{ + my $strModule = shift; + + if (!defined($hTestDefHash->{$strModule})) + { + confess &log(ASSERT, "unable to find module ${strModule}"); + } + + return $hTestDefHash->{$strModule}; +} + +push @EXPORT, qw(testDefModule); + + +#################################################################################################################################### +# testDefModuleTestList +#################################################################################################################################### +sub testDefModuleTestList +{ + my $strModule = shift; + + if (!defined($hModuleTest->{$strModule})) + { + confess &log(ASSERT, "unable to find module ${strModule}"); + } + + return @{$hModuleTest->{$strModule}}; +} + +push @EXPORT, qw(testDefModuleTestList); + +#################################################################################################################################### +# testDefModuleTest +#################################################################################################################################### +sub testDefModuleTest +{ + my $strModule = shift; + my $strModuleTest = shift; + + if (!defined($hTestDefHash->{$strModule}{$strModuleTest})) + { + confess &log(ASSERT, "unable to find module ${strModule}, test ${strModuleTest}"); + } + + return $hTestDefHash->{$strModule}{$strModuleTest}; +} + +push @EXPORT, qw(testDefModuleTest); + +#################################################################################################################################### +# testDefCoverageType +#################################################################################################################################### +sub testDefCoverageType +{ + return $hCoverageType; +} + +push @EXPORT, qw(testDefCoverageType); + +#################################################################################################################################### +# testDefCoverageList +#################################################################################################################################### +sub testDefCoverageList +{ + return $hCoverageList; +} + +push @EXPORT, qw(testDefCoverageList); 1; diff --git a/test/lib/pgBackRestTest/Common/JobTest.pm b/test/lib/pgBackRestTest/Common/JobTest.pm index 502e9d885..d47306ff3 100644 --- a/test/lib/pgBackRestTest/Common/JobTest.pm +++ b/test/lib/pgBackRestTest/Common/JobTest.pm @@ -28,6 +28,7 @@ use pgBackRestTest::Common::ContainerTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::ListTest; use pgBackRestTest::Common::RunTest; +use pgBackRestTest::Common::VmTest; #################################################################################################################################### # new @@ -56,7 +57,6 @@ sub new $self->{strLogLevel}, $self->{bLogForce}, $self->{bShowOutputAsync}, - $self->{bCoverage}, $self->{bNoCleanup}, $self->{iRetry}, ) = @@ -76,7 +76,6 @@ sub new {name => 'strLogLevel'}, {name => 'bLogForce'}, {name => 'bShowOutputAsync'}, - {name => 'bCoverage'}, {name => 'bNoCleanup'}, {name => 'iRetry'}, ); @@ -166,7 +165,7 @@ sub run # Create command my $strCommand = ($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') . - ($self->{bCoverage} ? testRunExe( + (vmCoverage($self->{oTest}->{&TEST_VM}) ? testRunExe( abs_path($0), dirname($self->{strCoveragePath}), $self->{strBackRestBase}, $self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME}, defined($self->{oTest}->{&TEST_RUN}) ? $self->{oTest}->{&TEST_RUN} : 'all') : abs_path($0)) . @@ -180,7 +179,6 @@ sub run (defined($self->{oTest}->{&TEST_PROCESS}) ? ' --process-max=' . $self->{oTest}->{&TEST_PROCESS} : '') . ($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') . ' --pgsql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} . - ($self->{bCoverage} ? ' --coverage' : '') . ($self->{bLogForce} ? ' --log-force' : '') . ($self->{bDryRun} ? ' --dry-run' : '') . ($self->{bDryRun} ? ' --vm-out' : '') . diff --git a/test/lib/pgBackRestTest/Common/ListTest.pm b/test/lib/pgBackRestTest/Common/ListTest.pm index 4ef875fdc..6125b2dbe 100644 --- a/test/lib/pgBackRestTest/Common/ListTest.pm +++ b/test/lib/pgBackRestTest/Common/ListTest.pm @@ -51,9 +51,8 @@ sub testListGet my $iyModuleTestRun = shift; my $strDbVersion = shift; my $iProcessMax = shift; - my $bCoverage = shift; + my $bCoverageOnly = shift; - my $oTestDef = testDefGet(); my $oyVm = vmGet(); my $oyTestRun = []; @@ -71,13 +70,17 @@ sub testListGet foreach my $strTestOS (@stryTestOS) { - foreach my $oModule (@{$$oTestDef{&TESTDEF_MODULE}}) + foreach my $strModule (testDefModuleList()) { - if (@{$stryModule} == 0 || grep(/^$$oModule{&TESTDEF_MODULE_NAME}$/i, @{$stryModule})) + my $hModule = testDefModule($strModule); + + if (@{$stryModule} == 0 || grep(/^$strModule$/i, @{$stryModule})) { - foreach my $oTest (@{$$oModule{test}}) + foreach my $strModuleTest (testDefModuleTestList($strModule)) { - if (@{$stryModuleTest} == 0 || grep(/^$$oTest{&TESTDEF_TEST_NAME}$/i, @{$stryModuleTest})) + my $hTest = testDefModuleTest($strModule, $strModuleTest); + + if (@{$stryModuleTest} == 0 || grep(/^$strModuleTest$/i, @{$stryModuleTest})) { my $iDbVersionMin = -1; my $iDbVersionMax = -1; @@ -91,7 +94,7 @@ sub testListGet $strDbVersionKey = &VM_DB_MINIMAL; } - if (defined($$oTest{&TESTDEF_TEST_DB}) && $$oTest{&TESTDEF_TEST_DB}) + if (defined($hTest->{&TESTDEF_DB}) && $hTest->{&TESTDEF_DB}) { $iDbVersionMin = 0; $iDbVersionMax = @{$$oyVm{$strTestOS}{$strDbVersionKey}} - 1; @@ -107,10 +110,10 @@ sub testListGet { # Individual tests will be each be run in a separate container. This is the default. my $bTestIndividual = - !defined($$oTest{&TESTDEF_TEST_INDIVIDUAL}) || $$oTest{&TESTDEF_TEST_INDIVIDUAL} ? true : false; + !defined($hTest->{&TESTDEF_INDIVIDUAL}) || $hTest->{&TESTDEF_INDIVIDUAL} ? true : false; my $iTestRunMin = $bTestIndividual ? 1 : -1; - my $iTestRunMax = $bTestIndividual ? $$oTest{&TESTDEF_TEST_TOTAL} : -1; + my $iTestRunMax = $bTestIndividual ? $hTest->{&TESTDEF_TOTAL} : -1; for (my $iTestRunIdx = $iTestRunMin; $iTestRunIdx <= $iTestRunMax; $iTestRunIdx++) { @@ -119,16 +122,12 @@ sub testListGet $bTestIndividual && @{$iyModuleTestRun} != 0 && !grep(/^$iTestRunIdx$/i, @{$iyModuleTestRun})); - # Skip this run if coverage is requested and this test does not provide coverage - next if ( - $bCoverage && - (($bTestIndividual && !defined($oTest->{&TESTDEF_TEST_COVERAGE}{$iTestRunIdx})) || - (!$bTestIndividual && !defined($oTest->{&TESTDEF_TEST_COVERAGE}{&TESTDEF_TEST_ALL}))) && - !defined($oModule->{&TESTDEF_TEST_COVERAGE})); + # Skip this run if only coverage tests are requested and this test does not provide coverage + next if ($bCoverageOnly && !defined($hTest->{&TESTDEF_COVERAGE})); my $iyProcessMax = [defined($iProcessMax) ? $iProcessMax : 1]; - if (defined($$oTest{&TESTDEF_TEST_PROCESS}) && $$oTest{&TESTDEF_TEST_PROCESS} && + if (defined($hTest->{&TESTDEF_PROCESS}) && $hTest->{&TESTDEF_PROCESS} && !defined($iProcessMax) && $bFirstDbVersion) { $iyProcessMax = [1, 4]; @@ -153,12 +152,12 @@ sub testListGet my $oTestRun = { &TEST_VM => $strTestOS, - &TEST_CONTAINER => defined($oTest->{&TESTDEF_TEST_CONTAINER}) ? - $oTest->{&TESTDEF_TEST_CONTAINER} : $oModule->{&TESTDEF_TEST_CONTAINER}, + &TEST_CONTAINER => defined($hTest->{&TESTDEF_CONTAINER}) ? + $hTest->{&TESTDEF_CONTAINER} : $hModule->{&TESTDEF_CONTAINER}, &TEST_PGSQL_BIN => $strPgSqlBin, &TEST_PERL_ARCH_PATH => $$oyVm{$strTestOS}{&VMDEF_PERL_ARCH_PATH}, - &TEST_MODULE => $$oModule{&TESTDEF_MODULE_NAME}, - &TEST_NAME => $$oTest{&TESTDEF_TEST_NAME}, + &TEST_MODULE => $strModule, + &TEST_NAME => $strModuleTest, &TEST_RUN => $iTestRunIdx == -1 ? (@{$iyModuleTestRun} == 0 ? undef : $iyModuleTestRun) : [$iTestRunIdx], diff --git a/test/lib/pgBackRestTest/Common/RunTest.pm b/test/lib/pgBackRestTest/Common/RunTest.pm index 98b5a1dec..d600f4978 100644 --- a/test/lib/pgBackRestTest/Common/RunTest.pm +++ b/test/lib/pgBackRestTest/Common/RunTest.pm @@ -22,6 +22,7 @@ use pgBackRest::Common::Wait; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::LogTest; +use pgBackRestTest::Common::VmTest; #################################################################################################################################### # Constant to use when bogus data is required @@ -118,7 +119,6 @@ sub process $self->{bDryRun}, $self->{bCleanup}, $self->{bLogForce}, - $self->{bCoverage}, $self->{strPgUser}, $self->{strBackRestUser}, $self->{strGroup}, @@ -141,7 +141,6 @@ sub process {name => 'bDryRun'}, {name => 'bCleanup'}, {name => 'bLogForce'}, - {name => 'bCoverage'}, {name => 'strPgUser'}, {name => 'strBackRestUser'}, {name => 'strGroup'}, @@ -157,12 +156,11 @@ sub process $self->cleanModule(); # Make sure the correct number of tests ran - my $hModule = testDefModuleGet($self->{strModule}); - my $hModuleTest = testDefModuleTestGet($hModule, $self->{strModuleTest}); + my $hModuleTest = testDefModuleTest($self->{strModule}, $self->{strModuleTest}); - if ($hModuleTest->{&TESTDEF_TEST_TOTAL} != $self->runCurrent()) + if ($hModuleTest->{&TESTDEF_TOTAL} != $self->runCurrent()) { - confess &log(ASSERT, "expected $hModuleTest->{&TESTDEF_TEST_TOTAL} tests to run but $self->{iRun} ran"); + confess &log(ASSERT, "expected $hModuleTest->{&TESTDEF_TOTAL} tests to run but $self->{iRun} ran"); } # Return from function and log return values if any @@ -205,14 +203,7 @@ sub begin # Else get the default expect setting else { - my $hModule = testDefModuleGet($self->{strModule}); - my $hModuleTest = testDefModuleTestGet($hModule, $self->{strModuleTest}); - $self->{bExpect} = - defined($hModuleTest->{&TESTDEF_EXPECT}) ? - ($hModuleTest->{&TESTDEF_EXPECT} ? true : false) : - (defined($hModule->{&TESTDEF_EXPECT}) ? - ($hModule->{&TESTDEF_EXPECT} ? true : false) : - false); + $self->{bExpect} = (testDefModuleTest($self->{strModule}, $self->{strModuleTest}))->{&TESTDEF_EXPECT}; } # Increment the run counter; @@ -494,33 +485,9 @@ sub testRunExe my $bLog = shift; # Limit Perl modules tested to what is defined in the test coverage (if it exists) + my $hTestCoverage = (testDefModuleTest($strModule, $strTest))->{&TESTDEF_COVERAGE}; my $strPerlModule; my $strPerlModuleLog; - my $hTestCoverage; - my $hTestDef = testDefGet(); - - foreach my $hTestModule (@{$hTestDef->{&TESTDEF_MODULE}}) - { - if ($hTestModule->{&TESTDEF_MODULE_NAME} eq $strModule) - { - $hTestCoverage = $hTestModule->{&TESTDEF_TEST_COVERAGE}; - - foreach my $hTest (@{$hTestModule->{&TESTDEF_TEST}}) - { - if (defined($strTest) && $hTest->{&TESTDEF_TEST_NAME} eq $strTest) - { - $hTestCoverage = - defined($hTest->{&TESTDEF_TEST_COVERAGE}{$iRun}) ? $hTest->{&TESTDEF_TEST_COVERAGE}{$iRun} : - $hTest->{&TESTDEF_TEST_COVERAGE}{&TESTDEF_TEST_ALL}; - - if (!defined($hTestCoverage)) - { - $hTestCoverage = $hTestModule->{&TESTDEF_TEST_COVERAGE}; - } - } - } - } - } if (defined($hTestCoverage)) { @@ -535,9 +502,8 @@ sub testRunExe if (defined($strPerlModule)) { $strExe = - 'perl -MDevel::Cover=-silent,1,-dir,' . $strCoveragePath . ',-subs_only,1' . - ",-select${strPerlModule},+inc," . $strBackRestBasePath . - ',-coverage,statement,branch,condition,path,subroutine' . " ${strExe}"; + "perl -MDevel::Cover=-silent,1,-dir,${strCoveragePath},-select${strPerlModule},+inc,${strBackRestBasePath}" . + ",-coverage,statement,branch,condition,path,subroutine ${strExe}"; if (defined($bLog) && $bLog) { @@ -559,7 +525,7 @@ sub backrestExe {return shift->{strBackRestExe}} sub backrestExeOriginal {return shift->{strBackRestExeOriginal}} sub backrestUser {return shift->{strBackRestUser}} sub basePath {return shift->{strBasePath}} -sub coverage {return shift->{bCoverage}} +sub coverage {vmBaseTest(shift->{strVm}, VM_OS_BASE_DEBIAN)} sub dataPath {return shift->basePath() . '/test/data'} sub doCleanup {return shift->{bCleanup}} sub doExpect {return shift->{bExpect}} diff --git a/test/lib/pgBackRestTest/Common/VmTest.pm b/test/lib/pgBackRestTest/Common/VmTest.pm index de160a164..625a97c83 100644 --- a/test/lib/pgBackRestTest/Common/VmTest.pm +++ b/test/lib/pgBackRestTest/Common/VmTest.pm @@ -275,4 +275,29 @@ sub vmGet push @EXPORT, qw(vmGet); +#################################################################################################################################### +# vmBaseTest +#################################################################################################################################### +sub vmBaseTest +{ + my $strVm = shift; + my $strDistroTest = shift; + + return $oyVm->{$strVm}{&VM_OS_BASE} eq $strDistroTest ? true : false; +} + +push @EXPORT, qw(vmBaseTest); + +#################################################################################################################################### +# vmCoverage +#################################################################################################################################### +sub vmCoverage +{ + my $strVm = shift; + + return $strVm eq VM_ALL ? false : vmBaseTest($strVm, VM_OS_BASE_DEBIAN); +} + +push @EXPORT, qw(vmCoverage); + 1; diff --git a/test/test.pl b/test/test.pl index 5745ac4bb..4027fb29a 100755 --- a/test/test.pl +++ b/test/test.pl @@ -17,6 +17,7 @@ $SIG{__DIE__} = sub { Carp::confess @_ }; use File::Basename qw(dirname); use Getopt::Long qw(GetOptions); use Cwd qw(abs_path cwd); +use JSON::PP; use Pod::Usage qw(pod2usage); use POSIX qw(ceil strftime); use Time::HiRes qw(gettimeofday); @@ -39,6 +40,7 @@ use BackRestDoc::Custom::DocCustomRelease; use pgBackRestTest::Common::ContainerTest; use pgBackRestTest::Common::CiTest; +use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::HostGroupTest; use pgBackRestTest::Common::JobTest; @@ -69,11 +71,11 @@ test.pl [options] --log-force force overwrite of current test log files --no-lint disable static source code analysis --build-only compile the C library / packages and run tests only - --coverage perform coverage analysis + --coverage-only only run coverage tests (as a subset of selected tests) --smart perform libc/package builds only when source timestamps have changed --no-package do not build packages --no-ci-config don't overwrite the current continuous integration config - --dev --no-lint --smart --no-package --vm-out --process-max=1 --retry=0 --ci-no-config + --dev --no-lint --smart --no-package --vm-out --process-max=1 Configuration Options: --psql-bin path to the psql executables (e.g. /usr/lib/postgresql/9.3/bin/) @@ -113,13 +115,13 @@ my $bHelp = false; my $bQuiet = false; my $strDbVersion = 'minimal'; my $bLogForce = false; -my $strVm = VM_ALL; +my $strVm; my $strVmHost = VM_HOST_DEFAULT; my $bVmBuild = false; my $bVmForce = false; my $bNoLint = false; my $bBuildOnly = false; -my $bCoverage = false; +my $bCoverageOnly = false; my $bSmart = false; my $bNoPackage = false; my $bNoCiConfig = false; @@ -151,7 +153,7 @@ GetOptions ('q|quiet' => \$bQuiet, 'build-only' => \$bBuildOnly, 'no-package' => \$bNoPackage, 'no-ci-config' => \$bNoCiConfig, - 'coverage' => \$bCoverage, + 'coverage-only' => \$bCoverageOnly, 'smart' => \$bSmart, 'dev' => \$bDev, 'retry=s' => \$iRetry) @@ -190,7 +192,6 @@ eval $bNoLint = true; $bSmart = true; $bNoPackage = true; - $bNoCiConfig = true; $bVmOut = true; $iProcessMax = 1; } @@ -231,18 +232,27 @@ eval $strTestPath = cwd() . '/test'; } - # Coverage can only be run with u16 containers due to version compatibility issues - if ($bCoverage) + if ($bCoverageOnly) { - if ($strVm eq VM_ALL) + if (!defined($strVm)) { &log(INFO, "Set --vm=${strVmHost} for coverage testing"); $strVm = $strVmHost; } - elsif ($strVm ne $strVmHost) + elsif ($strVm eq VM_ALL) { - confess &log(ERROR, "only --vm=${strVmHost} can be used for coverage testing"); + confess &log(ERROR, "select a single Debian-based VM for coverage testing"); } + elsif (!vmCoverage($strVm)) + { + confess &log(ERROR, "only Debian-based VMs can be used for coverage testing"); + } + } + + # If VM is not defined then set it to all + if (!defined($strVm)) + { + $strVm = VM_ALL; } # Get the base backrest path @@ -572,7 +582,7 @@ eval # Determine which tests to run #----------------------------------------------------------------------------------------------------------------------- my $oyTestRun = testListGet( - $strVm, \@stryModule, \@stryModuleTest, \@iyModuleTestRun, $strDbVersion, $iProcessMax, $bCoverage); + $strVm, \@stryModule, \@stryModuleTest, \@iyModuleTestRun, $strDbVersion, $iProcessMax, $bCoverageOnly); if (@{$oyTestRun} == 0) { @@ -650,8 +660,7 @@ eval { my $oJob = new pgBackRestTest::Common::JobTest( $strBackRestBase, $strTestPath, $strCoveragePath, $$oyTestRun[$iTestIdx], $bDryRun, $bVmOut, $iVmIdx, - $iVmMax, $iTestIdx, $iTestMax, $strLogLevel, $bLogForce, $bShowOutputAsync, $bCoverage, $bNoCleanup, - $iRetry); + $iVmMax, $iTestIdx, $iTestMax, $strLogLevel, $bLogForce, $bShowOutputAsync, $bNoCleanup, $iRetry); $iTestIdx++; if ($oJob->run()) @@ -665,18 +674,133 @@ eval } while ($iVmTotal > 0); - # Write out coverage info + # Write out coverage info and test coverage #----------------------------------------------------------------------------------------------------------------------- - if ($bCoverage) + my $iUncoveredCodeModuleTotal = 0; + + if (vmCoverage($strVm) && !$bDryRun) { - &log(INFO, 'Writing coverage report'); + &log(INFO, 'writing coverage report'); executeTest("rm -rf ${strBackRestBase}/test/coverage"); executeTest("cp -rp ${strCoveragePath} ${strCoveragePath}_temp"); - executeTest("cover -report json -outputdir ${strBackRestBase}/test/coverage ${strCoveragePath}_temp"); - executeTest("rm -rf ${strCoveragePath}_temp"); - executeTest("cp -rp ${strCoveragePath} ${strCoveragePath}_temp"); - executeTest("cover -outputdir ${strBackRestBase}/test/coverage ${strCoveragePath}_temp"); - executeTest("rm -rf ${strCoveragePath}_temp"); + executeTest( + LIB_COVER_EXE . " -report json -outputdir ${strBackRestBase}/test/coverage ${strCoveragePath}_temp", + {bSuppressStdErr => true}); + executeTest("sudo rm -rf ${strCoveragePath}_temp"); + executeTest("sudo cp -rp ${strCoveragePath} ${strCoveragePath}_temp"); + executeTest( + LIB_COVER_EXE . " -outputdir ${strBackRestBase}/test/coverage ${strCoveragePath}_temp", + {bSuppressStdErr => true}); + executeTest("sudo rm -rf ${strCoveragePath}_temp"); + + # Determine which modules were covered (only check coverage if all tests were successful) + #----------------------------------------------------------------------------------------------------------------------- + if ($iTestFail == 0) + { + my $hModuleTest; # Everything that was run + + # Build a hash of all modules, tests, and runs that were executed + foreach my $hTestRun (@{$oyTestRun}) + { + # Get coverage for the module + my $strModule = $hTestRun->{&TEST_MODULE}; + my $hModule = testDefModule($strModule); + + # Get coverage for the test + my $strTest = $hTestRun->{&TEST_NAME}; + my $hTest = testDefModuleTest($strModule, $strTest); + + # If no tests are listed it means all of them were run + if (@{$hTestRun->{&TEST_RUN}} == 0) + { + $hModuleTest->{$strModule}{$strTest} = true; + } + } + + # Load the results of coverage testing from JSON + my $oJSON = JSON::PP->new()->allow_nonref(); + my $hCoverageResult = $oJSON->decode(fileStringRead("${strBackRestBase}/test/coverage/cover.json")); + + # Now compare against code modules that should have full coverage + my $hCoverageList = testDefCoverageList(); + my $hCoverageType = testDefCoverageType(); + my $hCoverageActual; + + foreach my $strCodeModule (sort(keys(%{$hCoverageList}))) + { + if (@{$hCoverageList->{$strCodeModule}} > 0) + { + my $iCoverageTotal = 0; + + foreach my $hTest (@{$hCoverageList->{$strCodeModule}}) + { + if (!defined($hModuleTest->{$hTest->{strModule}}{$hTest->{strTest}})) + { + next; + } + + $iCoverageTotal++; + } + + if (@{$hCoverageList->{$strCodeModule}} == $iCoverageTotal) + { + $hCoverageActual->{$strCodeModule} = $hCoverageType->{$strCodeModule}; + } + } + } + + if (keys(%{$hCoverageActual}) > 0) + { + &log(INFO, 'test coverage for: ' . join(', ', sort(keys(%{$hCoverageActual})))); + } + else + { + &log(INFO, 'no code modules had all tests run required for coverage'); + } + + foreach my $strCodeModule (sort(keys(%{$hCoverageActual}))) + { + # Get summary results (??? Need to fix this for coverage testing on bin/pgbackrest since .pm is required) + my $hCoverageResultAll = + $hCoverageResult->{'summary'} + {"${strBackRestBase}/lib/" . BACKREST_NAME . "/${strCodeModule}.pm"}{total}; + + if (!defined($hCoverageResultAll)) + { + confess &log(ERROR, "unable to find coverage results for ${strCodeModule}"); + } + + # Check that all code has been covered + if ($hCoverageActual->{$strCodeModule} == TESTDEF_COVERAGE_FULL) + { + my $iUncoveredLines = + $hCoverageResultAll->{total} - $hCoverageResultAll->{covered} - $hCoverageResultAll->{uncoverable}; + + if ($iUncoveredLines != 0) + { + &log(ERROR, "code module ${strCodeModule} is not fully covered"); + $iUncoveredCodeModuleTotal++; + } + } + # Else test how much partial coverage where was + else + { + my $iCoveragePercent = int( + ($hCoverageResultAll->{covered} + $hCoverageResultAll->{uncoverable}) * 100 / + $hCoverageResultAll->{total}); + + if ($iCoveragePercent == 100) + { + &log(ERROR, "code module ${strCodeModule} has 100% coverage but is not marked fully covered"); + $iUncoveredCodeModuleTotal++; + } + else + { + &log(INFO, "code module ${strCodeModule} has (expected) partial coverage of ${iCoveragePercent}%"); + } + } + } + } } # Print test info and exit @@ -687,12 +811,16 @@ eval } else { - &log(INFO, 'TESTS COMPLETED ' . ($iTestFail == 0 ? 'SUCCESSFULLY' : "WITH ${iTestFail} FAILURE(S)") . - ($iTestRetry == 0 ? '' : ", ${iTestRetry} RETRY(IES)") . - ' (' . (time() - $lStartTime) . 's)'); + &log(INFO, + 'TESTS COMPLETED ' . ($iTestFail == 0 ? 'SUCCESSFULLY' . + ($iUncoveredCodeModuleTotal == 0 ? '' : " WITH ${iUncoveredCodeModuleTotal} MODULE(S) MISSING COVERAGE") : + "WITH ${iTestFail} FAILURE(S)") . ($iTestRetry == 0 ? '' : ", ${iTestRetry} RETRY(IES)") . + ' (' . (time() - $lStartTime) . 's)'); + + exit 1 if ($iTestFail > 0 || $iUncoveredCodeModuleTotal > 0); } - exit $iTestFail == 0 ? 0 : 1; + exit 0; } ################################################################################################################################ @@ -713,7 +841,6 @@ eval $strDbVersion ne 'minimal' ? $strDbVersion: undef, # Db version $stryModule[0], $stryModuleTest[0], \@iyModuleTestRun, # Module info $iProcessMax, $bVmOut, $bDryRun, $bNoCleanup, $bLogForce, # Test options - $bCoverage, # Test options TEST_USER, BACKREST_USER, TEST_GROUP); # User/group info if (!$bNoCleanup)