#!/usr/bin/perl #################################################################################################################################### # test.pl - pgBackRest Unit Tests #################################################################################################################################### #################################################################################################################################### # Perl includes #################################################################################################################################### use strict; use warnings FATAL => qw(all); use Carp qw(confess longmess); use English '-no_match_vars'; # Convert die to confess to capture the stack trace $SIG{__DIE__} = sub { Carp::confess @_ }; use Digest::SHA qw(sha1_hex); use File::Basename qw(dirname); use File::stat; use Getopt::Long qw(GetOptions); use Cwd qw(abs_path cwd); use JSON::PP; use Pod::Usage qw(pod2usage); use POSIX qw(ceil strftime); use Time::HiRes qw(gettimeofday); use lib dirname($0) . '/lib'; use lib dirname(dirname($0)) . '/lib'; use lib dirname(dirname($0)) . '/build/lib'; use lib dirname(dirname($0)) . '/doc/lib'; use pgBackRestDoc::Common::Exception; use pgBackRestDoc::Common::Log; use pgBackRestDoc::Common::String; use pgBackRestDoc::ProjectInfo; use pgBackRestTest::Common::BuildTest; use pgBackRestTest::Common::CodeCountTest; use pgBackRestTest::Common::ContainerTest; use pgBackRestTest::Common::CoverageTest; use pgBackRestTest::Common::DefineTest; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::HostGroupTest; use pgBackRestTest::Common::JobTest; use pgBackRestTest::Common::ListTest; use pgBackRestTest::Common::RunTest; use pgBackRestTest::Common::Storage; use pgBackRestTest::Common::StoragePosix; use pgBackRestTest::Common::VmTest; use pgBackRestTest::Common::Wait; #################################################################################################################################### # Usage #################################################################################################################################### =head1 NAME test.pl - pgBackRest Unit Tests =head1 SYNOPSIS test.pl [options] Test Options: --module test module to execute --test execute the specified test in a module --run execute only the specified test run --dry-run show only the tests that would be executed but don't execute them --no-cleanup don't cleanup after the last test is complete - useful for debugging --clean clean working and result paths for a completely fresh build --clean-only execute --clean and exit --pg-version version of postgres to test (all, defaults to minimal) --build-only build the binary (and honor --build-package) but don't run tests --build-package build the package --build-max max processes to use for builds (default 4) --c-only only run C tests --container-only only run tests that must be run in a container --no-performance do not run performance tests --gen-only only run auto-generation --min-gen only run required code generation --gen-check check that auto-generated files are correct (used in CI to detect changes) --code-count generate code counts --no-valgrind don't run valgrind on C unit tests (saves time) --no-coverage don't run coverage on C unit tests (saves time) --no-coverage-report run coverage but don't generate coverage report (saves time) --no-optimize don't do compile optimization for C (saves compile time) --backtrace enable backtrace when available (adds stack trace line numbers -- very slow) --profile generate profile info --no-debug don't generate a debug build --scale scale performance tests --tz test with the specified timezone --debug-test-trace test stack trace for low-level functions (slow, esp w/valgrind, may cause timeouts) Report Options: --coverage-summary generate a coverage summary report for the documentation --coverage-only only run coverage tests (as a subset of selected tests) for the documentation Configuration Options: --psql-bin path to the psql executables (e.g. /usr/lib/postgresql/9.3/bin/) --test-path path where tests are executed (defaults to ./test) --log-level log level to use for test harness (and Perl tests) (defaults to INFO) --log-level-test log level to use for C tests (defaults to OFF) --log-level-test-file log level to use for file logging in integration tests (defaults to TRACE) --no-log-timestamp suppress timestamps, timings, etc. Used to generate documentation. --make-cmd gnu-compatible make command (defaults to make) --quiet, -q equivalent to --log-level=off VM Options: --vm docker container to build/test (e.g. rh7) --vm-build build Docker containers --vm-force force a rebuild of Docker containers --vm-out Show VM output (default false) --vm-max max VMs to run in parallel (default 1) General Options: --version display version and exit --help display usage and exit =cut #################################################################################################################################### # Command line parameters #################################################################################################################################### my $bClean; my $bCleanOnly; my $strLogLevel = lc(INFO); my $strLogLevelTest = lc(OFF); my $strLogLevelTestFile = lc(DEBUG); my $bNoLogTimestamp = false; my $bVmOut = false; my @stryModule; my @stryModuleTest; my @iyModuleTestRun; my $iVmMax = 1; my $iVmId = undef; my $bDryRun = false; my $bNoCleanup = false; my $strPgSqlBin; my $strTestPath; my $strMakeCmd = 'make'; my $bVersion = false; my $bHelp = false; my $bQuiet = false; my $strPgVersion = 'minimal'; my $strVm; my $bVmBuild = false; my $bVmForce = false; my $bBuildOnly = false; my $bBuildPackage = false; my $iBuildMax = 4; my $bCoverageOnly = false; my $bCoverageSummary = false; my $bNoCoverage = false; my $bNoCoverageReport = false; my $bCOnly = false; my $bContainerOnly = false; my $bNoPerformance = false; my $bGenOnly = false; my $bGenCheck = false; my $bMinGen = false; my $bCodeCount = false; my $bBackTrace = false; my $bProfile = false; my $bNoValgrind = false; my $bNoOptimize = false; my $bNoDebug = false; my $iScale = 1; my $bDebugTestTrace = false; my $iRetry = 0; my $strTimeZone = undef; my @cmdOptions = @ARGV; GetOptions ('q|quiet' => \$bQuiet, 'version' => \$bVersion, 'help' => \$bHelp, 'clean' => \$bClean, 'clean-only' => \$bCleanOnly, 'pgsql-bin=s' => \$strPgSqlBin, 'test-path=s' => \$strTestPath, 'make-cmd=s' => \$strMakeCmd, 'log-level=s' => \$strLogLevel, 'log-level-test=s' => \$strLogLevelTest, 'log-level-test-file=s' => \$strLogLevelTestFile, 'no-log-timestamp' => \$bNoLogTimestamp, 'vm=s' => \$strVm, 'vm-out' => \$bVmOut, 'vm-build' => \$bVmBuild, 'vm-force' => \$bVmForce, 'module=s@' => \@stryModule, 'test=s@' => \@stryModuleTest, 'run=s@' => \@iyModuleTestRun, 'vm-id=s' => \$iVmId, 'vm-max=s' => \$iVmMax, 'dry-run' => \$bDryRun, 'no-cleanup' => \$bNoCleanup, 'pg-version=s' => \$strPgVersion, 'build-only' => \$bBuildOnly, 'build-package' => \$bBuildPackage, 'build-max=s' => \$iBuildMax, 'coverage-only' => \$bCoverageOnly, 'coverage-summary' => \$bCoverageSummary, 'no-coverage' => \$bNoCoverage, 'no-coverage-report' => \$bNoCoverageReport, 'c-only' => \$bCOnly, 'container-only' => \$bContainerOnly, 'no-performance' => \$bNoPerformance, 'gen-only' => \$bGenOnly, 'gen-check' => \$bGenCheck, 'min-gen' => \$bMinGen, 'code-count' => \$bCodeCount, 'backtrace' => \$bBackTrace, 'profile' => \$bProfile, 'no-valgrind' => \$bNoValgrind, 'no-optimize' => \$bNoOptimize, 'no-debug', => \$bNoDebug, 'scale=s' => \$iScale, 'tz=s', => \$strTimeZone, 'debug-test-trace', => \$bDebugTestTrace, 'retry=s' => \$iRetry) or pod2usage(2); #################################################################################################################################### # Run in eval block to catch errors #################################################################################################################################### eval { # Record the start time my $lStartTime = time(); # Display version and exit if requested if ($bVersion || $bHelp) { syswrite(*STDOUT, PROJECT_NAME . ' ' . PROJECT_VERSION . " Test Engine\n"); if ($bHelp) { syswrite(*STDOUT, "\n"); pod2usage(); } exit 0; } if (@ARGV > 0) { syswrite(*STDOUT, "invalid parameter\n\n"); pod2usage(); } ################################################################################################################################ # Disable code generation on dry-run ################################################################################################################################ if ($bDryRun) { $bMinGen = true; } ################################################################################################################################ # Update options for --coverage-summary ################################################################################################################################ if ($bCoverageSummary) { $bCoverageOnly = true; $bCOnly = true; } ################################################################################################################################ # Update options for --profile ################################################################################################################################ if ($bProfile) { $bNoValgrind = true; $bNoCoverage = true; } ################################################################################################################################ # Setup ################################################################################################################################ # Set a neutral umask so tests work as expected umask(0); # Set console log level if ($bQuiet) { $strLogLevel = 'off'; } logLevelSet(uc($strLogLevel), uc($strLogLevel), OFF, !$bNoLogTimestamp); &log(INFO, 'test begin on ' . hostArch() . " - log level ${strLogLevel}"); if (@stryModuleTest != 0 && @stryModule != 1) { confess "Only one --module can be provided when --test is specified"; } if (@iyModuleTestRun != 0 && @stryModuleTest != 1) { confess "Only one --test can be provided when --run is specified"; } # Set test path if not explicitly set if (!defined($strTestPath)) { $strTestPath = cwd() . '/test'; } my $oStorageTest = new pgBackRestTest::Common::Storage( $strTestPath, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); if ($bCoverageOnly) { if (!defined($strVm)) { confess &log(ERROR, "select a VM for coverage testing"); } elsif ($strVm eq VM_ALL) { confess &log(ERROR, "select a single VM for coverage testing"); } } # If VM is not defined then set it to all if (!defined($strVm)) { $strVm = VM_ALL; } # Else make sure vm is valid elsif ($strVm ne VM_ALL) { vmValid($strVm); } # Get the base backrest path my $strBackRestBase = dirname(dirname(abs_path($0))); my $strVagrantPath = "${strBackRestBase}/test/.vagrant"; my $oStorageBackRest = new pgBackRestTest::Common::Storage( $strBackRestBase, new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false})); # Check that the test path is not in the git repo path if (index("${strTestPath}/", "${strBackRestBase}/") != -1) { confess &log( ERROR, "test path '${strTestPath}' may not be in the repo path '${strBackRestBase}'\n" . "HINT: was test.pl run in '${strBackRestBase}'?\n" . "HINT: use --test-path to set a test path\n" . "HINT: run test.pl from outside the repo, e.g. 'pgbackrest/test/test.pl'"); } ################################################################################################################################ # Clean working and result paths ################################################################################################################################ if ($bClean || $bCleanOnly) { &log(INFO, "clean working (${strTestPath}) and result (${strBackRestBase}/test/result) paths"); if ($oStorageTest->pathExists($strTestPath)) { executeTest("find ${strTestPath} -mindepth 1 -print0 | xargs -0 rm -rf"); } if ($oStorageTest->pathExists("${strBackRestBase}/test/result")) { executeTest("find ${strBackRestBase}/test/result -mindepth 1 -print0 | xargs -0 rm -rf"); } # Exit when clean-only exit 0 if $bCleanOnly; } ################################################################################################################################ # Build Docker containers ################################################################################################################################ if ($bVmBuild) { containerBuild($oStorageBackRest, $strVm, $bVmForce); exit 0; } ################################################################################################################################ # Load test definition ################################################################################################################################ testDefLoad(${$oStorageBackRest->get("test/define.yaml")}); ################################################################################################################################ # Start VM and run ################################################################################################################################ if (!defined($iVmId)) { # Clean any existing files in the src path that might interfere with the vpath build. This is kosher because the user should # be expecting us to do builds in the src path during testing. Instead we clean the src path and do the builds elsewhere. #--------------------------------------------------------------------------------------------------------------------------- executeTest("make -C ${strBackRestBase}/src -f Makefile.in clean-all"); # Auto-generate configure files unless --min-gen specified #--------------------------------------------------------------------------------------------------------------------------- if (!$bMinGen) { &log(INFO, "autogenerate configure"); # Auto-generate version for configure.ac script #----------------------------------------------------------------------------------------------------------------------- my $strConfigureAcOld = ${$oStorageTest->get("${strBackRestBase}/src/build/configure.ac")}; my $strConfigureAcNew; foreach my $strLine (split("\n", $strConfigureAcOld)) { if ($strLine =~ /^AC_INIT\(/) { $strLine = 'AC_INIT([' . PROJECT_NAME . '], [' . PROJECT_VERSION . '])'; } $strConfigureAcNew .= "${strLine}\n"; } # Save into the src dir my @stryBuilt; my $strBuilt = 'src/build/configure.ac'; if (buildPutDiffers($oStorageBackRest, "${strBackRestBase}/${strBuilt}", $strConfigureAcNew)) { push(@stryBuilt, $strBuilt); } # Error when checking that files have already been generated but they change if ($bGenCheck && @stryBuilt) { confess &log( ERROR, "unexpected autogeneration of version in configure.ac script: " . join(', ', @stryBuilt) . ":\n" . trim(executeTest("git -C ${strBackRestBase} diff"))); } &log(INFO, " autogenerated version in configure.ac script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); # Auto-generate configure script #----------------------------------------------------------------------------------------------------------------------- # Set build file @stryBuilt = (); $strBuilt = 'src/configure'; # Get configure.ac and configure to see if anything has changed my $strConfigureAc = ${$oStorageBackRest->get('src/build/configure.ac')}; my $strConfigureAcHash = sha1_hex($strConfigureAc); my $rstrConfigure = $oStorageBackRest->get($oStorageBackRest->openRead($strBuilt, {bIgnoreMissing => true})); # Check if configure needs to be regenerated if (!defined($rstrConfigure) || !defined($$rstrConfigure) || $strConfigureAcHash ne substr($$rstrConfigure, length($$rstrConfigure) - 41, 40)) { # Generate aclocal.m4 my $strAcLocal = executeTest("cd ${strBackRestBase}/src/build && aclocal --OUT=-"); $strAcLocal = trim($strAcLocal) . "\n"; buildPutDiffers($oStorageBackRest, "${strBackRestBase}/src/build/aclocal.m4", $strAcLocal); # Generate configure my $strConfigure = executeTest("cd ${strBackRestBase}/src/build && autoconf --output=-"); $strConfigure = trim($strConfigure) . "\n\n# Generated from src/build/configure.ac sha1 ${strConfigureAcHash}\n"; # Remove cache created by autconf executeTest("rm -rf ${strBackRestBase}/src/build/autom4te.cache"); # Remove unused options from help my $strDirList = "sbin|libexec|sysconf|sharedstate|localstate|runstate|lib|include|oldinclude|dataroot|data|info" . "|locale|man|doc|html|dvi|pdf|ps"; $strConfigure =~ s/^ --(${strDirList})*dir=DIR.*\n//mg; # Save into the src dir $oStorageBackRest->put( $oStorageBackRest->openWrite("${strBackRestBase}/${strBuilt}", {strMode => '0755'}), $strConfigure); # Add to built list push(@stryBuilt, $strBuilt); } # Error when checking that files have already been generated but they change if ($bGenCheck && @stryBuilt) { confess &log( ERROR, "unexpected autogeneration of configure script: " . join(', ', @stryBuilt) . ":\n" . trim(executeTest("git -C ${strBackRestBase} diff"))); } &log(INFO, " autogenerated configure script: " . (@stryBuilt ? join(', ', @stryBuilt) : 'no changes')); } # Make a copy of the repo to track which files have been changed #--------------------------------------------------------------------------------------------------------------------------- my $strRepoCachePath = "${strTestPath}/repo"; # Create the repo path -- this should hopefully prevent obvious rsync errors below $oStorageTest->pathCreate($strRepoCachePath, {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); # Copy the repo executeTest( "git -C ${strBackRestBase} ls-files -c --others --exclude-standard |" . " rsync -rtW --delete --files-from=- --exclude=test/result" . # This option is not supported on MacOS. The eventual plan is to remove the need for it. (trim(`uname`) ne 'Darwin' ? ' --ignore-missing-args' : '') . " ${strBackRestBase}/ ${strRepoCachePath}"); # Auto-generate code files (if --min-gen specified then do minimum required) #--------------------------------------------------------------------------------------------------------------------------- my $strBuildPath = "${strTestPath}/build"; &log(INFO, (!-e $strBuildPath ? 'clean ' : '') . 'autogenerate code'); # Auto-generate version for root meson.build script my $strMesonBuildOld = ${$oStorageTest->get("${strBackRestBase}/meson.build")}; my $strMesonBuildNew; foreach my $strLine (split("\n", $strMesonBuildOld)) { if ($strLine =~ /^ version\: '/) { $strLine = " version: '" . PROJECT_VERSION . "',"; } $strMesonBuildNew .= "${strLine}\n"; } buildPutDiffers($oStorageBackRest, "${strBackRestBase}/meson.build", $strMesonBuildNew); # Setup build if it does not exist if (!-e $strBuildPath) { executeTest("meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug ${strBuildPath} ${strBackRestBase}"); } # Build code executeTest( "ninja -C ${strBuildPath}" . ($bMinGen ? '' : ' src/build-config src/build-error') . ' src/build-postgres' . ($bMinGen ? '' : " && ${strBuildPath}/src/build-config ${strBackRestBase}/src") . ($bMinGen ? '' : " && ${strBuildPath}/src/build-error ${strBackRestBase}/src") . " && cd $strRepoCachePath/src && ${strBuildPath}/src/build-postgres"); if ($bGenOnly) { exit 0; } # Generate code counts #--------------------------------------------------------------------------------------------------------------------------- if ($bCodeCount) { &log(INFO, "classify code files"); codeCountScan($oStorageBackRest, $strBackRestBase); exit 0; } # Clean up #--------------------------------------------------------------------------------------------------------------------------- my $iTestFail = 0; my $iTestRetry = 0; my $oyProcess = []; my $strCodePath = "${strBackRestBase}/test/result/coverage/raw"; if (!$bDryRun || $bVmOut) { &log(INFO, "cleanup old data" . ($strVm ne VM_NONE ? " and containers" : '')); if ($strVm ne VM_NONE) { containerRemove('test-([0-9]+|build)'); } for (my $iVmIdx = 0; $iVmIdx < 8; $iVmIdx++) { push(@{$oyProcess}, undef); } executeTest( "chmod 700 -R ${strTestPath}/test-* 2>&1 || true && rm -rf ${strTestPath}/temp ${strTestPath}/test-*" . " ${strTestPath}/data-*"); $oStorageTest->pathCreate("${strTestPath}/temp", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); # Remove old lcov dirs -- do it this way so the dirs stay open in finder/explorer, etc. executeTest("rm -rf ${strBackRestBase}/test/result/coverage/lcov/*"); # Overwrite the C coverage report so it will load but not show old coverage $oStorageTest->pathCreate( "${strBackRestBase}/test/result/coverage", {strMode => '0770', bIgnoreExists => true, bCreateParent => true}); $oStorageBackRest->put( "${strBackRestBase}/test/result/coverage/coverage.html", "