1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
pgbackrest/test/lib/pgBackRestTest/Common/JobTest.pm
David Steele f0250dab4b Move async forking and more error handling to C.
The Perl process was exiting directly when called but that interfered with proper locking for the forked async process. Now Perl returns results to the C process which handles all errors, including signals.
2018-04-12 20:42:26 -04:00

607 lines
25 KiB
Perl

####################################################################################################################################
# JobTest.pm - Run a test job and monitor progress
####################################################################################################################################
package pgBackRestTest::Common::JobTest;
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname basename);
use POSIX qw(ceil);
use Time::HiRes qw(gettimeofday);
use pgBackRest::DbVersion;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::DefineTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::ListTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Common::VmTest;
####################################################################################################################################
# Build flags from the last build. When the build flags change test files must be rebuilt
####################################################################################################################################
my $rhBuildFlags = undef;
####################################################################################################################################
# new
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{oStorageTest},
$self->{strBackRestBase},
$self->{strTestPath},
$self->{strCoveragePath},
$self->{oTest},
$self->{bDryRun},
$self->{bVmOut},
$self->{iVmIdx},
$self->{iVmMax},
$self->{iTestIdx},
$self->{iTestMax},
$self->{strLogLevel},
$self->{bLogForce},
$self->{bShowOutputAsync},
$self->{bNoCleanup},
$self->{iRetry},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'oStorageTest'},
{name => 'strBackRestBase'},
{name => 'strTestPath'},
{name => 'strCoveragePath'},
{name => 'oTest'},
{name => 'bDryRun'},
{name => 'bVmOut'},
{name => 'iVmIdx'},
{name => 'iVmMax'},
{name => 'iTestIdx'},
{name => 'iTestMax'},
{name => 'strLogLevel'},
{name => 'bLogForce'},
{name => 'bShowOutputAsync'},
{name => 'bNoCleanup'},
{name => 'iRetry'},
);
# Set try to 0
$self->{iTry} = 0;
# Setup the path where gcc coverage will be performed
$self->{strGCovPath} = "$self->{strTestPath}/gcov-$self->{oTest}->{&TEST_VM}-$self->{iVmIdx}";
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self, trace => true}
);
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
(my $strOperation) = logDebugParam (__PACKAGE__ . '->run', \@_,);
# Start the test timer
my $fTestStartTime = gettimeofday();
# Was the job run?
my $bRun = false;
# Should the job be run?
$self->{iTry}++;
if ($self->{iTry} <= ($self->{iRetry} + 1))
{
if ($self->{iTry} != 1 && $self->{iTry} == ($self->{iRetry} + 1))
{
$self->{strLogLevel} = lc(DEBUG);
}
my $strTest = sprintf(
'P%0' . length($self->{iVmMax}) . 'd-T%0' . length($self->{iTestMax}) . 'd/%0' .
length($self->{iTestMax}) . "d - ", $self->{iVmIdx} + 1, $self->{iTestIdx} + 1, $self->{iTestMax}) .
'vm=' . $self->{oTest}->{&TEST_VM} .
', module=' . $self->{oTest}->{&TEST_MODULE} .
', test=' . $self->{oTest}->{&TEST_NAME} .
(defined($self->{oTest}->{&TEST_RUN}) ? ', run=' . join(',', sort(@{$self->{oTest}->{&TEST_RUN}})) : '') .
(defined($self->{oTest}->{&TEST_DB}) ? ', db=' . $self->{oTest}->{&TEST_DB} : '') .
($self->{iTry} > 1 ? ' (retry ' . ($self->{iTry} - 1) . ')' : '');
my $strImage = 'test-' . $self->{iVmIdx};
my $strDbVersion = (defined($self->{oTest}->{&TEST_DB}) ? $self->{oTest}->{&TEST_DB} : PG_VERSION_94);
$strDbVersion =~ s/\.//;
&log($self->{bDryRun} && !$self->{bVmOut} || $self->{bShowOutputAsync} ? INFO : DETAIL, "${strTest}" .
(!($self->{bDryRun} || !$self->{bVmOut}) || $self->{bShowOutputAsync} ? "\n" : ''));
my $strVmTestPath = '/home/' . TEST_USER . "/test/${strImage}";
my $strHostTestPath = "$self->{strTestPath}/${strImage}";
# Don't create the container if this is a dry run unless output from the VM is required. Ouput can be requested
# to get more information about the specific tests that will be run.
if (!$self->{bDryRun} || $self->{bVmOut})
{
# Create host test directory
$self->{oStorageTest}->pathCreate($strHostTestPath, {strMode => '0770'});
# Create gcov directory
my $bGCovExists = true;
if ($self->{oTest}->{&TEST_C} && !$self->{oStorageTest}->pathExists($self->{strGCovPath}))
{
$self->{oStorageTest}->pathCreate($self->{strGCovPath}, {strMode => '0770'});
$bGCovExists = false;
}
if ($self->{oTest}->{&TEST_CONTAINER})
{
executeTest(
'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" .
" -v $self->{strCoveragePath}:$self->{strCoveragePath} " .
" -v ${strHostTestPath}:${strVmTestPath}" .
($self->{oTest}->{&TEST_C} ? " -v $self->{strGCovPath}:$self->{strGCovPath}" : '') .
" -v $self->{strBackRestBase}:$self->{strBackRestBase} " .
containerRepo() . ':' . $self->{oTest}->{&TEST_VM} .
"-test",
{bSuppressStdErr => true});
# If testing C code copy source files to the test directory
if ($self->{oTest}->{&TEST_C})
{
if (!$bGCovExists)
{
executeTest("cp -r $self->{strBackRestBase}/src/* $self->{strGCovPath}");
}
if (!$bGCovExists || defined($rhBuildFlags->{$self->{iVmIdx}}) != defined($self->{oTest}->{&TEST_CDEF}) ||
defined($rhBuildFlags->{$self->{iVmIdx}}) &&
$rhBuildFlags->{$self->{iVmIdx}} ne $self->{oTest}->{&TEST_CDEF})
{
executeTest("cp -r $self->{strBackRestBase}/test/src/* $self->{strGCovPath}");
}
$rhBuildFlags->{$self->{iVmIdx}} = $self->{oTest}->{&TEST_CDEF};
}
# If testing Perl code (or C code that calls Perl code) install bin and Perl C Library
if (!$self->{oTest}->{&TEST_C} || $self->{oTest}->{&TEST_PERL_REQ})
{
jobInstallC($self->{strBackRestBase}, $self->{oTest}->{&TEST_VM}, $strImage);
}
}
}
# Create run parameters
my $strCommandRunParam = '';
foreach my $iRunIdx (@{$self->{oTest}->{&TEST_RUN}})
{
$strCommandRunParam .= ' --run=' . $iRunIdx;
}
# Create command
my $strCommand;
if ($self->{oTest}->{&TEST_C})
{
$strCommand =
'docker exec -i -u ' . TEST_USER . " ${strImage} bash -l -c '" .
"cd $self->{strGCovPath} && " .
"make -s 2>&1 && " .
($self->{oTest}->{&TEST_VM} ne VM_CO6 ?
" valgrind -q --gen-suppressions=all --suppressions=$self->{strBackRestBase}/test/src/valgrind.suppress" .
" --leak-check=full --leak-resolution=high" : '') .
" ./test 2>&1'";
}
else
{
$strCommand =
($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') .
testRunExe(
vmCoverage($self->{oTest}->{&TEST_VM}), undef, abs_path($0), dirname($self->{strCoveragePath}),
$self->{strBackRestBase}, $self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME}) .
" --test-path=${strVmTestPath}" .
" --vm=$self->{oTest}->{&TEST_VM}" .
" --vm-id=$self->{iVmIdx}" .
" --module=" . $self->{oTest}->{&TEST_MODULE} .
' --test=' . $self->{oTest}->{&TEST_NAME} .
$strCommandRunParam .
(defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') .
($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') .
' --pgsql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} .
($self->{bLogForce} ? ' --log-force' : '') .
($self->{bDryRun} ? ' --dry-run' : '') .
($self->{bDryRun} ? ' --vm-out' : '') .
($self->{bNoCleanup} ? " --no-cleanup" : '');
}
&log(DETAIL, $strCommand);
if (!$self->{bDryRun} || $self->{bVmOut})
{
# If testing C code
if ($self->{oTest}->{&TEST_C})
{
my $hTest = (testDefModuleTest($self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME}));
my $hTestCoverage = $hTest->{&TESTDEF_COVERAGE};
my @stryCFile;
foreach my $strFile (sort(keys(%{$self->{oStorageTest}->manifest($self->{strGCovPath})})))
{
# Skip all files except .c files (including .auto.c)
next if $strFile !~ /(?<!\.auto)\.c$/;
# !!! Skip main for now until it can be rewritten
next if $strFile =~ /main\.c$/;
# Skip test.c -- it will be added manually at the end
next if $strFile =~ /test\.c$/;
if (!defined($hTestCoverage->{substr($strFile, 0, length($strFile) - 2)}) &&
$strFile !~ /^module\/[^\/]*\/.*Test\.c$/)
{
push(@stryCFile, "${strFile}");
}
}
# Generate list of C files to include for testing
my $strTestFile =
"module/$self->{oTest}->{&TEST_MODULE}/" . testRunName($self->{oTest}->{&TEST_NAME}, false) . 'Test.c';
my $strCInclude;
foreach my $strFile (sort(keys(%{$hTestCoverage})))
{
# Don't include the test file as it is already included below
next if $strFile =~ /Test$/;
# Don't any auto files as they are included in their companion C files
next if $strFile =~ /auto$/;
# Include the C file if it exists
my $strCIncludeFile = "${strFile}.c";
# If the C file does not exist use the header file instead
if (!$self->{oStorageTest}->exists("$self->{strGCovPath}/${strCIncludeFile}"))
{
# Error if code was expected
if ($hTestCoverage->{$strFile} ne TESTDEF_COVERAGE_NOCODE)
{
confess &log(ERROR, "unable to find source file '$self->{strGCovPath}/${strCIncludeFile}'");
}
$strCIncludeFile = "${strFile}.h";
}
$strCInclude .= (defined($strCInclude) ? "\n" : '') . "#include \"${strCIncludeFile}\"";
}
# Update C test file with test module
my $strTestC = ${$self->{oStorageTest}->get("$self->{strBackRestBase}/test/src/test.c")};
$strTestC =~ s/\{\[C\_INCLUDE\]\}/$strCInclude/g;
$strTestC =~ s/\{\[C\_TEST\_INCLUDE\]\}/\#include \"$strTestFile\"/g;
# Set globals
$strTestC =~ s/\{\[C\_TEST\_PATH\]\}/$strVmTestPath/g;
# Initialize tests
my $strTestInit;
for (my $iTestIdx = 1; $iTestIdx <= $hTest->{&TESTDEF_TOTAL}; $iTestIdx++)
{
my $bSelected = false;
if (!defined($self->{oTest}->{&TEST_RUN}) || @{$self->{oTest}->{&TEST_RUN}} == 0 ||
grep(/^$iTestIdx$/, @{$self->{oTest}->{&TEST_RUN}}))
{
$bSelected = true;
}
$strTestInit .=
(defined($strTestInit) ? "\n " : '') .
sprintf("testAdd(%3d, %8s);" , $iTestIdx, ($bSelected ? 'true' : 'false'));
}
$strTestC =~ s/\{\[C\_TEST\_LIST\]\}/$strTestInit/g;
# Save C test file
$self->{oStorageTest}->put("$self->{strGCovPath}/test.c", $strTestC);
# Build the Makefile
my $strMakefile =
"CC=gcc\n" .
"CFLAGS=-I. -std=c99 -fPIC -g \\\n" .
" -Werror -Wfatal-errors -Wall -Wextra -Wwrite-strings -Wno-clobbered -Wswitch-enum -Wconversion \\\n" .
($self->{oTest}->{&TEST_VM} eq VM_U16 ? " -Wformat-signedness \\\n" : '') .
# This warning appears to be broken on U12 even though the functionality is fine
($self->{oTest}->{&TEST_VM} eq VM_U12 || $self->{oTest}->{&TEST_VM} eq VM_CO6 ?
" -Wno-missing-field-initializers \\\n" : '') .
($self->{oTest}->{&TEST_VM} ne VM_CO6 && $self->{oTest}->{&TEST_VM} ne VM_U12 ?
" -Wpedantic \\\n" : '') .
" -Wformat=2 -Wformat-nonliteral \\\n" .
" `perl -MExtUtils::Embed -e ccopts`\n" .
"LDFLAGS=-lcrypto" . (vmCoverage($self->{oTest}->{&TEST_VM}) ? " -lgcov" : '') .
" `perl -MExtUtils::Embed -e ldopts`\n" .
'TESTFLAGS=' . ($self->{oTest}->{&TEST_DEBUG_UNIT_SUPPRESS} ? '' : "-DDEBUG_UNIT ") .
($self->{oTest}->{&TEST_CDEF} ? "$self->{oTest}->{&TEST_CDEF}" : '') .
"\n" .
"\nSRCS=" . join(' ', @stryCFile) . "\n" .
"OBJS=\$(SRCS:.c=.o)\n" .
"\n" .
"test: \$(OBJS) test.o\n" .
"\t\$(CC) -o test \$(OBJS) test.o \$(LDFLAGS)\n" .
"\n" .
"test.o: test.c\n" .
"\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) " .
(vmCoverage($self->{oTest}->{&TEST_VM}) ? '-fprofile-arcs -ftest-coverage -O0' : '-O2') .
" -c test.c\n" .
"\n" .
".c.o:\n" .
"\t\$(CC) \$(CFLAGS) \$(TESTFLAGS) -O2 -c \$< -o \$@\n";
$self->{oStorageTest}->put($self->{strGCovPath} . "/Makefile", $strMakefile);
}
my $oExec = new pgBackRestTest::Common::ExecuteTest(
$strCommand, {bSuppressError => true, bShowOutputAsync => $self->{bShowOutputAsync}});
$oExec->begin();
$self->{oProcess} =
{
exec => $oExec,
test => $strTest,
start_time => $fTestStartTime,
};
$bRun = true;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bRun', value => $bRun, trace => true}
);
}
####################################################################################################################################
# end
####################################################################################################################################
sub end
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
(my $strOperation) = logDebugParam (__PACKAGE__ . '->run', \@_,);
# Is the job done?
my $bDone = false;
my $bFail = false;
my $oExecDone = $self->{oProcess}{exec};
my $strTestDone = $self->{oProcess}{test};
my $iTestDoneIdx = $self->{oProcess}{idx};
my $iExitStatus = $oExecDone->end(undef, $self->{iVmMax} == 1);
if (defined($iExitStatus))
{
my $strImage = 'test-' . $self->{iVmIdx};
if ($self->{bShowOutputAsync})
{
syswrite(*STDOUT, "\n");
}
# If C library generate coverage info
if ($iExitStatus == 0 && $self->{oTest}->{&TEST_C} && vmCoverage($self->{oTest}->{&TEST_VM}))
{
# Generate a list of files to cover
my $hTestCoverage =
(testDefModuleTest($self->{oTest}->{&TEST_MODULE}, $self->{oTest}->{&TEST_NAME}))->{&TESTDEF_COVERAGE};
my @stryCoveredModule;
foreach my $strModule (sort(keys(%{$hTestCoverage})))
{
# Skip modules that have no code
next if ($hTestCoverage->{$strModule} eq TESTDEF_COVERAGE_NOCODE);
push (@stryCoveredModule, testRunName(basename($strModule), false) . ".c.gcov");
}
# Generate coverage reports for the modules
my $strLCovExe = "lcov --config-file=$self->{strBackRestBase}/test/src/lcov.conf";
my $strLCovOut = $self->{strGCovPath} . '/test.lcov';
executeTest(
'docker exec -i -u ' . TEST_USER . " ${strImage} " .
"${strLCovExe} --capture --directory=$self->{strGCovPath} --o=${strLCovOut}");
# Generate coverage report for each module
foreach my $strModule (sort(keys(%{$hTestCoverage})))
{
my $strModuleName = testRunName($strModule, false);
my $strModuleOutName = $strModuleName;
my $bTest = false;
if ($strModuleOutName =~ /^module/mg)
{
$strModuleOutName =~ s/^module/test/mg;
$bTest = true;
}
# Generate lcov reports
my $strModulePath = $self->{strBackRestBase} . "/test/.vagrant/code/${strModuleOutName}";
my $strLCovFile = "${strModulePath}.lcov";
my $strLCovTotal = $self->{strBackRestBase} . "/test/.vagrant/code/all.lcov";
executeTest(
'docker exec -i -u ' . TEST_USER . " ${strImage} " .
"${strLCovExe} --extract=${strLCovOut} */${strModuleName}.c --o=${strLCovFile}");
# Update source file
my $strCoverage = ${$self->{oStorageTest}->get($strLCovFile)};
if (defined($strCoverage))
{
if ($hTestCoverage->{$strModule} eq TESTDEF_COVERAGE_NOCODE)
{
confess &log(ERROR, "module '${strModule}' is marked 'no code' but has code");
}
# Get coverage info
my $iTotalLines = (split(':', ($strCoverage =~ m/^LF:.*$/mg)[0]))[1] + 0;
my $iCoveredLines = (split(':', ($strCoverage =~ m/^LH:.*$/mg)[0]))[1] + 0;
my $iTotalBranches = 0;
my $iCoveredBranches = 0;
if ($strCoverage =~ /^BRF\:$/mg && $strCoverage =~ /^BRH\:$/mg)
{
$iTotalBranches = (split(':', ($strCoverage =~ m/^BRF:.*$/mg)[0]))[1] + 0;
$iCoveredBranches = (split(':', ($strCoverage =~ m/^BRH:.*$/mg)[0]))[1] + 0;
}
# Report coverage if this is not a test or if the test does not have complete coverage
if (!$bTest || $iTotalLines != $iCoveredLines || $iTotalBranches != $iCoveredBranches)
{
# Fix source file name
$strCoverage =~ s/^SF\:.*$/SF:$strModulePath\.c/mg;
# Save coverage file
$self->{oStorageTest}->put($strLCovFile, $strCoverage);
if ($self->{oStorageTest}->exists(${strLCovTotal}))
{
executeTest(
'docker exec -i -u ' . TEST_USER . " ${strImage} " .
"${strLCovExe} --add-tracefile=${strLCovFile} --add-tracefile=${strLCovTotal} --o=${strLCovTotal}");
}
else
{
$self->{oStorageTest}->copy($strLCovFile, $strLCovTotal)
}
}
else
{
$self->{oStorageTest}->remove($strLCovFile);
}
}
else
{
if ($hTestCoverage->{$strModule} ne TESTDEF_COVERAGE_NOCODE)
{
confess &log(ERROR, "module '${strModule}' is marked 'code' but has no code");
}
}
}
}
# Record elapsed time
my $fTestElapsedTime = ceil((gettimeofday() - $self->{oProcess}{start_time}) * 100) / 100;
# Output error
if ($iExitStatus != 0)
{
&log(ERROR, "${strTestDone} (err${iExitStatus}-${fTestElapsedTime}s)" .
(defined($oExecDone->{strOutLog}) && !$self->{bShowOutputAsync} ?
":\n\n" . trim($oExecDone->{strOutLog}) . "\n" : ''), undef, undef, 4);
$bFail = true;
}
# Output success
else
{
&log(INFO, "${strTestDone} (${fTestElapsedTime}s)".
($self->{bVmOut} && !$self->{bShowOutputAsync} ?
":\n\n" . trim($oExecDone->{strOutLog}) . "\n" : ''), undef, undef, 4);
}
if (!$self->{bNoCleanup})
{
my $strHostTestPath = "$self->{strTestPath}/${strImage}";
containerRemove("test-$self->{iVmIdx}");
executeTest("sudo rm -rf ${strHostTestPath}");
}
$bDone = true;
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bDone', value => $bDone, trace => true},
{name => 'bFail', value => $bFail, trace => true}
);
}
####################################################################################################################################
# Install C binary and library
####################################################################################################################################
sub jobInstallC
{
my $strBasePath = shift;
my $strVm = shift;
my $strImage = shift;
# Install Perl C Library
my $oVm = vmGet();
my $strBuildPath = "${strBasePath}/test/.vagrant";
my $strBuildLibCPath = "$strBuildPath/libc/${strVm}/libc";
my $strBuildBinPath = "$strBuildPath/bin/${strVm}/src";
my $strPerlAutoPath = $oVm->{$strVm}{&VMDEF_PERL_ARCH_PATH} . '/auto/pgBackRest/LibC';
my $strPerlModulePath = $oVm->{$strVm}{&VMDEF_PERL_ARCH_PATH} . '/pgBackRest';
executeTest(
"docker exec -i -u root ${strImage} bash -c '" .
"mkdir -p -m 755 ${strPerlAutoPath} && " .
# "cp ${strBuildLibCPath}/blib/arch/auto/pgBackRest/LibC/LibC.bs ${strPerlAutoPath} && " .
"cp ${strBuildLibCPath}/blib/arch/auto/pgBackRest/LibC/LibC.so ${strPerlAutoPath} && " .
"cp ${strBuildLibCPath}/blib/lib/auto/pgBackRest/LibC/autosplit.ix ${strPerlAutoPath} && " .
"mkdir -p -m 755 ${strPerlModulePath} && " .
"cp ${strBuildLibCPath}/blib/lib/pgBackRest/LibC.pm ${strPerlModulePath} && " .
"cp ${strBuildLibCPath}/blib/lib/pgBackRest/LibCAuto.pm ${strPerlModulePath} && " .
"cp ${strBuildBinPath}/" . BACKREST_EXE . ' /usr/bin/' . BACKREST_EXE . ' && ' .
'chmod 755 /usr/bin/' . BACKREST_EXE . ' && ' .
"ln -s ${strBasePath}/lib/pgBackRest /usr/share/perl5/pgBackRest'");
}
push(@EXPORT, qw(jobInstallC));
1;