2017-02-21 15:59:23 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# JobTest.pm - Run a test job and monitor progress
|
|
|
|
####################################################################################################################################
|
|
|
|
package pgBackRestTest::Common::JobTest;
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# Perl includes
|
|
|
|
####################################################################################################################################
|
|
|
|
use strict;
|
|
|
|
use warnings FATAL => qw(all);
|
|
|
|
use Carp qw(confess);
|
|
|
|
use English '-no_match_vars';
|
|
|
|
|
|
|
|
use Cwd qw(abs_path);
|
|
|
|
use Exporter qw(import);
|
|
|
|
our @EXPORT = qw();
|
|
|
|
use File::Basename qw(dirname);
|
|
|
|
use POSIX qw(ceil);
|
|
|
|
use Time::HiRes qw(gettimeofday);
|
|
|
|
|
|
|
|
use pgBackRest::DbVersion;
|
|
|
|
use pgBackRest::Common::Exception;
|
|
|
|
use pgBackRest::Common::Log;
|
|
|
|
use pgBackRest::Common::String;
|
|
|
|
use pgBackRest::FileCommon;
|
|
|
|
|
|
|
|
use pgBackRestTest::Common::ContainerTest;
|
|
|
|
use pgBackRestTest::Common::ExecuteTest;
|
|
|
|
use pgBackRestTest::Common::ListTest;
|
2017-02-21 18:47:45 +02:00
|
|
|
use pgBackRestTest::Common::RunTest;
|
2017-04-10 18:31:30 +02:00
|
|
|
use pgBackRestTest::Common::VmTest;
|
2017-02-21 15:59:23 +02:00
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# new
|
|
|
|
####################################################################################################################################
|
|
|
|
sub new
|
|
|
|
{
|
|
|
|
my $class = shift; # Class name
|
|
|
|
|
|
|
|
# Create the class hash
|
|
|
|
my $self = {};
|
|
|
|
bless $self, $class;
|
|
|
|
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
|
|
(
|
|
|
|
my $strOperation,
|
|
|
|
$self->{strBackRestBase},
|
|
|
|
$self->{strTestPath},
|
|
|
|
$self->{strCoveragePath},
|
|
|
|
$self->{oTest},
|
|
|
|
$self->{bDryRun},
|
|
|
|
$self->{bVmOut},
|
|
|
|
$self->{iVmIdx},
|
|
|
|
$self->{iVmMax},
|
|
|
|
$self->{iTestIdx},
|
|
|
|
$self->{iTestMax},
|
|
|
|
$self->{strLogLevel},
|
|
|
|
$self->{bLogForce},
|
|
|
|
$self->{bShowOutputAsync},
|
|
|
|
$self->{bNoCleanup},
|
2017-02-22 05:10:02 +02:00
|
|
|
$self->{iRetry},
|
2017-02-21 15:59:23 +02:00
|
|
|
) =
|
|
|
|
logDebugParam
|
|
|
|
(
|
|
|
|
__PACKAGE__ . '->new', \@_,
|
|
|
|
{name => 'strBackRestBase'},
|
|
|
|
{name => 'strTestPath'},
|
|
|
|
{name => 'strCoveragePath'},
|
|
|
|
{name => 'oTest'},
|
|
|
|
{name => 'bDryRun'},
|
|
|
|
{name => 'bVmOut'},
|
|
|
|
{name => 'iVmIdx'},
|
|
|
|
{name => 'iVmMax'},
|
|
|
|
{name => 'iTestIdx'},
|
|
|
|
{name => 'iTestMax'},
|
|
|
|
{name => 'strLogLevel'},
|
|
|
|
{name => 'bLogForce'},
|
|
|
|
{name => 'bShowOutputAsync'},
|
|
|
|
{name => 'bNoCleanup'},
|
2017-02-22 05:10:02 +02:00
|
|
|
{name => 'iRetry'},
|
2017-02-21 15:59:23 +02:00
|
|
|
);
|
|
|
|
|
|
|
|
# Set try to 0
|
|
|
|
$self->{iTry} = 0;
|
|
|
|
|
|
|
|
# Return from function and log return values if any
|
|
|
|
return logDebugReturn
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
{name => 'self', value => $self, trace => true}
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# run
|
|
|
|
####################################################################################################################################
|
|
|
|
sub run
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
|
|
(my $strOperation) = logDebugParam (__PACKAGE__ . '->run', \@_,);
|
|
|
|
|
|
|
|
# Was the job run?
|
|
|
|
my $bRun = false;
|
|
|
|
|
|
|
|
# Should the job be run?
|
|
|
|
$self->{iTry}++;
|
|
|
|
|
2017-02-22 05:10:02 +02:00
|
|
|
if ($self->{iTry} <= ($self->{iRetry} + 1))
|
2017-02-21 15:59:23 +02:00
|
|
|
{
|
2017-02-22 05:10:02 +02:00
|
|
|
if ($self->{iTry} != 1 && $self->{iTry} == ($self->{iRetry} + 1))
|
2017-02-21 15:59:23 +02:00
|
|
|
{
|
|
|
|
$self->{strLogLevel} = lc(DEBUG);
|
|
|
|
}
|
|
|
|
|
|
|
|
my $strTest = sprintf('P%0' . length($self->{iVmMax}) . 'd-T%0' . length($self->{iTestMax}) . 'd/%0' .
|
|
|
|
length($self->{iTestMax}) . "d - ", $self->{iVmIdx} + 1, $self->{iTestIdx} + 1, $self->{iTestMax}) .
|
|
|
|
'vm=' . $self->{oTest}->{&TEST_VM} .
|
|
|
|
', module=' . $self->{oTest}->{&TEST_MODULE} .
|
|
|
|
', test=' . $self->{oTest}->{&TEST_NAME} .
|
|
|
|
(defined($self->{oTest}->{&TEST_RUN}) ? ', run=' . join(',', @{$self->{oTest}->{&TEST_RUN}}) : '') .
|
|
|
|
(defined($self->{oTest}->{&TEST_PROCESS}) ? ', process-max=' . $self->{oTest}->{&TEST_PROCESS} : '') .
|
|
|
|
(defined($self->{oTest}->{&TEST_DB}) ? ', db=' . $self->{oTest}->{&TEST_DB} : '') .
|
|
|
|
($self->{iTry} > 1 ? ' (retry ' . ($self->{iTry} - 1) . ')' : '');
|
|
|
|
|
|
|
|
my $strImage = 'test-' . $self->{iVmIdx};
|
|
|
|
my $strDbVersion = (defined($self->{oTest}->{&TEST_DB}) ? $self->{oTest}->{&TEST_DB} : PG_VERSION_94);
|
|
|
|
$strDbVersion =~ s/\.//;
|
|
|
|
|
|
|
|
&log($self->{bDryRun} && !$self->{bVmOut} || $self->{bShowOutputAsync} ? INFO : DETAIL, "${strTest}" .
|
|
|
|
($self->{bDryRun} || $self->{bShowOutputAsync} ? "\n" : ''));
|
|
|
|
|
|
|
|
my $strVmTestPath = '/home/' . TEST_USER . "/test/${strImage}";
|
|
|
|
my $strHostTestPath = "$self->{strTestPath}/${strImage}";
|
|
|
|
|
|
|
|
# Don't create the container if this is a dry run unless output from the VM is required. Ouput can be requested
|
|
|
|
# to get more information about the specific tests that will be run.
|
|
|
|
if (!$self->{bDryRun} || $self->{bDryRun})
|
|
|
|
{
|
|
|
|
# Create host test directory
|
|
|
|
filePathCreate($strHostTestPath, '0770');
|
|
|
|
|
|
|
|
if ($self->{oTest}->{&TEST_CONTAINER})
|
|
|
|
{
|
|
|
|
executeTest(
|
|
|
|
'docker run -itd -h ' . $self->{oTest}->{&TEST_VM} . "-test --name=${strImage}" .
|
|
|
|
" -v $self->{strCoveragePath}:$self->{strCoveragePath} " .
|
|
|
|
" -v ${strHostTestPath}:${strVmTestPath}" .
|
|
|
|
" -v $self->{strBackRestBase}:$self->{strBackRestBase} " .
|
|
|
|
containerRepo() . ':' . $self->{oTest}->{&TEST_VM} .
|
|
|
|
"-loop-test-pre",
|
|
|
|
{bSuppressStdErr => true});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# Create run parameters
|
|
|
|
my $strCommandRunParam = '';
|
|
|
|
|
|
|
|
foreach my $iRunIdx (@{$self->{oTest}->{&TEST_RUN}})
|
|
|
|
{
|
|
|
|
$strCommandRunParam .= ' --run=' . $iRunIdx;
|
|
|
|
}
|
|
|
|
|
|
|
|
# Create command
|
|
|
|
my $strCommand =
|
|
|
|
($self->{oTest}->{&TEST_CONTAINER} ? 'docker exec -i -u ' . TEST_USER . " ${strImage} " : '') .
|
2017-04-10 18:31:30 +02:00
|
|
|
(vmCoverage($self->{oTest}->{&TEST_VM}) ? testRunExe(
|
2017-02-21 15:59:23 +02:00
|
|
|
abs_path($0), dirname($self->{strCoveragePath}), $self->{strBackRestBase}, $self->{oTest}->{&TEST_MODULE},
|
|
|
|
$self->{oTest}->{&TEST_NAME}, defined($self->{oTest}->{&TEST_RUN}) ? $self->{oTest}->{&TEST_RUN} : 'all') :
|
|
|
|
abs_path($0)) .
|
|
|
|
" --test-path=${strVmTestPath}" .
|
|
|
|
" --vm=$self->{oTest}->{&TEST_VM}" .
|
|
|
|
" --vm-id=$self->{iVmIdx}" .
|
|
|
|
" --module=" . $self->{oTest}->{&TEST_MODULE} .
|
|
|
|
' --test=' . $self->{oTest}->{&TEST_NAME} .
|
|
|
|
$strCommandRunParam .
|
|
|
|
(defined($self->{oTest}->{&TEST_DB}) ? ' --db-version=' . $self->{oTest}->{&TEST_DB} : '') .
|
|
|
|
(defined($self->{oTest}->{&TEST_PROCESS}) ? ' --process-max=' . $self->{oTest}->{&TEST_PROCESS} : '') .
|
|
|
|
($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') .
|
|
|
|
' --pgsql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} .
|
|
|
|
($self->{bLogForce} ? ' --log-force' : '') .
|
|
|
|
($self->{bDryRun} ? ' --dry-run' : '') .
|
|
|
|
($self->{bDryRun} ? ' --vm-out' : '') .
|
|
|
|
($self->{bNoCleanup} ? " --no-cleanup" : '');
|
|
|
|
|
|
|
|
&log(DETAIL, $strCommand);
|
|
|
|
|
|
|
|
if (!$self->{bDryRun} || $self->{bVmOut})
|
|
|
|
{
|
|
|
|
my $fTestStartTime = gettimeofday();
|
|
|
|
|
|
|
|
# Set permissions on the Docker test directory. This can be removed once users/groups are sync'd between
|
|
|
|
# Docker and the host VM.
|
|
|
|
if ($self->{oTest}->{&TEST_CONTAINER})
|
|
|
|
{
|
|
|
|
executeTest("docker exec ${strImage} chown " . TEST_USER . ':' . TEST_GROUP . " -R ${strVmTestPath}");
|
|
|
|
}
|
|
|
|
|
|
|
|
my $oExec = new pgBackRestTest::Common::ExecuteTest(
|
|
|
|
$strCommand,
|
|
|
|
{bSuppressError => true, bShowOutputAsync => $self->{bShowOutputAsync}});
|
|
|
|
|
|
|
|
$oExec->begin();
|
|
|
|
|
|
|
|
$self->{oProcess} =
|
|
|
|
{
|
|
|
|
exec => $oExec,
|
|
|
|
test => $strTest,
|
|
|
|
# idx => $self->{iTestIdx},
|
|
|
|
# container => $self->{oTest}->{&TEST_CONTAINER},
|
|
|
|
start_time => $fTestStartTime
|
|
|
|
};
|
|
|
|
|
|
|
|
$bRun = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# Return from function and log return values if any
|
|
|
|
return logDebugReturn
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
{name => 'bRun', value => $bRun, trace => true}
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# end
|
|
|
|
####################################################################################################################################
|
|
|
|
sub end
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
|
|
(my $strOperation) = logDebugParam (__PACKAGE__ . '->run', \@_,);
|
|
|
|
|
|
|
|
# Is the job done?
|
|
|
|
my $bDone = false;
|
|
|
|
my $bFail = false;
|
|
|
|
|
|
|
|
my $oExecDone = $self->{oProcess}{exec};
|
|
|
|
my $strTestDone = $self->{oProcess}{test};
|
|
|
|
my $iTestDoneIdx = $self->{oProcess}{idx};
|
|
|
|
|
|
|
|
my $iExitStatus = $oExecDone->end(undef, $self->{iVmMax} == 1);
|
|
|
|
|
|
|
|
if (defined($iExitStatus))
|
|
|
|
{
|
|
|
|
if ($self->{bShowOutputAsync})
|
|
|
|
{
|
|
|
|
syswrite(*STDOUT, "\n");
|
|
|
|
}
|
|
|
|
|
|
|
|
my $fTestElapsedTime = ceil((gettimeofday() - $self->{oProcess}{start_time}) * 100) / 100;
|
|
|
|
|
|
|
|
if ($iExitStatus != 0)
|
|
|
|
{
|
|
|
|
&log(ERROR, "${strTestDone} (err${iExitStatus}-${fTestElapsedTime}s)" .
|
|
|
|
(defined($oExecDone->{strOutLog}) && !$self->{bShowOutputAsync} ?
|
|
|
|
":\n\n" . trim($oExecDone->{strOutLog}) . "\n" : ''), undef, undef, 4);
|
|
|
|
$bFail = true;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
&log(INFO, "${strTestDone} (${fTestElapsedTime}s)".
|
|
|
|
($self->{bVmOut} && !$self->{bShowOutputAsync} ?
|
|
|
|
":\n\n" . trim($oExecDone->{strOutLog}) . "\n" : ''), undef, undef, 4);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!$self->{bNoCleanup})
|
|
|
|
{
|
|
|
|
my $strImage = 'test-' . $self->{iVmIdx};
|
|
|
|
my $strHostTestPath = "$self->{strTestPath}/${strImage}";
|
|
|
|
|
|
|
|
containerRemove("test-$self->{iVmIdx}");
|
|
|
|
executeTest("sudo rm -rf ${strHostTestPath}");
|
|
|
|
}
|
|
|
|
|
|
|
|
$bDone = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
# Return from function and log return values if any
|
|
|
|
return logDebugReturn
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
{name => 'bDone', value => $bDone, trace => true},
|
|
|
|
{name => 'bFail', value => $bFail, trace => true}
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
1;
|