db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Integration test improvements for disk and memory efficiency.

Browse Source 
Set log-level-file=off when more that one test will run.  In this case is it impossible to see the logs anyway since they will be automatically cleaned up after the test.  This improves performance pretty dramatically since trace-level logging is expensive.  If a singe integration test is run then log-level-file is trace by default but can be changed with the --log-level-test-file option.

Reduce buffer-size to 64k to save memory during testing and allow more processes to run in parallel.

Update log replacement rules so that these options can change without affecting expect logs.
This commit is contained in:
David Steele
2019-12-17 15:23:07 -05:00
parent ccea30b8d8
commit 977ec2e307
23 changed files with 533 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/lib/pgBackRestTest/Common/JobTest.pm
+3
View File
@@ -65,6 +65,7 @@ sub new
$self->{iTestMax},
$self->{strLogLevel},
$self->{strLogLevelTest},
$self->{strLogLevelTestFile},
$self->{bLogForce},
$self->{bShowOutputAsync},
$self->{bNoCleanup},
@@ -97,6 +98,7 @@ sub new
{name => 'iTestMax'},
{name => 'strLogLevel'},
{name => 'strLogLevelTest'},
{name => 'strLogLevelTestFile'},
{name => 'bLogForce'},
{name => 'bShowOutputAsync'},
{name => 'bNoCleanup'},
@@ -279,6 +281,7 @@ sub run
$strCommandRunParam .
(defined($self->{oTest}->{&TEST_DB}) ? ' --pg-version=' . $self->{oTest}->{&TEST_DB} : '') .
($self->{strLogLevel} ne lc(INFO) ? " --log-level=$self->{strLogLevel}" : '') .
($self->{strLogLevelTestFile} ne lc(TRACE) ? " --log-level-test-file=$self->{strLogLevelTestFile}" : '') .
' --pgsql-bin=' . $self->{oTest}->{&TEST_PGSQL_BIN} .
($self->{strTimeZone} ? " --tz='$self->{strTimeZone}'" : '') .
($self->{bLogForce} ? ' --log-force' : '') .
test/lib/pgBackRestTest/Common/LogTest.pm
+28 -12
View File
@@ -21,6 +21,11 @@ use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
####################################################################################################################################
# Expect log cache for supplemental files so they are only added to expect logs when they have changed
####################################################################################################################################
my $hyExpectCache = undef;
####################################################################################################################################
# new
####################################################################################################################################
@@ -148,15 +153,7 @@ sub supplementalAdd
my $strComment = shift;
my $strContent = shift;
my $strHeader = "+ supplemental file: " . $self->regExpReplaceAll($strFileName);
if (defined($strComment))
{
$self->{strLog} .= "\n" . $self->regExpReplaceAll($strComment) . "\n" . ('=' x '132') . "\n";
}
$self->{strLog} .= "\n${strHeader}\n" . ('-' x length($strHeader)) . "\n";
# Get content if it is not defined
if (!defined($strContent))
{
open(my $hFile, '<', $strFileName)
@@ -164,14 +161,27 @@ sub supplementalAdd
while (my $strLine = readline($hFile))
{
$self->{strLog} .= $self->regExpReplaceAll($strLine);
$strContent .= $strLine;
}
close($hFile);
}
else
# Only ouput when the content of this file has changed
if (!defined($hyExpectCache->{$strFileName}) || $hyExpectCache->{$strFileName} ne $strContent)
{
if (defined($strContent) && length($strContent) > 0)
# $hyExpectCache->{$strFileName} = $strContent;
my $strHeader = "+ supplemental file: " . $self->regExpReplaceAll($strFileName);
if (defined($strComment))
{
$self->{strLog} .= "\n" . $self->regExpReplaceAll($strComment) . "\n" . ('=' x '132') . "\n";
}
$self->{strLog} .= "\n${strHeader}\n" . ('-' x length($strHeader)) . "\n";
if (length($strContent) > 0)
{
foreach my $strLine (split("\n", $strContent))
{
@@ -439,6 +449,12 @@ sub regExpReplaceAll
"${strTimestampRegExp} / ${strTimestampRegExp}\$", false);
$strLine = $self->regExpReplace($strLine, 'CHECKSUM', 'checksum=[\"]{0,1}[0-f]{40}', '[0-f]{40}$', false);
$strLine = $self->regExpReplace($strLine, 'LOG-LEVEL-FILE', 'log-level-file=[a-z]+', '[a-z]+$', false);
$strLine = $self->regExpReplace($strLine, 'LOG-SUBPROCESS', 'log-subprocess=[a-z]+', '[a-z]+$', false);
$strLine = $self->regExpReplace($strLine, '', ' --log-subprocess', undef, false);
$strLine = $self->regExpReplace($strLine, '', ' --no-log-subprocess', undef, false);
$strLine = $self->regExpReplace($strLine, 'BUFFER-SIZE', 'buffer-size=[0-9a-z]+', '[0-9a-z]+$', false);
$strLine = $self->regExpReplace($strLine, 'REMOTE-PROCESS-TERMINATED-MESSAGE',
'remote process terminated.*: (ssh.*|no output from terminated process)$',
'(ssh.*|no output from terminated process)$', false);
test/lib/pgBackRestTest/Common/RunTest.pm
+3
View File
@@ -125,6 +125,7 @@ sub process
$self->{bDryRun},
$self->{bCleanup},
$self->{bLogForce},
$self->{strLogLevelTestFile},
$self->{strPgUser},
$self->{strGroup},
) =
@@ -146,6 +147,7 @@ sub process
{name => 'bDryRun'},
{name => 'bCleanup'},
{name => 'bLogForce'},
{name => 'strLogLevelTestFile'},
{name => 'strPgUser'},
{name => 'strGroup'},
);
@@ -547,6 +549,7 @@ sub dataPath {return shift->basePath() . '/test/data'}
sub doCleanup {return shift->{bCleanup}}
sub doExpect {return shift->{bExpect}}
sub doLogForce {return shift->{bLogForce}}
sub logLevelTestFile {return shift->{strLogLevelTestFile}}
sub group {return shift->{strGroup}}
sub isDryRun {return shift->{bDryRun}}
sub expect {return shift->{oExpect}}