mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-01-18 04:58:51 +02:00
Cleanup, work on test framework.
This commit is contained in:
David Steele
parent
a849f2d2fd
commit
2d9f027e3a
@ -6,13 +6,13 @@
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use threads;
|
||||
use strict;
|
||||
use warnings;
|
||||
use threads;
|
||||
use Carp;
|
||||
|
||||
use File::Basename;
|
||||
use Getopt::Long;
|
||||
use Carp;
|
||||
|
||||
use lib dirname($0) . "/../lib";
|
||||
use BackRest::Utility;
|
||||
@ -75,7 +75,7 @@ my $strType; # Type of backup: full, differential (diff), incremental
|
||||
# Test parameters - not for general use
|
||||
my $bNoFork = false; # Prevents the archive process from forking when local archiving is enabled
|
||||
my $bTest = false; # Enters test mode - not harmful in anyway, but adds special logging and pauses for unit testing
|
||||
my $iTestDelay = 5; # Amount of time to delay after hitting a test point (the default would not be enough for manual tests)
|
||||
my $iTestDelay = 2; # Amount of time to delay after hitting a test point (the default would not be enough for manual tests)
|
||||
|
||||
GetOptions ("config=s" => \$strConfigFile,
|
||||
"stanza=s" => \$strStanza,
|
||||
@ -87,11 +87,8 @@ GetOptions ("config=s" => \$strConfigFile,
|
||||
"test-delay=s" => \$iTestDelay)
|
||||
or confess("Error in command line arguments\n");
|
||||
|
||||
# Test delay should be between 1 and 600 seconds
|
||||
if (!($iTestDelay >= 1 && $iTestDelay <= 600))
|
||||
{
|
||||
confess &log(ERROR, 'test-delay must be between 1 and 600 seconds');
|
||||
}
|
||||
# Set test parameters
|
||||
test_set($bTest, $iTestDelay);
|
||||
|
||||
####################################################################################################################################
|
||||
# Global variables
|
||||
@ -99,7 +96,6 @@ if (!($iTestDelay >= 1 && $iTestDelay <= 600))
|
||||
my %oConfig; # Configuration hash
|
||||
my $oRemote; # Remote object
|
||||
my $strRemote; # Defines which side is remote, DB or BACKUP
|
||||
my %oConfig; # Config file loaded into a hash
|
||||
|
||||
####################################################################################################################################
|
||||
# CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false)
|
||||
@ -213,6 +209,11 @@ $SIG{TERM} = \&safe_exit;
|
||||
$SIG{HUP} = \&safe_exit;
|
||||
$SIG{INT} = \&safe_exit;
|
||||
|
||||
####################################################################################################################################
|
||||
# START EVAL BLOCK TO CATCH ERRORS AND STOP THREADS
|
||||
####################################################################################################################################
|
||||
eval {
|
||||
|
||||
####################################################################################################################################
|
||||
# START MAIN
|
||||
####################################################################################################################################
|
||||
@ -643,3 +644,14 @@ if ($strOperation eq OP_EXPIRE)
|
||||
}
|
||||
|
||||
remote_exit(0);
|
||||
|
||||
####################################################################################################################################
|
||||
# START EVAL BLOCK TO CATCH ERRORS AND STOP THREADS
|
||||
####################################################################################################################################
|
||||
};
|
||||
|
||||
if ($@)
|
||||
{
|
||||
remote_exit();
|
||||
confess $@;
|
||||
}
|
||||
|
||||
@ -4,10 +4,10 @@
|
||||
package BackRest::Backup;
|
||||
|
||||
use threads;
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Carp;
|
||||
|
||||
use File::Basename;
|
||||
use File::Path qw(remove_tree);
|
||||
use Scalar::Util qw(looks_like_number);
|
||||
@ -35,8 +35,6 @@ my $iThreadThreshold = 10;
|
||||
my $iSmallFileThreshold = 65536;
|
||||
my $bArchiveRequired;
|
||||
my $iThreadTimeout;
|
||||
my $bTest;
|
||||
my $iTestDelay;
|
||||
|
||||
# Thread variables
|
||||
my @oThread;
|
||||
@ -58,8 +56,6 @@ sub backup_init
|
||||
my $iThreadMaxParam = shift;
|
||||
my $bArchiveRequiredParam = shift;
|
||||
my $iThreadTimeoutParam = shift;
|
||||
my $bTestParam = shift;
|
||||
my $iTestDelayParam = shift;
|
||||
|
||||
$oDb = $oDbParam;
|
||||
$oFile = $oFileParam;
|
||||
@ -70,14 +66,6 @@ sub backup_init
|
||||
$iThreadMax = $iThreadMaxParam;
|
||||
$bArchiveRequired = $bArchiveRequiredParam;
|
||||
$iThreadTimeout = $iThreadTimeoutParam;
|
||||
$bTest = defined($bTestParam) ? $bTestParam : false;
|
||||
$iTestDelay = defined($bTestParam) ? $iTestDelayParam : undef;
|
||||
|
||||
# Make sure that a delay is specified in test mode
|
||||
if ($bTest && !defined($iTestDelay))
|
||||
{
|
||||
confess &log(ASSERT, "iTestDelay must be provided when bTest is true");
|
||||
}
|
||||
|
||||
if (!defined($iThreadMax))
|
||||
{
|
||||
@ -1364,6 +1352,7 @@ sub backup
|
||||
$oDb->tablespace_map_get(\%oTablespaceMap);
|
||||
|
||||
backup_manifest_build($oFile->{strCommandManifest}, $strDbClusterPath, \%oBackupManifest, \%oLastManifest, \%oTablespaceMap);
|
||||
&log(TEST, TEST_MANIFEST_BUILD);
|
||||
|
||||
# If the backup tmp path already exists, remove invalid files
|
||||
if (-e $strBackupTmpPath)
|
||||
|
||||
@ -4,11 +4,11 @@
|
||||
package BackRest::Db;
|
||||
|
||||
use threads;
|
||||
|
||||
use Moose;
|
||||
use strict;
|
||||
use warnings;
|
||||
use Carp;
|
||||
|
||||
use Moose;
|
||||
use Net::OpenSSH;
|
||||
use File::Basename;
|
||||
use IPC::System::Simple qw(capture);
|
||||
|
||||
@ -4,10 +4,10 @@
|
||||
package BackRest::Utility;
|
||||
|
||||
use threads;
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use Carp;
|
||||
|
||||
use Fcntl qw(:DEFAULT :flock);
|
||||
use File::Path qw(remove_tree);
|
||||
use File::Basename;
|
||||
@ -20,10 +20,11 @@ use Exporter qw(import);
|
||||
|
||||
our @EXPORT = qw(version_get
|
||||
data_hash_build trim common_prefix wait_for_file date_string_get file_size_format execute
|
||||
log log_file_set log_level_set
|
||||
log log_file_set log_level_set test_set
|
||||
lock_file_create lock_file_remove
|
||||
config_save config_load
|
||||
TRACE DEBUG ERROR ASSERT WARN INFO OFF true false);
|
||||
TRACE DEBUG ERROR ASSERT WARN INFO OFF true false
|
||||
TEST TEST_ENCLOSE TEST_MANIFEST_BUILD);
|
||||
|
||||
# Global constants
|
||||
use constant
|
||||
@ -59,6 +60,20 @@ $oLogLevelRank{ERROR}{rank} = 2;
|
||||
$oLogLevelRank{ASSERT}{rank} = 1;
|
||||
$oLogLevelRank{OFF}{rank} = 0;
|
||||
|
||||
####################################################################################################################################
|
||||
# TEST Constants and Variables
|
||||
####################################################################################################################################
|
||||
use constant
|
||||
{
|
||||
TEST => 'TEST',
|
||||
TEST_ENCLOSE => 'PgBaCkReStTeSt',
|
||||
TEST_MANIFEST_BUILD => 'MANIFEST_BUILD'
|
||||
};
|
||||
|
||||
# Test global variables
|
||||
my $bTest = false;
|
||||
my $iTestDelay = 2;
|
||||
|
||||
####################################################################################################################################
|
||||
# VERSION_GET
|
||||
####################################################################################################################################
|
||||
@ -321,6 +336,30 @@ sub log_file_set
|
||||
print $hLogFile "-------------------PROCESS START-------------------\n";
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# TEST_SET - set test parameters
|
||||
####################################################################################################################################
|
||||
sub test_set
|
||||
{
|
||||
my $bTestParam = shift;
|
||||
my $iTestDelayParam = shift;
|
||||
|
||||
$bTest = defined($bTestParam) ? $bTestParam : false;
|
||||
$iTestDelay = defined($bTestParam) ? $iTestDelayParam : $iTestDelay;
|
||||
|
||||
# Test delay should be between 1 and 600 seconds
|
||||
if (!($iTestDelay >= 1 && $iTestDelay <= 600))
|
||||
{
|
||||
confess &log(ERROR, 'test-delay must be between 1 and 600 seconds');
|
||||
}
|
||||
|
||||
# Make sure that a delay is specified in test mode
|
||||
if ($bTest && !defined($iTestDelay))
|
||||
{
|
||||
confess &log(ASSERT, "iTestDelay must be provided when bTest is true");
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# LOG_LEVEL_SET - set the log level for file and console
|
||||
####################################################################################################################################
|
||||
@ -361,7 +400,11 @@ sub log
|
||||
|
||||
my $strMessageFormat = $strMessage;
|
||||
|
||||
if (!defined($oLogLevelRank{"${strLevel}"}{rank}))
|
||||
if ($bTest && $strLevel eq TEST)
|
||||
{
|
||||
$strMessageFormat = TEST_ENCLOSE . '-' . $strMessageFormat . '-' . TEST_ENCLOSE;
|
||||
}
|
||||
elsif (!defined($oLogLevelRank{"${strLevel}"}{rank}))
|
||||
{
|
||||
confess &log(ASSERT, "log level ${strLevel} does not exist");
|
||||
}
|
||||
@ -393,7 +436,8 @@ sub log
|
||||
(" " x (7 - length($strLevel))) . "${strLevel}: ${strMessageFormat}" .
|
||||
(defined($iCode) ? " (code ${iCode})" : "") . "\n";
|
||||
|
||||
if ($oLogLevelRank{"${strLevel}"}{rank} <= $oLogLevelRank{"${strLogLevelConsole}"}{rank})
|
||||
if ($oLogLevelRank{"${strLevel}"}{rank} <= $oLogLevelRank{"${strLogLevelConsole}"}{rank} ||
|
||||
$bTest && $strLevel eq TEST)
|
||||
{
|
||||
print $strMessageFormat;
|
||||
}
|
||||
@ -411,6 +455,11 @@ sub log
|
||||
return BackRest::Exception->new(iCode => $iCode, strMessage => $strMessage);
|
||||
}
|
||||
|
||||
if ($bTest && $strLevel eq TEST)
|
||||
{
|
||||
sleep($iTestDelay);
|
||||
}
|
||||
|
||||
return $strMessage;
|
||||
}
|
||||
|
||||
|
||||
@ -499,7 +499,7 @@ sub BackRestTestBackup_Test
|
||||
|
||||
my $strCommand = BackRestTestCommon_CommandMainGet() . ' --config=' .
|
||||
($bRemote ? BackRestTestCommon_BackupPathGet() : BackRestTestCommon_DbPathGet()) .
|
||||
"/pg_backrest.conf --type=incr --stanza=${strStanza} backup";
|
||||
"/pg_backrest.conf --test --type=incr --stanza=${strStanza} backup";
|
||||
|
||||
BackRestTestCommon_Execute($strCommand, $bRemote);
|
||||
# exit 0;
|
||||
|
||||
@ -99,80 +99,53 @@ sub BackRestTestCommon_Execute
|
||||
$strCommand = "ssh ${strCommonUserBackRest}\@${strCommonHost} '${strCommand}'";
|
||||
}
|
||||
|
||||
# system($strCommand);
|
||||
# my $strError = '';
|
||||
# Create error and out file handles and buffers
|
||||
my $strErrorLog = '';
|
||||
my $hError;
|
||||
# open($hError, '>', BackRestTestCommon_TestPathGet() . '/stderr.log');# or confess "unable to open handle to stderr string: $!\n";
|
||||
# open($hError, '>', \$strError) or confess "unable to open handle to stderr string: $!\n";
|
||||
|
||||
# my $strOut = '';
|
||||
my $strOutLog = '';
|
||||
my $hOut;
|
||||
# open($hOut, '>', BackRestTestCommon_TestPathGet() . '/stdout.log');# or confess "unable to open handle to stderr string: $!\n";
|
||||
# open($hOut, '>', \$strOut) or confess "unable to open handle to stdout string: $!\n";
|
||||
|
||||
# Execute the command
|
||||
my $pId = open3(undef, $hOut, $hError, $strCommand);
|
||||
|
||||
# Create select objects
|
||||
my $oErrorSelect = IO::Select->new();
|
||||
$oErrorSelect->add($hError);
|
||||
my $oOutSelect = IO::Select->new();
|
||||
$oOutSelect->add($hOut);
|
||||
|
||||
# Wait for the process to finish and report any errors
|
||||
# my $iExitStatus;
|
||||
|
||||
# waitpid($pId, 0);
|
||||
|
||||
# While the process is running drain the stdout and stderr streams
|
||||
while(waitpid($pId, WNOHANG) == 0)
|
||||
{
|
||||
# print "stuck here\n";
|
||||
|
||||
# Drain the stderr stream
|
||||
if ($oErrorSelect->can_read(.1))
|
||||
{
|
||||
# print "read err\n";
|
||||
|
||||
while (my $strLine = readline($hError))
|
||||
{
|
||||
# print "out: ${strLine}";
|
||||
$strErrorLog .= $strLine;
|
||||
}
|
||||
}
|
||||
|
||||
# Drain the stdout stream
|
||||
if ($oOutSelect->can_read(.1))
|
||||
{
|
||||
# print "read out begin\n";
|
||||
|
||||
while (my $strLine = readline($hOut))
|
||||
{
|
||||
# print "out: ${strLine}";
|
||||
$strOutLog .= $strLine;
|
||||
}
|
||||
|
||||
# print "read out end\n";
|
||||
}
|
||||
}
|
||||
|
||||
# print "got out\n";
|
||||
|
||||
# Check the exit status and output an error if needed
|
||||
my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
|
||||
# $iExitStatus = $iExitStatus >> 8;
|
||||
|
||||
if ($iExitStatus != 0 && !$bSuppressError)
|
||||
{
|
||||
print "${strErrorLog}";
|
||||
|
||||
confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus);
|
||||
confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus .
|
||||
($strOutLog ne '' ? "STDOUT:\n${strOutLog}" : '') .
|
||||
($strErrorLog ne '' ? "STDERR:\n${strErrorLog}" : ''));
|
||||
}
|
||||
|
||||
# print "${strOutLog}\n";
|
||||
|
||||
# close($hError);
|
||||
# close($hOut);
|
||||
|
||||
# while (my $strLine = readline($hOut))
|
||||
# {
|
||||
# print $strLine;
|
||||
# }
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
@ -220,10 +193,8 @@ sub BackRestTestCommon_ConfigCreate
|
||||
my $iThreadMax = shift;
|
||||
my $bArchiveLocal = shift;
|
||||
my $bCompressAsync = shift;
|
||||
# my $oParamHashRef = shift;
|
||||
|
||||
my %oParamHash;
|
||||
# tie %oParamHash, 'Config::IniFiles';
|
||||
|
||||
if (defined($strRemote))
|
||||
{
|
||||
@ -243,7 +214,7 @@ sub BackRestTestCommon_ConfigCreate
|
||||
$oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
|
||||
}
|
||||
|
||||
$oParamHash{'global:log'}{'level-console'} = 'trace';
|
||||
$oParamHash{'global:log'}{'level-console'} = 'error';
|
||||
$oParamHash{'global:log'}{'level-file'} = 'trace';
|
||||
|
||||
if ($strLocal eq REMOTE_BACKUP)
|
||||
@ -298,14 +269,6 @@ sub BackRestTestCommon_ConfigCreate
|
||||
$oParamHash{'global:backup'}{'thread-max'} = $iThreadMax;
|
||||
}
|
||||
|
||||
# foreach my $strSection (keys $oParamHashRef)
|
||||
# {
|
||||
# foreach my $strKey (keys ${$oParamHashRef}{$strSection})
|
||||
# {
|
||||
# $oParamHash{$strSection}{$strKey} = ${$oParamHashRef}{$strSection}{$strKey};
|
||||
# }
|
||||
# }
|
||||
|
||||
# Write out the configuration file
|
||||
my $strFile = BackRestTestCommon_TestPathGet() . '/pg_backrest.conf';
|
||||
config_save($strFile, \%oParamHash);
|
||||
|
||||
@ -78,17 +78,6 @@ sub BackRestTestUtility_Test
|
||||
# Create the test directory
|
||||
if ($bCreate)
|
||||
{
|
||||
# $oFile = BackRest::File->new();
|
||||
|
||||
# # Create the file object
|
||||
# $oFile = (BackRest::File->new
|
||||
# (
|
||||
# # strStanza => $strStanza,
|
||||
# # strBackupPath => BackRestTestCommon_BackupPathGet(),
|
||||
# # strRemote => $bRemote ? 'backup' : undef,
|
||||
# # oRemote => $bRemote ? $oRemote : undef
|
||||
# ))->clone();
|
||||
|
||||
BackRestTestUtility_Create();
|
||||
|
||||
$bCreate = false;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user