You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-11-06 08:49:29 +02:00
v0.60: better version support and WAL improvements
* Pushing duplicate WAL now generates an error. This worked before only if checksums were disabled. * Database System IDs are used to make sure that all WAL in an archive matches up. This should help prevent misconfigurations that send WAL from multiple clusters to the same archive. * Regression tests working back to PostgreSQL 8.3. * Improved threading model by starting threads early and terminating them late.
This commit is contained in:
David Steele
@@ -25,11 +25,13 @@ use BackRest::Config;
|
||||
use BackRest::Manifest;
|
||||
use BackRest::File;
|
||||
use BackRest::Remote;
|
||||
use BackRest::Archive;
|
||||
|
||||
use BackRestTest::CommonTest;
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw(BackRestTestBackup_Test);
|
||||
our @EXPORT = qw(BackRestTestBackup_Test BackRestTestBackup_Create BackRestTestBackup_Drop BackRestTestBackup_ClusterStop
|
||||
BackRestTestBackup_PgSelectOne BackRestTestBackup_PgExecute);
|
||||
|
||||
my $strTestPath;
|
||||
my $strHost;
|
||||
@@ -43,15 +45,43 @@ my $hDb;
|
||||
####################################################################################################################################
|
||||
sub BackRestTestBackup_PgConnect
|
||||
{
|
||||
my $iWaitSeconds = shift;
|
||||
|
||||
# Disconnect user session
|
||||
BackRestTestBackup_PgDisconnect();
|
||||
|
||||
# Connect to the db (whether it is local or remote)
|
||||
$hDb = DBI->connect('dbi:Pg:dbname=postgres;port=' . BackRestTestCommon_DbPortGet .
|
||||
';host=' . BackRestTestCommon_DbPathGet(),
|
||||
BackRestTestCommon_UserGet(),
|
||||
undef,
|
||||
{AutoCommit => 0, RaiseError => 1});
|
||||
# Default
|
||||
$iWaitSeconds = defined($iWaitSeconds) ? $iWaitSeconds : 30;
|
||||
|
||||
# Record the start time
|
||||
my $lTime = time();
|
||||
|
||||
do
|
||||
{
|
||||
# Connect to the db (whether it is local or remote)
|
||||
eval
|
||||
{
|
||||
$hDb = DBI->connect('dbi:Pg:dbname=postgres;port=' . BackRestTestCommon_DbPortGet .
|
||||
';host=' . BackRestTestCommon_DbPathGet(),
|
||||
BackRestTestCommon_UserGet(),
|
||||
undef,
|
||||
{AutoCommit => 0, RaiseError => 1});
|
||||
};
|
||||
|
||||
if (!$@)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
# If waiting then sleep before trying again
|
||||
if (defined($iWaitSeconds))
|
||||
{
|
||||
hsleep(.1);
|
||||
}
|
||||
}
|
||||
while ($lTime > time() - $iWaitSeconds);
|
||||
|
||||
confess &log(ERROR, "unable to connect to Postgres after ${iWaitSeconds} second(s)");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
@@ -195,7 +225,7 @@ sub BackRestTestBackup_ClusterStop
|
||||
BackRestTestBackup_PgDisconnect();
|
||||
|
||||
# Drop the cluster
|
||||
BackRestTestCommon_ClusterStop
|
||||
BackRestTestCommon_ClusterStop($strPath, $bImmediate);
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
@@ -206,11 +236,13 @@ sub BackRestTestBackup_ClusterStart
|
||||
my $strPath = shift;
|
||||
my $iPort = shift;
|
||||
my $bHotStandby = shift;
|
||||
my $bArchive = shift;
|
||||
|
||||
# Set default
|
||||
$iPort = defined($iPort) ? $iPort : BackRestTestCommon_DbPortGet();
|
||||
$strPath = defined($strPath) ? $strPath : BackRestTestCommon_DbCommonPathGet();
|
||||
$bHotStandby = defined($bHotStandby) ? $bHotStandby : false;
|
||||
$bArchive = defined($bArchive) ? $bArchive : true;
|
||||
|
||||
# Make sure postgres is not running
|
||||
if (-e $strPath . '/postmaster.pid')
|
||||
@@ -223,12 +255,38 @@ sub BackRestTestBackup_ClusterStart
|
||||
' --config=' . BackRestTestCommon_DbPathGet() . '/pg_backrest.conf archive-push %p';
|
||||
|
||||
# Start the cluster
|
||||
BackRestTestCommon_Execute(BackRestTestCommon_PgSqlBinPathGet() . "/pg_ctl start -o \"-c port=${iPort}" .
|
||||
' -c checkpoint_segments=1' .
|
||||
" -c wal_level=hot_standby -c archive_mode=on -c archive_command='${strArchive}'" .
|
||||
($bHotStandby ? ' -c hot_standby=on' : '') .
|
||||
" -c unix_socket_directories='" . BackRestTestCommon_DbPathGet() . "'\" " .
|
||||
"-D ${strPath} -l ${strPath}/postgresql.log -w -s");
|
||||
my $strCommand = BackRestTestCommon_PgSqlBinPathGet() . "/pg_ctl start -o \"-c port=${iPort}" .
|
||||
' -c checkpoint_segments=1';
|
||||
|
||||
if ($bArchive)
|
||||
{
|
||||
if (BackRestTestCommon_DbVersion() >= '8.3')
|
||||
{
|
||||
$strCommand .= " -c archive_mode=on";
|
||||
}
|
||||
|
||||
$strCommand .= " -c archive_command='${strArchive}'";
|
||||
|
||||
if (BackRestTestCommon_DbVersion() >= '9.0')
|
||||
{
|
||||
$strCommand .= " -c wal_level=hot_standby";
|
||||
|
||||
if ($bHotStandby)
|
||||
{
|
||||
$strCommand .= ' -c hot_standby=on';
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
$strCommand .= " -c archive_mode=on -c wal_level=archive -c archive_command=true";
|
||||
}
|
||||
|
||||
$strCommand .= " -c unix_socket_director" . (BackRestTestCommon_DbVersion() < '9.3' ? "y='" : "ies='") .
|
||||
BackRestTestCommon_DbPathGet() . "'\" " .
|
||||
"-D ${strPath} -l ${strPath}/postgresql.log -s";
|
||||
|
||||
BackRestTestCommon_Execute($strCommand);
|
||||
|
||||
# Connect user session
|
||||
BackRestTestBackup_PgConnect();
|
||||
@@ -261,10 +319,14 @@ sub BackRestTestBackup_ClusterCreate
|
||||
{
|
||||
my $strPath = shift;
|
||||
my $iPort = shift;
|
||||
my $bArchive = shift;
|
||||
|
||||
# Defaults
|
||||
$strPath = defined($strPath) ? $strPath : BackRestTestCommon_DbCommonPathGet();
|
||||
|
||||
BackRestTestCommon_Execute(BackRestTestCommon_PgSqlBinPathGet() . "/initdb -D ${strPath} -A trust");
|
||||
|
||||
BackRestTestBackup_ClusterStart($strPath, $iPort);
|
||||
BackRestTestBackup_ClusterStart($strPath, $iPort, undef, $bArchive);
|
||||
|
||||
# Connect user session
|
||||
BackRestTestBackup_PgConnect();
|
||||
@@ -302,6 +364,7 @@ sub BackRestTestBackup_Create
|
||||
{
|
||||
my $bRemote = shift;
|
||||
my $bCluster = shift;
|
||||
my $bArchive = shift;
|
||||
|
||||
# Set defaults
|
||||
$bRemote = defined($bRemote) ? $bRemote : false;
|
||||
@@ -331,20 +394,12 @@ sub BackRestTestBackup_Create
|
||||
BackRestTestCommon_PathCreate(BackRestTestCommon_LocalPathGet());
|
||||
}
|
||||
|
||||
# Create the backup directory
|
||||
if ($bRemote)
|
||||
{
|
||||
BackRestTestCommon_Execute('mkdir -m 700 ' . BackRestTestCommon_RepoPathGet(), true);
|
||||
}
|
||||
else
|
||||
{
|
||||
BackRestTestCommon_PathCreate(BackRestTestCommon_RepoPathGet());
|
||||
}
|
||||
BackRestTestCommon_CreateRepo($bRemote);
|
||||
|
||||
# Create the cluster
|
||||
if ($bCluster)
|
||||
{
|
||||
BackRestTestBackup_ClusterCreate(BackRestTestCommon_DbCommonPathGet(), BackRestTestCommon_DbPortGet());
|
||||
BackRestTestBackup_ClusterCreate(undef, undef, $bArchive);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1388,6 +1443,8 @@ sub BackRestTestBackup_Test
|
||||
$strHost, # Host
|
||||
$strUserBackRest, # User
|
||||
BackRestTestCommon_CommandRemoteGet(), # Command
|
||||
$strStanza, # Stanza
|
||||
'', # Repo Path
|
||||
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
|
||||
@@ -1398,6 +1455,8 @@ sub BackRestTestBackup_Test
|
||||
undef, # Host
|
||||
undef, # User
|
||||
undef, # Command
|
||||
undef, # Stanza
|
||||
undef, # Repo Path
|
||||
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
|
||||
@@ -1425,10 +1484,9 @@ sub BackRestTestBackup_Test
|
||||
"rmt ${bRemote}, cmp ${bCompress}, " .
|
||||
"arc_async ${bArchiveAsync}")) {next}
|
||||
|
||||
# Create the test directory
|
||||
# Create the file object
|
||||
if ($bCreate)
|
||||
{
|
||||
# Create the file object
|
||||
$oFile = (new BackRest::File
|
||||
(
|
||||
$strStanza,
|
||||
@@ -1437,11 +1495,12 @@ sub BackRestTestBackup_Test
|
||||
$bRemote ? $oRemote : $oLocal
|
||||
))->clone();
|
||||
|
||||
BackRestTestBackup_Create($bRemote, false);
|
||||
|
||||
$bCreate = false;
|
||||
}
|
||||
|
||||
# Create the test directory
|
||||
BackRestTestBackup_Create($bRemote, false);
|
||||
|
||||
BackRestTestCommon_ConfigCreate('db',
|
||||
($bRemote ? BACKUP : undef),
|
||||
$bCompress,
|
||||
@@ -1452,7 +1511,7 @@ sub BackRestTestBackup_Test
|
||||
undef);
|
||||
|
||||
my $strCommand = BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_DbPathGet() .
|
||||
'/pg_backrest.conf --stanza=db archive-push';
|
||||
'/pg_backrest.conf --no-fork --stanza=db archive-push';
|
||||
|
||||
# Loop through backups
|
||||
for (my $iBackup = 1; $iBackup <= 3; $iBackup++)
|
||||
@@ -1487,6 +1546,54 @@ sub BackRestTestBackup_Test
|
||||
|
||||
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}");
|
||||
|
||||
if ($iArchive == $iBackup)
|
||||
{
|
||||
# load the archive info file so it can be munged for testing
|
||||
my $strInfoFile = $oFile->path_get(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE);
|
||||
my %oInfo;
|
||||
BackRestTestCommon_iniLoad($strInfoFile, \%oInfo, $bRemote);
|
||||
my $strDbVersion = $oInfo{database}{version};
|
||||
my $ullDbSysId = $oInfo{database}{'system-id'};
|
||||
|
||||
# Break the database version
|
||||
$oInfo{database}{version} = '8.0';
|
||||
BackRestTestCommon_iniSave($strInfoFile, \%oInfo, $bRemote);
|
||||
|
||||
&log(INFO, ' test db version mismatch error');
|
||||
|
||||
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}", undef, undef, undef,
|
||||
ERROR_ARCHIVE_MISMATCH);
|
||||
|
||||
# Break the database version
|
||||
$oInfo{database}{version} = $strDbVersion;
|
||||
$oInfo{database}{'system-id'} = '5000900090001855000';
|
||||
BackRestTestCommon_iniSave($strInfoFile, \%oInfo, $bRemote);
|
||||
|
||||
&log(INFO, ' test db system-id mismatch error');
|
||||
|
||||
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}", undef, undef, undef,
|
||||
ERROR_ARCHIVE_MISMATCH);
|
||||
|
||||
# Move settings back to original
|
||||
$oInfo{database}{'system-id'} = $ullDbSysId;
|
||||
BackRestTestCommon_iniSave($strInfoFile, \%oInfo, $bRemote);
|
||||
|
||||
# Now it should break on archive duplication
|
||||
&log(INFO, ' test archive duplicate error');
|
||||
|
||||
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}", undef, undef, undef,
|
||||
ERROR_ARCHIVE_DUPLICATE);
|
||||
|
||||
if ($bArchiveAsync && $bRemote)
|
||||
{
|
||||
my $strDuplicateWal = BackRestTestCommon_LocalPathGet() . "/archive/${strStanza}/out/" .
|
||||
"${strArchiveFile}-1c7e00fd09b9dd11fc2966590b3e3274645dd031";
|
||||
|
||||
unlink ($strDuplicateWal)
|
||||
or confess "unable to remove duplicate WAL segment created for testing: ${strDuplicateWal}";
|
||||
}
|
||||
}
|
||||
|
||||
# Build the archive name to check for at the destination
|
||||
my $strArchiveCheck = "${strArchiveFile}-${strArchiveChecksum}";
|
||||
|
||||
@@ -1626,11 +1733,8 @@ sub BackRestTestBackup_Test
|
||||
}
|
||||
else
|
||||
{
|
||||
if (BackRestTestCommon_Execute($strCommand . " 000000090000000900000009 ${strXlogPath}/RECOVERYXLOG",
|
||||
false, true) != 1)
|
||||
{
|
||||
confess 'archive-get should return 1 when archive log is not present';
|
||||
}
|
||||
BackRestTestCommon_Execute($strCommand . " 000000090000000900000009 ${strXlogPath}/RECOVERYXLOG",
|
||||
undef, undef, undef, 1);
|
||||
}
|
||||
|
||||
$bCreate = true;
|
||||
@@ -2097,8 +2201,7 @@ sub BackRestTestBackup_Test
|
||||
# Create the test directory
|
||||
if ($bCreate)
|
||||
{
|
||||
BackRestTestBackup_Create($bRemote);
|
||||
$bCreate = false;
|
||||
BackRestTestBackup_Create($bRemote, false);
|
||||
}
|
||||
|
||||
# Create db config
|
||||
@@ -2124,6 +2227,13 @@ sub BackRestTestBackup_Test
|
||||
undef); # compress-async
|
||||
}
|
||||
|
||||
# Create the cluster
|
||||
if ($bCreate)
|
||||
{
|
||||
BackRestTestBackup_ClusterCreate();
|
||||
$bCreate = false;
|
||||
}
|
||||
|
||||
# Static backup parameters
|
||||
my $bSynthetic = false;
|
||||
my $fTestDelay = .1;
|
||||
@@ -2210,7 +2320,11 @@ sub BackRestTestBackup_Test
|
||||
|
||||
BackRestTestBackup_PgExecute("update test set message = '$strNameMessage'", false, true);
|
||||
BackRestTestBackup_PgSwitchXlog();
|
||||
BackRestTestBackup_PgExecute("select pg_create_restore_point('${strNameTarget}')", false, false);
|
||||
|
||||
if (BackRestTestCommon_DbVersion() >= 9.1)
|
||||
{
|
||||
BackRestTestBackup_PgExecute("select pg_create_restore_point('${strNameTarget}')", false, false);
|
||||
}
|
||||
|
||||
&log(INFO, " name target is ${strNameTarget}");
|
||||
|
||||
@@ -2233,7 +2347,7 @@ sub BackRestTestBackup_Test
|
||||
$strComment = 'postmaster running';
|
||||
$iExpectedExitStatus = ERROR_POSTMASTER_RUNNING;
|
||||
|
||||
BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
BackRestTestBackup_Restore($oFile, 'latest', $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
|
||||
@@ -2243,7 +2357,7 @@ sub BackRestTestBackup_Test
|
||||
$strComment = 'path not empty';
|
||||
$iExpectedExitStatus = ERROR_RESTORE_PATH_NOT_EMPTY;
|
||||
|
||||
BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
BackRestTestBackup_Restore($oFile, 'latest', $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
|
||||
@@ -2255,7 +2369,7 @@ sub BackRestTestBackup_Test
|
||||
$strComment = undef;
|
||||
$iExpectedExitStatus = undef;
|
||||
|
||||
BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
BackRestTestBackup_Restore($oFile, 'latest', $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
|
||||
@@ -2269,7 +2383,7 @@ sub BackRestTestBackup_Test
|
||||
$strType = RECOVERY_TYPE_XID;
|
||||
$strTarget = $strXidTarget;
|
||||
$bTargetExclusive = undef;
|
||||
$bTargetResume = true;
|
||||
$bTargetResume = BackRestTestCommon_DbVersion() >= 9.1 ? true : undef;
|
||||
$strTargetTimeline = undef;
|
||||
$oRecoveryHashRef = undef;
|
||||
$strComment = undef;
|
||||
@@ -2279,7 +2393,7 @@ sub BackRestTestBackup_Test
|
||||
|
||||
BackRestTestBackup_ClusterStop();
|
||||
|
||||
BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
BackRestTestBackup_Restore($oFile, $strIncrBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
|
||||
@@ -2313,7 +2427,7 @@ sub BackRestTestBackup_Test
|
||||
$oFile->move(PATH_ABSOLUTE, BackRestTestCommon_TestPathGet() . '/recovery.conf',
|
||||
PATH_ABSOLUTE, BackRestTestCommon_DbCommonPathGet() . '/recovery.conf');
|
||||
|
||||
BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
BackRestTestBackup_Restore($oFile, 'latest', $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
|
||||
@@ -2364,7 +2478,7 @@ sub BackRestTestBackup_Test
|
||||
|
||||
BackRestTestBackup_ClusterStop();
|
||||
|
||||
BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
BackRestTestBackup_Restore($oFile, $strIncrBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
|
||||
@@ -2373,52 +2487,57 @@ sub BackRestTestBackup_Test
|
||||
|
||||
# Restore (restore type = name)
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
$bDelta = true;
|
||||
$bForce = true;
|
||||
$strType = RECOVERY_TYPE_NAME;
|
||||
$strTarget = $strNameTarget;
|
||||
$bTargetExclusive = undef;
|
||||
$bTargetResume = undef;
|
||||
$strTargetTimeline = undef;
|
||||
$oRecoveryHashRef = undef;
|
||||
$strComment = undef;
|
||||
$iExpectedExitStatus = undef;
|
||||
if (BackRestTestCommon_DbVersion() >= 9.1)
|
||||
{
|
||||
$bDelta = true;
|
||||
$bForce = true;
|
||||
$strType = RECOVERY_TYPE_NAME;
|
||||
$strTarget = $strNameTarget;
|
||||
$bTargetExclusive = undef;
|
||||
$bTargetResume = undef;
|
||||
$strTargetTimeline = undef;
|
||||
$oRecoveryHashRef = undef;
|
||||
$strComment = undef;
|
||||
$iExpectedExitStatus = undef;
|
||||
|
||||
&log(INFO, " testing recovery type = ${strType}");
|
||||
&log(INFO, " testing recovery type = ${strType}");
|
||||
|
||||
BackRestTestBackup_ClusterStop();
|
||||
BackRestTestBackup_ClusterStop();
|
||||
|
||||
BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
BackRestTestBackup_Restore($oFile, 'latest', $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
|
||||
BackRestTestBackup_ClusterStart();
|
||||
BackRestTestBackup_PgSelectOneTest('select message from test', $strNameMessage);
|
||||
BackRestTestBackup_ClusterStart();
|
||||
BackRestTestBackup_PgSelectOneTest('select message from test', $strNameMessage);
|
||||
}
|
||||
|
||||
# Restore (restore type = default, timeline = 3)
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
$bDelta = true;
|
||||
$bForce = false;
|
||||
$strType = RECOVERY_TYPE_DEFAULT;
|
||||
$strTarget = undef;
|
||||
$bTargetExclusive = undef;
|
||||
$bTargetResume = undef;
|
||||
$strTargetTimeline = 3;
|
||||
$oRecoveryHashRef = {'standy-mode' => 'on'};
|
||||
$oRecoveryHashRef = undef;
|
||||
$strComment = undef;
|
||||
$iExpectedExitStatus = undef;
|
||||
if (BackRestTestCommon_DbVersion() >= 8.4)
|
||||
{
|
||||
$bDelta = true;
|
||||
$bForce = false;
|
||||
$strType = RECOVERY_TYPE_DEFAULT;
|
||||
$strTarget = undef;
|
||||
$bTargetExclusive = undef;
|
||||
$bTargetResume = undef;
|
||||
$strTargetTimeline = 3;
|
||||
$oRecoveryHashRef = BackRestTestCommon_DbVersion() >= 9.0 ? {'standby-mode' => 'on'} : undef;
|
||||
$strComment = undef;
|
||||
$iExpectedExitStatus = undef;
|
||||
|
||||
&log(INFO, " testing recovery type = ${strType}");
|
||||
&log(INFO, " testing recovery type = ${strType}");
|
||||
|
||||
BackRestTestBackup_ClusterStop();
|
||||
BackRestTestBackup_ClusterStop();
|
||||
|
||||
BackRestTestBackup_Restore($oFile, $strFullBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
BackRestTestBackup_Restore($oFile, $strIncrBackup, $strStanza, $bRemote, undef, undef, $bDelta, $bForce,
|
||||
$strType, $strTarget, $bTargetExclusive, $bTargetResume, $strTargetTimeline,
|
||||
$oRecoveryHashRef, $strComment, $iExpectedExitStatus);
|
||||
|
||||
BackRestTestBackup_ClusterStart(undef, undef, true);
|
||||
BackRestTestBackup_PgSelectOneTest('select message from test', $strTimelineMessage, 120);
|
||||
BackRestTestBackup_ClusterStart(undef, undef, true);
|
||||
BackRestTestBackup_PgSelectOneTest('select message from test', $strTimelineMessage, 120);
|
||||
}
|
||||
|
||||
$bCreate = true;
|
||||
}
|
||||
@@ -2449,8 +2568,8 @@ sub BackRestTestBackup_Test
|
||||
(
|
||||
$strStanza,
|
||||
BackRestTestCommon_RepoPathGet(),
|
||||
undef,
|
||||
undef
|
||||
NONE,
|
||||
$oLocal
|
||||
))->clone();
|
||||
|
||||
# Create the test database
|
||||
@@ -2576,8 +2695,8 @@ sub BackRestTestBackup_Test
|
||||
(
|
||||
$strStanza,
|
||||
BackRestTestCommon_RepoPathGet(),
|
||||
undef,
|
||||
undef
|
||||
NONE,
|
||||
$oLocal
|
||||
))->clone();
|
||||
|
||||
# Create the test database
|
||||
|
||||
@@ -21,9 +21,11 @@ use File::Copy qw(move);
|
||||
|
||||
use lib dirname($0) . '/../lib';
|
||||
use BackRest::Utility;
|
||||
use BackRest::Config;
|
||||
use BackRest::Remote;
|
||||
use BackRest::File;
|
||||
use BackRest::Manifest;
|
||||
use BackRest::Db;
|
||||
|
||||
use Exporter qw(import);
|
||||
our @EXPORT = qw(BackRestTestCommon_Create BackRestTestCommon_Drop BackRestTestCommon_Setup BackRestTestCommon_ExecuteBegin
|
||||
@@ -37,7 +39,8 @@ our @EXPORT = qw(BackRestTestCommon_Create BackRestTestCommon_Drop BackRestTestC
|
||||
BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_DataPathGet
|
||||
BackRestTestCommon_RepoPathGet BackRestTestCommon_LocalPathGet BackRestTestCommon_DbPathGet
|
||||
BackRestTestCommon_DbCommonPathGet BackRestTestCommon_ClusterStop BackRestTestCommon_DbTablespacePathGet
|
||||
BackRestTestCommon_DbPortGet);
|
||||
BackRestTestCommon_DbPortGet BackRestTestCommon_iniLoad BackRestTestCommon_iniSave BackRestTestCommon_DbVersion
|
||||
BackRestTestCommon_CommandPsqlGet BackRestTestCommon_DropRepo BackRestTestCommon_CreateRepo);
|
||||
|
||||
my $strPgSqlBin;
|
||||
my $strCommonStanza;
|
||||
@@ -56,6 +59,7 @@ my $strCommonDbPath;
|
||||
my $strCommonDbCommonPath;
|
||||
my $strCommonDbTablespacePath;
|
||||
my $iCommonDbPort;
|
||||
my $strCommonDbVersion;
|
||||
my $iModuleTestRun;
|
||||
my $bDryRun;
|
||||
my $bNoCleanup;
|
||||
@@ -68,7 +72,6 @@ my $hOut;
|
||||
my $pId;
|
||||
my $strCommand;
|
||||
|
||||
|
||||
####################################################################################################################################
|
||||
# BackRestTestCommon_ClusterStop
|
||||
####################################################################################################################################
|
||||
@@ -89,6 +92,41 @@ sub BackRestTestCommon_ClusterStop
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# BackRestTestCommon_DropRepo
|
||||
####################################################################################################################################
|
||||
sub BackRestTestCommon_DropRepo
|
||||
{
|
||||
# Remove the backrest private directory
|
||||
while (-e BackRestTestCommon_RepoPathGet())
|
||||
{
|
||||
BackRestTestCommon_PathRemove(BackRestTestCommon_RepoPathGet(), true, true);
|
||||
BackRestTestCommon_PathRemove(BackRestTestCommon_RepoPathGet(), false, true);
|
||||
hsleep(.1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
####################################################################################################################################
|
||||
# BackRestTestCommon_CreateRepo
|
||||
####################################################################################################################################
|
||||
sub BackRestTestCommon_CreateRepo
|
||||
{
|
||||
my $bRemote = shift;
|
||||
|
||||
BackRestTestCommon_DropRepo();
|
||||
|
||||
# Create the backup directory
|
||||
if ($bRemote)
|
||||
{
|
||||
BackRestTestCommon_Execute('mkdir -m 700 ' . BackRestTestCommon_RepoPathGet(), true);
|
||||
}
|
||||
else
|
||||
{
|
||||
BackRestTestCommon_PathCreate(BackRestTestCommon_RepoPathGet());
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# BackRestTestCommon_Drop
|
||||
####################################################################################################################################
|
||||
@@ -98,12 +136,7 @@ sub BackRestTestCommon_Drop
|
||||
BackRestTestCommon_ClusterStop(BackRestTestCommon_DbCommonPathGet(), true);
|
||||
|
||||
# Remove the backrest private directory
|
||||
while (-e BackRestTestCommon_RepoPathGet())
|
||||
{
|
||||
BackRestTestCommon_PathRemove(BackRestTestCommon_RepoPathGet(), true, true);
|
||||
BackRestTestCommon_PathRemove(BackRestTestCommon_RepoPathGet(), false, true);
|
||||
hsleep(.1);
|
||||
}
|
||||
BackRestTestCommon_DropRepo();
|
||||
|
||||
# Remove the test directory
|
||||
BackRestTestCommon_PathRemove(BackRestTestCommon_TestPathGet());
|
||||
@@ -468,6 +501,73 @@ sub BackRestTestCommon_Setup
|
||||
$iModuleTestRun = $iModuleTestRunParam;
|
||||
$bDryRun = $bDryRunParam;
|
||||
$bNoCleanup = $bNoCleanupParam;
|
||||
|
||||
BackRestTestCommon_Execute($strPgSqlBin . '/postgres --version');
|
||||
|
||||
# Get the Postgres version
|
||||
my @stryVersionToken = split(/ /, $strOutLog);
|
||||
@stryVersionToken = split(/\./, $stryVersionToken[2]);
|
||||
$strCommonDbVersion = $stryVersionToken[0] . '.' . $stryVersionToken[1];
|
||||
|
||||
# Don't run unit tests for unsupported versions
|
||||
my $strVersionSupport = versionSupport();
|
||||
|
||||
if ($strCommonDbVersion < ${$strVersionSupport}[0])
|
||||
{
|
||||
confess "currently only version ${$strVersionSupport}[0] and up are supported";
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# BackRestTestCommon_iniLoad
|
||||
####################################################################################################################################
|
||||
sub BackRestTestCommon_iniLoad
|
||||
{
|
||||
my $strFileName = shift;
|
||||
my $oIniRef = shift;
|
||||
my $bRemote = shift;
|
||||
|
||||
# Defaults
|
||||
$bRemote = defined($bRemote) ? $bRemote : false;
|
||||
|
||||
if ($bRemote)
|
||||
{
|
||||
BackRestTestCommon_Execute("chmod g+x " . BackRestTestCommon_RepoPathGet(), $bRemote);
|
||||
}
|
||||
|
||||
ini_load($strFileName, $oIniRef);
|
||||
|
||||
if ($bRemote)
|
||||
{
|
||||
BackRestTestCommon_Execute("chmod g-x " . BackRestTestCommon_RepoPathGet(), $bRemote);
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# BackRestTestCommon_iniSave
|
||||
####################################################################################################################################
|
||||
sub BackRestTestCommon_iniSave
|
||||
{
|
||||
my $strFileName = shift;
|
||||
my $oIniRef = shift;
|
||||
my $bRemote = shift;
|
||||
|
||||
# Defaults
|
||||
$bRemote = defined($bRemote) ? $bRemote : false;
|
||||
|
||||
if ($bRemote)
|
||||
{
|
||||
BackRestTestCommon_Execute("chmod g+x " . BackRestTestCommon_RepoPathGet(), $bRemote);
|
||||
BackRestTestCommon_Execute("chmod g+w " . $strFileName, $bRemote);
|
||||
}
|
||||
|
||||
ini_save($strFileName, $oIniRef);
|
||||
|
||||
if ($bRemote)
|
||||
{
|
||||
BackRestTestCommon_Execute("chmod g-w " . $strFileName, $bRemote);
|
||||
BackRestTestCommon_Execute("chmod g-x " . BackRestTestCommon_RepoPathGet(), $bRemote);
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
@@ -562,11 +662,11 @@ sub BackRestTestCommon_ConfigRecovery
|
||||
}
|
||||
|
||||
# Rewrite remap section
|
||||
delete($oConfig{"${strStanza}:recovery:option"});
|
||||
delete($oConfig{"${strStanza}:restore:recovery-setting"});
|
||||
|
||||
foreach my $strOption (sort(keys $oRecoveryHashRef))
|
||||
{
|
||||
$oConfig{"${strStanza}:recovery:option"}{$strOption} = ${$oRecoveryHashRef}{$strOption};
|
||||
$oConfig{"${strStanza}:restore:recovery-setting"}{$strOption} = ${$oRecoveryHashRef}{$strOption};
|
||||
}
|
||||
|
||||
# Resave the config file
|
||||
@@ -614,7 +714,7 @@ sub BackRestTestCommon_ConfigCreate
|
||||
$oParamHash{$strCommonStanza}{'db-user'} = $strCommonUser;
|
||||
}
|
||||
|
||||
$oParamHash{'global:log'}{'log-level-console'} = 'error';
|
||||
$oParamHash{'global:log'}{'log-level-console'} = 'debug';
|
||||
$oParamHash{'global:log'}{'log-level-file'} = 'trace';
|
||||
|
||||
if ($strLocal eq BACKUP)
|
||||
@@ -627,7 +727,7 @@ sub BackRestTestCommon_ConfigCreate
|
||||
|
||||
if (defined($strRemote))
|
||||
{
|
||||
$oParamHash{'global:log'}{'log-level-console'} = 'trace';
|
||||
# $oParamHash{'global:log'}{'log-level-console'} = 'trace';
|
||||
|
||||
# if ($bArchiveAsync)
|
||||
# {
|
||||
@@ -718,6 +818,11 @@ sub BackRestTestCommon_StanzaGet
|
||||
return $strCommonStanza;
|
||||
}
|
||||
|
||||
sub BackRestTestCommon_CommandPsqlGet
|
||||
{
|
||||
return $strCommonCommandPsql;
|
||||
}
|
||||
|
||||
sub BackRestTestCommon_CommandMainGet
|
||||
{
|
||||
return $strCommonCommandMain;
|
||||
@@ -793,4 +898,9 @@ sub BackRestTestCommon_DbPortGet
|
||||
return $iCommonDbPort;
|
||||
}
|
||||
|
||||
sub BackRestTestCommon_DbVersion
|
||||
{
|
||||
return $strCommonDbVersion;
|
||||
}
|
||||
|
||||
1;
|
||||
|
||||
127
test/lib/BackRestTest/CompareTest.pm
Executable file
127
test/lib/BackRestTest/CompareTest.pm
Executable file
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/perl
|
||||
####################################################################################################################################
|
||||
# CompareTest.pl - Performance comparison tests between rsync and backrest
|
||||
####################################################################################################################################
|
||||
package BackRestTest::CompareTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# Perl includes
|
||||
####################################################################################################################################
|
||||
use strict;
|
||||
use warnings FATAL => qw(all);
|
||||
use Carp qw(confess);
|
||||
|
||||
use File::Basename qw(dirname);
|
||||
use Time::HiRes qw(gettimeofday);
|
||||
use File::stat;
|
||||
use Exporter qw(import);
|
||||
|
||||
use lib dirname($0) . '/../lib';
|
||||
use BackRest::Utility;
|
||||
use BackRestTest::CommonTest;
|
||||
use BackRestTest::BackupTest;
|
||||
|
||||
####################################################################################################################################
|
||||
# Exports
|
||||
####################################################################################################################################
|
||||
our @EXPORT = qw(BackRestTestCompare_Test);
|
||||
|
||||
####################################################################################################################################
|
||||
# BackRestTestCompare_BuildDb
|
||||
####################################################################################################################################
|
||||
sub BackRestTestCompare_BuildDb
|
||||
{
|
||||
my $iTableTotal = shift;
|
||||
my $iTableSize = shift;
|
||||
|
||||
&log(INFO, "build database: " . file_size_format($iTableTotal * $iTableSize * 1024 * 1024));
|
||||
|
||||
for (my $iTableIdx = 0; $iTableIdx < $iTableTotal; $iTableIdx++)
|
||||
{
|
||||
my $strSourceFile = BackRestTestCommon_DataPathGet() . "/test.table.bin";
|
||||
my $strTableFile = BackRestTestCommon_DbCommonPathGet() . "/test-${iTableIdx}";
|
||||
|
||||
for (my $iTableSizeIdx = 0; $iTableSizeIdx < $iTableSize; $iTableSizeIdx++)
|
||||
{
|
||||
BackRestTestCommon_Execute("cat ${strSourceFile} >> ${strTableFile}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# BackRestTestCompare_Test
|
||||
####################################################################################################################################
|
||||
sub BackRestTestCompare_Test
|
||||
{
|
||||
my $strTest = shift;
|
||||
|
||||
#-------------------------------------------------------------------------------------------------------------------------------
|
||||
# Test rsync
|
||||
#-------------------------------------------------------------------------------------------------------------------------------
|
||||
if ($strTest eq 'all' || $strTest eq 'rsync')
|
||||
{
|
||||
my $iRun = 0;
|
||||
my $bRemote = false;
|
||||
|
||||
&log(INFO, "Test rsync\n");
|
||||
|
||||
# Increment the run, log, and decide whether this unit test should be run
|
||||
if (!BackRestTestCommon_Run(++$iRun,
|
||||
"rmt ${bRemote}")) {next}
|
||||
|
||||
# Create the cluster and paths
|
||||
BackRestTestBackup_Create($bRemote, false);
|
||||
BackRestTestCommon_PathCreate(BackRestTestCommon_DbCommonPathGet() . '/pg_tblspc');
|
||||
|
||||
BackRestTestCompare_BuildDb(48, 10);
|
||||
BackRestTestCommon_Execute('sync');
|
||||
|
||||
for (my $bRemote = true; $bRemote <= true; $bRemote++)
|
||||
{
|
||||
for (my $bRsync = true; $bRsync >= false; $bRsync--)
|
||||
{
|
||||
my $strCommand;
|
||||
BackRestTestCommon_CreateRepo($bRemote);
|
||||
|
||||
&log(INFO, ($bRsync ? 'rsync' : 'backrest') . " test");
|
||||
|
||||
if ($bRsync)
|
||||
{
|
||||
$strCommand = 'rsync --compress-level=6 -zvlhprtogHS --delete ' .
|
||||
($bRemote ? BackRestTestCommon_UserGet . '@' . BackRestTestCommon_HostGet . ':' : '') .
|
||||
BackRestTestCommon_DbCommonPathGet() . '/ ' . BackRestTestCommon_RepoPathGet() . ';' .
|
||||
'gzip -r "' . BackRestTestCommon_RepoPathGet() . '"';
|
||||
}
|
||||
else
|
||||
{
|
||||
$strCommand = BackRestTestCommon_CommandMainGet() .
|
||||
' --stanza=main' .
|
||||
($bRemote ? ' "--db-host=' . BackRestTestCommon_HostGet . '"' .
|
||||
' "--db-user=' . BackRestTestCommon_UserGet . '"' : '') .
|
||||
# ' --log-level-file=debug' .
|
||||
' --no-start-stop' .
|
||||
# ' --no-compress' .
|
||||
' --thread-max=4' .
|
||||
' "--db-path=' . BackRestTestCommon_DbCommonPathGet() . '"' .
|
||||
' "--repo-path=' . BackRestTestCommon_RepoPathGet() . '"' .
|
||||
' --type=full backup';
|
||||
}
|
||||
|
||||
my $fTimeBegin = gettimeofday();
|
||||
BackRestTestCommon_Execute($strCommand, $bRemote);
|
||||
BackRestTestCommon_Execute('sync');
|
||||
my $fTimeEnd = gettimeofday();
|
||||
|
||||
&log(INFO, " time = " . (int(($fTimeEnd - $fTimeBegin) * 100) / 100));
|
||||
}
|
||||
}
|
||||
|
||||
if (BackRestTestCommon_Cleanup())
|
||||
{
|
||||
&log(INFO, 'cleanup');
|
||||
BackRestTestBackup_Drop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
1;
|
||||
@@ -507,6 +507,25 @@ sub BackRestTestConfig_Test
|
||||
optionTestExpect(OPTION_RESTORE_RECOVERY_SETTING, 'db.domain.net', 'primary-conn-info');
|
||||
}
|
||||
|
||||
if (BackRestTestCommon_Run(++$iRun, OP_RESTORE . ' values passed to ' . OP_ARCHIVE_GET))
|
||||
{
|
||||
optionSetTest($oOption, OPTION_STANZA, $strStanza);
|
||||
optionSetTest($oOption, OPTION_DB_PATH, '/db path/main');
|
||||
optionSetTest($oOption, OPTION_REPO_PATH, '/repo');
|
||||
optionSetTest($oOption, OPTION_BACKUP_HOST, 'db.mydomain.com');
|
||||
|
||||
configLoadExpect($oOption, OP_RESTORE);
|
||||
|
||||
my $strCommand = operationWrite(OP_ARCHIVE_GET);
|
||||
my $strExpectedCommand = "$0 --backup-host=db.mydomain.com \"--db-path=/db path/main\"" .
|
||||
" --repo-path=/repo --stanza=main " . OP_ARCHIVE_GET;
|
||||
|
||||
if ($strCommand ne $strExpectedCommand)
|
||||
{
|
||||
confess "expected command '${strExpectedCommand}' but got '${strCommand}'";
|
||||
}
|
||||
}
|
||||
|
||||
if (BackRestTestCommon_Run(++$iRun, OP_BACKUP . ' valid value ' . OPTION_COMMAND_PSQL))
|
||||
{
|
||||
optionSetTest($oOption, OPTION_STANZA, $strStanza);
|
||||
@@ -749,6 +768,18 @@ sub BackRestTestConfig_Test
|
||||
optionTestExpect(OPTION_RESTORE_RECOVERY_SETTING, '/path/to/pg_backrest.pl', 'archive-command');
|
||||
}
|
||||
|
||||
if (BackRestTestCommon_Run(++$iRun, OP_RESTORE . ' option ' . OPTION_RESTORE_RECOVERY_SETTING))
|
||||
{
|
||||
$oConfig = {};
|
||||
$$oConfig{$strStanza . ':' . &CONFIG_SECTION_RESTORE_RECOVERY_SETTING}{'standby-mode'} = 'on';
|
||||
ini_save($strConfigFile, $oConfig);
|
||||
|
||||
optionSetTest($oOption, OPTION_STANZA, $strStanza);
|
||||
optionSetTest($oOption, OPTION_CONFIG, $strConfigFile);
|
||||
|
||||
configLoadExpect($oOption, OP_ARCHIVE_GET);
|
||||
}
|
||||
|
||||
if (BackRestTestCommon_Run(++$iRun, OP_BACKUP . ' option ' . OPTION_DB_PATH))
|
||||
{
|
||||
$oConfig = {};
|
||||
|
||||
@@ -97,6 +97,8 @@ sub BackRestTestFile_Test
|
||||
$strHost, # Host
|
||||
$strUser, # User
|
||||
BackRestTestCommon_CommandRemoteGet(), # Command
|
||||
$strStanza, # Stanza
|
||||
'', # Repo Path
|
||||
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
|
||||
@@ -107,6 +109,8 @@ sub BackRestTestFile_Test
|
||||
undef, # Host
|
||||
undef, # User
|
||||
undef, # Command
|
||||
undef, # Stanza
|
||||
undef, # Repo Path
|
||||
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
|
||||
@@ -455,7 +459,7 @@ sub BackRestTestFile_Test
|
||||
&log(DEBUG, "begin ${lTimeBegin}, check ${lTimeBeginCheck}, end " . time());
|
||||
|
||||
# Current time should have advanced by 1 second
|
||||
if (time() == int($lTimeBegin))
|
||||
if (int(time()) == int($lTimeBegin))
|
||||
{
|
||||
confess "time was not advanced by 1 second";
|
||||
}
|
||||
|
||||
@@ -46,6 +46,8 @@ sub BackRestTestUtility_Test
|
||||
undef, # Host
|
||||
undef, # User
|
||||
undef, # Command
|
||||
undef, # Stanza
|
||||
undef, # Repo Path
|
||||
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
|
||||
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
|
||||
|
||||
Reference in New Issue
Block a user