db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-06-20 01:17:49 +02:00
Code Issues Releases Activity

Storage and IO layer refactor:

Browse Source 
Refactor storage layer to allow for new repository filesystems using drivers. (Reviewed by Cynthia Shang.)
Refactor IO layer to allow for new compression formats, checksum types, and other capabilities using filters. (Reviewed by Cynthia Shang.)
This commit is contained in:
David Steele
2017-06-09 17:51:41 -04:00
parent 7e982f05f5
commit de7fc37f88
183 changed files with 17880 additions and 14734 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/lib/pgBackRestTest/Env/ExpireEnvTest.pm
+51 -25
View File
@@ -11,6 +11,8 @@ use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(basename);
use pgBackRest::Archive::ArchiveInfo;
use pgBackRest::Backup::Common;
use pgBackRest::Backup::Info;
@@ -18,16 +20,17 @@ use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Stanza;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# new
@@ -45,7 +48,7 @@ sub new
my $strOperation,
$self->{oHostBackup},
$self->{strBackRestExe},
$self->{oFile},
$self->{oStorageRepo},
$self->{oLogTest},
$self->{oRunTest},
) =
@@ -54,7 +57,7 @@ sub new
__PACKAGE__ . '->new', \@_,
{name => 'oHostBackup', required => false, trace => true},
{name => 'strBackRestExe', trace => true},
{name => 'oFile', trace => true},
{name => 'oStorageRepo', trace => true},
{name => 'oLogTest', required => false, trace => true},
{name => 'oRunTest', required => false, trace => true},
);
@@ -107,8 +110,8 @@ sub stanzaSet
$$oStanza{iCatalogVersion} = $oStanzaCreate->{oDb}{iCatalogVersion};
$$oStanza{iControlVersion} = $oStanzaCreate->{oDb}{iControlVersion};
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{oFile}->pathGet(PATH_BACKUP_ARCHIVE));
my $oBackupInfo = new pgBackRest::Backup::Info($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER));
my $oArchiveInfo = new pgBackRest::Archive::ArchiveInfo($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE));
my $oBackupInfo = new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP));
if ($bStanzaUpgrade)
{
@@ -122,9 +125,9 @@ sub stanzaSet
}
# Get the archive and directory paths for the stanza
$$oStanza{strArchiveClusterPath} = $self->{oFile}->pathGet(PATH_BACKUP_ARCHIVE) . '/' . ($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oFile}->pathGet(PATH_BACKUP_CLUSTER);
filePathCreate($$oStanza{strArchiveClusterPath}, undef, undef, true);
$$oStanza{strArchiveClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE) . '/' . ($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP);
storageRepo()->pathCreate($$oStanza{strArchiveClusterPath}, {bCreateParent => true});
$self->{oStanzaHash}{$strStanza} = $oStanza;
@@ -159,7 +162,7 @@ sub stanzaCreate
my $strDbPath = optionGet(OPTION_DB_PATH);
# Create the test path for pg_control
filePathCreate(($strDbPath . '/' . DB_PATH_GLOBAL), undef, true);
storageTest()->pathCreate(($strDbPath . '/' . DB_PATH_GLOBAL), {bIgnoreExists => true});
# Copy pg_control for stanza-create
executeTest(
@@ -199,7 +202,7 @@ sub stanzaUpgrade
$strDbVersionTemp =~ s/\.//;
# Remove pg_control
fileRemove(optionGet(OPTION_DB_PATH) . '/' . DB_FILE_PGCONTROL);
storageTest()->remove(optionGet(OPTION_DB_PATH) . '/' . DB_FILE_PGCONTROL);
# Copy pg_control for stanza-upgrade
executeTest(
@@ -257,7 +260,7 @@ sub backupCreate
$lTimestamp);
my $strBackupClusterSetPath = "$$oStanza{strBackupClusterPath}/${strBackupLabel}";
filePathCreate($strBackupClusterSetPath);
storageRepo()->pathCreate($strBackupClusterSetPath);
&log(INFO, "create backup ${strBackupLabel}");
@@ -296,9 +299,6 @@ sub backupCreate
$oManifest->save();
$$oStanza{oManifest} = $oManifest;
# Create the compressed history manifest
$self->{oFile}->compress(PATH_BACKUP_ABSOLUTE, $strManifestFile, false);
# Add the backup to info
my $oBackupInfo = new pgBackRest::Backup::Info($$oStanza{strBackupClusterPath}, false);
@@ -401,10 +401,10 @@ sub archiveCreate
do
{
my $strPath = "$$oStanza{strArchiveClusterPath}/" . substr($strArchive, 0, 16);
filePathCreate($strPath, undef, true);
storageRepo()->pathCreate($strPath, {bIgnoreExists => true});
my $strFile = "${strPath}/${strArchive}-0000000000000000000000000000000000000000" . ($iArchiveIdx % 2 == 0 ? '.gz' : '');
testFileCreate($strFile, 'ARCHIVE');
storageRepo()->put($strFile, 'ARCHIVE');
$iArchiveIdx++;
@@ -449,16 +449,42 @@ sub supplementalLog
if (defined($self->{oLogTest}))
{
$self->{oLogTest}->supplementalAdd($self->{oHostBackup}->repoPath() .
"/backup/${strStanza}/backup.info", $$oStanza{strBackupDescription});
$self->{oLogTest}->supplementalAdd(
$self->{oHostBackup}->repoPath() . "/backup/${strStanza}/backup.info", $$oStanza{strBackupDescription},
${storageRepo->get($self->{oHostBackup}->repoPath() . "/backup/${strStanza}/backup.info")});
executeTest(
'ls ' . $self->{oHostBackup}->repoPath() . "/backup/${strStanza} | grep -v \"backup.*\"",
{oLogTest => $self->{oLogTest}});
# Output backup list
$self->{oLogTest}->logAdd(
'ls ' . $self->{oHostBackup}->repoPath() . "/backup/${strStanza} | grep -v \"backup.*\"", undef,
join("\n", grep(!/^backup\.info.*$/i, storageRepo()->list("backup/${strStanza}"))));
executeTest(
'ls -R ' . $self->{oHostBackup}->repoPath() . "/archive/${strStanza} | grep -v \"archive.info\"",
{oLogTest => $self->{oLogTest}});
# Output archive manifest
my $rhManifest = storageRepo()->manifest(STORAGE_REPO_ARCHIVE);
my $strManifest;
my $strPrefix = '';
foreach my $strEntry (sort(keys(%{$rhManifest})))
{
# Skip files
next if $strEntry eq ARCHIVE_INFO_FILE || $strEntry eq ARCHIVE_INFO_FILE . INI_COPY_EXT;
if ($rhManifest->{$strEntry}->{type} eq 'd')
{
$strEntry = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE) . ($strEntry eq '.' ? '' : "/${strEntry}");
# &log(WARN, "DIR $strEntry");
$strManifest .= (defined($strManifest) ? "\n" : '') . "${strEntry}:\n";
$strPrefix = $strEntry;
}
else
{
# &log(WARN, "FILE $strEntry");
$strManifest .= basename($strEntry) . "\n";
}
}
$self->{oLogTest}->logAdd(
'ls -R ' . $self->{oHostBackup}->repoPath() . "/archive/${strStanza} | grep -v \"archive.info\"", undef, $strManifest);
}
return logDebugReturn($strOperation);
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
+106 -103
View File
@@ -16,14 +16,16 @@ use Exporter qw(import);
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Archive::ArchiveInfo;
use pgBackRest::Backup::Common;
use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Posix::Driver;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBaseTest;
@@ -85,16 +87,19 @@ sub new
my $self = $class->SUPER::new($strName, {strImage => $strImage, strUser => $strUser});
bless $self, $class;
# Set parameters
# Create repo path
$self->{strRepoPath} = $self->testRunGet()->testPath() . "/$$oParam{strBackupDestination}/" . HOST_PATH_REPO;
if ($$oParam{strBackupDestination} eq $self->nameGet())
{
$self->{strLogPath} = $self->repoPath() . '/' . HOST_PATH_LOG;
$self->{strLockPath} = $self->repoPath() . '/' . HOST_PATH_LOCK;
filePathCreate($self->repoPath(), '0770');
storageTest()->pathCreate($self->repoPath(), {strMode => '0770'});
}
# Set log/lock paths
$self->{strLogPath} = $self->testPath() . '/' . HOST_PATH_LOG;
$self->{strLockPath} = $self->testPath() . '/' . HOST_PATH_LOCK;
# Set conf file
$self->{strBackRestConfig} = $self->testPath() . '/' . BACKREST_CONF;
# Set LogTest object
@@ -109,18 +114,6 @@ sub new
# Default hardlink to false
$self->{bHardLink} = false;
# Create the local file object
$self->{oFile} = new pgBackRest::File(
$self->stanza(),
$self->repoPath(),
new pgBackRest::Protocol::Common::Common
(
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
HOST_PROTOCOL_TIMEOUT # Protocol timeout
));
# Create a placeholder hash for file munging
$self->{hInfoFile} = {};
@@ -173,7 +166,7 @@ sub backupBegin
(defined($oExpectedManifest) ? " --no-online" : '') .
(defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') .
(defined($$oParam{bStandby}) && $$oParam{bStandby} ? " --backup-standby" : '') .
(defined($oParam->{bRepoLink}) && !$oParam->{bRepoLink} ? ' --no-repo-link' : '') .
(defined($oParam->{strRepoType}) ? " --repo-type=$oParam->{strRepoType}" : '') .
($strType ne 'incr' ? " --type=${strType}" : '') .
' --stanza=' . (defined($oParam->{strStanza}) ? $oParam->{strStanza} : $self->stanza()) . ' backup' .
(defined($strTest) ? " --test --test-delay=${fTestDelay} --test-point=" . lc($strTest) . '=y' : ''),
@@ -242,84 +235,75 @@ sub backupEnd
if (!$self->synthetic())
{
$oExpectedManifest = iniParse(
fileStringRead($self->repoPath() . '/backup/' . $self->stanza() . "/${strBackup}/" . FILE_MANIFEST));
${storageRepo()->get(storageRepo()->pathGet('backup/' . $self->stanza() . "/${strBackup}/" . FILE_MANIFEST))});
}
# Make sure tablespace links are correct
if ($strType eq BACKUP_TYPE_FULL || $self->hardLink())
if (($strType eq BACKUP_TYPE_FULL || $self->hardLink()) && $self->hasLink())
{
my $hTablespaceManifest = $self->{oFile}->manifest(
PATH_BACKUP_CLUSTER, "${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC);
my $hTablespaceManifest = storageRepo()->manifest(
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC);
# Remove . and ..
delete($hTablespaceManifest->{'.'});
delete($hTablespaceManifest->{'..'});
# Only check links if repo links are disabled
if (!defined($oParam->{bRepoLink}) || $oParam->{bRepoLink})
# Iterate file links
for my $strFile (sort(keys(%{$hTablespaceManifest})))
{
# Iterate file links
for my $strFile (sort(keys(%{$hTablespaceManifest})))
# Make sure the link is in the expected manifest
my $hManifestTarget =
$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"};
if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK ||
$hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile)
{
# Make sure the link is in the expected manifest
my $hManifestTarget =
$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{&MANIFEST_TARGET_PGTBLSPC . "/${strFile}"};
if (!defined($hManifestTarget) || $hManifestTarget->{&MANIFEST_SUBKEY_TYPE} ne MANIFEST_VALUE_LINK ||
$hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID} ne $strFile)
{
confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id");
}
# Make sure the link really is a link
if ($hTablespaceManifest->{$strFile}{type} ne 'l')
{
confess &log(ERROR, "'${strFile}' in tablespace directory is not a link");
}
# Make sure the link destination is correct
my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}";
if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination)
{
confess &log(ERROR,
"'${strFile}' link should reference '${strLinkDestination}' but actually references " .
"'$hTablespaceManifest->{$strFile}{link_destination}'");
}
confess &log(ERROR, "'${strFile}' is not in expected manifest as a link with the correct tablespace id");
}
# Iterate manifest targets
for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}})))
# Make sure the link really is a link
if ($hTablespaceManifest->{$strFile}{type} ne 'l')
{
my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget};
my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID};
confess &log(ERROR, "'${strFile}' in tablespace directory is not a link");
}
# Make sure the target exists as a link on disk
if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) &&
!defined($hTablespaceManifest->{$strTablespaceId}))
{
confess &log(ERROR,
"target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'");
}
# Make sure the link destination is correct
my $strLinkDestination = '../../' . MANIFEST_TARGET_PGTBLSPC . "/${strFile}";
if ($hTablespaceManifest->{$strFile}{link_destination} ne $strLinkDestination)
{
confess &log(ERROR,
"'${strFile}' link should reference '${strLinkDestination}' but actually references " .
"'$hTablespaceManifest->{$strFile}{link_destination}'");
}
}
# Else make sure the tablespace directory is empty
elsif (keys(%{$hTablespaceManifest}) > 0)
# Iterate manifest targets
for my $strTarget (sort(keys(%{$oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}})))
{
confess &log(ERROR, DB_PATH_PGTBLSPC . ' directory should be empty when --no-' . OPTION_REPO_LINK);
my $hManifestTarget = $oExpectedManifest->{&MANIFEST_SECTION_BACKUP_TARGET}{$strTarget};
my $strTablespaceId = $hManifestTarget->{&MANIFEST_SUBKEY_TABLESPACE_ID};
# Make sure the target exists as a link on disk
if ($hManifestTarget->{&MANIFEST_SUBKEY_TYPE} eq MANIFEST_VALUE_LINK && defined($strTablespaceId) &&
!defined($hTablespaceManifest->{$strTablespaceId}))
{
confess &log(ERROR,
"target '${strTarget}' does not have a link at '" . DB_PATH_PGTBLSPC. "/${strTablespaceId}'");
}
}
}
# Else there should not be a tablespace directory at all
elsif ($self->{oFile}->exists(PATH_BACKUP_CLUSTER, "${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))
elsif (storageRepo()->pathExists(STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))
{
confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory');
}
# Check that latest link exists unless repo links are disabled
my $strLatestLink = $self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, LINK_LATEST);
my $bLatestLinkExists = fileExists($strLatestLink);
my $strLatestLink = storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . LINK_LATEST);
my $bLatestLinkExists = storageRepo()->exists($strLatestLink);
if (!defined($oParam->{bRepoLink}) || $oParam->{bRepoLink})
if ((!defined($oParam->{strRepoType}) || $oParam->{strRepoType} eq REPO_TYPE_POSIX) && $self->hasLink())
{
my $strLatestLinkDestination = readlink($strLatestLink);
@@ -330,7 +314,7 @@ sub backupEnd
}
elsif ($bLatestLinkExists)
{
confess &log(ERROR, "'" . LINK_LATEST . "' link should not exist when --no-" . OPTION_REPO_LINK);
confess &log(ERROR, "'" . LINK_LATEST . "' link should not exist");
}
# Only do compare for synthetic backups since for real backups the expected manifest *is* the actual manifest.
@@ -368,8 +352,12 @@ sub backupEnd
if ($self->synthetic() && $bManifestCompare)
{
$self->{oLogTest}->supplementalAdd($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST));
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), undef,
${storageRepo->get(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST)});
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
}
}
@@ -441,7 +429,7 @@ sub backupCompare
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_LABEL} = $strBackup;
my $oActualManifest = new pgBackRest::Manifest(
$self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST));
storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST));
${$oExpectedManifest}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_TIMESTAMP_START} =
$oActualManifest->get(MANIFEST_SECTION_BACKUP, &MANIFEST_KEY_TIMESTAMP_START);
@@ -463,7 +451,7 @@ sub backupCompare
my $lRepoSize =
$oActualManifest->test(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REFERENCE) ?
$oActualManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFileKey, MANIFEST_SUBKEY_REPO_SIZE, false) :
(fileStat($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/${strFileKey}.gz")))->size;
(storageTest->info(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/${strFileKey}.gz")))->size;
if (defined($lRepoSize) &&
$lRepoSize != $oExpectedManifest->{&MANIFEST_SECTION_TARGET_FILE}{$strFileKey}{&MANIFEST_SUBKEY_SIZE})
@@ -495,13 +483,13 @@ sub backupCompare
my $strTestPath = $self->testPath();
fileStringWrite("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent}));
fileStringWrite("${strTestPath}/expected.manifest", iniRender($oExpectedManifest));
storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent}));
storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifest));
executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
fileRemove("${strTestPath}/expected.manifest");
fileRemove("${strTestPath}/actual.manifest");
storageTest()->remove("${strTestPath}/expected.manifest");
storageTest()->remove("${strTestPath}/actual.manifest");
# Return from function and log return values if any
return logDebugReturn($strOperation);
@@ -587,9 +575,8 @@ sub backupLast
{
my $self = shift;
my @stryBackup = $self->{oFile}->list(
PATH_BACKUP_CLUSTER, undef,
{strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'});
my @stryBackup = storageRepo()->list(
STORAGE_REPO_BACKUP, {strExpression => '[0-9]{8}-[0-9]{6}F(_[0-9]{8}-[0-9]{6}(D|I)){0,1}', strSortOrder => 'reverse'});
if (!defined($stryBackup[0]))
{
@@ -755,15 +742,19 @@ sub stanzaCreate
# If the info file was created, then add it to the expect log
if (defined($self->{oLogTest}) && $self->synthetic() &&
fileExists($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info'))
storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO))
{
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
}
if (defined($self->{oLogTest}) && $self->synthetic() &&
fileExists($self->repoPath() . '/archive/' . $self->stanza() . '/archive.info'))
storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE))
{
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/archive/' . $self->stanza() . '/archive.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
}
# Return from function and log return values if any
@@ -808,15 +799,19 @@ sub stanzaUpgrade
# If the info file was created, then add it to the expect log
if (defined($self->{oLogTest}) && $self->synthetic() &&
fileExists($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info'))
storageRepo()->exists('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO))
{
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/backup/' . $self->stanza() . '/backup.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('backup/' . $self->stanza() . qw{/} . FILE_BACKUP_INFO), undef,
${storageRepo()->get(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)});
}
if (defined($self->{oLogTest}) && $self->synthetic() &&
fileExists($self->repoPath() . '/archive/' . $self->stanza() . '/archive.info'))
storageRepo()->exists('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE))
{
$self->{oLogTest}->supplementalAdd($self->repoPath() . '/archive/' . $self->stanza() . '/archive.info');
$self->{oLogTest}->supplementalAdd(
storageRepo()->pathGet('archive/' . $self->stanza() . qw{/} . ARCHIVE_INFO_FILE), undef,
${storageRepo()->get(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE)});
}
# Return from function and log return values if any
@@ -924,10 +919,12 @@ sub configCreate
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_LEVEL_FILE} = lc(TRACE);
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_LEVEL_STDERR} = lc(OFF);
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_PATH} = $self->repoPath();
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_LOG_PATH} = $self->logPath();
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_LOCK_PATH} = $self->lockPath();
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_PROTOCOL_TIMEOUT} = 60;
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_DB_TIMEOUT} = 45;
if ($self->processMax() > 1)
{
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_PROCESS_MAX} = $self->processMax();
@@ -940,6 +937,8 @@ sub configCreate
if ($self->isHostBackup())
{
$oParamHash{&CONFIG_SECTION_GLOBAL}{&OPTION_REPO_PATH} = $self->repoPath();
if (defined($$oParam{bHardlink}) && $$oParam{bHardlink})
{
$self->{bHardLink} = true;
@@ -1033,7 +1032,7 @@ sub configCreate
}
# Write out the configuration file
fileStringWrite($self->backrestConfig(), iniRender(\%oParamHash, true));
storageTest()->put($self->backrestConfig(), iniRender(\%oParamHash, true));
# Modify the file permissions so it can be read/saved by all test users
executeTest('sudo chmod 660 ' . $self->backrestConfig());
@@ -1065,7 +1064,7 @@ sub manifestMunge
{name => 'bCache', default => true},
);
$self->infoMunge($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST), $hParam, $bCache);
$self->infoMunge(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $hParam, $bCache);
# Return from function and log return values if any
return logDebugReturn($strOperation);
@@ -1092,7 +1091,7 @@ sub manifestRestore
{name => 'bSave', default => true},
);
$self->infoRestore($self->{oFile}->pathGet(PATH_BACKUP_CLUSTER, "${strBackup}/" . FILE_MANIFEST), $bSave);
$self->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strBackup}/" . FILE_MANIFEST), $bSave);
# Return from function and log return values if any
return logDebugReturn($strOperation);
@@ -1128,12 +1127,13 @@ sub infoMunge
# If the original file content does not exist then load it
if (!defined($self->{hInfoFile}{$strFileName}))
{
$self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini($strFileName);
$self->{hInfoFile}{$strFileName} = new pgBackRest::Common::Ini($strFileName, {oStorage => storageRepo()});
}
# Make a copy of the original file contents
my $oMungeIni = new pgBackRest::Common::Ini(
$strFileName, {bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent})});
$strFileName,
{bLoad => false, strContent => iniRender($self->{hInfoFile}{$strFileName}->{oContent}), oStorage => storageRepo()});
# Load params
foreach my $strSection (keys(%{$hParam}))
@@ -1157,15 +1157,15 @@ sub infoMunge
}
# Modify the file/directory permissions so it can be saved
executeTest("sudo rm -f ${strFileName} && sudo chmod 770 " . dirname($strFileName));
executeTest("sudo rm -f ${strFileName}* && sudo chmod 770 " . dirname($strFileName));
# Save the munged data to the file
$oMungeIni->save();
# Fix permissions
executeTest(
"sudo chmod 640 ${strFileName} && sudo chmod 750 " . dirname($strFileName) .
' && sudo chown ' . $self->userGet() . " ${strFileName}");
executeTest(
"sudo chmod 640 ${strFileName}* && sudo chmod 750 " . dirname($strFileName) .
' && sudo chown ' . $self->userGet() . " ${strFileName}*");
# Clear the cache is requested
if (!$bCache)
@@ -1207,16 +1207,18 @@ sub infoRestore
if ($bSave)
{
# Modify the file/directory permissions so it can be saved
executeTest("sudo rm -f ${strFileName} && sudo chmod 770 " . dirname($strFileName));
executeTest("sudo rm -f ${strFileName}* && sudo chmod 770 " . dirname($strFileName));
# Save the munged data to the file
$self->{hInfoFile}{$strFileName}->{bModified} = true;
$self->{hInfoFile}{$strFileName}->save();
# Fix permissions
executeTest("sudo chmod 640 ${strFileName} && sudo chmod 750 " . dirname($strFileName));
executeTest(
"sudo chmod 640 ${strFileName} && sudo chmod 750 " . dirname($strFileName) .
' && sudo chown ' . $self->userGet() . " ${strFileName}*");
}
}
}
else
{
confess &log(ASSERT, "There is no original data cached for $strFileName. infoMunge must be called first.");
@@ -1236,6 +1238,7 @@ sub backrestConfig {return shift->{strBackRestConfig}}
sub backupDestination {return shift->{strBackupDestination}}
sub backrestExe {return testRunGet()->backrestExe()}
sub hardLink {return shift->{bHardLink}}
sub hasLink {storageRepo()->driver()->className() eq STORAGE_POSIX_DRIVER}
sub isHostBackup {my $self = shift; return $self->backupDestination() eq $self->nameGet()}
sub isHostDbMaster {return shift->nameGet() eq HOST_DB_MASTER}
sub isHostDbStandby {return shift->nameGet() eq HOST_DB_STANDBY}
test/lib/pgBackRestTest/Env/Host/HostBaseTest.pm
+4 -2
View File
@@ -17,7 +17,7 @@ use Exporter qw(import);
use File::Basename qw(dirname);
use pgBackRest::Common::Log;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Common::ContainerTest;
@@ -36,6 +36,8 @@ use constant HOST_DB_STANDBY => 'db-stand
push @EXPORT, qw(HOST_DB_STANDBY);
use constant HOST_BACKUP => 'backup';
push @EXPORT, qw(HOST_BACKUP);
use constant HOST_S3 => 's3-server';
push @EXPORT, qw(HOST_S3);
####################################################################################################################################
# new
@@ -59,7 +61,7 @@ sub new
);
my $strTestPath = testRunGet()->testPath() . ($strName eq HOST_BASE ? '' : "/${strName}");
filePathCreate($strTestPath, '0770');
storageTest()->pathCreate($strTestPath, {strMode => '0770'});
# Create the host
my $strProjectPath = dirname(dirname(abs_path($0)));
test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm
+36 -50
View File
@@ -25,15 +25,15 @@ use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# Host defaults
@@ -84,20 +84,10 @@ sub new
$self->{strDbBasePath} = $self->dbPath() . '/' . HOST_PATH_DB_BASE;
$self->{strTablespacePath} = $self->dbPath() . '/tablespace';
filePathCreate($self->dbBasePath(), undef, undef, true);
storageTest()->pathCreate($self->dbBasePath(), {strMode => '0700', bCreateParent => true});
if ($$oParam{strBackupDestination} ne $self->nameGet())
{
$self->{strSpoolPath} = $self->testPath() . '/' . HOST_PATH_SPOOL;
$self->{strLogPath} = $self->spoolPath() . '/' . HOST_PATH_LOG;
$self->{strLockPath} = $self->spoolPath() . '/' . HOST_PATH_LOCK;
filePathCreate($self->spoolPath());
}
else
{
$self->{strSpoolPath} = $self->repoPath();
}
$self->{strSpoolPath} = $self->testPath() . '/' . HOST_PATH_SPOOL;
storageTest()->pathCreate($self->spoolPath());
# Initialize linkRemap Hashes
$self->{hLinkRemap} = {};
@@ -143,16 +133,10 @@ sub archivePush
{
$strSourceFile = "${strXlogPath}/" . uc(sprintf('0000000100000001%08x', $iArchiveNo));
$self->{oFile}->copy(
PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false, # Destination is not compressed
undef, undef, undef, # Unused params
true); # Create path if it does not exist
storageTest()->copy($strArchiveTestFile, storageTest()->openWrite($strSourceFile, {bPathCreate => true}));
filePathCreate("${strXlogPath}/archive_status/", undef, true, true);
fileStringWrite("${strXlogPath}/archive_status/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)) . '.ready');
storageTest()->pathCreate("${strXlogPath}/archive_status/", {bIgnoreExists => true, bCreateParent => true});
storageTest()->put("${strXlogPath}/archive_status/" . uc(sprintf('0000000100000001%08x', $iArchiveNo)) . '.ready');
}
$self->executeSimple(
@@ -160,7 +144,7 @@ sub archivePush
' --config=' . $self->backrestConfig() .
' --log-level-console=warn --archive-queue-max=' . int(2 * PG_WAL_SIZE) .
' --stanza=' . $self->stanza() .
(defined($iExpectedError) && $iExpectedError == ERROR_HOST_CONNECT ? ' --backup-host=bogus' : '') .
(defined($iExpectedError) && $iExpectedError == ERROR_FILE_READ ? ' --backup-host=bogus' : '') .
($bAsync ? '' : ' --no-archive-async') .
" archive-push" . (defined($strSourceFile) ? " ${strSourceFile}" : ''),
{iExpectedExitStatus => $iExpectedError, oLogTest => $self->{oLogTest}, bLogOutput => $self->synthetic()});
@@ -182,7 +166,7 @@ sub configRecovery
my $strStanza = $self->stanza();
# Load db config file
my $oConfig = iniParse(fileStringRead($self->backrestConfig()), {bRelaxed => true});
my $oConfig = iniParse(${storageTest->get($self->backrestConfig())}, {bRelaxed => true});
# Rewrite recovery options
my @stryRecoveryOption;
@@ -198,7 +182,7 @@ sub configRecovery
}
# Save db config file
fileStringWrite($self->backrestConfig(), iniRender($oConfig, true));
storageTest()->put($self->backrestConfig(), iniRender($oConfig, true));
}
####################################################################################################################################
@@ -214,7 +198,7 @@ sub configRemap
my $strStanza = $self->stanza();
# Load db config file
my $oConfig = iniParse(fileStringRead($self->backrestConfig()), {bRelaxed => true});
my $oConfig = iniParse(${storageTest()->get($self->backrestConfig())}, {bRelaxed => true});
# Load backup config file
my $oRemoteConfig;
@@ -224,7 +208,7 @@ sub configRemap
if (defined($oHostBackup))
{
$oRemoteConfig = iniParse(fileStringRead($oHostBackup->backrestConfig()), {bRelaxed => true});
$oRemoteConfig = iniParse(${storageTest()->get($oHostBackup->backrestConfig())}, {bRelaxed => true});
}
# Rewrite recovery section
@@ -262,7 +246,7 @@ sub configRemap
}
# Save db config file
fileStringWrite($self->backrestConfig(), iniRender($oConfig, true));
storageTest()->put($self->backrestConfig(), iniRender($oConfig, true));
# Save backup config file (but not is this is the standby which is not the source of backups)
if (defined($oHostBackup))
@@ -271,7 +255,7 @@ sub configRemap
executeTest(
'sudo chmod 660 ' . $oHostBackup->backrestConfig() . ' && sudo chmod 770 ' . dirname($oHostBackup->backrestConfig()));
fileStringWrite($oHostBackup->backrestConfig(), iniRender($oRemoteConfig, true));
storageTest()->put($oHostBackup->backrestConfig(), iniRender($oRemoteConfig, true));
# Fix permissions
executeTest(
@@ -369,9 +353,10 @@ sub restore
{
# Load the manifest
my $oExpectedManifest = new pgBackRest::Manifest(
$self->{oFile}->pathGet(
PATH_BACKUP_CLUSTER, ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . '/' . FILE_MANIFEST),
true);
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} . ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) . qw{/} .
FILE_MANIFEST),
true);
$oExpectedManifestRef = $oExpectedManifest->{oContent};
@@ -481,17 +466,17 @@ sub restoreCompare
{
my $oExpectedManifest =
new pgBackRest::Manifest(
$self->{oFile}->pathGet(
PATH_BACKUP_CLUSTER,
($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) .
'/'. FILE_MANIFEST), true);
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} . ($strBackup eq 'latest' ? $oHostBackup->backupLast() : $strBackup) .
'/'. FILE_MANIFEST),
true);
$oLastManifest =
new pgBackRest::Manifest(
$self->{oFile}->pathGet(
PATH_BACKUP_CLUSTER,
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} .
'/' . FILE_MANIFEST), true);
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . qw{/} .
${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP}{&MANIFEST_KEY_PRIOR} . qw{/} . FILE_MANIFEST),
true);
}
# Generate the tablespace map for real backups
@@ -553,7 +538,7 @@ sub restoreCompare
$$oExpectedManifestRef{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_CATALOG});
$oActualManifest->build(
$self->{oFile}, ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}, $strDbClusterPath,
storageTest(), ${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION}, $strDbClusterPath,
$oLastManifest, false, $oTablespaceMap);
my $strSectionPath = $oActualManifest->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH);
@@ -614,13 +599,14 @@ sub restoreCompare
if ($oActualManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) != 0)
{
my $oStat = fileStat($oActualManifest->dbPathGet($strSectionPath, $strName));
my $oStat = storageTest()->info($oActualManifest->dbPathGet($strSectionPath, $strName));
if ($oStat->blocks > 0 || S_ISLNK($oStat->mode))
{
my ($strHash) = storageTest()->hashSize($oActualManifest->dbPathGet($strSectionPath, $strName));
$oActualManifest->set(
MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM,
$self->{oFile}->hash(PATH_DB_ABSOLUTE, $oActualManifest->dbPathGet($strSectionPath, $strName)));
MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM, $strHash);
}
else
{
@@ -723,13 +709,13 @@ sub restoreCompare
$self->manifestDefault($oExpectedManifestRef);
fileStringWrite("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent}));
fileStringWrite("${strTestPath}/expected.manifest", iniRender($oExpectedManifestRef));
storageTest()->put("${strTestPath}/actual.manifest", iniRender($oActualManifest->{oContent}));
storageTest()->put("${strTestPath}/expected.manifest", iniRender($oExpectedManifestRef));
executeTest("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest");
fileRemove("${strTestPath}/expected.manifest");
fileRemove("${strTestPath}/actual.manifest");
storageTest()->remove("${strTestPath}/expected.manifest");
storageTest()->remove("${strTestPath}/actual.manifest");
}
####################################################################################################################################
test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm
+11 -11
View File
@@ -23,8 +23,8 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::DbVersion;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;
@@ -222,7 +222,7 @@ sub manifestFileCreate
my $strPathFile = $self->dbFileCreate($oManifestRef, $strTarget, $strFile, $strContent, $lTime, $strMode);
# Stat the file
my $oStat = fileStat($strPathFile);
my $oStat = storageTest()->info($strPathFile);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_FILE}{$strManifestKey}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
@@ -336,7 +336,7 @@ sub manifestLinkCreate
my $strDbFile = $self->dbLinkCreate($oManifestRef, $strPath, $strFile, $strDestination);
# Stat the link
my $oStat = fileStat($strDbFile);
my $oStat = storageTest()->info($strDbFile);
# Check for errors in stat
if (!defined($oStat))
@@ -358,7 +358,7 @@ sub manifestLinkCreate
(defined(dirname($strPath)) ? dirname($strPath) : '') . "/${strDestination}";
}
$oStat = fileStat($strDestinationFile);
$oStat = storageTest()->info($strDestinationFile);
my $strSection = MANIFEST_SECTION_TARGET_PATH;
@@ -367,7 +367,7 @@ sub manifestLinkCreate
$strSection = MANIFEST_SECTION_TARGET_FILE;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_SIZE} = $oStat->size;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_TIMESTAMP} = $oStat->mtime;
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM} = fileHash($strDestinationFile);
(${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_CHECKSUM}) = storageTest()->hashSize($strDestinationFile);
${$oManifestRef}{$strSection}{$strManifestKey}{&MANIFEST_SUBKEY_MASTER} =
defined($bMaster) ? ($bMaster ? JSON::PP::true : JSON::PP::false) : JSON::PP::false;
@@ -467,7 +467,7 @@ sub manifestPathCreate
my $strManifestKey = $self->manifestKeyGet($oManifestRef, $strPath, $strSubPath);
# Create the db path
my $strDbPath = $self->dbPathCreate($oManifestRef, $strPath, $strSubPath, $strMode);
my $strDbPath = $self->dbPathCreate($oManifestRef, $strPath, $strSubPath, defined($strMode) ? $strMode : '0700');
# Stat the file
my $oStat = lstat($strDbPath);
@@ -554,7 +554,7 @@ sub manifestTablespaceCreate
# Load linked path into manifest
my $strLinkPath = $self->tablespacePath($iOid);
my $strTarget = MANIFEST_TARGET_PGTBLSPC . "/${iOid}";
my $oStat = fileStat($strLinkPath);
my $oStat = storageTest()->info($strLinkPath);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{$strTarget}{&MANIFEST_SUBKEY_USER} = getpwuid($oStat->uid);
@@ -576,11 +576,11 @@ sub manifestTablespaceCreate
if (!-e $strTablespacePath)
{
filePathCreate($strTablespacePath, $strMode);
storageTest()->pathCreate($strTablespacePath, {strMode => defined($strMode) ? $strMode : '0700'});
}
# Load tablespace path into manifest
$oStat = fileStat($strTablespacePath);
$oStat = storageTest()->info($strTablespacePath);
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGTBLSPC} =
${$oManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_TARGET_PGDATA};
@@ -597,7 +597,7 @@ sub manifestTablespaceCreate
or confess "unable to link ${strLink} to ${strLinkPath}";
# Load link into the manifest
$oStat = fileStat($strLink);
$oStat = storageTest()->info($strLink);
my $strLinkTarget = MANIFEST_TARGET_PGDATA . "/${strTarget}";
${$oManifestRef}{&MANIFEST_SECTION_TARGET_LINK}{$strLinkTarget}{&MANIFEST_SUBKEY_GROUP} = getgrgid($oStat->gid);
@@ -677,7 +677,7 @@ sub dbPathCreate
# Create the path
if (!(-e $strFinalPath))
{
filePathCreate($strFinalPath, $strMode);
storageTest()->pathCreate($strFinalPath, {strMode => $strMode});
}
return $strFinalPath;
test/lib/pgBackRestTest/Env/Host/HostDbTest.pm
+4 -4
View File
@@ -21,8 +21,8 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::DbVersion;
use pgBackRest::FileCommon;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
use pgBackRestTest::Env::Host::HostBackupTest;
@@ -123,7 +123,7 @@ sub new
}
# Create pg_xlog directory
filePathCreate($self->dbPath() . '/pg_xlog');
storageTest()->pathCreate($self->dbPath() . '/pg_xlog', {strMode => '0700'});
# Return from function and log return values if any
return logDebugReturn
@@ -476,13 +476,13 @@ sub clusterStop
}
# Grep for errors in postgresql.log
if (!$bIgnoreLogError && fileExists($self->pgLogFile()))
if (!$bIgnoreLogError && storageTest()->exists($self->pgLogFile()))
{
$self->executeSimple('grep ERROR ' . $self->pgLogFile(), {iExpectedExitStatus => 1});
}
# Remove the log file
fileRemove($self->pgLogFile(), true);
storageTest()->remove($self->pgLogFile(), {bIgnoreMissing => true});
}
####################################################################################################################################
test/lib/pgBackRestTest/Env/HostEnvTest.pm
+16 -41
View File
@@ -13,11 +13,11 @@ use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
use Storable qw(dclone);
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::File;
use pgBackRest::FileCommon;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Env::Host::HostBaseTest;
@@ -25,6 +25,7 @@ use pgBackRestTest::Env::Host::HostDbCommonTest;
use pgBackRestTest::Env::Host::HostDbTest;
use pgBackRestTest::Env::Host::HostDbSyntheticTest;
use pgBackRestTest::Common::HostGroupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# Constants
@@ -46,16 +47,6 @@ use constant WAL_VERSION_95 => '95';
use constant WAL_VERSION_95_SYS_ID => 6392579261579036436;
push @EXPORT, qw(WAL_VERSION_95_SYS_ID);
####################################################################################################################################
# initModule
####################################################################################################################################
sub initModule
{
# Set file and path modes
pathModeDefaultSet('0700');
fileModeDefaultSet('0600');
}
####################################################################################################################################
# setup
####################################################################################################################################
@@ -115,21 +106,6 @@ sub setup
$oHostGroup->hostAdd($oHostDbStandby);
}
# Create the local file object
my $oFile =
new pgBackRest::File
(
$oHostDbMaster->stanza(),
$oHostDbMaster->repoPath(),
new pgBackRest::Protocol::Common::Common
(
OPTION_DEFAULT_BUFFER_SIZE, # Buffer size
OPTION_DEFAULT_COMPRESS_LEVEL, # Compress level
OPTION_DEFAULT_COMPRESS_LEVEL_NETWORK, # Compress network level
HOST_PROTOCOL_TIMEOUT # Protocol timeout
)
);
# Create db master config
$oHostDbMaster->configCreate({
strBackupSource => $$oConfigParam{strBackupSource},
@@ -160,7 +136,14 @@ sub setup
bArchiveAsync => $$oConfigParam{bArchiveAsync}});
}
return $oHostDbMaster, $oHostDbStandby, $oHostBackup, $oFile;
# Set options needed for storage helper
my $oOption = {};
$self->optionSetTest($oOption, OPTION_DB_PATH, $oHostDbMaster->dbBasePath());
$self->optionSetTest($oOption, OPTION_REPO_PATH, $oHostBackup->repoPath());
$self->optionSetTest($oOption, OPTION_STANZA, $self->stanza());
$self->testResult(sub {$self->configLoadExpect(dclone($oOption), CMD_ARCHIVE_PUSH)}, '', 'config load');
return $oHostDbMaster, $oHostDbStandby, $oHostBackup;
}
####################################################################################################################################
@@ -171,7 +154,6 @@ sub setup
sub archiveGenerate
{
my $self = shift;
my $oFile = shift;
my $strXlogPath = shift;
my $iSourceNo = shift;
my $iArchiveNo = shift;
@@ -184,10 +166,7 @@ sub archiveGenerate
my $strSourceFile = "${strXlogPath}/${strArchiveFile}";
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false); # Destination is not compressed
storageTest()->copy($strArchiveTestFile, $strSourceFile);
return $strArchiveFile, $strSourceFile;
}
@@ -215,7 +194,6 @@ sub walSegment
sub walGenerate
{
my $self = shift;
my $oFile = shift;
my $strWalPath = shift;
my $strPgVersion = shift;
my $iSourceNo = shift;
@@ -225,12 +203,9 @@ sub walGenerate
my $strWalFile = "${strWalPath}/${strWalSegment}" . (defined($bPartial) && $bPartial ? '.partial' : '');
my $strArchiveTestFile = $self->dataPath() . "/backup.wal${iSourceNo}_${strPgVersion}.bin";
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strWalFile, # Destination file
false, # Source is not compressed
false); # Destination is not compressed
storageTest()->copy($strArchiveTestFile, $strWalFile);
fileStringWrite("${strWalPath}/archive_status/${strWalSegment}.ready");
storageTest()->put("${strWalPath}/archive_status/${strWalSegment}.ready");
return $strWalFile;
}
@@ -246,8 +221,8 @@ sub walRemove
my $strWalPath = shift;
my $strWalFile = shift;
fileRemove("$self->{strWalPath}/${strWalFile}");
fileRemove("$self->{strWalPath}/archive_status/${strWalFile}.ready");
storageTest()->remove("$self->{strWalPath}/${strWalFile}");
storageTest()->remove("$self->{strWalPath}/archive_status/${strWalFile}.ready");
}
1;