db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Remove obsolete methods in pgBackRest::Storage::Storage module.

Browse Source 
All the methods in this module will need to be implemented via the command-line in order to get rid of LibC, so the first step is to reduce the code in the module as much as possible.

First remove storageDb() and use storageTest() instead.  Then create storageTest() using pgBackRestTest::Common::Storage which has no dependencies on LibC.  Now the only storage using the LibC interface is storageRepo().

Remove all link functions since those operations cannot be performed on a repo unless it is Posix, in which case the LibC interface is not needed.  Same for owner().

Remove pathSync() because syncs are not required in the tests.  No test data is reused after a crash.

Path create/exists functions should never be explicitly performed on a repo so remove those.  File exists can be implemented by calling info() instead.

Remove encryption detection functions which were only used by Backup/Archive::Info reconstruct() which are now obsolete.

Remove all filters except pgBackRest::Storage::Filter::CipherBlock since they are not being used.  That also means there are no filters returning results so remove all the result code.

Move hashSize() and pathAbsolute() into pgBackRest::Storage::Base where they can be shared between pgBackRest::Storage::Storage and pgBackRestTest::Common::Storage.
This commit is contained in:
David Steele
2020-03-06 14:10:09 -05:00
parent 00647c7109
commit 02aa03d1a2
21 changed files with 113 additions and 1273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/lib/pgBackRestTest/Common/FileTest.pm
+1 -1
View File
@@ -168,7 +168,7 @@ sub forceStorageMode
);
# Mode commands are ignored on S3
if ($oStorage->type() ne STORAGE_S3)
if (!$oStorage->can('type') || $oStorage->type() ne STORAGE_S3)
{
executeTest('chmod ' . ($bRecurse ? '-R ' : '') . "${strMode} " . $oStorage->pathGet($strPathExp));
}
test/lib/pgBackRestTest/Common/RunTest.pm
+4 -3
View File
@@ -19,14 +19,14 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Storage;
use pgBackRest::Version;
use pgBackRestTest::Common::BuildTest;
use pgBackRestTest::Common::DefineTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::LogTest;
use pgBackRestTest::Common::Storage;
use pgBackRestTest::Common::StoragePosix;
use pgBackRestTest::Common::VmTest;
####################################################################################################################################
@@ -156,7 +156,8 @@ sub process
$self->{bFirstTest} = true;
# Initialize test storage
$oStorage = new pgBackRest::Storage::Storage(STORAGE_LOCAL, {strPath => $self->testPath()});
$oStorage = new pgBackRestTest::Common::Storage(
$self->testPath(), new pgBackRestTest::Common::StoragePosix({bFileSync => false, bPathSync => false}));
# Generate backrest exe
$self->{strBackRestExe} = defined($self->{strBackRestExeC}) ? $self->{strBackRestExeC} : $self->{strBackRestExeHelper};
test/lib/pgBackRestTest/Env/ExpireEnvTest.pm
+1 -14
View File
@@ -221,7 +221,7 @@ sub stanzaSet
new pgBackRest::Archive::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE), false,
{bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_ARCHIVE : undef});
$oBackupInfo =
new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP), false, false,
new pgBackRest::Backup::Info($self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP), false,
{bIgnoreMissing => true, strCipherPassSub => $bEncrypted ? ENCRYPTION_KEY_MANIFEST : undef});
}
# Else get the info data from disk
@@ -260,7 +260,6 @@ sub stanzaSet
# Get the archive and directory paths for the stanza
$$oStanza{strArchiveClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_ARCHIVE) . '/' . ($oArchiveInfo->archiveId());
$$oStanza{strBackupClusterPath} = $self->{oStorageRepo}->pathGet(STORAGE_REPO_BACKUP);
storageRepo()->pathCreate($$oStanza{strArchiveClusterPath}, {bCreateParent => true});
$self->{oStanzaHash}{$strStanza} = $oStanza;
@@ -301,15 +300,6 @@ sub stanzaCreate
$self->controlGenerate($strDbPath, $strDbVersion);
executeTest('chmod 600 ' . $strDbPath . '/' . DB_FILE_PGCONTROL);
# Create the stanza repo paths if they don't exist
if (!cfgOptionTest(CFGOPT_REPO_TYPE, CFGOPTVAL_REPO_TYPE_S3))
{
storageTest()->pathCreate(
cfgOption(CFGOPT_REPO_PATH) . "/archive/$strStanza", {bIgnoreExists => true, bCreateParent => true});
storageTest()->pathCreate(
cfgOption(CFGOPT_REPO_PATH) . "/backup/$strStanza", {bIgnoreExists => true, bCreateParent => true});
}
# Create the stanza and set the local stanza object
$self->stanzaSet($strStanza, $strDbVersion, false);
@@ -398,7 +388,6 @@ sub backupCreate
$lTimestamp);
my $strBackupClusterSetPath = "$$oStanza{strBackupClusterPath}/${strBackupLabel}";
storageRepo()->pathCreate($strBackupClusterSetPath);
&log(INFO, "create backup ${strBackupLabel}");
@@ -558,8 +547,6 @@ sub archiveCreate
do
{
my $strPath = "$$oStanza{strArchiveClusterPath}/" . substr($strArchive, 0, 16);
storageRepo()->pathCreate($strPath, {bIgnoreExists => true});
my $strFile = "${strPath}/${strArchive}-0000000000000000000000000000000000000000" . ($iArchiveIdx % 2 == 0 ? '.gz' : '');
storageRepo()->put($strFile, 'ARCHIVE', {strCipherPass => $strCipherPass});
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
+5 -8
View File
@@ -102,12 +102,6 @@ sub new
$self->{strRepoPath} = '/';
}
# Create the repo-path if on a local filesystem
if ($$oParam{strBackupDestination} eq $self->nameGet() && $oParam->{bRepoLocal})
{
storageTest()->pathCreate($self->repoPath(), {strMode => '0770'});
}
# Set log/lock paths
$self->{strLogPath} = $self->testPath() . '/' . HOST_PATH_LOG;
storageTest()->pathCreate($self->{strLogPath}, {strMode => '0770'});
@@ -309,8 +303,11 @@ sub backupEnd
}
}
}
# Else there should not be a tablespace directory at all
elsif (storageRepo()->pathExists(STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC))
# Else there should not be a tablespace directory at all. This is only valid for storage that supports links.
elsif (storageRepo()->capability(STORAGE_CAPABILITY_LINK) &&
storageTest()->pathExists(
storageRepo()->pathGet(
STORAGE_REPO_BACKUP . "/${strBackup}/" . MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC)))
{
confess &log(ERROR, 'backup must be full or hard-linked to have ' . DB_PATH_PGTBLSPC . ' directory');
}
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
+3 -3
View File
@@ -142,7 +142,7 @@ sub run
'184473f470864e067ee3a22e64b47b0a1c356f29', $lTime, undef, true);
# Load sample page
my $tBasePage = ${storageLocal()->get($self->dataPath() . '/page.bin')};
my $tBasePage = ${storageTest()->get($self->dataPath() . '/page.bin')};
my $iBasePageChecksum = 0x1B99;
# Create base path
@@ -477,7 +477,7 @@ sub run
strOptionalParam => '--force --' . cfgOptionName(CFGOPT_CHECKSUM_PAGE) . ($bDeltaBackup ? ' --delta' : '')});
# Remove postmaster.pid so restore will succeed (the rest will be cleaned up by the delta)
storageDb->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID);
storageTest->remove($oHostDbMaster->dbBasePath() . '/' . DB_FILE_POSTMASTERPID);
# Restore - tests various mode, extra files/paths, missing files/paths
#---------------------------------------------------------------------------------------------------------------------------
@@ -680,7 +680,7 @@ sub run
$oManifest{&MANIFEST_SECTION_TARGET_FILE}{'pg_data/changetime.txt'}{&MANIFEST_SUBKEY_TIMESTAMP} = $lTime - 100;
# Change the content of the changecontent file to be the same size but leave the timestamp the same on the file
storageDb()->put($oHostDbMaster->dbBasePath() . '/changecontent.txt', 'CHGCONT');
storageTest()->put($oHostDbMaster->dbBasePath() . '/changecontent.txt', 'CHGCONT');
utime($lTime, $lTime, $oHostDbMaster->dbBasePath() . '/changecontent.txt')
or confess &log(ERROR, "unable to set time for file ".$oHostDbMaster->dbBasePath() . '/changecontent.txt');
test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm
+5 -5
View File
@@ -180,9 +180,9 @@ sub run
$strCommandGet . " ${strSourceFile} ${strWalPath}/RECOVERYXLOG", {oLogTest => $self->expect()});
# Check that the destination file exists
if (storageDb()->exists("${strWalPath}/RECOVERYXLOG"))
if (storageTest()->exists("${strWalPath}/RECOVERYXLOG"))
{
my ($strActualChecksum) = storageDb()->hashSize("${strWalPath}/RECOVERYXLOG");
my ($strActualChecksum) = storageTest()->hashSize("${strWalPath}/RECOVERYXLOG");
if ($strActualChecksum ne $strArchiveChecksum)
{
@@ -344,9 +344,9 @@ sub run
{oLogTest => $self->expect()});
# Check that the destination file exists
if (storageDb()->exists("${strWalPath}/RECOVERYXLOG"))
if (storageTest()->exists("${strWalPath}/RECOVERYXLOG"))
{
my ($strActualChecksum) = storageDb()->hashSize("${strWalPath}/RECOVERYXLOG");
my ($strActualChecksum) = storageTest()->hashSize("${strWalPath}/RECOVERYXLOG");
if ($strActualChecksum ne $strArchiveChecksum)
{
@@ -369,7 +369,7 @@ sub run
$strCommandGet . " --archive-async 00000002.history ${strWalPath}/00000002.history",
{oLogTest => $self->expect()});
if (${storageDb()->get("${strWalPath}/00000002.history")} ne 'HISTORYDATA')
if (${storageTest()->get("${strWalPath}/00000002.history")} ne 'HISTORYDATA')
{
confess 'history contents do not match original';
}
test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm
+5 -5
View File
@@ -83,7 +83,7 @@ sub run
strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_LOG_LEVEL_FILE) . '=info'});
# Generate pg_control for stanza-create
storageDb()->pathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true});
storageTest()->pathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true});
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_93);
# Fail stanza upgrade before stanza-create has been performed
@@ -119,7 +119,7 @@ sub run
# Create the wal path
my $strWalPath = $oHostDbMaster->dbBasePath() . '/pg_xlog';
storageDb()->pathCreate("${strWalPath}/archive_status", {bCreateParent => true});
storageTest()->pathCreate("${strWalPath}/archive_status", {bCreateParent => true});
# Stanza Create fails - missing archive.info from non-empty archive dir
#--------------------------------------------------------------------------------------------------------------------------
@@ -158,7 +158,7 @@ sub run
# Upgrade the DB by copying new pg_control
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94);
forceStorageMode(storageDb(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
forceStorageMode(storageTest(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
# Fail on attempt to push an archive
$oHostDbMaster->archivePush($strWalPath, $strArchiveTestFile, 1, ERROR_ARCHIVE_MISMATCH);
@@ -184,7 +184,7 @@ sub run
# Copy the new pg_control back so the tests can continue with the upgraded stanza
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94);
forceStorageMode(storageDb(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
forceStorageMode(storageTest(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
# After stanza upgrade, make sure archives are pushed to the new db verion-id directory (9.4-2)
#--------------------------------------------------------------------------------------------------------------------------
@@ -206,7 +206,7 @@ sub run
#--------------------------------------------------------------------------------------------------------------------------
# Copy pg_control for 9.5
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_95);
forceStorageMode(storageDb(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
forceStorageMode(storageTest(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
$oHostBackup->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
+7 -7
View File
@@ -324,7 +324,7 @@ sub run
'fail on backup info file missing from non-empty dir', {iExpectedExitStatus => ERROR_PATH_NOT_EMPTY});
# Change the database version by copying a new pg_control file to a new pg-path to use for db mismatch test
storageLocal()->pathCreate(
storageTest()->pathCreate(
$oHostDbMaster->dbPath() . '/testbase/' . DB_PATH_GLOBAL,
{strMode => '0700', bIgnoreExists => true, bCreateParent => true});
$self->controlGenerate(
@@ -774,7 +774,7 @@ sub run
# Version <= 8.4 always places a PG_VERSION file in the tablespace
if ($oHostDbMaster->pgVersion() <= PG_VERSION_84)
{
if (!storageLocal()->exists("${strTablespacePath}/" . DB_FILE_PGVERSION))
if (!storageTest()->exists("${strTablespacePath}/" . DB_FILE_PGVERSION))
{
confess &log(ASSERT, "unable to find '" . DB_FILE_PGVERSION . "' in tablespace path '${strTablespacePath}'");
}
@@ -792,14 +792,14 @@ sub run
'/PG_' . $oHostDbMaster->pgVersion() . qw{_} . $oBackupInfo->get(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_CATALOG);
# Check that path exists
if (!storageLocal()->pathExists($strTablespacePath))
if (!storageTest()->pathExists($strTablespacePath))
{
confess &log(ASSERT, "unable to find tablespace path '${strTablespacePath}'");
}
}
# Make sure there are some files in the tablespace path (exclude PG_VERSION if <= 8.4 since that was tested above)
if (grep(!/^PG\_VERSION$/i, storageLocal()->list($strTablespacePath)) == 0)
if (grep(!/^PG\_VERSION$/i, storageTest()->list($strTablespacePath)) == 0)
{
confess &log(ASSERT, "no files found in tablespace path '${strTablespacePath}'");
}
@@ -869,7 +869,7 @@ sub run
# Save recovery file to test so we can use it in the next test
$strRecoveryFile = $oHostDbMaster->pgVersion() >= PG_VERSION_12 ? 'postgresql.auto.conf' : DB_FILE_RECOVERYCONF;
storageLocal()->copy(
storageTest()->copy(
$oHostDbMaster->dbBasePath() . qw{/} . $strRecoveryFile, $self->testPath() . qw{/} . $strRecoveryFile);
$oHostDbMaster->clusterStart();
@@ -891,12 +891,12 @@ sub run
executeTest('rm -rf ' . $oHostDbMaster->tablespacePath(1) . "/*");
# Restore recovery file that was saved in last test
storageLocal()->move($self->testPath . "/${strRecoveryFile}", $oHostDbMaster->dbBasePath() . "/${strRecoveryFile}");
storageTest()->move($self->testPath . "/${strRecoveryFile}", $oHostDbMaster->dbBasePath() . "/${strRecoveryFile}");
# Also touch recovery.signal when required
if ($oHostDbMaster->pgVersion() >= PG_VERSION_12)
{
storageDb()->put($oHostDbMaster->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL);
storageTest()->put($oHostDbMaster->dbBasePath() . "/" . DB_FILE_RECOVERYSIGNAL);
}
$oHostDbMaster->restore(