db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-10-30 23:37:45 +02:00
Code Issues Releases Activity

The stanza-create/update/delete commands are implemented entirely in C.

Browse Source 
Contributed by Cynthia Shang.
This commit is contained in:
Cynthia Shang
2019-08-21 16:26:28 -04:00
committed by David Steele
parent 53f27da3a6
commit c733319063
33 changed files with 2037 additions and 3400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
View File

@@ -1119,23 +1119,15 @@ sub run
$strType = CFGOPTVAL_BACKUP_TYPE_INCR;
$oHostDbMaster->manifestReference(\%oManifest, $strBackup);
# Delete the backup.info and make sure the backup fails - the user must then run a stanza-create --force. If backup.info is
# encrypted is cannot be deleted, so copy it to old instead.
# Delete the backup.info and make sure the backup fails.
my $strBackupInfoFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO;
my $strBackupInfoCopyFile = STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO . INI_COPY_EXT;
my $strBackupInfoOldFile = "${strBackupInfoFile}.old";
my $strBackupInfoCopyOldFile = "${strBackupInfoCopyFile}.old";
if ($bEncrypt)
{
# Save the backup.info and copy files so they can be restored later
forceStorageMove(storageRepo(), $strBackupInfoFile, $strBackupInfoOldFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strBackupInfoCopyFile, $strBackupInfoCopyOldFile, {bRecurse => false});
}
else
{
forceStorageRemove(storageRepo(), $strBackupInfoFile);
forceStorageRemove(storageRepo(), $strBackupInfoCopyFile);
}
$oHostDbMaster->manifestFileCreate(
\%oManifest, MANIFEST_TARGET_PGDATA, 'base/16384/17000', 'BASEUPDT', '9a53d532e27785e681766c98516a5e93f096a501',
@@ -1152,17 +1144,9 @@ sub run
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
}
# Use force to create the stanza (this is expected to fail for encrypted repos)
$oHostBackup->stanzaCreate('create required data for stanza',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE),
iExpectedExitStatus => $bEncrypt ? ERROR_FILE_MISSING : undef});
# Copy encrypted backup info files back so testing can proceed
if ($bEncrypt)
{
forceStorageMove(storageRepo(), $strBackupInfoOldFile, $strBackupInfoFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strBackupInfoCopyOldFile, $strBackupInfoCopyFile, {bRecurse => false});
}
# Copy backup info files back so testing can proceed
forceStorageMove(storageRepo(), $strBackupInfoOldFile, $strBackupInfoFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strBackupInfoCopyOldFile, $strBackupInfoCopyFile, {bRecurse => false});
# Perform the backup
$strBackup =$oHostBackup->backup($strType, 'update files', {oExpectedManifest => \%oManifest});

2
test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm
View File

@@ -143,7 +143,7 @@ sub run
#---------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaCreate(
'stanza create',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' push first WAL');

224
test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm
View File

@@ -79,8 +79,8 @@ sub run
true, $self->expect(), {bHostBackup => $bRemote, bS3 => $bS3, bRepoEncrypt => $bEncrypt});
# Create the stanza
$oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_OPEN,
strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('fail on missing control file', {iExpectedExitStatus => ERROR_FILE_MISSING,
strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_LOG_LEVEL_FILE) . '=info'});
# Generate pg_control for stanza-create
storageDb()->pathCreate(($oHostDbMaster->dbBasePath() . '/' . DB_PATH_GLOBAL), {bCreateParent => true});
@@ -106,8 +106,9 @@ sub run
# Change the database version by copying a new pg_control file
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_94);
$oHostBackup->stanzaCreate('fail on database mismatch without force option',
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaCreate('fail on database mismatch and warn force option deprecated',
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) .
' --' . cfgOptionName(CFGOPT_FORCE)});
# Restore pg_control
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_93);
@@ -130,17 +131,9 @@ sub run
' --stanza=db archive-push';
$oHostDbMaster->executeSimple($strCommand . " ${strSourceFile}", {oLogTest => $self->expect()});
# With data existing in the archive dir, remove the info files and confirm failure
if ($bEncrypt)
{
forceStorageMove(storageRepo(), $strArchiveInfoFile, $strArchiveInfoOldFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strArchiveInfoCopyFile, $strArchiveInfoCopyOldFile, {bRecurse => false});
}
else
{
forceStorageRemove(storageRepo(), $strArchiveInfoFile);
forceStorageRemove(storageRepo(), $strArchiveInfoCopyFile);
}
# With data existing in the archive dir, move the info files and confirm failure
forceStorageMove(storageRepo(), $strArchiveInfoFile, $strArchiveInfoOldFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strArchiveInfoCopyFile, $strArchiveInfoCopyOldFile, {bRecurse => false});
if (!$bEncrypt)
{
@@ -148,96 +141,9 @@ sub run
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
}
# Stanza Create fails using force - failure to unzip compressed file
#--------------------------------------------------------------------------------------------------------------------------
# S3 doesn't support filesystem-style permissions so skip these tests
if (!$bS3)
{
# Change the permissions of the archive file so it cannot be read
forceStorageMode(
storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_93 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' .
COMPRESS_EXT,
'220');
# Force creation of the info file but fail on gunzip
$oHostBackup->stanzaCreate('gunzip fail on forced stanza-create',
{iExpectedExitStatus => ERROR_FILE_OPEN, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' .
cfgOptionName(CFGOPT_FORCE)});
# Change permissions back
forceStorageMode(
storageRepo(), STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_93 . '-1/' . substr($strArchiveFile, 0, 16) . '/*.' .
COMPRESS_EXT,
'640');
}
# Stanza Create succeeds when using force - recreates archive.info from compressed archive file
#--------------------------------------------------------------------------------------------------------------------------
# Force creation of archive info from the gz file
$oHostBackup->stanzaCreate('force create archive.info from gz file',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE),
iExpectedExitStatus => $bEncrypt ? ERROR_FILE_MISSING : undef});
if (!$bEncrypt)
{
$self->testResult(sub {storageRepo()->exists($strArchiveInfoFile)}, true, " archive.info file was created");
}
# Stanza Create succeeds when using force - recreates archive.info from uncompressed archive file
#--------------------------------------------------------------------------------------------------------------------------
# Unzip the archive file and recreate the archive.info file from it
my $strArchiveTest = PG_VERSION_93 . "-1/${strArchiveFile}-" . $self->walGenerateContentChecksum(PG_VERSION_93);
if (!$bEncrypt)
{
forceStorageMode(
storageRepo(), dirname(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveTest}.gz")), 'g+w',
{bRecursive => true});
storageRepo()->copy(
storageRepo()->openRead(
STORAGE_REPO_ARCHIVE . "/${strArchiveTest}.gz",
{rhyFilter => [{strClass => STORAGE_FILTER_GZIP, rxyParam => [STORAGE_DECOMPRESS, false]}]}),
STORAGE_REPO_ARCHIVE . "/${strArchiveTest}");
$oHostBackup->stanzaCreate('force create archive.info from uncompressed file',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
}
# Stanza Create succeeds when using force - missing archive file
#--------------------------------------------------------------------------------------------------------------------------
# Remove the uncompressed WAL archive file and archive.info
if (!$bEncrypt)
{
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . "/${strArchiveTest}");
forceStorageRemove(storageRepo(), $strArchiveInfoFile);
forceStorageRemove(storageRepo(), $strArchiveInfoCopyFile);
$oHostBackup->stanzaCreate('force with missing WAL archive file',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
}
# Stanza Create succeeds when using force - missing archive directory
#--------------------------------------------------------------------------------------------------------------------------
# Remove the WAL archive directory
if (!$bEncrypt)
{
forceStorageRemove(
storageRepo(),
STORAGE_REPO_ARCHIVE . qw{/} . PG_VERSION_93 . '-1/' . substr($strArchiveFile, 0, 16), {bRecurse => true});
forceStorageRemove(storageRepo(), $strArchiveInfoFile);
forceStorageRemove(storageRepo(), $strArchiveInfoCopyFile);
$oHostBackup->stanzaCreate('force with missing WAL archive directory',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
}
# Encrypted info files could not be reconstructed above so just copy them back
if ($bEncrypt)
{
forceStorageMove(storageRepo(), $strArchiveInfoOldFile, $strArchiveInfoFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strArchiveInfoCopyOldFile, $strArchiveInfoCopyFile, {bRecurse => false});
}
# Restore info files from copy
forceStorageMove(storageRepo(), $strArchiveInfoOldFile, $strArchiveInfoFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strArchiveInfoCopyOldFile, $strArchiveInfoCopyFile, {bRecurse => false});
# Just before upgrading push one last WAL on the old version to ensure it can be retrieved later
#--------------------------------------------------------------------------------------------------------------------------
@@ -259,6 +165,9 @@ sub run
# Perform a successful stanza upgrade noting additional history lines in info files for new version of the database
#--------------------------------------------------------------------------------------------------------------------------
# Save a pre-upgrade copy of archive info fo testing db-id mismatch
forceStorageMove(storageRepo(), $strArchiveInfoCopyFile, $strArchiveInfoCopyOldFile, {bRecurse => false});
$oHostBackup->stanzaUpgrade('successful upgrade creates additional history', {strOptionalParam => '--no-' .
cfgOptionName(CFGOPT_ONLINE)});
@@ -286,104 +195,41 @@ sub run
'000000010000000100000001-' . $self->walGenerateContentChecksum(PG_VERSION_94) . '.' . COMPRESS_EXT,
'check that WAL is in the archive at -2');
# Create a DB history mismatch between the info files
#--------------------------------------------------------------------------------------------------------------------------
# Remove the archive info file and force reconstruction
if (!$bEncrypt)
{
forceStorageMove(storageRepo(), $strArchiveInfoFile, $strArchiveInfoOldFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strArchiveInfoCopyFile, $strArchiveInfoCopyOldFile, {bRecurse => false});
forceStorageRemove(storageRepo(), $strArchiveInfoFile);
forceStorageRemove(storageRepo(), $strArchiveInfoCopyFile);
$oHostBackup->stanzaCreate('use force to recreate the stanza producing mismatched info history but same current db-id',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
}
# Create a DB-ID mismatch between the info files
#--------------------------------------------------------------------------------------------------------------------------
if (!$bEncrypt)
{
forceStorageMove(storageRepo(), $strBackupInfoFile, $strBackupInfoOldFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strBackupInfoCopyFile, $strBackupInfoCopyOldFile, {bRecurse => false});
forceStorageRemove(storageRepo(), $strBackupInfoFile);
forceStorageRemove(storageRepo(), $strBackupInfoCopyFile);
$oHostBackup->stanzaCreate('use force to recreate the stanza producing mismatched db-id',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE)});
}
# Confirm successful backup at db-1 although archive at db-2 and format error thrown by expire (if not encrypted) since
# archive.info history and backup.info history are mismatched
#--------------------------------------------------------------------------------------------------------------------------
# Create the tablespace directory and perform a backup
#--------------------------------------------------------------------------------------------------------------------------
storageTest()->pathCreate($oHostDbMaster->dbBasePath() . '/' . DB_PATH_PGTBLSPC);
$oHostBackup->backup(
'full', 'create first full backup ',
{iExpectedExitStatus => $bEncrypt ? undef : ERROR_FORMAT,
strOptionalParam => '--repo1-retention-full=2 --no-' . cfgOptionName(CFGOPT_ONLINE)}, false);
{strOptionalParam => '--repo1-retention-full=2 --no-' . cfgOptionName(CFGOPT_ONLINE)}, false);
# Stanza Create fails when not using force - no backup.info but backup exists
# Upgrade the stanza
#--------------------------------------------------------------------------------------------------------------------------
if ($bEncrypt)
{
forceStorageMove(storageRepo(), $strBackupInfoFile, $strBackupInfoOldFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strBackupInfoCopyFile, $strBackupInfoCopyOldFile, {bRecurse => false});
}
else
{
forceStorageRemove(storageRepo(), $strBackupInfoFile);
forceStorageRemove(storageRepo(), $strBackupInfoCopyFile);
}
$oHostBackup->stanzaCreate('fail no force to recreate the stanza from backups',
{iExpectedExitStatus => ERROR_FILE_MISSING, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
# Stanza Create succeeds using force - reconstruct backup.info from backup
#--------------------------------------------------------------------------------------------------------------------------
$oHostBackup->stanzaCreate('use force to recreate the stanza from backups',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE) . ' --' . cfgOptionName(CFGOPT_FORCE),
iExpectedExitStatus => $bEncrypt ? ERROR_FILE_MISSING : undef});
# Copy old backup.info files back to avoid history mismatch
forceStorageMove(storageRepo(), $strBackupInfoOldFile, $strBackupInfoFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strBackupInfoCopyOldFile, $strBackupInfoCopyFile, {bRecurse => false});
# Copy pg_control for 9.5
$self->controlGenerate($oHostDbMaster->dbBasePath(), PG_VERSION_95);
forceStorageMode(storageDb(), $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL, '600');
# Test archive dir version XX.Y-Z ensuring sort order of db ids is reconstructed correctly from the directory db-id value.
# Not testing with encryption since unnecessary and copying files around in this case is burdensome.
#--------------------------------------------------------------------------------------------------------------------------
if (!$bEncrypt)
{
# Create the 10-3 directory and copy a WAL file to it (with a different system id than what it will be upgraded to)
forceStorageMode(storageRepo(), STORAGE_REPO_ARCHIVE, '770');
storageRepo()->pathCreate(STORAGE_REPO_ARCHIVE . '/10-3/0000000100000001', {bCreateParent => true});
storageRepo()->put(
storageRepo()->openWrite(
STORAGE_REPO_ARCHIVE . '/10-3/0000000100000001/000000010000000100000001',
{strCipherPass => $oHostBackup->cipherPassArchive()}),
$self->walGenerateContent(PG_VERSION_10));
forceStorageOwner(storageRepo(), STORAGE_REPO_ARCHIVE . '/10-3', $oHostBackup->userGet(), {bRecurse => true});
# Make sure the archive.info has the history in the db-id order such that 10 is before 9.5.
$oHostBackup->stanzaUpgrade(
'successfully upgrade with XX.Y-Z',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
$oHostBackup->stanzaUpgrade('successfully upgrade', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
# Remove the 10-3 directory and copy old archive.info file back. Recreate so archive.info and backup.info files match.
forceStorageRemove(storageRepo(), STORAGE_REPO_ARCHIVE . '/10-3', {bRecurse => true});
forceStorageRemove(storageRepo(), $strArchiveInfoFile);
forceStorageRemove(storageRepo(), $strArchiveInfoCopyFile);
forceStorageMove(storageRepo(), $strArchiveInfoOldFile, $strArchiveInfoFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strArchiveInfoCopyOldFile, $strArchiveInfoCopyFile, {bRecurse => false});
}
# Copy archive.info and restore really old version
forceStorageMove(storageRepo(), $strArchiveInfoFile, $strArchiveInfoOldFile, {bRecurse => false});
forceStorageRemove(storageRepo(), $strArchiveInfoCopyFile, {bRecurse => false});
forceStorageMove(storageRepo(), $strArchiveInfoCopyOldFile, $strArchiveInfoFile, {bRecurse => false});
# Confirm versions
my $oAchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet('archive/' . $self->stanza()));
my $oBackupInfo = new pgBackRest::Backup::Info(storageRepo()->pathGet('backup/' . $self->stanza()));
$self->testResult(sub {$oAchiveInfo->test(INFO_ARCHIVE_SECTION_DB, INFO_ARCHIVE_KEY_DB_VERSION, undef,
PG_VERSION_93)}, true, 'archive at old pg version');
$self->testResult(sub {$oBackupInfo->test(INFO_BACKUP_SECTION_DB, INFO_BACKUP_KEY_DB_VERSION, undef,
PG_VERSION_95)}, true, 'backup at new pg version');
$oHostBackup->stanzaUpgrade(
'successfully upgrade - no info file mismatch',
{strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
'upgrade fails with mismatched db-ids',
{iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
# Restore archive.info
forceStorageMove(storageRepo(), $strArchiveInfoOldFile, $strArchiveInfoFile, {bRecurse => false});
# Push a WAL and create a backup in the new DB to confirm diff changed to full and info command displays the JSON correctly
#--------------------------------------------------------------------------------------------------------------------------