db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-06-25 00:16:54 +02:00
Code Issues Releases Activity

The archive-get command is implemented entirely in C.

Browse Source 
This new implementation should behave exactly like the old Perl code with the exception of a few updated log messages.

Remove as much of the Perl code as possible without breaking other commands.
This commit is contained in:
David Steele
2019-02-27 23:03:02 +02:00
parent 9367cc461c
commit db4b447be8
27 changed files with 560 additions and 905 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
test/lib/pgBackRestTest/Module/Command/CommandArchiveGetPerlTest.pm
View File

@ -14,9 +14,7 @@ use Carp qw(confess);
use Storable qw(dclone);
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Get::Async;
use pgBackRest::Archive::Get::File;
use pgBackRest::Archive::Get::Get;
use pgBackRest::Archive::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
@ -191,190 +189,6 @@ sub run
substr($strWalSegment, 0, 16) . "/" . $strArchiveFile)})}, $strFileHash,
'check correct WAL archiveID when in multiple locations');
}
################################################################################################################################
if ($self->begin("Archive::Get::Get::get() sync"))
{
# archive.info missing
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {archiveGetFile($strWalSegment, $strDestinationFile, false)},
ERROR_FILE_MISSING,
ARCHIVE_INFO_FILE . " does not exist but is required to push/get WAL segments\n" .
"HINT: is archive_command configured in postgresql.conf?\n" .
"HINT: has a stanza-create been performed?\n" .
"HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.");
# Create and save archive.info file
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true});
$oArchiveInfo->create(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), false);
$oArchiveInfo->dbSectionSet(PG_VERSION_93, $self->dbSysId(PG_VERSION_93), $oArchiveInfo->dbHistoryIdGet(false) + 1);
$oArchiveInfo->dbSectionSet(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), $oArchiveInfo->dbHistoryIdGet(false) + 10);
$oArchiveInfo->save();
# file not found
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {archiveGetFile($strWalSegment, $strDestinationFile, false)}, 1,
"unable to find ${strWalSegment} in the archive");
# file found but is not a WAL segment
#---------------------------------------------------------------------------------------------------------------------------
$strArchivePath = $self->{strArchivePath} . "/" . PG_VERSION_94 . "-1/";
storageRepo()->pathCreate($strArchivePath);
storageRepo()->put($strArchivePath . BOGUS, BOGUS);
my $strBogusHash = cryptoHashOne('sha1', BOGUS);
# Create path to copy file
storageRepo()->pathCreate($strDestinationPath);
$self->testResult(sub {archiveGetFile(BOGUS, $strDestinationFile, false)}, 0,
"non-WAL segment copied");
# Confirm the correct file is copied
$self->testResult(sub {cryptoHashOne('sha1', ${storageRepo()->get($strDestinationFile)})}, $strBogusHash,
' check correct non-WAL copied from older archiveId');
# create same WAL segment in same DB but different archives and different hash values. Confirm latest one copied.
#---------------------------------------------------------------------------------------------------------------------------
my $strWalMajorPath = "${strArchivePath}/" . substr($strWalSegment, 0, 16);
my $strWalSegmentName = "${strWalSegment}-${strFileHash}";
# Put zero byte file in old archive
storageRepo()->pathCreate($strWalMajorPath);
storageRepo()->put("${strWalMajorPath}/${strWalSegmentName}");
# Create newest archive path
$strArchivePath = $self->{strArchivePath} . "/" . PG_VERSION_94 . "-12/";
$strWalMajorPath = "${strArchivePath}/" . substr($strWalSegment, 0, 16);
$strWalSegmentName = "${strWalSegment}-${strFileHash}";
# Store with actual data that will match the hash check
storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true});
storageRepo()->put("${strWalMajorPath}/${strWalSegmentName}", $strFileContent);
$self->testResult(sub {archiveGetFile($strWalSegmentName, $strDestinationFile, false)}, 0,
"WAL segment copied");
# Confirm the correct file is copied
$self->testResult(sub {cryptoHashOne('sha1', ${storageRepo()->get($strDestinationFile)})}, $strFileHash,
' check correct WAL copied when in multiple locations');
# Get files from an older DB version to simulate restoring from an old backup set to a database that is of that same version
#---------------------------------------------------------------------------------------------------------------------------
# Create same WAL name in older DB archive but with different data to ensure it is copied
$strArchivePath = $self->{strArchivePath} . "/" . PG_VERSION_93 . "-2/";
$strWalMajorPath = "${strArchivePath}/" . substr($strWalSegment, 0, 16);
$strWalSegmentName = "${strWalSegment}-${strFileHash}";
my $strWalContent = 'WALTESTDATA';
my $strWalHash = cryptoHashOne('sha1', $strWalContent);
# Store with actual data that will match the hash check
storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true});
storageRepo()->put("${strWalMajorPath}/${strWalSegmentName}", $strWalContent);
# Remove the destination file to ensure it is copied
storageTest()->remove($strDestinationFile);
# Overwrite current pg_control file with older version
$self->controlGenerate($self->{strDbPath}, PG_VERSION_93);
$self->testResult(sub {archiveGetFile($strWalSegmentName, $strDestinationFile, false)}, 0,
"WAL segment copied from older db backupset to same version older db");
# Confirm the correct file is copied
$self->testResult(sub {cryptoHashOne('sha1', ${storageRepo()->get($strDestinationFile)})}, $strWalHash,
' check correct WAL copied from older db');
}
################################################################################################################################
if ($self->begin("Archive::Get::Get::get() async"))
{
# Test error in local process when stanza has not been created
#---------------------------------------------------------------------------------------------------------------------------
my @stryWal = ('000000010000000A0000000A', '000000010000000A0000000B');
my $oGetAsync = new pgBackRest::Archive::Get::Async(
$self->{strSpoolPath}, $self->backrestExe(), \@stryWal);
$self->optionTestSet(CFGOPT_SPOOL_PATH, $self->{strRepoPath});
$self->configTestLoad(CFGCMD_ARCHIVE_GET_ASYNC);
$oGetAsync->process();
my $strErrorMessage =
"55\n" .
"raised from local-1 process: archive.info does not exist but is required to push/get WAL segments\n" .
"HINT: is archive_command configured in postgresql.conf?\n" .
"HINT: has a stanza-create been performed?\n" .
"HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.";
$self->testResult(
sub {storageSpool()->list(STORAGE_SPOOL_ARCHIVE_IN)},
"(000000010000000A0000000A.error, 000000010000000A0000000B.error)", 'error files created');
$self->testResult(
${storageSpool()->get(STORAGE_SPOOL_ARCHIVE_IN . "/000000010000000A0000000A.error")}, $strErrorMessage,
"check error file contents");
storageSpool()->remove(STORAGE_SPOOL_ARCHIVE_IN . "/000000010000000A0000000A.error");
$self->testResult(
${storageSpool()->get(STORAGE_SPOOL_ARCHIVE_IN . "/000000010000000A0000000B.error")}, $strErrorMessage,
"check error file contents");
storageSpool()->remove(STORAGE_SPOOL_ARCHIVE_IN . "/000000010000000A0000000B.error");
# Create archive info file
#---------------------------------------------------------------------------------------------------------------------------
my $oArchiveInfo = new pgBackRest::Archive::Info($self->{strArchivePath}, false, {bIgnoreMissing => true});
$oArchiveInfo->create(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), true);
my $strArchiveId = $oArchiveInfo->archiveId();
# Transfer first file
#---------------------------------------------------------------------------------------------------------------------------
my $strWalPath = "$self->{strRepoPath}/archive/db/${strArchiveId}/000000010000000A";
storageRepo()->pathCreate($strWalPath, {bCreateParent => true});
$self->walGenerate($strWalPath, PG_VERSION_94, 1, "000000010000000A0000000A", false, true, false);
$oGetAsync->processQueue();
$self->testResult(
sub {storageSpool()->list(STORAGE_SPOOL_ARCHIVE_IN)},
"(000000010000000A0000000A, 000000010000000A0000000B.ok)", 'WAL and OK file');
# Transfer second file
#---------------------------------------------------------------------------------------------------------------------------
@stryWal = ('000000010000000A0000000B');
storageSpool()->remove(STORAGE_SPOOL_ARCHIVE_IN . "/000000010000000A0000000B.ok");
$self->walGenerate($strWalPath, PG_VERSION_94, 1, "000000010000000A0000000B", false, true, false);
$oGetAsync->processQueue();
$self->testResult(
sub {storageSpool()->list(STORAGE_SPOOL_ARCHIVE_IN)},
"(000000010000000A0000000A, 000000010000000A0000000B)", 'WAL files');
# Error on main process
#---------------------------------------------------------------------------------------------------------------------------
@stryWal = ('000000010000000A0000000C');
storageTest()->put(storageTest()->openWrite($self->{strLockPath} . "/db.stop", {bPathCreate => true}), undef);
$oGetAsync->processQueue();
$self->testResult(
sub {storageSpool()->list(STORAGE_SPOOL_ARCHIVE_IN)},
"(000000010000000A0000000A, 000000010000000A0000000B, 000000010000000A0000000C.error)", 'WAL files and error file');
# Set protocol timeout low
#---------------------------------------------------------------------------------------------------------------------------
$self->optionTestSet(CFGOPT_PROTOCOL_TIMEOUT, 30);
$self->optionTestSet(CFGOPT_DB_TIMEOUT, 29);
$self->configTestLoad(CFGCMD_ARCHIVE_GET_ASYNC);
$oGetAsync->process();
}
}
1;