db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Refactor archive common functions in preparation for parallel async archive-get.

Browse Source
This commit is contained in:
David Steele
2018-04-29 10:16:59 -04:00
parent 721432f6ae
commit 89d3476e32
25 changed files with 575 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/pgBackRest/Archive/Base.pm
-102
View File
@@ -54,106 +54,4 @@ sub new
);
}
####################################################################################################################################
# getCheck - Given a specific database version and system-id, find a file in the archive. If no database info was passed, the
# current database will be used.
####################################################################################################################################
sub getCheck
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$ullDbSysId,
$strFile,
$bCheck,
) =
logDebugParam
(
__PACKAGE__ . '->getCheck', \@_,
{name => 'strDbVersion', required => false},
{name => 'ullDbSysId', required => false},
{name => 'strFile', required => false},
{name => 'bCheck', required => false, default => true},
);
my @stryArchiveId = ();
my $strArchiveId;
my $strArchiveFile;
my $strCipherPass;
# If the dbVersion/dbSysId are not passed, then we need to retrieve the database information
if (!defined($strDbVersion) || !defined($ullDbSysId) )
{
# get DB info for comparison
($strDbVersion, my $iControlVersion, my $iCatalogVersion, $ullDbSysId) = dbMasterGet()->info();
}
# Get db info from the repo
if (!isRepoLocal())
{
($strArchiveId, $strArchiveFile, $strCipherPass) = protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP)->cmdExecute(
OP_ARCHIVE_GET_CHECK, [$strDbVersion, $ullDbSysId, $strFile, $bCheck], true);
}
else
{
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), true);
# Check that the archive info is compatible with the database if required (not required for archive-get)
if ($bCheck)
{
push(@stryArchiveId, $oArchiveInfo->check($strDbVersion, $ullDbSysId));
}
# Else if the database version and system-id are in the info history list then get a list of corresponding archiveIds
else
{
@stryArchiveId = $oArchiveInfo->archiveIdList($strDbVersion, $ullDbSysId);
}
# Default the returned archiveId to the newest in the event the WAL segment is not found then the most recent archiveID will
# be returned. If none were found, then the preceding calls will error.
$strArchiveId = $stryArchiveId[0];
# If a file was passed to look for, then look for the file starting in the newest matching archiveId to the oldest
if (defined($strFile))
{
foreach my $strId (@stryArchiveId)
{
# Then if it is a WAL segment, try to find it
if (walIsSegment($strFile))
{
$strArchiveFile = walSegmentFind(storageRepo(), $strId, $strFile);
}
# Else if not a WAL segment, see if it exists in the archive dir
elsif (storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/${strId}/${strFile}"))
{
$strArchiveFile = $strFile;
}
# If the file was found, then return the archiveId where it was found
if (defined($strArchiveFile))
{
$strArchiveId = $strId;
last;
}
}
}
# Get the encryption passphrase to read/write files (undefined if the repo is not encrypted)
$strCipherPass = $oArchiveInfo->cipherPassSub();
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strArchiveId', value => $strArchiveId},
{name => 'strArchiveFile', value => $strArchiveFile},
{name => 'strCipherPass', value => $strCipherPass, redact => true}
);
}
1;
lib/pgBackRest/Archive/Common.pm
+65
View File
@@ -44,6 +44,14 @@ use constant PG_WAL_SYSTEM_ID_OFFSET_LT_93 => 12;
use constant PG_WAL_SEGMENT_SIZE => 16777216;
push @EXPORT, qw(PG_WAL_SEGMENT_SIZE);
####################################################################################################################################
# WAL status constants
####################################################################################################################################
use constant WAL_STATUS_ERROR => 'error';
push @EXPORT, qw(WAL_STATUS_ERROR);
use constant WAL_STATUS_OK => 'ok';
push @EXPORT, qw(WAL_STATUS_OK);
####################################################################################################################################
# PostgreSQL WAL magic
####################################################################################################################################
@@ -425,4 +433,61 @@ sub walIsPartial
push @EXPORT, qw(walIsPartial);
####################################################################################################################################
# archiveAsyncStatusWrite
#
# Write out a status file to the spool path with information about the success or failure of an archive push.
####################################################################################################################################
sub archiveAsyncStatusWrite
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strType,
$strSpoolPath,
$strWalFile,
$iCode,
$strMessage,
) =
logDebugParam
(
__PACKAGE__ . '::archiveAsyncStatusWrite', \@_,
{name => 'strType'},
{name => 'strSpoolPath'},
{name => 'strWalFile'},
{name => 'iCode', required => false},
{name => 'strMessage', required => false},
);
# Remove any error file exists unless a new one will be written
if ($strType ne WAL_STATUS_ERROR)
{
# Remove the error file, if any
storageLocal()->remove("${strSpoolPath}/${strWalFile}.error", {bIgnoreMissing => true});
}
# Write the status file
my $strStatus;
if (defined($iCode))
{
if (!defined($strMessage))
{
confess &log(ASSERT, 'strMessage must be set when iCode is set');
}
$strStatus = "${iCode}\n${strMessage}";
}
elsif ($strType eq WAL_STATUS_ERROR)
{
confess &log(ASSERT, 'error status must have iCode and strMessage set');
}
storageLocal()->put(
storageLocal()->openWrite("${strSpoolPath}/${strWalFile}.${strType}", {bAtomic => true}), $strStatus);
}
push @EXPORT, qw(archiveAsyncStatusWrite);
1;
lib/pgBackRest/Archive/Get/File.pm
+202
View File
@@ -0,0 +1,202 @@
####################################################################################################################################
# ARCHIVE GET FILE MODULE
####################################################################################################################################
package pgBackRest::Archive::Get::File;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(basename dirname);
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Info;
use pgBackRest::Db;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Filter::Gzip;
use pgBackRest::Storage::Filter::Sha;
use pgBackRest::Storage::Helper;
####################################################################################################################################
# Given a specific database version and system-id, find a file in the archive. If no database info was passed, the current database
# will be used.
####################################################################################################################################
sub archiveGetCheck
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strDbVersion,
$ullDbSysId,
$strFile,
$bCheck,
) =
logDebugParam
(
__PACKAGE__ . '::archiveGetCheck', \@_,
{name => 'strDbVersion', required => false},
{name => 'ullDbSysId', required => false},
{name => 'strFile', required => false},
{name => 'bCheck', required => false, default => true},
);
my @stryArchiveId = ();
my $strArchiveId;
my $strArchiveFile;
my $strCipherPass;
# If the dbVersion/dbSysId are not passed, then we need to retrieve the database information
if (!defined($strDbVersion) || !defined($ullDbSysId) )
{
# get DB info for comparison
($strDbVersion, my $iControlVersion, my $iCatalogVersion, $ullDbSysId) = dbMasterGet()->info();
}
# Get db info from the repo
if (!isRepoLocal())
{
($strArchiveId, $strArchiveFile, $strCipherPass) = protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP)->cmdExecute(
OP_ARCHIVE_GET_CHECK, [$strDbVersion, $ullDbSysId, $strFile, $bCheck], true);
}
else
{
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), true);
# Check that the archive info is compatible with the database if required (not required for archive-get)
if ($bCheck)
{
push(@stryArchiveId, $oArchiveInfo->check($strDbVersion, $ullDbSysId));
}
# Else if the database version and system-id are in the info history list then get a list of corresponding archiveIds
else
{
@stryArchiveId = $oArchiveInfo->archiveIdList($strDbVersion, $ullDbSysId);
}
# Default the returned archiveId to the newest in the event the WAL segment is not found then the most recent archiveID will
# be returned. If none were found, then the preceding calls will error.
$strArchiveId = $stryArchiveId[0];
# If a file was passed to look for, then look for the file starting in the newest matching archiveId to the oldest
if (defined($strFile))
{
foreach my $strId (@stryArchiveId)
{
# Then if it is a WAL segment, try to find it
if (walIsSegment($strFile))
{
$strArchiveFile = walSegmentFind(storageRepo(), $strId, $strFile);
}
# Else if not a WAL segment, see if it exists in the archive dir
elsif (storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/${strId}/${strFile}"))
{
$strArchiveFile = $strFile;
}
# If the file was found, then return the archiveId where it was found
if (defined($strArchiveFile))
{
$strArchiveId = $strId;
last;
}
}
}
# Get the encryption passphrase to read/write files (undefined if the repo is not encrypted)
$strCipherPass = $oArchiveInfo->cipherPassSub();
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strArchiveId', value => $strArchiveId},
{name => 'strArchiveFile', value => $strArchiveFile},
{name => 'strCipherPass', value => $strCipherPass, redact => true}
);
}
push @EXPORT, qw(archiveGetCheck);
####################################################################################################################################
# archiveGetFile
#
# Copy a file from the archive.
####################################################################################################################################
sub archiveGetFile
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strSourceArchive,
$strDestinationFile
) =
logDebugParam
(
__PACKAGE__ . '::archiveGetFile', \@_,
{name => 'strSourceArchive'},
{name => 'strDestinationFile'}
);
lockStopTest();
# Get the repo storage
my $oStorageRepo = storageRepo();
# Construct absolute path to the WAL file when it is relative
$strDestinationFile = walPath($strDestinationFile, cfgOption(CFGOPT_PG_PATH, false), cfgCommandName(cfgCommandGet()));
# Get the wal segment filename
my ($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(undef, undef, $strSourceArchive, false);
# If there are no matching archive files then there are two possibilities:
# 1) The end of the archive stream has been reached, this is normal and a 1 will be returned
# 2) There is a hole in the archive stream and a hard error should be returned. However, holes are possible due to async
# archiving - so when to report a hole? Since a hard error will cause PG to terminate, for now treat as case #1.
my $iResult = 0;
if (!defined($strArchiveFile))
{
&log(INFO, "unable to find ${strSourceArchive} in the archive");
$iResult = 1;
}
else
{
# Determine if the source file is already compressed
my $bSourceCompressed = $strArchiveFile =~ ('^.*\.' . COMPRESS_EXT . '$') ? true : false;
# Copy the archive file to the requested location
# If the file is encrypted, then the passphrase from the info file is required to open the archive file in the repo
$oStorageRepo->copy(
$oStorageRepo->openRead(
STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strArchiveFile}", {bProtocolCompress => !$bSourceCompressed,
strCipherPass => defined($strCipherPass) ? $strCipherPass : undef}),
storageDb()->openWrite(
$strDestinationFile,
{rhyFilter => $bSourceCompressed ?
[{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef}));
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iResult', value => $iResult}
);
}
push @EXPORT, qw(archiveGetFile);
1;
lib/pgBackRest/Archive/Get/Get.pm
+2 -103
View File
@@ -19,6 +19,7 @@ use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Get::File;
use pgBackRest::Archive::Info;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
@@ -77,109 +78,7 @@ sub process
return logDebugReturn
(
$strOperation,
{name => 'iResult', value => $self->get($strSourceArchive, $strDestinationFile), trace => true}
);
}
####################################################################################################################################
# get
####################################################################################################################################
sub get
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strSourceArchive,
$strDestinationFile
) =
logDebugParam
(
__PACKAGE__ . '->get', \@_,
{name => 'strSourceArchive'},
{name => 'strDestinationFile'}
);
lockStopTest();
# Get the repo storage
my $oStorageRepo = storageRepo();
# Construct absolute path to the WAL file when it is relative
$strDestinationFile = walPath($strDestinationFile, cfgOption(CFGOPT_PG_PATH, false), cfgCommandName(cfgCommandGet()));
# Get the wal segment filename
my ($strArchiveId, $strArchiveFile, $strCipherPass) = $self->getCheck(undef, undef, $strSourceArchive, false);
# If there are no matching archive files then there are two possibilities:
# 1) The end of the archive stream has been reached, this is normal and a 1 will be returned
# 2) There is a hole in the archive stream and a hard error should be returned. However, holes are possible due to async
# archiving - so when to report a hole? Since a hard error will cause PG to terminate, for now treat as case #1.
my $iResult = 0;
if (!defined($strArchiveFile))
{
&log(INFO, "unable to find ${strSourceArchive} in the archive");
$iResult = 1;
}
else
{
# Determine if the source file is already compressed
my $bSourceCompressed = $strArchiveFile =~ ('^.*\.' . COMPRESS_EXT . '$') ? true : false;
# Copy the archive file to the requested location
# If the file is encrypted, then the passphrase from the info file is required to open the archive file in the repo
$oStorageRepo->copy(
$oStorageRepo->openRead(
STORAGE_REPO_ARCHIVE . "/${strArchiveId}/${strArchiveFile}", {bProtocolCompress => !$bSourceCompressed,
strCipherPass => defined($strCipherPass) ? $strCipherPass : undef}),
storageDb()->openWrite(
$strDestinationFile,
{rhyFilter => $bSourceCompressed ?
[{strClass => STORAGE_FILTER_GZIP, rxyParam => [{strCompressType => STORAGE_DECOMPRESS}]}] : undef}));
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'iResult', value => $iResult}
);
}
####################################################################################################################################
# getArchiveId
#
# CAUTION: Only to be used by commands where the DB Version and DB System ID are not important such that the pg-path is not valid
# for the command (i.e. expire command). Since this function will not check validity of the database version call getCheck()
# instead.
####################################################################################################################################
sub getArchiveId
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->getArchiveId');
my $strArchiveId;
if (!isRepoLocal())
{
$strArchiveId = protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP)->cmdExecute(OP_ARCHIVE_GET_ARCHIVE_ID, undef, true);
}
else
{
$strArchiveId = (new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), true))->archiveId();
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strArchiveId', value => $strArchiveId, trace => true}
{name => 'iResult', value => archiveGetFile($strSourceArchive, $strDestinationFile), trace => true},
);
}
lib/pgBackRest/Archive/Push/Async.pm
+9 -63
View File
@@ -169,8 +169,9 @@ sub processQueue
# If error then write out an error file
if (defined($hJob->{oException}))
{
$self->walStatusWrite(
WAL_STATUS_ERROR, $strWalFile, $hJob->{oException}->code(), $hJob->{oException}->message());
archiveAsyncStatusWrite(
WAL_STATUS_ERROR, $self->{strSpoolPath}, $strWalFile, $hJob->{oException}->code(),
$hJob->{oException}->message());
$iErrorTotal++;
@@ -181,8 +182,8 @@ sub processQueue
# Else write success
else
{
$self->walStatusWrite(
WAL_STATUS_OK, $strWalFile, defined($strWarning) ? 0 : undef,
archiveAsyncStatusWrite(
WAL_STATUS_OK, $self->{strSpoolPath}, $strWalFile, defined($strWarning) ? 0 : undef,
defined($strWarning) ? $strWarning : undef);
$iOkTotal++;
@@ -200,8 +201,8 @@ sub processQueue
{
foreach my $strDropFile (@{$stryDropList})
{
$self->walStatusWrite(
WAL_STATUS_OK, $strDropFile, 0,
archiveAsyncStatusWrite(
WAL_STATUS_OK, $self->{strSpoolPath}, $strDropFile, 0,
"dropped WAL file ${strDropFile} because archive queue exceeded " .
cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) . ' bytes');
@@ -224,8 +225,8 @@ sub processQueue
# Error all queued jobs
foreach my $strWalFile (@{$stryWalFile})
{
$self->walStatusWrite(
WAL_STATUS_ERROR, $strWalFile, $iCode, $strMessage);
archiveAsyncStatusWrite(
WAL_STATUS_ERROR, $self->{strSpoolPath}, $strWalFile, $iCode, $strMessage);
}
}
}
@@ -240,59 +241,4 @@ sub processQueue
);
}
####################################################################################################################################
# walStatusWrite
#
# Write out a status file to the spool path with information about the success or failure of an archive push.
####################################################################################################################################
sub walStatusWrite
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strType,
$strWalFile,
$iCode,
$strMessage,
) =
logDebugParam
(
__PACKAGE__ . '->writeStatus', \@_,
{name => 'strType'},
{name => 'strWalFile'},
{name => 'iCode', required => false},
{name => 'strMessage', required => false},
);
# Remove any error file exists unless a new one will be written
if ($strType ne WAL_STATUS_ERROR)
{
# Remove the error file, if any
storageSpool()->remove("$self->{strSpoolPath}/${strWalFile}.error", {bIgnoreMissing => true});
}
# Write the status file
my $strStatus;
if (defined($iCode))
{
if (!defined($strMessage))
{
confess &log(ASSERT, 'strMessage must be set when iCode is set');
}
$strStatus = "${iCode}\n${strMessage}";
}
elsif ($strType eq WAL_STATUS_ERROR)
{
confess &log(ASSERT, 'error status must have iCode and strMessage set');
}
storageSpool()->put(
storageSpool()->openWrite("$self->{strSpoolPath}/${strWalFile}.${strType}", {bAtomic => true}), $strStatus);
}
1;
lib/pgBackRest/Archive/Push/File.pm
+2 -2
View File
@@ -26,8 +26,8 @@ use pgBackRest::Storage::Helper;
####################################################################################################################################
# archivePushCheck
#
# Check that a WAL segment does not already exist in the archive be pushing. Files that are not segments (e.g. .history, .backup)
# will always be reported as not present and will be overwritten by archivePushFile().
# Check that a WAL segment does not already exist in the archive before pushing. Files that are not segments (e.g. .history,
# .backup) will always be reported as not present and will be overwritten by archivePushFile().
####################################################################################################################################
sub archivePushCheck
{
lib/pgBackRest/Archive/Push/Push.pm
-8
View File
@@ -24,14 +24,6 @@ use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
####################################################################################################################################
# WAL status constants
####################################################################################################################################
use constant WAL_STATUS_ERROR => 'error';
push @EXPORT, qw(WAL_STATUS_ERROR);
use constant WAL_STATUS_OK => 'ok';
push @EXPORT, qw(WAL_STATUS_OK);
####################################################################################################################################
# process
#
lib/pgBackRest/Check/Check.pm
+2 -2
View File
@@ -9,7 +9,7 @@ use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Get::Get;
use pgBackRest::Archive::Get::File;
use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
@@ -150,7 +150,7 @@ sub process
eval
{
# Check that the archive info file is written and is valid for the current database of the stanza
($strArchiveId) = new pgBackRest::Archive::Get::Get()->getCheck();
($strArchiveId) = archiveGetCheck();
return true;
}
or do
lib/pgBackRest/Protocol/Helper.pm
-2
View File
@@ -23,8 +23,6 @@ use constant OP_BACKUP_FILE => 'backupF
push @EXPORT, qw(OP_BACKUP_FILE);
# Archive Module
use constant OP_ARCHIVE_GET_ARCHIVE_ID => 'archiveId';
push @EXPORT, qw(OP_ARCHIVE_GET_ARCHIVE_ID);
use constant OP_ARCHIVE_GET_CHECK => 'archiveCheck';
push @EXPORT, qw(OP_ARCHIVE_GET_CHECK);
use constant OP_ARCHIVE_PUSH_CHECK => 'archivePushCheck';
lib/pgBackRest/Protocol/Remote/Minion.pm
+2 -4
View File
@@ -14,7 +14,7 @@ use pgBackRest::Backup::File;
use pgBackRest::Common::Log;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Wait;
use pgBackRest::Archive::Get::Get;
use pgBackRest::Archive::Get::File;
use pgBackRest::Archive::Push::File;
use pgBackRest::Check::Check;
use pgBackRest::Config::Config;
@@ -70,7 +70,6 @@ sub init
# Create objects
my $oStorage = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_DB) ? storageDb() : storageRepo();
my $oArchiveGet = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_BACKUP) ? new pgBackRest::Archive::Get::Get() : undef;
my $oCheck = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_BACKUP) ? new pgBackRest::Check::Check() : undef;
my $oInfo = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_BACKUP) ? new pgBackRest::Info() : undef;
my $oDb = cfgOptionTest(CFGOPT_TYPE, CFGOPTVAL_REMOTE_TYPE_DB) ? new pgBackRest::Db() : undef;
@@ -79,8 +78,7 @@ sub init
my $hCommandMap =
{
# ArchiveGet commands
&OP_ARCHIVE_GET_ARCHIVE_ID => sub {$oArchiveGet->getArchiveId()},
&OP_ARCHIVE_GET_CHECK => sub {$oArchiveGet->getCheck(@{shift()})},
&OP_ARCHIVE_GET_CHECK => sub {archiveGetCheck(@{shift()})},
# ArchivePush commands
&OP_ARCHIVE_PUSH_CHECK => sub {archivePushCheck(@{shift()})},
src/Makefile
+1
View File
@@ -54,6 +54,7 @@ DESTDIR =
# List of required source files. main.c should always be listed last and the rest in alpha order.
####################################################################################################################################
SRCS = \
command/archive/common.c \
command/archive/push/push.c \
command/help/help.c \
command/command.c \
src/command/archive/common.c
+96
View File
@@ -0,0 +1,96 @@
/***********************************************************************************************************************************
Archive Push Command
***********************************************************************************************************************************/
#include <string.h>
#include "command/archive/common.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/wait.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Check for ok/error status files in the spool in/out directory
***********************************************************************************************************************************/
bool
archiveAsyncStatus(const String *walSegment, bool confessOnError)
{
bool result = false;
MEM_CONTEXT_TEMP_BEGIN()
{
StringList *fileList = storageListP(
storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT), .expression = strNewFmt("^%s\\.(ok|error)$", strPtr(walSegment)));
if (fileList != NULL && strLstSize(fileList) > 0)
{
// If more than one status file was found then assert - this could be a bug in the async process
if (strLstSize(fileList) != 1)
{
THROW(
AssertError, "multiple status files found in '%s' for WAL segment '%s'",
strPtr(storagePathNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT))), strPtr(walSegment));
}
// Get the status file content
const String *statusFile = strLstGet(fileList, 0);
String *content = strNewBuf(
storageGetNP(storageNewReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile)))));
// Get the code and message if the file has content
int code = 0;
const String *message = NULL;
if (strSize(content) != 0)
{
// Find the line feed after the error code -- should be the first one
const char *linefeedPtr = strchr(strPtr(content), '\n');
// Error if linefeed not found
if (linefeedPtr == NULL)
THROW(FormatError, "%s content must have at least two lines", strPtr(statusFile));
// Error if message is zero-length
if (strlen(linefeedPtr + 1) == 0)
THROW(FormatError, "%s message must be > 0", strPtr(statusFile));
// Get contents
code = varIntForce(varNewStr(strNewN(strPtr(content), (size_t)(linefeedPtr - strPtr(content)))));
message = strTrim(strNew(linefeedPtr + 1));
}
// Process OK files
if (strEndsWithZ(statusFile, ".ok"))
{
// If there is content in the status file it is a warning
if (strSize(content) != 0)
{
// If error code is not success, then this was a renamed .error file
if (code != 0)
{
message = strNewFmt(
"WAL segment '%s' was not pushed due to error [%d] and was manually skipped: %s", strPtr(walSegment),
code, strPtr(message));
}
LOG_WARN(strPtr(message));
}
result = true;
}
else if (confessOnError)
{
// Error status files must have content
if (strSize(content) == 0)
THROW(AssertError, "status file '%s' has no content", strPtr(statusFile));
// Throw error using the code passed in the file
THROW_CODE(code, strPtr(message));
}
}
}
MEM_CONTEXT_TEMP_END();
return result;
}
src/command/archive/common.h
+14
View File
@@ -0,0 +1,14 @@
/***********************************************************************************************************************************
Archive Common
***********************************************************************************************************************************/
#ifndef COMMAND_ARCHIVE_COMMON_H
#define COMMAND_ARCHIVE_COMMON_H
#include "common/type/string.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
bool archiveAsyncStatus(const String *walSegment, bool confessOnError);
#endif
src/command/archive/push/push.c
+2 -96
View File
@@ -1,113 +1,19 @@
/***********************************************************************************************************************************
Archive Push Command
***********************************************************************************************************************************/
#include <libgen.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#include "command/archive/common.h"
#include "command/command.h"
#include "common/fork.h"
#include "common/error.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/regExp.h"
#include "common/wait.h"
#include "config/config.h"
#include "config/load.h"
#include "perl/exec.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Check for ok/error status files in the spool out directory
***********************************************************************************************************************************/
static bool
walStatus(const String *walSegment, bool confessOnError)
{
bool result = false;
MEM_CONTEXT_TEMP_BEGIN()
{
StringList *fileList = storageListP(
storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT), .expression = strNewFmt("^%s\\.(ok|error)$", strPtr(walSegment)));
if (fileList != NULL && strLstSize(fileList) > 0)
{
// If more than one status file was found then assert - this could be a bug in the async process
if (strLstSize(fileList) != 1)
{
THROW(
AssertError, "multiple status files found in '%s' for WAL segment '%s'",
strPtr(storagePathNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT))), strPtr(walSegment));
}
// Get the status file content
const String *statusFile = strLstGet(fileList, 0);
String *content = strNewBuf(
storageGetNP(storageNewReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile)))));
// Get the code and message if the file has content
int code = 0;
const String *message = NULL;
if (strSize(content) != 0)
{
// Find the line feed after the error code -- should be the first one
const char *linefeedPtr = strchr(strPtr(content), '\n');
// Error if linefeed not found
if (linefeedPtr == NULL)
THROW(FormatError, "%s content must have at least two lines", strPtr(statusFile));
// Error if message is zero-length
if (strlen(linefeedPtr + 1) == 0)
THROW(FormatError, "%s message must be > 0", strPtr(statusFile));
// Get contents
code = varIntForce(varNewStr(strNewN(strPtr(content), (size_t)(linefeedPtr - strPtr(content)))));
message = strTrim(strNew(linefeedPtr + 1));
}
// Process OK files
if (strEndsWithZ(statusFile, ".ok"))
{
// If there is content in the status file it is a warning
if (strSize(content) != 0)
{
// If error code is not success, then this was a renamed .error file
if (code != 0)
{
message = strNewFmt(
"WAL segment '%s' was not pushed due to error [%d] and was manually skipped: %s", strPtr(walSegment),
code, strPtr(message));
}
LOG_WARN(strPtr(message));
}
result = true;
}
else if (confessOnError)
{
// Error status files must have content
if (strSize(content) == 0)
THROW(AssertError, "status file '%s' has no content", strPtr(statusFile));
// Throw error using the code passed in the file
THROW_CODE(code, strPtr(message));
}
}
}
MEM_CONTEXT_TEMP_END();
return result;
}
/***********************************************************************************************************************************
Push a WAL segment to the repository
***********************************************************************************************************************************/
@@ -139,7 +45,7 @@ cmdArchivePush()
{
// Check if the WAL segment has been pushed. Errors will not be confessed on the first try to allow the async
// process a chance to fix them.
pushed = walStatus(walSegment, confessOnError);
pushed = archiveAsyncStatus(walSegment, confessOnError);
// If the WAL segment has not already been pushed then start the async process to push it. There's no point in
// forking the async process off more than once so track that as well. Use an archive lock to prevent more than
test/define.yaml
+11 -3
View File
@@ -497,16 +497,23 @@ unit:
- name: archive
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: common
total: 1
coverage:
command/archive/common: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: common-perl
total: 4
total: 5
coverage:
Archive/Common: partial
# ----------------------------------------------------------------------------------------------------------------------------
- name: push-perl
total: 8
total: 7
coverage:
Archive/Push/Async: full
@@ -516,7 +523,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: push
total: 2
total: 1
perlReq: true
coverage:
@@ -535,6 +542,7 @@ unit:
coverage:
Archive/Base: partial
Archive/Get/File: partial
Archive/Get/Get: partial
# ********************************************************************************************************************************
test/expect/mock-archive-001.log
+4 -4
View File
@@ -116,12 +116,12 @@ P00 INFO: archive-push command end: completed successfully
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001
P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001
P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = <false>
P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true>
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = <redacted>, strCipherType = aes-256-cbc, strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true
P00 DEBUG: Db->new(): iRemoteIdx = 1
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
@@ -143,13 +143,13 @@ P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStor
P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000001-[0-f]{40}(\.gz){0,1}$, strPathExp = <REPO:ARCHIVE>/9.4-1/0000000100000001, strSortOrder = <forward>
P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113)
P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113
P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113, strArchiveId = 9.4-1, strCipherPass = <redacted>
P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113, strArchiveId = 9.4-1, strCipherPass = <redacted>
P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = <false>, rhyFilter = [undef], strCipherPass = <redacted>, xFileExp = <REPO:ARCHIVE>/9.4-1/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true>
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
P00 DEBUG: Storage::Local->openWrite(): bAtomic = <false>, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = [undef], strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::Get::Get->get=>: iResult = 0
P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0
P00 DEBUG: Main::mainCleanup(): iExitCode = 0
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
test/expect/mock-archive-002.log
+4 -4
View File
@@ -100,7 +100,7 @@ P00 INFO: archive-push command end: completed successfully
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001
P00 INFO: get WAL segment 000000010000000100000001
P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001
P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000001
P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = <false>
P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
@@ -110,7 +110,7 @@ P00 DEBUG: Protocol::Helper::protocolParam=>: strRemoteCommand = [BACKREST-
P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 3, iCompressLevelNetwork = 1, iProtocolTimeout = 60, iSshPort = [undef], strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-1]
P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 3, iCompressLevelNetwork = 1, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no pgbackrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --stanza=db --type=backup remote', strId = remote process on 'backup', strName = remote
P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object]
P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000001, ullDbSysId = [undef]
P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true
P00 DEBUG: Db->new(): iRemoteIdx = 1
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
@@ -118,13 +118,13 @@ P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201409291, iDbControlVersion = 942, strDbVersion = 9.4, ullDbSysId = 1000000000000000094
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol
P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113, strArchiveId = 9.4-1, strCipherPass = [undef]
P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113, strArchiveId = 9.4-1, strCipherPass = [undef]
P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = <REPO:ARCHIVE>/9.4-1/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true>
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
P00 DEBUG: Storage::Local->openWrite(): bAtomic = <false>, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = [undef], strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::Get::Get->get=>: iResult = 0
P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0
P00 DEBUG: Main::mainCleanup(): iExitCode = 0
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup
test/expect/mock-stanza-001.log
+4 -4
View File
@@ -518,12 +518,12 @@ db-version="9.4"
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002
P00 INFO: get WAL segment 000000010000000100000002
P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002
P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002
P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = <false>
P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true>
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/repo, strTempExtension = pgbackrest.tmp
P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef]
P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef]
P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true
P00 DEBUG: Db->new(): iRemoteIdx = 1
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
@@ -545,13 +545,13 @@ P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStor
P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000002-[0-f]{40}(\.gz){0,1}$, strPathExp = <REPO:ARCHIVE>/9.3-1/0000000100000001, strSortOrder = <forward>
P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz)
P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz
P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = [undef]
P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = [undef]
P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = <false>, rhyFilter = [undef], strCipherPass = [undef], xFileExp = <REPO:ARCHIVE>/9.3-1/000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true>
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
P00 DEBUG: Storage::Local->openWrite(): bAtomic = <false>, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::Get::Get->get=>: iResult = 0
P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0
P00 DEBUG: Main::mainCleanup(): iExitCode = 0
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
test/expect/mock-stanza-002.log
+4 -4
View File
@@ -384,7 +384,7 @@ db-version="9.4"
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-2] --stanza=db
P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002
P00 INFO: get WAL segment 000000010000000100000002
P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002
P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002
P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = <false>
P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
@@ -394,7 +394,7 @@ P00 DEBUG: Protocol::Helper::protocolParam=>: strRemoteCommand = [BACKREST-
P00 DEBUG: Protocol::Remote::Master->new(): iBufferMax = 4194304, iCompressLevel = 3, iCompressLevelNetwork = 1, iProtocolTimeout = 60, iSshPort = [undef], strCommand = [BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --stanza=db --type=backup remote, strCommandSSH = ssh, strHost = backup, strUser = [USER-2]
P00 DEBUG: Protocol::Command::Master->new(): iBufferMax = 4194304, iCompressLevel = 3, iCompressLevelNetwork = 1, iProtocolTimeout = 60, strCommand = ssh -o LogLevel=error -o Compression=no -o PasswordAuthentication=no pgbackrest@backup '[BACKREST-BIN] --buffer-size=4194304 --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --stanza=db --type=backup remote', strId = remote process on 'backup', strName = remote
P00 DEBUG: Protocol::Storage::Remote->new(): oProtocol = [object]
P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef]
P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef]
P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true
P00 DEBUG: Db->new(): iRemoteIdx = 1
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
@@ -402,13 +402,13 @@ P00 DEBUG: Db->info(): strDbPath = <[TEST_PATH]/db-master/db/base>
P00 DEBUG: Db->info=>: iDbCatalogVersion = 201306121, iDbControlVersion = 937, strDbVersion = 9.3, ullDbSysId = 1000000000000000093
P00 DEBUG: Protocol::Helper::protocolGet(): bCache = <true>, iProcessIdx = [undef], iRemoteIdx = <1>, strBackRestBin = [undef], strCommand = <archive-get>, strRemoteType = backup
P00 DEBUG: Protocol::Helper::protocolGet: found cached protocol
P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = <redacted>
P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = <redacted>
P00 DEBUG: Protocol::Storage::Remote->openRead(): rhParam = [hash], strFileExp = <REPO:ARCHIVE>/9.3-1/000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true>
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
P00 DEBUG: Storage::Local->openWrite(): bAtomic = <false>, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::Get::Get->get=>: iResult = 0
P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0
P00 DEBUG: Main::mainCleanup(): iExitCode = 0
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy: found cached protocol: iRemoteIdx = 1, strRemoteType = backup
test/expect/mock-stanza-003.log
+4 -4
View File
@@ -493,12 +493,12 @@ db-version="9.4"
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000002, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-ssl --repo1-type=s3 --stanza=db
P00 DEBUG: Archive::Get::Get->process(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002
P00 INFO: get WAL segment 000000010000000100000002
P00 DEBUG: Archive::Get::Get->get(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002
P00 DEBUG: Archive::Get::File::archiveGetFile(): strDestinationFile = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG, strSourceArchive = 000000010000000100000002
P00 DEBUG: Common::Lock::lockStopTest(): bStanzaStopRequired = <false>
P00 DEBUG: Common::Lock::lockStopTest=>: bStopExists = false
P00 DEBUG: Storage::S3::Request->new(): bVerifySsl = false, iPort = [undef], lBufferMax = 4194304, strAccessKeyId = <redacted>, strBucket = pgbackrest-dev, strCaFile = [undef], strCaPath = [undef], strEndPoint = s3.amazonaws.com, strHost = [undef], strRegion = us-east-1, strSecretAccessKey = <redacted>
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [hash], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = /, strTempExtension = pgbackrest.tmp
P00 DEBUG: Archive::Base->getCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef]
P00 DEBUG: Archive::Get::File::archiveGetCheck(): bCheck = false, strDbVersion = [undef], strFile = 000000010000000100000002, ullDbSysId = [undef]
P00 DEBUG: Db::dbObjectGet(): bMasterOnly = true
P00 DEBUG: Db->new(): iRemoteIdx = 1
P00 DEBUG: Db::dbObjectGet=>: iDbMasterIdx = 1, iDbStandbyIdx = [undef], oDbMaster = [object], oDbStandby = [undef]
@@ -520,13 +520,13 @@ P00 DEBUG: Archive::Common::walSegmentFind(): iWaitSeconds = [undef], oStor
P00 DEBUG: Storage::Local->list(): bIgnoreMissing = true, strExpression = ^000000010000000100000002-[0-f]{40}(\.gz){0,1}$, strPathExp = <REPO:ARCHIVE>/9.3-1/0000000100000001, strSortOrder = <forward>
P00 DEBUG: Storage::Local->list=>: stryFileList = (000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz)
P00 DEBUG: Archive::Common::walSegmentFind=>: strWalFileName = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz
P00 DEBUG: Archive::Base->getCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = [undef]
P00 DEBUG: Archive::Get::File::archiveGetCheck=>: strArchiveFile = 000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz, strArchiveId = 9.3-1, strCipherPass = [undef]
P00 DEBUG: Storage::Local->openRead(): bIgnoreMissing = <false>, rhyFilter = [undef], strCipherPass = [undef], xFileExp = <REPO:ARCHIVE>/9.3-1/000000010000000100000002-488ba4b8b98acc510bce86b8f16e3c1ed9886a29.gz
P00 DEBUG: Storage::Posix::Driver->new(): bFileSync = <true>, bPathSync = <true>
P00 DEBUG: Storage::Local->new(): bAllowTemp = <true>, hRule = [undef], lBufferMax = 4194304, oDriver = [object], strCipherPassUser = [undef], strCipherType = [undef], strDefaultFileMode = <0640>, strDefaultPathMode = <0750>, strPathBase = [TEST_PATH]/db-master/db/base, strTempExtension = pgbackrest.tmp
P00 DEBUG: Storage::Local->openWrite(): bAtomic = <false>, bPathCreate = <false>, lTimestamp = [undef], rhyFilter = ({rxyParam => ({strCompressType => decompress}), strClass => pgBackRest::Storage::Filter::Gzip}), strCipherPass = [undef], strGroup = [undef], strMode = <0640>, strUser = [undef], xFileExp = [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
P00 DEBUG: Storage::Base->copy(): xDestinationFile = [object], xSourceFile = [object]
P00 DEBUG: Archive::Get::Get->get=>: iResult = 0
P00 DEBUG: Archive::Get::File::archiveGetFile=>: iResult = 0
P00 DEBUG: Main::mainCleanup(): iExitCode = 0
P00 DEBUG: Protocol::Helper::protocolDestroy(): bComplete = true, iRemoteIdx = [undef], strRemoteType = [undef]
P00 DEBUG: Protocol::Helper::protocolDestroy=>: iExitStatus = 0
test/lib/pgBackRestTest/Module/Archive/ArchiveCommonPerlTest.pm
+40
View File
@@ -156,6 +156,46 @@ sub run
$self->testResult(
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
}
################################################################################################################################
if ($self->begin("archiveAsyncStatusWrite()"))
{
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
# Create the spool path
my $strSpoolPath = $self->testPath() . "/spool/out";
$self->storageTest()->pathCreate($strSpoolPath, {bIgnoreExists => true, bCreateParent => true});
#---------------------------------------------------------------------------------------------------------------------------
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
# Generate a normal ok
archiveAsyncStatusWrite(WAL_STATUS_OK, $strSpoolPath, $strSegment);
#---------------------------------------------------------------------------------------------------------------------------
# Generate a valid warning ok
archiveAsyncStatusWrite(WAL_STATUS_OK, $strSpoolPath, $strSegment, 0, 'Test Warning');
#---------------------------------------------------------------------------------------------------------------------------
# Generate an invalid error
$self->testException(
sub {archiveAsyncStatusWrite(WAL_STATUS_ERROR, $strSpoolPath, $strSegment)}, ERROR_ASSERT,
"error status must have iCode and strMessage set");
#---------------------------------------------------------------------------------------------------------------------------
# Generate an invalid error
$self->testException(
sub {archiveAsyncStatusWrite(WAL_STATUS_ERROR, $strSpoolPath, $strSegment, ERROR_ASSERT)},
ERROR_ASSERT, "strMessage must be set when iCode is set");
#---------------------------------------------------------------------------------------------------------------------------
# Generate a valid error
archiveAsyncStatusWrite(
WAL_STATUS_ERROR, $strSpoolPath, $strSegment, ERROR_ARCHIVE_DUPLICATE,
"WAL segment ${strSegment} already exists in the archive");
}
}
1;
test/lib/pgBackRestTest/Module/Archive/ArchiveGetPerlTest.pm
+16 -17
View File
@@ -15,6 +15,7 @@ use Storable qw(dclone);
use Digest::SHA qw(sha1_hex);
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Get::File;
use pgBackRest::Archive::Get::Get;
use pgBackRest::Archive::Info;
use pgBackRest::Common::Exception;
@@ -91,8 +92,6 @@ sub run
################################################################################################################################
if ($self->begin("Archive::Base::getCheck()"))
{
my $oArchiveBase = new pgBackRest::Archive::Base();
# Create and save archive.info file
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true});
@@ -104,39 +103,39 @@ sub run
# db-version, db-sys-id passed but combination doesn't exist in archive.info history
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oArchiveBase->getCheck(
$self->testException(sub {archiveGetCheck(
PG_VERSION_95, $self->dbSysId(PG_VERSION_94), undef, false)}, ERROR_UNKNOWN,
"unable to retrieve the archive id for database version '" . PG_VERSION_95 . "' and system-id '" .
$self->dbSysId(PG_VERSION_94) . "'");
# db-version, db-sys-id and wal passed all undefined
#---------------------------------------------------------------------------------------------------------------------------
my ($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck(undef, undef, undef, false);
my ($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(undef, undef, undef, false);
$self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)},
true, 'undef db-version, db-sys-id and wal returns only current db archive-id');
# db-version defined, db-sys-id and wal undefined
#---------------------------------------------------------------------------------------------------------------------------
($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck(PG_VERSION_92, undef, undef, false);
($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(PG_VERSION_92, undef, undef, false);
$self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)},
true, 'old db-version, db-sys-id and wal undefined returns only current db archive-id');
# db-version undefined, db-sys-id defined and wal undefined
#---------------------------------------------------------------------------------------------------------------------------
($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck(
($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(
undef, $self->dbSysId(PG_VERSION_93), undef, false);
$self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)},
true, 'undef db-version, old db-sys-id and wal undef returns only current db archive-id');
# old db-version, db-sys-id and wal undefined, check = true (default)
#---------------------------------------------------------------------------------------------------------------------------
($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck(PG_VERSION_92, undef, undef);
($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(PG_VERSION_92, undef, undef);
$self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)},
true, 'old db-version, db-sys-id and wal undefined, check = true returns only current db archive-id');
# old db-version, old db-sys-id and wal undefined, check = true (default)
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {$oArchiveBase->getCheck(
$self->testException(sub {archiveGetCheck(
PG_VERSION_93, $self->dbSysId(PG_VERSION_93), undef)}, ERROR_ARCHIVE_MISMATCH,
"WAL segment version " . PG_VERSION_93 . " does not match archive version " . PG_VERSION_94 . "\n" .
"WAL segment system-id " . $self->dbSysId(PG_VERSION_93) . " does not match archive system-id " .
@@ -151,7 +150,7 @@ sub run
storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true});
storageRepo()->put("${strWalMajorPath}/${strWalSegmentName}");
($strArchiveId, $strArchiveFile, $strCipherPass) = $oArchiveBase->getCheck(undef, undef, $strWalSegment, false);
($strArchiveId, $strArchiveFile, $strCipherPass) = archiveGetCheck(undef, undef, $strWalSegment, false);
$self->testResult(sub {($strArchiveId eq PG_VERSION_94 . '-13') && !defined($strArchiveFile) && !defined($strCipherPass)},
true, 'undef db-version, db-sys-id with a requested wal not in current db archive returns only current db archive-id');
@@ -159,7 +158,7 @@ sub run
# Pass db-version and db-sys-id where WAL is actually located
#---------------------------------------------------------------------------------------------------------------------------
($strArchiveId, $strArchiveFile, $strCipherPass) =
$oArchiveBase->getCheck(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), $strWalSegment, false);
archiveGetCheck(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), $strWalSegment, false);
$self->testResult(
sub {($strArchiveId eq PG_VERSION_92 . '-1') && ($strArchiveFile eq $strWalSegmentName) && !defined($strCipherPass)},
@@ -176,7 +175,7 @@ sub run
storageRepo()->put("${strWalMajorPath}/${strWalSegmentName}", $strFileContent);
($strArchiveId, $strArchiveFile, $strCipherPass) =
$oArchiveBase->getCheck(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), $strWalSegment, false);
archiveGetCheck(PG_VERSION_92, $self->dbSysId(PG_VERSION_92), $strWalSegment, false);
# Using the returned values, confirm the correct file is read
$self->testResult(sub {sha1_hex(${storageRepo()->get($self->{strArchivePath} . "/" . $strArchiveId . "/" .
@@ -189,7 +188,7 @@ sub run
{
# archive.info missing
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {new pgBackRest::Archive::Get::Get()->get($strWalSegment, $strDestinationFile)},
$self->testException(sub {archiveGetFile($strWalSegment, $strDestinationFile)},
ERROR_FILE_MISSING,
ARCHIVE_INFO_FILE . " does not exist but is required to push/get WAL segments\n" .
"HINT: is archive_command configured in postgresql.conf?\n" .
@@ -206,7 +205,7 @@ sub run
# file not found
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {new pgBackRest::Archive::Get::Get()->get($strWalSegment, $strDestinationFile)}, 1,
$self->testResult(sub {archiveGetFile($strWalSegment, $strDestinationFile)}, 1,
"unable to find ${strWalSegment} in the archive");
# file found but is not a WAL segment
@@ -220,7 +219,7 @@ sub run
# Create path to copy file
storageRepo()->pathCreate($strDestinationPath);
$self->testResult(sub {new pgBackRest::Archive::Get::Get()->get(BOGUS, $strDestinationFile)}, 0,
$self->testResult(sub {archiveGetFile(BOGUS, $strDestinationFile)}, 0,
"non-WAL segment copied");
# Confirm the correct file is copied
@@ -245,14 +244,14 @@ sub run
storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true});
storageRepo()->put("${strWalMajorPath}/${strWalSegmentName}", $strFileContent);
$self->testResult(sub {new pgBackRest::Archive::Get::Get()->get($strWalSegmentName, $strDestinationFile)}, 0,
$self->testResult(sub {archiveGetFile($strWalSegmentName, $strDestinationFile)}, 0,
"WAL segment copied");
# Confirm the correct file is copied
$self->testResult(sub {sha1_hex(${storageRepo()->get($strDestinationFile)})}, $strFileHash,
' check correct WAL copied when in multiple locations');
# get files from an older DB version to simulate restoring from an old backup set to a database that is of that same version
# Get files from an older DB version to simulate restoring from an old backup set to a database that is of that same version
#---------------------------------------------------------------------------------------------------------------------------
# Create same WAL name in older DB archive but with different data to ensure it is copied
$strArchivePath = $self->{strArchivePath} . "/" . PG_VERSION_93 . "-2/";
@@ -272,7 +271,7 @@ sub run
# Overwrite current pg_control file with older version
$self->controlGenerate($self->{strDbPath}, PG_VERSION_93);
$self->testResult(sub {new pgBackRest::Archive::Get::Get()->get($strWalSegmentName, $strDestinationFile)}, 0,
$self->testResult(sub {archiveGetFile($strWalSegmentName, $strDestinationFile)}, 0,
"WAL segment copied from older db backupset to same version older db");
# Confirm the correct file is copied
test/lib/pgBackRestTest/Module/Archive/ArchivePushPerlTest.pm
+1 -41
View File
@@ -323,47 +323,6 @@ sub run
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
}
################################################################################################################################
if ($self->begin("ArchivePushAsync->walStatusWrite()"))
{
my $oPush = new pgBackRest::Archive::Push::Push();
my $oPushAsync = new pgBackRest::Archive::Push::Async($self->{strWalPath}, $self->{strSpoolPath});
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$oPushAsync->initServer();
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
#---------------------------------------------------------------------------------------------------------------------------
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
# Generate a normal ok
$oPushAsync->walStatusWrite(WAL_STATUS_OK, $strSegment);
#---------------------------------------------------------------------------------------------------------------------------
# Generate a valid warning ok
$oPushAsync->walStatusWrite(WAL_STATUS_OK, $strSegment, 0, 'Test Warning');
#---------------------------------------------------------------------------------------------------------------------------
# Generate an invalid error
$self->testException(
sub {$oPushAsync->walStatusWrite(WAL_STATUS_ERROR, $strSegment)}, ERROR_ASSERT,
"error status must have iCode and strMessage set");
#---------------------------------------------------------------------------------------------------------------------------
# Generate an invalid error
$self->testException(
sub {$oPushAsync->walStatusWrite(WAL_STATUS_ERROR, $strSegment, ERROR_ASSERT)}, ERROR_ASSERT,
"strMessage must be set when iCode is set");
#---------------------------------------------------------------------------------------------------------------------------
# Generate a valid error
$oPushAsync->walStatusWrite(
WAL_STATUS_ERROR, $strSegment, ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strSegment} already exists in the archive");
}
################################################################################################################################
if ($self->begin("ArchivePushAsync->process()"))
{
@@ -668,6 +627,7 @@ sub run
$self->optionTestClear(CFGOPT_SPOOL_PATH);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
}
################################################################################################################################
if ($self->begin("ArchivePushFile::archivePushFile - encryption"))
{
test/src/module/archive/commonTest.c
+90
View File
@@ -0,0 +1,90 @@
/***********************************************************************************************************************************
Test Archive Common
***********************************************************************************************************************************/
#include <unistd.h>
#include "common/harnessConfig.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun()
{
// *****************************************************************************************************************************
if (testBegin("archiveAsyncStatus()"))
{
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAdd(argList, strNewFmt("--spool-path=%s", testPath()));
strLstAddZ(argList, "--archive-async");
strLstAddZ(argList, "--archive-timeout=1");
strLstAddZ(argList, "--stanza=db");
strLstAddZ(argList, "archive-push");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// -------------------------------------------------------------------------------------------------------------------------
String *segment = strNew("000000010000000100000001");
TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), false, "directory and status file not present");
// -------------------------------------------------------------------------------------------------------------------------
mkdir(strPtr(strNewFmt("%s/archive", testPath())), 0750);
mkdir(strPtr(strNewFmt("%s/archive/db", testPath())), 0750);
mkdir(strPtr(strNewFmt("%s/archive/db/out", testPath())), 0750);
TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), false, "status file not present");
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew(BOGUS_STR)));
TEST_ERROR(
archiveAsyncStatus(segment, false), FormatError, "000000010000000100000001.ok content must have at least two lines");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew(BOGUS_STR "\n")));
TEST_ERROR(archiveAsyncStatus(segment, false), FormatError, "000000010000000100000001.ok message must be > 0");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew(BOGUS_STR "\nmessage")));
TEST_ERROR(archiveAsyncStatus(segment, false), FormatError, "unable to convert str 'BOGUS' to int");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew("0\nwarning")));
TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), true, "ok file with warning");
testLogResult("P00 WARN: warning");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew("25\nerror")));
TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), true, "error status renamed to ok");
testLogResult(
"P00 WARN: WAL segment '000000010000000100000001' was not pushed due to error [25] and was manually skipped: error");
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
bufNewStr(strNew("")));
TEST_ERROR(
archiveAsyncStatus(segment, false), AssertError,
strPtr(
strNewFmt(
"multiple status files found in '%s/archive/db/out' for WAL segment '000000010000000100000001'", testPath())));
unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment)))));
TEST_ERROR(archiveAsyncStatus(segment, true), AssertError, "status file '000000010000000100000001.error' has no content");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
bufNewStr(strNew("25\nmessage")));
TEST_ERROR(archiveAsyncStatus(segment, true), AssertError, "message");
TEST_RESULT_BOOL(archiveAsyncStatus(segment, false), false, "suppress error");
unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment)))));
}
}
test/src/module/archive/pushTest.c
-81
View File
@@ -1,11 +1,6 @@
/***********************************************************************************************************************************
Test Archive Push Command
***********************************************************************************************************************************/
#include <stdlib.h>
#include "config/load.h"
#include "version.h"
#include "common/harnessConfig.h"
/***********************************************************************************************************************************
@@ -14,82 +9,6 @@ Test Run
void
testRun()
{
// *****************************************************************************************************************************
if (testBegin("walStatus()"))
{
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAdd(argList, strNewFmt("--spool-path=%s", testPath()));
strLstAddZ(argList, "--archive-async");
strLstAddZ(argList, "--archive-timeout=1");
strLstAddZ(argList, "--stanza=db");
strLstAddZ(argList, "archive-push");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// -------------------------------------------------------------------------------------------------------------------------
String *segment = strNew("000000010000000100000001");
TEST_RESULT_BOOL(walStatus(segment, false), false, "directory and status file not present");
// -------------------------------------------------------------------------------------------------------------------------
mkdir(strPtr(strNewFmt("%s/archive", testPath())), 0750);
mkdir(strPtr(strNewFmt("%s/archive/db", testPath())), 0750);
mkdir(strPtr(strNewFmt("%s/archive/db/out", testPath())), 0750);
TEST_RESULT_BOOL(walStatus(segment, false), false, "status file not present");
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew(BOGUS_STR)));
TEST_ERROR(walStatus(segment, false), FormatError, "000000010000000100000001.ok content must have at least two lines");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew(BOGUS_STR "\n")));
TEST_ERROR(walStatus(segment, false), FormatError, "000000010000000100000001.ok message must be > 0");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew(BOGUS_STR "\nmessage")));
TEST_ERROR(walStatus(segment, false), FormatError, "unable to convert str 'BOGUS' to int");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew("0\nwarning")));
TEST_RESULT_BOOL(walStatus(segment, false), true, "ok file with warning");
testLogResult("P00 WARN: warning");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew("25\nerror")));
TEST_RESULT_BOOL(walStatus(segment, false), true, "error status renamed to ok");
testLogResult(
"P00 WARN: WAL segment '000000010000000100000001' was not pushed due to error [25] and was manually skipped: error");
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
bufNewStr(strNew("")));
TEST_ERROR(
walStatus(segment, false), AssertError,
strPtr(
strNewFmt(
"multiple status files found in '%s/archive/db/out' for WAL segment '000000010000000100000001'", testPath())));
unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment)))));
TEST_ERROR(walStatus(segment, true), AssertError, "status file '000000010000000100000001.error' has no content");
storagePutNP(
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
bufNewStr(strNew("25\nmessage")));
TEST_ERROR(walStatus(segment, true), AssertError, "message");
TEST_RESULT_BOOL(walStatus(segment, false), false, "suppress error");
unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment)))));
}
// *****************************************************************************************************************************
if (testBegin("cmdArchivePush()"))
{