db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-05 00:28:52 +02:00
Code Issues Releases Activity

The archive-push command is implemented entirely in C.

Browse Source 
This new implementation should behave exactly like the old Perl code with the exception of updated log messages.

Remove as much of the Perl code as possible without breaking other commands.
This commit is contained in:
David Steele
2019-03-29 13:26:33 +00:00
parent 251dbede8f
commit 1b48684713
37 changed files with 1430 additions and 2766 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
doc/xml/release.xml
View File

@ -28,6 +28,12 @@
</release-item>
</release-bug-list>
<release-improvement-list>
<release-item>
<p>The <cmd>archive-push</cmd> command is implemented entirely in C.</p>
</release-item>
</release-improvement-list>
<release-development-list>
<release-item>
<p>Add separate <cmd>archive-push-async</cmd> command.</p>

57
lib/pgBackRest/Archive/Base.pm
View File

@ -1,57 +0,0 @@
####################################################################################################################################
# ARCHIVE MODULE
####################################################################################################################################
package pgBackRest::Archive::Base;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use pgBackRest::Archive::Info;
use pgBackRest::Archive::Common;
use pgBackRest::Db;
use pgBackRest::DbVersion;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
####################################################################################################################################
# constructor
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Create the class hash
my $self = {};
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strBackRestBin},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strBackRestBin', default => projectBin(), trace => true},
);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
1;

230
lib/pgBackRest/Archive/Common.pm
View File

@ -44,32 +44,6 @@ use constant PG_WAL_SYSTEM_ID_OFFSET_LT_93 => 12;
use constant PG_WAL_SEGMENT_SIZE => 16777216;
push @EXPORT, qw(PG_WAL_SEGMENT_SIZE);
####################################################################################################################################
# WAL status constants
####################################################################################################################################
use constant WAL_STATUS_ERROR => 'error';
push @EXPORT, qw(WAL_STATUS_ERROR);
use constant WAL_STATUS_OK => 'ok';
push @EXPORT, qw(WAL_STATUS_OK);
####################################################################################################################################
# PostgreSQL WAL magic
####################################################################################################################################
my $oWalMagicHash =
{
hex('0xD062') => PG_VERSION_83,
hex('0xD063') => PG_VERSION_84,
hex('0xD064') => PG_VERSION_90,
hex('0xD066') => PG_VERSION_91,
hex('0xD071') => PG_VERSION_92,
hex('0xD075') => PG_VERSION_93,
hex('0xD07E') => PG_VERSION_94,
hex('0xD087') => PG_VERSION_95,
hex('0xD093') => PG_VERSION_96,
hex('0xD097') => PG_VERSION_10,
hex('0xD098') => PG_VERSION_11,
};
####################################################################################################################################
# lsnNormalize
#
@ -177,95 +151,6 @@ sub lsnFileRange
push @EXPORT, qw(lsnFileRange);
####################################################################################################################################
# walInfo
#
# Retrieve information such as db version and system identifier from a WAL segment.
####################################################################################################################################
sub walInfo
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strWalFile,
) =
logDebugParam
(
__PACKAGE__ . '::walInfo', \@_,
{name => 'strWalFile'}
);
# Open the WAL segment and read magic number
#-------------------------------------------------------------------------------------------------------------------------------
my $hFile;
my $tBlock;
sysopen($hFile, $strWalFile, O_RDONLY)
or confess &log(ERROR, "unable to open ${strWalFile}", ERROR_FILE_OPEN);
# Read magic
sysread($hFile, $tBlock, 2) == 2
or confess &log(ERROR, "unable to read wal magic");
my $iMagic = unpack('S', $tBlock);
# Map the WAL magic number to the version of PostgreSQL.
#
# The magic number can be found in src/include/access/xlog_internal.h The offset can be determined by counting bytes in the
# XLogPageHeaderData struct, though this value rarely changes.
#-------------------------------------------------------------------------------------------------------------------------------
my $strDbVersion = $$oWalMagicHash{$iMagic};
if (!defined($strDbVersion))
{
confess &log(ERROR, "unexpected WAL magic 0x" . sprintf("%X", $iMagic) . "\n" .
'HINT: is this version of PostgreSQL supported?',
ERROR_VERSION_NOT_SUPPORTED);
}
# Map the WAL PostgreSQL version to the system identifier offset. The offset can be determined by counting bytes in the
# XLogPageHeaderData struct, though this value rarely changes.
#-------------------------------------------------------------------------------------------------------------------------------
my $iSysIdOffset = $strDbVersion >= PG_VERSION_93 ? PG_WAL_SYSTEM_ID_OFFSET_GTE_93 : PG_WAL_SYSTEM_ID_OFFSET_LT_93;
# Check flags to be sure the long header is present (this is an extra check to be sure the system id exists)
#-------------------------------------------------------------------------------------------------------------------------------
sysread($hFile, $tBlock, 2) == 2
or confess &log(ERROR, "unable to read wal info");
my $iFlag = unpack('S', $tBlock);
# Make sure that the long header is present or there won't be a system id
$iFlag & 2
or confess &log(ERROR, "expected long header in flags " . sprintf("%x", $iFlag));
# Get the system id
#-------------------------------------------------------------------------------------------------------------------------------
sysseek($hFile, $iSysIdOffset, SEEK_CUR)
or confess &log(ERROR, "unable to read padding");
sysread($hFile, $tBlock, 8) == 8
or confess &log(ERROR, "unable to read database system identifier");
length($tBlock) == 8
or confess &log(ERROR, "block is incorrect length");
close($hFile);
my $ullDbSysId = unpack('Q', $tBlock);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strDbVersion', value => $strDbVersion},
{name => 'ullDbSysId', value => $ullDbSysId}
);
}
push @EXPORT, qw(walInfo);
####################################################################################################################################
# walSegmentFind
#
@ -345,54 +230,6 @@ sub walSegmentFind
push @EXPORT, qw(walSegmentFind);
####################################################################################################################################
# walPath
#
# Generates the location of the wal directory using a relative wal path and the supplied db path.
####################################################################################################################################
sub walPath
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strWalFile,
$strDbPath,
$strCommand,
) =
logDebugParam
(
__PACKAGE__ . '::walPath', \@_,
{name => 'strWalFile', trace => true},
{name => 'strDbPath', trace => true, required => false},
{name => 'strCommand', trace => true},
);
if (index($strWalFile, '/') != 0)
{
if (!defined($strDbPath))
{
confess &log(ERROR,
"option '" . cfgOptionName(CFGOPT_PG_PATH) . "' must be specified when relative wal paths are used\n" .
"HINT: Is \%f passed to ${strCommand} instead of \%p?\n" .
"HINT: PostgreSQL may pass relative paths even with \%p depending on the environment.",
ERROR_OPTION_REQUIRED);
}
$strWalFile = "${strDbPath}/${strWalFile}";
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strWalFile', value => $strWalFile, trace => true}
);
}
push @EXPORT, qw(walPath);
####################################################################################################################################
# walIsSegment
#
@ -441,71 +278,4 @@ sub walIsPartial
push @EXPORT, qw(walIsPartial);
####################################################################################################################################
# archiveAsyncStatusWrite
#
# Write out a status file to the spool path with information about the success or failure of an archive push.
####################################################################################################################################
sub archiveAsyncStatusWrite
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strType,
$strSpoolPath,
$strWalFile,
$iCode,
$strMessage,
$bIgnoreErrorOnOk,
) =
logDebugParam
(
__PACKAGE__ . '::archiveAsyncStatusWrite', \@_,
{name => 'strType'},
{name => 'strSpoolPath'},
{name => 'strWalFile'},
{name => 'iCode', required => false},
{name => 'strMessage', required => false},
{name => 'bIgnoreErrorOnOk', required => false, default => false},
);
# Remove any error file exists unless a new one will be written
if ($strType ne WAL_STATUS_ERROR)
{
# Remove the error file, if any
storageLocal()->remove("${strSpoolPath}/${strWalFile}.error", {bIgnoreMissing => true});
}
# If an error will be written but an ok file already exists this may be expected and will be indicated by bIgnoreErrorOnOk set
# to true. In this case just return without writing the error file.
if (!($strType eq WAL_STATUS_ERROR && $bIgnoreErrorOnOk && storageLocal()->exists("${strSpoolPath}/${strWalFile}.ok")))
{
# Write the status file
my $strStatus;
if (defined($iCode))
{
if (!defined($strMessage))
{
confess &log(ASSERT, 'strMessage must be set when iCode is set');
}
$strStatus = "${iCode}\n${strMessage}";
}
elsif ($strType eq WAL_STATUS_ERROR)
{
confess &log(ASSERT, 'error status must have iCode and strMessage set');
}
storageLocal()->put(
storageLocal()->openWrite("${strSpoolPath}/${strWalFile}.${strType}", {bAtomic => true}), $strStatus);
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
push @EXPORT, qw(archiveAsyncStatusWrite);
1;

238
lib/pgBackRest/Archive/Push/Async.pm
View File

@ -1,238 +0,0 @@
####################################################################################################################################
# ARCHIVE PUSH ASYNC MODULE
####################################################################################################################################
package pgBackRest::Archive::Push::Async;
use parent 'pgBackRest::Archive::Push::Push';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Info;
use pgBackRest::Archive::Push::Push;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Db;
use pgBackRest::DbVersion;
use pgBackRest::LibC qw(:lock);
use pgBackRest::Protocol::Local::Process;
use pgBackRest::Protocol::Helper;
use pgBackRest::Storage::Helper;
use pgBackRest::Version;
####################################################################################################################################
# constructor
####################################################################################################################################
sub new
{
my $class = shift; # Class name
# Init object
my $self = $class->SUPER::new();
bless $self, $class;
# Assign function parameters, defaults, and log debug info
(
my $strOperation,
$self->{strWalPath},
$self->{strSpoolPath},
$self->{strBackRestBin},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'strWalPath'},
{name => 'strSpoolPath'},
{name => 'strBackRestBin', default => projectBin()},
);
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'self', value => $self}
);
}
####################################################################################################################################
# initServer
#
# Create the spool directory and initialize the archive process.
####################################################################################################################################
sub initServer
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->initServer');
# Create the spool path
storageSpool()->pathCreate($self->{strSpoolPath}, {bIgnoreExists => true, bCreateParent => true});
# Initialize the archive process
$self->{oArchiveProcess} = new pgBackRest::Protocol::Local::Process(
CFGOPTVAL_LOCAL_TYPE_BACKUP, cfgOption(CFGOPT_PROTOCOL_TIMEOUT) < 60 ? cfgOption(CFGOPT_PROTOCOL_TIMEOUT) / 2 : 30,
$self->{strBackRestBin}, false);
$self->{oArchiveProcess}->hostAdd(1, cfgOption(CFGOPT_PROCESS_MAX));
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# process
#
# Setup the server and process the queue. This function is separate from processQueue() for testing purposes.
####################################################################################################################################
sub process
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->process');
# Open the log file
logFileSet(storageLocal(), cfgOption(CFGOPT_LOG_PATH) . '/' . cfgOption(CFGOPT_STANZA) . '-archive-push-async');
# There is no loop here because it seems wise to let the async process exit periodically. As the queue grows each async
# execution will naturally run longer. This behavior is also far easier to test.
$self->initServer();
$self->processQueue();
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# processQueue
#
# Push WAL to the archive.
####################################################################################################################################
sub processQueue
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->processQueue');
# If queue max is exceeded then drop all WAL in the queue
my $iDropTotal = 0;
if (cfgOptionTest(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX))
{
my $stryDropList = $self->dropList($self->readyList());
if (@{$stryDropList} > 0)
{
foreach my $strDropFile (@{$stryDropList})
{
archiveAsyncStatusWrite(
WAL_STATUS_OK, $self->{strSpoolPath}, $strDropFile, 0,
"dropped WAL file ${strDropFile} because archive queue exceeded " .
cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) . ' bytes');
$iDropTotal++;
}
}
}
# Get jobs to process
my $stryWalFile = $self->readyList();
# Queue the jobs
foreach my $strWalFile (@{$stryWalFile})
{
$self->{oArchiveProcess}->queueJob(
1, 'default', $strWalFile, OP_ARCHIVE_PUSH_FILE,
[$self->{strWalPath}, $strWalFile, cfgOption(CFGOPT_COMPRESS), cfgOption(CFGOPT_COMPRESS_LEVEL)]);
}
# Process jobs if there are any
my $iOkTotal = 0;
my $iErrorTotal = 0;
if ($self->{oArchiveProcess}->jobTotal() > 0)
{
&log(INFO,
'push ' . @{$stryWalFile} . ' WAL file(s) to archive: ' .
${$stryWalFile}[0] . (@{$stryWalFile} > 1 ? "...${$stryWalFile}[-1]" : ''));
eval
{
# Check for a stop lock
lockStopTest();
# Hold a lock when the repo is remote to be sure no other process is pushing WAL
!isRepoLocal() && protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP);
while (my $hyJob = $self->{oArchiveProcess}->process())
{
# Send keep alives to protocol
protocolKeepAlive();
foreach my $hJob (@{$hyJob})
{
my $strWalFile = @{$hJob->{rParam}}[1];
my $strWarning = @{$hJob->{rResult}}[0];
# If error then write out an error file
if (defined($hJob->{oException}))
{
archiveAsyncStatusWrite(
WAL_STATUS_ERROR, $self->{strSpoolPath}, $strWalFile, $hJob->{oException}->code(),
$hJob->{oException}->message());
$iErrorTotal++;
&log(WARN,
"could not push WAL file ${strWalFile} to archive (will be retried): [" .
$hJob->{oException}->code() . "] " . $hJob->{oException}->message());
}
# Else write success
else
{
archiveAsyncStatusWrite(
WAL_STATUS_OK, $self->{strSpoolPath}, $strWalFile, defined($strWarning) ? 0 : undef,
defined($strWarning) ? $strWarning : undef);
$iOkTotal++;
&log(DETAIL, "pushed WAL file ${strWalFile} to archive", undef, undef, undef, $hJob->{iProcessId});
}
}
}
return 1;
}
or do
{
# Get error info
my $iCode = exceptionCode($EVAL_ERROR);
my $strMessage = exceptionMessage($EVAL_ERROR);
# Error all queued jobs except for the ones that have already succeeded
foreach my $strWalFile (@{$stryWalFile})
{
archiveAsyncStatusWrite(
WAL_STATUS_ERROR, $self->{strSpoolPath}, $strWalFile, $iCode, $strMessage, true);
}
}
}
return logDebugReturn
(
$strOperation,
{name => 'iNewTotal', value => scalar(@{$stryWalFile})},
{name => 'iDropTotal', value => $iDropTotal},
{name => 'iOkTotal', value => $iOkTotal},
{name => 'iErrorTotal', value => $iErrorTotal}
);
}
1;

213
lib/pgBackRest/Archive/Push/File.pm
View File

@ -1,213 +0,0 @@
####################################################################################################################################
# ARCHIVE PUSH FILE MODULE
####################################################################################################################################
package pgBackRest::Archive::Push::File;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(basename dirname);
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Filter::Gzip;
use pgBackRest::Storage::Filter::Sha;
use pgBackRest::Storage::Helper;
####################################################################################################################################
# archivePushCheck
#
# Check that a WAL segment does not already exist in the archive before pushing. Files that are not segments (e.g. .history,
# .backup) will always be reported as not present and will be overwritten by archivePushFile().
####################################################################################################################################
sub archivePushCheck
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strArchiveFile,
$strDbVersion,
$ullDbSysId,
$strWalFile,
) =
logDebugParam
(
__PACKAGE__ . '::archivePushCheck', \@_,
{name => 'strArchiveFile'},
{name => 'strDbVersion', required => false},
{name => 'ullDbSysId', required => false},
{name => 'strWalFile', required => false},
);
# Set operation and debug strings
my $oStorageRepo = storageRepo();
my $strArchiveId;
my $strChecksum;
my $strCipherPass = undef;
# WAL file is segment?
my $bWalSegment = walIsSegment($strArchiveFile);
if (!isRepoLocal())
{
# Execute the command
($strArchiveId, $strChecksum, $strCipherPass) = protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP)->cmdExecute(
OP_ARCHIVE_PUSH_CHECK, [$strArchiveFile, $strDbVersion, $ullDbSysId], true);
}
else
{
my $oArchiveInfo = new pgBackRest::Archive::Info($oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE));
# If a segment check db version and system-id
if ($bWalSegment)
{
# If the info file exists check db version and system-id else error
$strArchiveId = $oArchiveInfo->check($strDbVersion, $ullDbSysId);
# Check if the WAL segment already exists in the archive
my $strFoundFile = walSegmentFind($oStorageRepo, $strArchiveId, $strArchiveFile);
if (defined($strFoundFile))
{
$strChecksum = substr($strFoundFile, length($strArchiveFile) + 1, 40);
}
}
# Else just get the archive id
else
{
$strArchiveId = $oArchiveInfo->archiveId();
}
# Get the encryption passphrase to read/write files (undefined if the repo is not encrypted)
$strCipherPass = $oArchiveInfo->cipherPassSub();
}
my $strWarning;
if (defined($strChecksum) && !cfgCommandTest(CFGCMD_REMOTE))
{
my ($strChecksumNew) = storageDb()->hashSize($strWalFile);
if ($strChecksumNew ne $strChecksum)
{
confess &log(ERROR, "WAL segment " . basename($strWalFile) . " already exists in the archive", ERROR_ARCHIVE_DUPLICATE);
}
$strWarning =
"WAL segment " . basename($strWalFile) . " already exists in the archive with the same checksum\n" .
"HINT: this is valid in some recovery scenarios but may also indicate a problem.";
&log(WARN, $strWarning);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strArchiveId', value => $strArchiveId},
{name => 'strChecksum', value => $strChecksum},
{name => 'strCipherPass', value => $strCipherPass, redact => true},
{name => 'strWarning', value => $strWarning},
);
}
push @EXPORT, qw(archivePushCheck);
####################################################################################################################################
# archivePushFile
#
# Copy a file from the WAL directory to the archive.
####################################################################################################################################
sub archivePushFile
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strWalPath,
$strWalFile,
$bCompress,
$iCompressLevel,
) =
logDebugParam
(
__PACKAGE__ . '::archivePushFile', \@_,
{name => 'strWalPath'},
{name => 'strWalFile'},
{name => 'bCompress'},
{name => 'iCompressLevel'},
);
# Get cluster info from the WAL
my $oStorageRepo = storageRepo();
my $strDbVersion;
my $ullDbSysId;
if (walIsSegment($strWalFile))
{
($strDbVersion, $ullDbSysId) = walInfo("${strWalPath}/${strWalFile}");
}
# Check if the WAL already exists in the repo
my ($strArchiveId, $strChecksum, $strCipherPass, $strWarning) = archivePushCheck(
$strWalFile, $strDbVersion, $ullDbSysId, walIsSegment($strWalFile) ? "${strWalPath}/${strWalFile}" : undef);
# Only copy the WAL segment if checksum is not defined. If checksum is defined it means that the WAL segment already exists
# in the repository with the same checksum (else there would have been an error on checksum mismatch).
if (!defined($strChecksum))
{
my $strArchiveFile = "${strArchiveId}/${strWalFile}";
# If a WAL segment
if (walIsSegment($strWalFile))
{
# Get hash
my ($strSourceHash) = storageDb()->hashSize("${strWalPath}/${strWalFile}");
$strArchiveFile .= "-${strSourceHash}";
# Add compress extension
if ($bCompress)
{
$strArchiveFile .= qw{.} . COMPRESS_EXT;
}
}
# Add compression
my $rhyFilter;
if (walIsSegment($strWalFile) && $bCompress)
{
push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{iLevel => $iCompressLevel}]});
}
# Copy. If the file is encrypted, then the key from the info file is required to open the archive file in the repo.
$oStorageRepo->copy(
storageDb()->openRead("${strWalPath}/${strWalFile}", {rhyFilter => $rhyFilter}),
$oStorageRepo->openWrite(
STORAGE_REPO_ARCHIVE . "/${strArchiveFile}",
{bPathCreate => true, bAtomic => true, bProtocolCompress => !walIsSegment($strWalFile) || !$bCompress,
strCipherPass => $strCipherPass}));
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strWarning', value => $strWarning}
);
}
push @EXPORT, qw(archivePushFile);
1;

214
lib/pgBackRest/Archive/Push/Push.pm
View File

@ -1,214 +0,0 @@
####################################################################################################################################
# ARCHIVE PUSH MODULE
####################################################################################################################################
package pgBackRest::Archive::Push::Push;
use parent 'pgBackRest::Archive::Base';
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(basename dirname);
use pgBackRest::Archive::Common;
use pgBackRest::DbVersion;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
####################################################################################################################################
# process
#
# Push a WAL segment. The WAL can be pushed in sync or async mode.
####################################################################################################################################
sub process
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strWalPathFile,
$bAsync,
) =
logDebugParam
(
__PACKAGE__ . '->process', \@_,
{name => 'strWalPathFile', required => false},
{name => 'bAsync', required => true},
);
# Make sure the command happens on the db side
if (!isDbLocal())
{
confess &log(ERROR, cfgCommandName(CFGCMD_ARCHIVE_PUSH) . ' operation must run on db host', ERROR_HOST_INVALID);
}
if (!defined($strWalPathFile))
{
confess &log(ERROR, 'WAL file to push required', ERROR_PARAM_REQUIRED);
}
# Extract WAL path and file
my $strWalPath = dirname(walPath($strWalPathFile, cfgOption(CFGOPT_PG_PATH, false), cfgCommandName(cfgCommandGet())));
my $strWalFile = basename($strWalPathFile);
# Start the async process and wait for WAL to complete
if ($bAsync)
{
# Load module dynamically
require pgBackRest::Archive::Push::Async;
(new pgBackRest::Archive::Push::Async(
$strWalPath, storageSpool()->pathGet(STORAGE_SPOOL_ARCHIVE_OUT), $self->{strBackRestBin}))->process();
}
# Else push synchronously
else
{
# Check for a stop lock
lockStopTest();
# Load module dynamically
require pgBackRest::Archive::Push::File;
pgBackRest::Archive::Push::File->import();
# Drop file if queue max has been exceeded
$self->{strWalPath} = $strWalPath;
if (cfgOptionTest(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) && @{$self->dropList($self->readyList())} > 0)
{
&log(WARN,
"dropped WAL file ${strWalFile} because archive queue exceeded " . cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) . ' bytes');
}
# Else push the WAL file
else
{
archivePushFile($strWalPath, $strWalFile, cfgOption(CFGOPT_COMPRESS), cfgOption(CFGOPT_COMPRESS_LEVEL));
&log(INFO, "pushed WAL segment ${strWalFile}");
}
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# readyList
#
# Determine which WAL PostgreSQL has marked as ready to be archived. This is the heart of the "look ahead" functionality in async
# archiving.
####################################################################################################################################
sub readyList
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->readyList');
# Read the .ok files
my $hOkFile = {};
if (defined($self->{strSpoolPath}))
{
foreach my $strOkFile (storageSpool()->list($self->{strSpoolPath}, {strExpression => '\.ok$', bIgnoreMissing => true}))
{
$strOkFile = substr($strOkFile, 0, length($strOkFile) - length('.ok'));
$hOkFile->{$strOkFile} = true;
}
}
# Read the .ready files
my $strWalStatusPath = "$self->{strWalPath}/archive_status";
my @stryReadyFile = storageDb()->list($strWalStatusPath, {strExpression => '\.ready$'});
# Generate a list of new files
my @stryNewReadyFile;
my $hReadyFile = {};
foreach my $strReadyFile (@stryReadyFile)
{
# Remove .ready extension
$strReadyFile = substr($strReadyFile, 0, length($strReadyFile) - length('.ready'));
# Add the file if it is not already queued or previously processed
if (!defined($hOkFile->{$strReadyFile}))
{
# Push onto list of new files
push(@stryNewReadyFile, $strReadyFile);
}
# Add to the ready hash for speed finding removed files
$hReadyFile->{$strReadyFile} = true;
}
# Remove .ok files that are no longer in .ready state
foreach my $strOkFile (sort(keys(%{$hOkFile})))
{
if (!defined($hReadyFile->{$strOkFile}))
{
storageSpool()->remove("$self->{strSpoolPath}/${strOkFile}.ok");
}
}
return logDebugReturn
(
$strOperation,
{name => 'stryWalFile', value => \@stryNewReadyFile, ref => true}
);
}
####################################################################################################################################
# dropList
#
# Determine if the queue of WAL ready to be archived has grown larger the the user-configurable setting. If so, return the list of
# WAL that should be dropped to allow PostgreSQL to continue running. For the moment this is the entire list of ready WAL,
# otherwise the process may archive small spurts of WAL when at queue max which is not likely to be useful.
####################################################################################################################################
sub dropList
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$stryWalFile,
) =
logDebugParam
(
__PACKAGE__ . '->dropList', \@_,
{name => 'stryReadyList'},
);
# Determine the total size of the WAL
my $iTotalSize = 0;
for my $strWalFile (@{$stryWalFile})
{
$iTotalSize += (storageDb()->info("$self->{strWalPath}/${strWalFile}"))->size();
}
# If total size exceeds queue max then drop all files
my $stryDropFile = [];
if ($iTotalSize > cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX))
{
$stryDropFile = $stryWalFile;
}
return logDebugReturn
(
$strOperation,
{name => 'stryDropFile', value => $stryDropFile, ref => true}
);
}
1;

24
lib/pgBackRest/Main.pm
View File

@ -69,31 +69,9 @@ sub main
cfgCommandSet(cfgCommandId($strCommand));
}
# Process archive-push command
# --------------------------------------------------------------------------------------------------------------------------
if (cfgCommandTest(CFGCMD_ARCHIVE_PUSH))
{
# Load module dynamically
require pgBackRest::Archive::Push::Push;
pgBackRest::Archive::Push::Push->import();
new pgBackRest::Archive::Push::Push()->process($stryCommandArg[0], false);
}
# Process archive-push-async command
# --------------------------------------------------------------------------------------------------------------------------
elsif (cfgCommandTest(CFGCMD_ARCHIVE_PUSH_ASYNC))
{
# Load module dynamically
require pgBackRest::Archive::Push::Push;
pgBackRest::Archive::Push::Push->import();
new pgBackRest::Archive::Push::Push()->process($stryCommandArg[0], true);
}
# Process remote command
# --------------------------------------------------------------------------------------------------------------------------
elsif (cfgCommandTest(CFGCMD_REMOTE))
if (cfgCommandTest(CFGCMD_REMOTE))
{
# Set log levels
cfgOptionSet(CFGOPT_LOG_LEVEL_STDERR, PROTOCOL, true);

6
lib/pgBackRest/Protocol/Helper.pm
View File

@ -25,12 +25,6 @@ use constant OP_BACKUP_FILE => 'backupF
# Archive Module
use constant OP_ARCHIVE_GET_CHECK => 'archiveCheck';
push @EXPORT, qw(OP_ARCHIVE_GET_CHECK);
use constant OP_ARCHIVE_PUSH_CHECK => 'archivePushCheck';
push @EXPORT, qw(OP_ARCHIVE_PUSH_CHECK);
# Archive File Module
use constant OP_ARCHIVE_PUSH_FILE => 'archivePushFile';
push @EXPORT, qw(OP_ARCHIVE_PUSH_FILE);
# Check Module
use constant OP_CHECK_BACKUP_INFO_CHECK => 'backupInfoCheck';

2
lib/pgBackRest/Protocol/Local/Minion.pm
View File

@ -8,7 +8,6 @@ use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use pgBackRest::Archive::Push::File;
use pgBackRest::Backup::File;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
@ -54,7 +53,6 @@ sub init
# Create anonymous subs for each command
my $hCommandMap =
{
&OP_ARCHIVE_PUSH_FILE => sub {archivePushFile(@{shift()})},
&OP_BACKUP_FILE => sub {backupFile(@{shift()})},
&OP_RESTORE_FILE => sub {restoreFile(@{shift()})},

4
lib/pgBackRest/Protocol/Remote/Minion.pm
View File

@ -15,7 +15,6 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::Io::Buffered;
use pgBackRest::Common::Wait;
use pgBackRest::Archive::Get::File;
use pgBackRest::Archive::Push::File;
use pgBackRest::Check::Check;
use pgBackRest::Config::Config;
use pgBackRest::Db;
@ -78,9 +77,6 @@ sub init
# ArchiveGet commands
&OP_ARCHIVE_GET_CHECK => sub {archiveGetCheck(@{shift()})},
# ArchivePush commands
&OP_ARCHIVE_PUSH_CHECK => sub {archivePushCheck(@{shift()})},
# Check commands
&OP_CHECK_BACKUP_INFO_CHECK => sub {$oCheck->backupInfoCheck(@{shift()})},

12
src/Makefile
View File

@ -63,6 +63,8 @@ SRCS = \
command/archive/get/file.c \
command/archive/get/get.c \
command/archive/get/protocol.c \
command/archive/push/file.c \
command/archive/push/protocol.c \
command/archive/push/push.c \
command/help/help.c \
command/info/info.c \
@ -213,7 +215,13 @@ command/archive/get/get.o: command/archive/get/get.c command/archive/common.h co
command/archive/get/protocol.o: command/archive/get/protocol.c command/archive/get/file.h command/archive/get/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h
$(CC) $(CFLAGS) -c command/archive/get/protocol.c -o command/archive/get/protocol.o
command/archive/push/push.o: command/archive/push/push.c command/archive/common.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h perl/exec.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h
command/archive/push/file.o: command/archive/push/file.c command/archive/common.h command/archive/push/file.h command/control/control.h common/assert.h common/compress/gzip/common.h common/compress/gzip/compress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h
$(CC) $(CFLAGS) -c command/archive/push/file.c -o command/archive/push/file.o
command/archive/push/protocol.o: command/archive/push/protocol.c command/archive/push/file.h command/archive/push/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h
$(CC) $(CFLAGS) -c command/archive/push/protocol.c -o command/archive/push/protocol.o
command/archive/push/push.o: command/archive/push/push.c command/archive/common.h command/archive/push/file.h command/archive/push/protocol.h command/command.h command/control/control.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/fork.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h info/infoArchive.h info/infoPg.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h
$(CC) $(CFLAGS) -c command/archive/push/push.c -o command/archive/push/push.o
command/command.o: command/command.c common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h
@ -228,7 +236,7 @@ command/help/help.o: command/help/help.c common/assert.h common/debug.h common/e
command/info/info.o: command/info/info.c command/archive/common.h command/info/info.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h perl/exec.h postgres/interface.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h
$(CC) $(CFLAGS) -c command/info/info.c -o command/info/info.o
command/local/local.o: command/local/local.c command/archive/get/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
command/local/local.o: command/local/local.c command/archive/get/protocol.h command/archive/push/protocol.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h
$(CC) $(CFLAGS) -c command/local/local.c -o command/local/local.o
command/remote/remote.o: command/remote/remote.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/handleRead.h common/io/handleWrite.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/protocol.h protocol/client.h protocol/command.h protocol/helper.h protocol/server.h storage/driver/remote/protocol.h

150
src/command/archive/push/file.c Normal file
View File

@ -0,0 +1,150 @@
/***********************************************************************************************************************************
Archive Push File
***********************************************************************************************************************************/
#include "command/archive/push/file.h"
#include "command/archive/common.h"
#include "command/control/control.h"
#include "common/compress/gzip/common.h"
#include "common/compress/gzip/compress.h"
#include "common/crypto/cipherBlock.h"
#include "common/crypto/hash.h"
#include "common/debug.h"
#include "common/io/filter/group.h"
#include "common/io/io.h"
#include "common/log.h"
#include "config/config.h"
#include "postgres/interface.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Copy a file from the source to the archive
***********************************************************************************************************************************/
String *
archivePushFile(
const String *walSource, const String *archiveId, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile,
CipherType cipherType, const String *cipherPass, bool compress, int compressLevel)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, walSource);
FUNCTION_LOG_PARAM(STRING, archiveId);
FUNCTION_LOG_PARAM(UINT, pgVersion);
FUNCTION_LOG_PARAM(UINT64, pgSystemId);
FUNCTION_LOG_PARAM(STRING, archiveFile);
FUNCTION_LOG_PARAM(ENUM, cipherType);
FUNCTION_TEST_PARAM(STRING, cipherPass);
FUNCTION_LOG_PARAM(BOOL, compress);
FUNCTION_LOG_PARAM(INT, compressLevel);
FUNCTION_LOG_END();
ASSERT(walSource != NULL);
ASSERT(archiveFile != NULL);
String *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
// Is this a WAL segment?
bool isSegment = walIsSegment(archiveFile);
// If this is a segment compare archive version and systemId to the WAL header
if (isSegment)
{
PgWal walInfo = pgWalFromFile(walSource);
if (walInfo.version != pgVersion || walInfo.systemId != pgSystemId)
{
THROW_FMT(
ArchiveMismatchError,
"WAL file '%s' version %s, system-id %" PRIu64 " do not match stanza version %s, system-id %" PRIu64,
strPtr(walSource), strPtr(pgVersionToStr(walInfo.version)), walInfo.systemId, strPtr(pgVersionToStr(pgVersion)),
pgSystemId);
}
}
// Set archive destination initially to the archive file, this will be updated later for wal segments
String *archiveDestination = strDup(archiveFile);
// Get wal segment checksum and compare it to what exists in the repo, if any
String *walSegmentFile = NULL;
if (isSegment)
{
// Generate a sha1 checksum for the wal segment. ??? Probably need a function in storage for this.
IoRead *read = storageFileReadIo(storageNewReadNP(storageLocal(), walSource));
IoFilterGroup *filterGroup = ioFilterGroupAdd(ioFilterGroupNew(), cryptoHashFilter(cryptoHashNew(HASH_TYPE_SHA1_STR)));
ioReadFilterGroupSet(read, filterGroup);
Buffer *buffer = bufNew(ioBufferSize());
ioReadOpen(read);
do
{
ioRead(read, buffer);
bufUsedZero(buffer);
}
while (!ioReadEof(read));
ioReadClose(read);
String *walSegmentChecksum = varStr(ioFilterGroupResult(filterGroup, CRYPTO_HASH_FILTER_TYPE_STR));
// If the wal segment already exists in the repo then compare checksums
walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile);
if (walSegmentFile != NULL)
{
String *walSegmentRepoChecksum = strSubN(walSegmentFile, strSize(archiveFile) + 1, HASH_TYPE_SHA1_SIZE_HEX);
if (strEq(walSegmentChecksum, walSegmentRepoChecksum))
{
memContextSwitch(MEM_CONTEXT_OLD());
result = strNewFmt(
"WAL file '%s' already exists in the archive with the same checksum"
"\nHINT: this is valid in some recovery scenarios but may also indicate a problem.",
strPtr(archiveFile));
memContextSwitch(MEM_CONTEXT_TEMP());
}
else
THROW_FMT(ArchiveDuplicateError, "WAL file '%s' already exists in the archive", strPtr(archiveFile));
}
// Append the checksum to the archive destination
strCatFmt(archiveDestination, "-%s", strPtr(walSegmentChecksum));
}
// Only copy if the file was not found in the archive
if (walSegmentFile == NULL)
{
StorageFileRead *source = storageNewReadNP(storageLocal(), walSource);
// Add filters
IoFilterGroup *filterGroup = ioFilterGroupNew();
// If the file will be compressed then add compression filter
if (isSegment && compress)
{
strCat(archiveDestination, "." GZIP_EXT);
ioFilterGroupAdd(filterGroup, gzipCompressFilter(gzipCompressNew(compressLevel, false)));
}
// If there is a cipher then add the encrypt filter
if (cipherType != cipherTypeNone)
{
ioFilterGroupAdd(
filterGroup,
cipherBlockFilter(
cipherBlockNew(cipherModeEncrypt, cipherType, bufNewStr(cipherPass), NULL)));
}
ioReadFilterGroupSet(storageFileReadIo(source), filterGroup);
// Copy the file
storageCopyNP(
source,
storageNewWriteNP(
storageRepoWrite(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(archiveDestination))));
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING, result);
}

18
src/command/archive/push/file.h Normal file
View File

@ -0,0 +1,18 @@
/***********************************************************************************************************************************
Archive Push File
***********************************************************************************************************************************/
#ifndef COMMAND_ARCHIVE_PUSH_FILE_H
#define COMMAND_ARCHIVE_PUSH_FILE_H
#include "common/crypto/common.h"
#include "common/type/string.h"
#include "storage/storage.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
String *archivePushFile(
const String *walSource, const String *archiveId, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile,
CipherType cipherType, const String *cipherPass, bool compress, int compressLevel);
#endif

57
src/command/archive/push/protocol.c Normal file
View File

@ -0,0 +1,57 @@
/***********************************************************************************************************************************
Archive Push Protocol Handler
***********************************************************************************************************************************/
#include "command/archive/push/file.h"
#include "command/archive/push/protocol.h"
#include "common/debug.h"
#include "common/io/io.h"
#include "common/log.h"
#include "common/memContext.h"
#include "config/config.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
STRING_EXTERN(PROTOCOL_COMMAND_ARCHIVE_PUSH_STR, PROTOCOL_COMMAND_ARCHIVE_PUSH);
/***********************************************************************************************************************************
Process protocol requests
***********************************************************************************************************************************/
bool
archivePushProtocol(const String *command, const VariantList *paramList, ProtocolServer *server)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STRING, command);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(PROTOCOL_SERVER, server);
FUNCTION_LOG_END();
ASSERT(command != NULL);
// Get the repo storage in case it is remote and encryption settings need to be pulled down
storageRepo();
// Attempt to satisfy the request -- we may get requests that are meant for other handlers
bool found = true;
MEM_CONTEXT_TEMP_BEGIN()
{
if (strEq(command, PROTOCOL_COMMAND_ARCHIVE_PUSH_STR))
{
protocolServerResponse(
server,
varNewStr(
archivePushFile(
varStr(varLstGet(paramList, 0)), varStr(varLstGet(paramList, 1)),
(unsigned int)varUInt64(varLstGet(paramList, 2)), varUInt64(varLstGet(paramList, 3)),
varStr(varLstGet(paramList, 4)), (CipherType)varUInt64(varLstGet(paramList, 5)),
varStr(varLstGet(paramList, 6)), varBool(varLstGet(paramList, 7)), varIntForce(varLstGet(paramList, 8)))));
}
else
found = false;
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, found);
}

22
src/command/archive/push/protocol.h Normal file
View File

@ -0,0 +1,22 @@
/***********************************************************************************************************************************
Archive Push Protocol Handler
***********************************************************************************************************************************/
#ifndef COMMAND_ARCHIVE_PUSH_PROTOCOL_H
#define COMMAND_ARCHIVE_PUSH_PROTOCOL_H
#include "common/type/string.h"
#include "common/type/variantList.h"
#include "protocol/server.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define PROTOCOL_COMMAND_ARCHIVE_PUSH "archivePush"
STRING_DECLARE(PROTOCOL_COMMAND_ARCHIVE_PUSH_STR);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
bool archivePushProtocol(const String *command, const VariantList *paramList, ProtocolServer *server);
#endif

415
src/command/archive/push/push.c
View File

@ -4,7 +4,10 @@ Archive Push Command
#include <unistd.h>
#include "command/archive/common.h"
#include "command/archive/push/file.h"
#include "command/archive/push/protocol.h"
#include "command/command.h"
#include "command/control/control.h"
#include "common/debug.h"
#include "common/fork.h"
#include "common/log.h"
@ -12,9 +15,235 @@ Archive Push Command
#include "common/wait.h"
#include "config/config.h"
#include "config/exec.h"
#include "perl/exec.h"
#include "info/infoArchive.h"
#include "postgres/interface.h"
#include "protocol/helper.h"
#include "protocol/parallel.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Ready file extension constants
***********************************************************************************************************************************/
#define STATUS_EXT_READY ".ready"
#define STATUS_EXT_READY_SIZE (sizeof(STATUS_EXT_READY) - 1)
/***********************************************************************************************************************************
Format the warning when a file is dropped
***********************************************************************************************************************************/
static String *
archivePushDropWarning(const String *walFile, int64_t queueMax)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, walFile);
FUNCTION_TEST_PARAM(INT64, queueMax);
FUNCTION_TEST_END();
FUNCTION_TEST_RETURN(
strNewFmt(
"dropped WAL file '%s' because archive queue exceeded %s", strPtr(walFile),
strPtr(strSizeFormat((uint64_t)queueMax))));
}
/***********************************************************************************************************************************
Determine if the WAL process list has become large enough to drop
***********************************************************************************************************************************/
static bool
archivePushDrop(const String *walPath, const StringList *const processList)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, walPath);
FUNCTION_LOG_PARAM(STRING_LIST, processList);
FUNCTION_LOG_END();
const uint64_t queueMax = (uint64_t)cfgOptionInt64(cfgOptArchivePushQueueMax);
uint64_t queueSize = 0;
bool result = false;
for (unsigned int processIdx = 0; processIdx < strLstSize(processList); processIdx++)
{
queueSize += storageInfoNP(
storagePg(), strNewFmt("%s/%s", strPtr(walPath), strPtr(strLstGet(processList, processIdx)))).size;
if (queueSize > queueMax)
{
result = true;
break;
}
}
FUNCTION_LOG_RETURN(BOOL, result);
}
/***********************************************************************************************************************************
Get the list of WAL files ready to be pushed according to PostgreSQL
***********************************************************************************************************************************/
static StringList *
archivePushReadyList(const String *walPath)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STRING, walPath);
FUNCTION_LOG_END();
ASSERT(walPath != NULL);
StringList *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
result = strLstNew();
// Read the ready files from the archive_status directory
StringList *readyListRaw = strLstSort(
storageListP(
storagePg(), strNewFmt("%s/" PG_PATH_ARCHIVE_STATUS, strPtr(walPath)),
.expression = strNew("\\" STATUS_EXT_READY "$"), .errorOnMissing = true),
sortOrderAsc);
for (unsigned int readyIdx = 0; readyIdx < strLstSize(readyListRaw); readyIdx++)
{
strLstAdd(
result,
strSubN(strLstGet(readyListRaw, readyIdx), 0, strSize(strLstGet(readyListRaw, readyIdx)) - STATUS_EXT_READY_SIZE));
}
strLstMove(result, MEM_CONTEXT_OLD());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING_LIST, result);
}
/***********************************************************************************************************************************
Determine which WAL files need to be pushed to the archive when in async mode
This is the heart of the "look ahead" functionality in async archiving. Any files in the out directory that do not end in ok are
removed and any ok files that do not have a corresponding ready file in archive_status (meaning it has been acknowledged by
PostgreSQL) are removed. Then all ready files that do not have a corresponding ok file (meaning it has already been processed) are
returned for processing.
***********************************************************************************************************************************/
static StringList *
archivePushProcessList(const String *walPath)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STRING, walPath);
FUNCTION_LOG_END();
StringList *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
// Create the spool out path if it does not already exist
storagePathCreateNP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT));
// Read the status files from the spool directory, then remove any files that do not end in ok and create a list of the
// ok files for further processing
StringList *statusList = strLstSort(
storageListP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT), .errorOnMissing = true), sortOrderAsc);
StringList *okList = strLstNew();
for (unsigned int statusIdx = 0; statusIdx < strLstSize(statusList); statusIdx++)
{
const String *statusFile = strLstGet(statusList, statusIdx);
if (strEndsWithZ(statusFile, STATUS_EXT_OK))
strLstAdd(okList, strSubN(statusFile, 0, strSize(statusFile) - STATUS_EXT_OK_SIZE));
else
{
storageRemoveP(
storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s", strPtr(statusFile)), .errorOnMissing = true);
}
}
// Read the ready files from the archive_status directory
StringList *readyList = archivePushReadyList(walPath);
// Remove ok files that are not in the ready list
StringList *okRemoveList = strLstMergeAnti(okList, readyList);
for (unsigned int okRemoveIdx = 0; okRemoveIdx < strLstSize(okRemoveList); okRemoveIdx++)
{
storageRemoveP(
storageSpoolWrite(),
strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s" STATUS_EXT_OK, strPtr(strLstGet(okRemoveList, okRemoveIdx))),
.errorOnMissing = true);
}
// Return all ready files that are not in the ok list
result = strLstMove(strLstMergeAnti(readyList, okList), MEM_CONTEXT_OLD());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING_LIST, result);
}
/***********************************************************************************************************************************
Check that pg_control and archive.info match and get the archive id and archive cipher passphrase (if present)
As much information as possible is collected here so that async archiving has as little work as possible to do for each file. Sync
archiving does not benefit but it makes sense to use the same function.
***********************************************************************************************************************************/
#define FUNCTION_LOG_ARCHIVE_PUSH_CHECK_RESULT_TYPE \
ArchivePushCheckResult
#define FUNCTION_LOG_ARCHIVE_PUSH_CHECK_RESULT_FORMAT(value, buffer, bufferSize) \
objToLog(&value, "ArchivePushCheckResult", buffer, bufferSize)
typedef struct ArchivePushCheckResult
{
unsigned int pgVersion; // PostgreSQL version
uint64_t pgSystemId; // PostgreSQL system id
unsigned int pgWalSegmentSize; // PostgreSQL WAL segment size
String *archiveId; // Archive id for current pg version
String *archiveCipherPass; // Archive cipher passphrase
} ArchivePushCheckResult;
static ArchivePushCheckResult
archivePushCheck(CipherType cipherType, const String *cipherPass)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(ENUM, cipherType);
FUNCTION_TEST_PARAM(STRING, cipherPass);
FUNCTION_LOG_END();
ArchivePushCheckResult result = {0};
MEM_CONTEXT_TEMP_BEGIN()
{
// Get info from pg_control
PgControl controlInfo = pgControlFromFile(cfgOptionStr(cfgOptPgPath));
// Attempt to load the archive info file
InfoArchive *info = infoArchiveNew(
storageRepo(), STRING_CONST(STORAGE_REPO_ARCHIVE "/" INFO_ARCHIVE_FILE), false, cipherType, cipherPass);
// Get archive id for the most recent version -- archive-push will only operate against the most recent version
String *archiveId = infoPgArchiveId(infoArchivePg(info), infoPgDataCurrentId(infoArchivePg(info)));
InfoPgData archiveInfo = infoPgData(infoArchivePg(info), infoPgDataCurrentId(infoArchivePg(info)));
// Ensure that the version and system identifier match
if (controlInfo.version != archiveInfo.version || controlInfo.systemId != archiveInfo.systemId)
{
THROW_FMT(
ArchiveMismatchError,
"PostgreSQL version %s, system-id %" PRIu64 " do not match stanza version %s, system-id %" PRIu64
"\nHINT: are you archiving to the correct stanza?",
strPtr(pgVersionToStr(controlInfo.version)), controlInfo.systemId, strPtr(pgVersionToStr(archiveInfo.version)),
archiveInfo.systemId);
}
memContextSwitch(MEM_CONTEXT_OLD());
result.pgVersion = controlInfo.version;
result.pgSystemId = controlInfo.systemId;
result.pgWalSegmentSize = controlInfo.walSegmentSize;
result.archiveId = strDup(archiveId);
result.archiveCipherPass = strDup(infoArchiveCipherPass(info));
memContextSwitch(MEM_CONTEXT_TEMP());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(ARCHIVE_PUSH_CHECK_RESULT, result);
}
/***********************************************************************************************************************************
Push a WAL segment to the repository
***********************************************************************************************************************************/
@ -23,6 +252,8 @@ cmdArchivePush(void)
{
FUNCTION_LOG_VOID(logLevelDebug);
ASSERT(cfgCommand() == cfgCmdArchivePush);
MEM_CONTEXT_TEMP_BEGIN()
{
// Make sure there is a parameter to retrieve the WAL segment from
@ -31,8 +262,12 @@ cmdArchivePush(void)
if (strLstSize(commandParam) != 1)
THROW(ParamRequiredError, "WAL segment to push required");
// Test for stop file
lockStopTest();
// Get the segment name
String *walSegment = strBase(strLstGet(commandParam, 0));
String *walFile = walPath(strLstGet(commandParam, 0), cfgOptionStr(cfgOptPgPath), strNew(cfgCommandName(cfgCommand())));
String *archiveFile = strBase(walFile);
if (cfgOptionBool(cfgOptArchiveAsync))
{
@ -47,7 +282,7 @@ cmdArchivePush(void)
{
// Check if the WAL segment has been pushed. Errors will not be confessed on the first try to allow the async
// process a chance to fix them.
pushed = archiveAsyncStatus(archiveModePush, walSegment, confessOnError);
pushed = archiveAsyncStatus(archiveModePush, archiveFile, confessOnError);
// If the WAL segment has not already been pushed then start the async process to push it. There's no point in
// forking the async process off more than once so track that as well. Use an archive lock to prevent more than
@ -64,7 +299,7 @@ cmdArchivePush(void)
// Generate command options
StringList *commandExec = cfgExecParam(cfgCmdArchivePushAsync, optionReplace);
strLstInsert(commandExec, 0, cfgExe());
strLstAdd(commandExec, strLstGet(commandParam, 0));
strLstAdd(commandExec, strPath(walFile));
// Release the lock and mark the async process as forked
lockRelease(true);
@ -92,15 +327,179 @@ cmdArchivePush(void)
if (!pushed)
{
THROW_FMT(
ArchiveTimeoutError, "unable to push WAL segment '%s' asynchronously after %lg second(s)",
strPtr(walSegment), cfgOptionDbl(cfgOptArchiveTimeout));
ArchiveTimeoutError, "unable to push WAL file '%s' to the archive asynchronously after %lg second(s)",
strPtr(archiveFile), cfgOptionDbl(cfgOptArchiveTimeout));
}
// Log success
LOG_INFO("pushed WAL segment %s asynchronously", strPtr(walSegment));
LOG_INFO("pushed WAL file '%s' to the archive asynchronously", strPtr(archiveFile));
}
else
perlExec();
{
// Get the repo storage in case it is remote and encryption settings need to be pulled down
storageRepo();
// Get archive info
ArchivePushCheckResult archiveInfo = archivePushCheck(
cipherType(cfgOptionStr(cfgOptRepoCipherType)), cfgOptionStr(cfgOptRepoCipherPass));
// Check if the push queue has been exceeded
if (cfgOptionTest(cfgOptArchivePushQueueMax) &&
archivePushDrop(strPath(walFile), archivePushReadyList(strPath(walFile))))
{
LOG_WARN(strPtr(archivePushDropWarning(archiveFile, cfgOptionInt64(cfgOptArchivePushQueueMax))));
}
// Else push the file
else
{
// Push the file to the archive
String *warning = archivePushFile(
walFile, archiveInfo.archiveId, archiveInfo.pgVersion, archiveInfo.pgSystemId, archiveFile,
cipherType(cfgOptionStr(cfgOptRepoCipherType)), archiveInfo.archiveCipherPass,
cfgOptionBool(cfgOptCompress), cfgOptionInt(cfgOptCompressLevel));
// If a warning was returned then log it
if (warning != NULL)
LOG_WARN(strPtr(warning));
// Log success
LOG_INFO("pushed WAL file '%s' to the archive", strPtr(archiveFile));
}
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Async version of archive push that runs in parallel for performance
***********************************************************************************************************************************/
void
cmdArchivePushAsync(void)
{
FUNCTION_LOG_VOID(logLevelDebug);
ASSERT(cfgCommand() == cfgCmdArchivePushAsync);
MEM_CONTEXT_TEMP_BEGIN()
{
// Make sure there is a parameter with the wal path
const StringList *commandParam = cfgCommandParam();
if (strLstSize(commandParam) != 1)
THROW(ParamRequiredError, "WAL path to push required");
const String *walPath = strLstGet(commandParam, 0);
TRY_BEGIN()
{
// Test for stop file
lockStopTest();
// Get a list of WAL files that are ready for processing
StringList *walFileList = archivePushProcessList(walPath);
// The archive-push-async command should not have been called unless there are WAL files to process
if (strLstSize(walFileList) == 0)
THROW(AssertError, "no WAL files to process");
LOG_INFO(
"push %u WAL file(s) to archive: %s%s", strLstSize(walFileList), strPtr(strLstGet(walFileList, 0)),
strLstSize(walFileList) == 1 ?
"" : strPtr(strNewFmt("...%s", strPtr(strLstGet(walFileList, strLstSize(walFileList) - 1)))));
// Drop files if queue max has been exceeded
if (cfgOptionTest(cfgOptArchivePushQueueMax) && archivePushDrop(walPath, walFileList))
{
for (unsigned int walFileIdx = 0; walFileIdx < strLstSize(walFileList); walFileIdx++)
{
const String *walFile = strLstGet(walFileList, walFileIdx);
const String *warning = archivePushDropWarning(walFile, cfgOptionInt64(cfgOptArchivePushQueueMax));
archiveAsyncStatusOkWrite(archiveModePush, walFile, warning);
LOG_WARN(strPtr(warning));
}
}
// Else continue processing
else
{
// Get the repo storage in case it is remote and encryption settings need to be pulled down
storageRepo();
// Get archive info
ArchivePushCheckResult archiveInfo = archivePushCheck(
cipherType(cfgOptionStr(cfgOptRepoCipherType)), cfgOptionStr(cfgOptRepoCipherPass));
// Create the parallel executor
ProtocolParallel *parallelExec = protocolParallelNew(
(TimeMSec)(cfgOptionDbl(cfgOptProtocolTimeout) * MSEC_PER_SEC) / 2);
for (unsigned int processIdx = 1; processIdx <= (unsigned int)cfgOptionInt(cfgOptProcessMax); processIdx++)
protocolParallelClientAdd(parallelExec, protocolLocalGet(protocolStorageTypeRepo, processIdx));
// Queue jobs in executor
for (unsigned int walFileIdx = 0; walFileIdx < strLstSize(walFileList); walFileIdx++)
{
protocolKeepAlive();
const String *walFile = strLstGet(walFileList, walFileIdx);
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_STR);
protocolCommandParamAdd(command, varNewStr(strNewFmt("%s/%s", strPtr(walPath), strPtr(walFile))));
protocolCommandParamAdd(command, varNewStr(archiveInfo.archiveId));
protocolCommandParamAdd(command, varNewUInt64(archiveInfo.pgVersion));
protocolCommandParamAdd(command, varNewUInt64(archiveInfo.pgSystemId));
protocolCommandParamAdd(command, varNewStr(walFile));
protocolCommandParamAdd(command, varNewUInt64(cipherType(cfgOptionStr(cfgOptRepoCipherType))));
protocolCommandParamAdd(command, varNewStr(archiveInfo.archiveCipherPass));
protocolCommandParamAdd(command, varNewBool(cfgOptionBool(cfgOptCompress)));
protocolCommandParamAdd(command, varNewInt(cfgOptionInt(cfgOptCompressLevel)));
protocolParallelJobAdd(parallelExec, protocolParallelJobNew(varNewStr(walFile), command));
}
// Process jobs
do
{
unsigned int completed = protocolParallelProcess(parallelExec);
for (unsigned int jobIdx = 0; jobIdx < completed; jobIdx++)
{
protocolKeepAlive();
// Get the job and job key
ProtocolParallelJob *job = protocolParallelResult(parallelExec);
const String *walFile = varStr(protocolParallelJobKey(job));
// The job was successful
if (protocolParallelJobErrorCode(job) == 0)
{
LOG_DETAIL("pushed WAL file '%s' to the archive", strPtr(walFile));
archiveAsyncStatusOkWrite(archiveModePush, walFile, varStr(protocolParallelJobResult(job)));
}
// Else the job errored
else
{
LOG_WARN(
"could not push WAL file '%s' to the archive (will be retried): [%d] %s", strPtr(walFile),
protocolParallelJobErrorCode(job), strPtr(protocolParallelJobErrorMessage(job)));
archiveAsyncStatusErrorWrite(
archiveModePush, walFile, protocolParallelJobErrorCode(job), protocolParallelJobErrorMessage(job));
}
}
}
while (!protocolParallelDone(parallelExec));
}
}
// On any global error write a single error file to cover all unprocessed files
CATCH_ANY()
{
archiveAsyncStatusErrorWrite(archiveModePush, NULL, errorCode(), strNew(errorMessage()));
RETHROW();
}
TRY_END();
}
MEM_CONTEXT_TEMP_END();

1
src/command/archive/push/push.h
View File

@ -8,5 +8,6 @@ Archive Push Command
Functions
***********************************************************************************************************************************/
void cmdArchivePush(void);
void cmdArchivePushAsync(void);
#endif

2
src/command/local/local.c
View File

@ -2,6 +2,7 @@
Local Command
***********************************************************************************************************************************/
#include "command/archive/get/protocol.h"
#include "command/archive/push/protocol.h"
#include "common/debug.h"
#include "common/io/handleRead.h"
#include "common/io/handleWrite.h"
@ -29,6 +30,7 @@ cmdLocal(int handleRead, int handleWrite)
ProtocolServer *server = protocolServerNew(name, PROTOCOL_SERVICE_LOCAL_STR, read, write);
protocolServerHandlerAdd(server, archiveGetProtocol);
protocolServerHandlerAdd(server, archivePushProtocol);
protocolServerProcess(server);
}
MEM_CONTEXT_TEMP_END();

13
src/main.c
View File

@ -65,7 +65,9 @@ main(int argListSize, const char *argList[])
// Local command. Currently only implements a subset.
// -------------------------------------------------------------------------------------------------------------------------
else if (cfgCommand() == cfgCmdLocal && strEqZ(cfgOptionStr(cfgOptCommand), cfgCommandName(cfgCmdArchiveGetAsync)))
else if (cfgCommand() == cfgCmdLocal &&
(strEqZ(cfgOptionStr(cfgOptCommand), cfgCommandName(cfgCmdArchiveGetAsync)) ||
strEqZ(cfgOptionStr(cfgOptCommand), cfgCommandName(cfgCmdArchivePushAsync))))
{
cmdLocal(STDIN_FILENO, STDOUT_FILENO);
}
@ -75,6 +77,8 @@ main(int argListSize, const char *argList[])
else if (cfgCommand() == cfgCmdRemote &&
(strEqZ(cfgOptionStr(cfgOptCommand), cfgCommandName(cfgCmdArchiveGet)) ||
strEqZ(cfgOptionStr(cfgOptCommand), cfgCommandName(cfgCmdArchiveGetAsync)) ||
strEqZ(cfgOptionStr(cfgOptCommand), cfgCommandName(cfgCmdArchivePush)) ||
strEqZ(cfgOptionStr(cfgOptCommand), cfgCommandName(cfgCmdArchivePushAsync)) ||
strEqZ(cfgOptionStr(cfgOptCommand), cfgCommandName(cfgCmdInfo))))
{
cmdRemote(STDIN_FILENO, STDOUT_FILENO);
@ -101,6 +105,13 @@ main(int argListSize, const char *argList[])
cmdArchivePush();
}
// Archive push async command
// -------------------------------------------------------------------------------------------------------------------------
else if (cfgCommand() == cfgCmdArchivePushAsync)
{
cmdArchivePushAsync();
}
// Backup command. Still executed in Perl but this implements running expire after backup.
// -------------------------------------------------------------------------------------------------------------------------
else if (cfgCommand() == cfgCmdBackup)

842
src/perl/embed.auto.c
View File

@ -9,59 +9,6 @@ Embedded Perl modules
***********************************************************************************************************************************/
static const EmbeddedModule embeddedModule[] =
{
{
.name = "pgBackRest/Archive/Base.pm",
.data =
"\n\n\n"
"package pgBackRest::Archive::Base;\n"
"\n"
"use strict;\n"
"use warnings FATAL => qw(all);\n"
"use Carp qw(confess);\n"
"\n"
"use Exporter qw(import);\n"
"our @EXPORT = qw();\n"
"use File::Basename qw(dirname);\n"
"\n"
"use pgBackRest::Archive::Info;\n"
"use pgBackRest::Archive::Common;\n"
"use pgBackRest::Db;\n"
"use pgBackRest::DbVersion;\n"
"use pgBackRest::Common::Exception;\n"
"use pgBackRest::Common::Log;\n"
"use pgBackRest::Common::Wait;\n"
"use pgBackRest::Config::Config;\n"
"use pgBackRest::Protocol::Helper;\n"
"use pgBackRest::Protocol::Storage::Helper;\n"
"use pgBackRest::Storage::Helper;\n"
"use pgBackRest::Version;\n"
"\n\n\n\n"
"sub new\n"
"{\n"
"my $class = shift;\n"
"\n\n"
"my $self = {};\n"
"bless $self, $class;\n"
"\n\n"
"(\n"
"my $strOperation,\n"
"$self->{strBackRestBin},\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '->new', \\@_,\n"
"{name => 'strBackRestBin', default => projectBin(), trace => true},\n"
");\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'self', value => $self}\n"
");\n"
"}\n"
"\n"
"1;\n"
},
{
.name = "pgBackRest/Archive/Common.pm",
.data =
@ -99,26 +46,6 @@ static const EmbeddedModule embeddedModule[] =
"\n\n\n\n"
"use constant PG_WAL_SEGMENT_SIZE => 16777216;\n"
"push @EXPORT, qw(PG_WAL_SEGMENT_SIZE);\n"
"\n\n\n\n"
"use constant WAL_STATUS_ERROR => 'error';\n"
"push @EXPORT, qw(WAL_STATUS_ERROR);\n"
"use constant WAL_STATUS_OK => 'ok';\n"
"push @EXPORT, qw(WAL_STATUS_OK);\n"
"\n\n\n\n"
"my $oWalMagicHash =\n"
"{\n"
"hex('0xD062') => PG_VERSION_83,\n"
"hex('0xD063') => PG_VERSION_84,\n"
"hex('0xD064') => PG_VERSION_90,\n"
"hex('0xD066') => PG_VERSION_91,\n"
"hex('0xD071') => PG_VERSION_92,\n"
"hex('0xD075') => PG_VERSION_93,\n"
"hex('0xD07E') => PG_VERSION_94,\n"
"hex('0xD087') => PG_VERSION_95,\n"
"hex('0xD093') => PG_VERSION_96,\n"
"hex('0xD097') => PG_VERSION_10,\n"
"hex('0xD098') => PG_VERSION_11,\n"
"};\n"
"\n\n\n\n\n\n"
"sub lsnNormalize\n"
"{\n"
@ -210,73 +137,6 @@ static const EmbeddedModule embeddedModule[] =
"}\n"
"\n"
"push @EXPORT, qw(lsnFileRange);\n"
"\n\n\n\n\n\n"
"sub walInfo\n"
"{\n"
"\n"
"my\n"
"(\n"
"$strOperation,\n"
"$strWalFile,\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '::walInfo', \\@_,\n"
"{name => 'strWalFile'}\n"
");\n"
"\n\n\n"
"my $hFile;\n"
"my $tBlock;\n"
"\n"
"sysopen($hFile, $strWalFile, O_RDONLY)\n"
"or confess &log(ERROR, \"unable to open ${strWalFile}\", ERROR_FILE_OPEN);\n"
"\n\n"
"sysread($hFile, $tBlock, 2) == 2\n"
"or confess &log(ERROR, \"unable to read wal magic\");\n"
"\n"
"my $iMagic = unpack('S', $tBlock);\n"
"\n\n\n\n\n\n"
"my $strDbVersion = $$oWalMagicHash{$iMagic};\n"
"\n"
"if (!defined($strDbVersion))\n"
"{\n"
"confess &log(ERROR, \"unexpected WAL magic 0x\" . sprintf(\"%X\", $iMagic) . \"\\n\" .\n"
"'HINT: is this version of PostgreSQL supported?',\n"
"ERROR_VERSION_NOT_SUPPORTED);\n"
"}\n"
"\n\n\n\n"
"my $iSysIdOffset = $strDbVersion >= PG_VERSION_93 ? PG_WAL_SYSTEM_ID_OFFSET_GTE_93 : PG_WAL_SYSTEM_ID_OFFSET_LT_93;\n"
"\n\n\n"
"sysread($hFile, $tBlock, 2) == 2\n"
"or confess &log(ERROR, \"unable to read wal info\");\n"
"\n"
"my $iFlag = unpack('S', $tBlock);\n"
"\n\n"
"$iFlag & 2\n"
"or confess &log(ERROR, \"expected long header in flags \" . sprintf(\"%x\", $iFlag));\n"
"\n\n\n"
"sysseek($hFile, $iSysIdOffset, SEEK_CUR)\n"
"or confess &log(ERROR, \"unable to read padding\");\n"
"\n"
"sysread($hFile, $tBlock, 8) == 8\n"
"or confess &log(ERROR, \"unable to read database system identifier\");\n"
"\n"
"length($tBlock) == 8\n"
"or confess &log(ERROR, \"block is incorrect length\");\n"
"\n"
"close($hFile);\n"
"\n"
"my $ullDbSysId = unpack('Q', $tBlock);\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'strDbVersion', value => $strDbVersion},\n"
"{name => 'ullDbSysId', value => $ullDbSysId}\n"
");\n"
"}\n"
"\n"
"push @EXPORT, qw(walInfo);\n"
"\n\n\n\n\n\n\n"
"sub walSegmentFind\n"
"{\n"
@ -345,48 +205,6 @@ static const EmbeddedModule embeddedModule[] =
"\n"
"push @EXPORT, qw(walSegmentFind);\n"
"\n\n\n\n\n\n"
"sub walPath\n"
"{\n"
"\n"
"my\n"
"(\n"
"$strOperation,\n"
"$strWalFile,\n"
"$strDbPath,\n"
"$strCommand,\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '::walPath', \\@_,\n"
"{name => 'strWalFile', trace => true},\n"
"{name => 'strDbPath', trace => true, required => false},\n"
"{name => 'strCommand', trace => true},\n"
");\n"
"\n"
"if (index($strWalFile, '/') != 0)\n"
"{\n"
"if (!defined($strDbPath))\n"
"{\n"
"confess &log(ERROR,\n"
"\"option '\" . cfgOptionName(CFGOPT_PG_PATH) . \"' must be specified when relative wal paths are used\\n\" .\n"
"\"HINT: Is \\%f passed to ${strCommand} instead of \\%p?\\n\" .\n"
"\"HINT: PostgreSQL may pass relative paths even with \\%p depending on the environment.\",\n"
"ERROR_OPTION_REQUIRED);\n"
"}\n"
"\n"
"$strWalFile = \"${strDbPath}/${strWalFile}\";\n"
"}\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'strWalFile', value => $strWalFile, trace => true}\n"
");\n"
"\n"
"}\n"
"\n"
"push @EXPORT, qw(walPath);\n"
"\n\n\n\n\n\n"
"sub walIsSegment\n"
"{\n"
"\n"
@ -424,64 +242,6 @@ static const EmbeddedModule embeddedModule[] =
"}\n"
"\n"
"push @EXPORT, qw(walIsPartial);\n"
"\n\n\n\n\n\n"
"sub archiveAsyncStatusWrite\n"
"{\n"
"\n"
"my\n"
"(\n"
"$strOperation,\n"
"$strType,\n"
"$strSpoolPath,\n"
"$strWalFile,\n"
"$iCode,\n"
"$strMessage,\n"
"$bIgnoreErrorOnOk,\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '::archiveAsyncStatusWrite', \\@_,\n"
"{name => 'strType'},\n"
"{name => 'strSpoolPath'},\n"
"{name => 'strWalFile'},\n"
"{name => 'iCode', required => false},\n"
"{name => 'strMessage', required => false},\n"
"{name => 'bIgnoreErrorOnOk', required => false, default => false},\n"
");\n"
"\n\n"
"if ($strType ne WAL_STATUS_ERROR)\n"
"{\n"
"\n"
"storageLocal()->remove(\"${strSpoolPath}/${strWalFile}.error\", {bIgnoreMissing => true});\n"
"}\n"
"\n\n\n"
"if (!($strType eq WAL_STATUS_ERROR && $bIgnoreErrorOnOk && storageLocal()->exists(\"${strSpoolPath}/${strWalFile}.ok\")))\n"
"{\n"
"\n"
"my $strStatus;\n"
"\n"
"if (defined($iCode))\n"
"{\n"
"if (!defined($strMessage))\n"
"{\n"
"confess &log(ASSERT, 'strMessage must be set when iCode is set');\n"
"}\n"
"\n"
"$strStatus = \"${iCode}\\n${strMessage}\";\n"
"}\n"
"elsif ($strType eq WAL_STATUS_ERROR)\n"
"{\n"
"confess &log(ASSERT, 'error status must have iCode and strMessage set');\n"
"}\n"
"\n"
"storageLocal()->put(\n"
"storageLocal()->openWrite(\"${strSpoolPath}/${strWalFile}.${strType}\", {bAtomic => true}), $strStatus);\n"
"}\n"
"\n\n"
"return logDebugReturn($strOperation);\n"
"}\n"
"\n"
"push @EXPORT, qw(archiveAsyncStatusWrite);\n"
"\n"
"1;\n"
},
@ -1108,578 +868,6 @@ static const EmbeddedModule embeddedModule[] =
"\n"
"1;\n"
},
{
.name = "pgBackRest/Archive/Push/Async.pm",
.data =
"\n\n\n"
"package pgBackRest::Archive::Push::Async;\n"
"use parent 'pgBackRest::Archive::Push::Push';\n"
"\n"
"use strict;\n"
"use warnings FATAL => qw(all);\n"
"use Carp qw(confess);\n"
"use English '-no_match_vars';\n"
"\n"
"use pgBackRest::Common::Exception;\n"
"use pgBackRest::Common::Lock;\n"
"use pgBackRest::Common::Log;\n"
"use pgBackRest::Archive::Common;\n"
"use pgBackRest::Archive::Info;\n"
"use pgBackRest::Archive::Push::Push;\n"
"use pgBackRest::Common::String;\n"
"use pgBackRest::Common::Wait;\n"
"use pgBackRest::Config::Config;\n"
"use pgBackRest::Db;\n"
"use pgBackRest::DbVersion;\n"
"use pgBackRest::LibC qw(:lock);\n"
"use pgBackRest::Protocol::Local::Process;\n"
"use pgBackRest::Protocol::Helper;\n"
"use pgBackRest::Storage::Helper;\n"
"use pgBackRest::Version;\n"
"\n\n\n\n"
"sub new\n"
"{\n"
"my $class = shift;\n"
"\n\n"
"my $self = $class->SUPER::new();\n"
"bless $self, $class;\n"
"\n\n"
"(\n"
"my $strOperation,\n"
"$self->{strWalPath},\n"
"$self->{strSpoolPath},\n"
"$self->{strBackRestBin},\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '->new', \\@_,\n"
"{name => 'strWalPath'},\n"
"{name => 'strSpoolPath'},\n"
"{name => 'strBackRestBin', default => projectBin()},\n"
");\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'self', value => $self}\n"
");\n"
"}\n"
"\n\n\n\n\n\n"
"sub initServer\n"
"{\n"
"my $self = shift;\n"
"\n\n"
"my ($strOperation) = logDebugParam(__PACKAGE__ . '->initServer');\n"
"\n\n"
"storageSpool()->pathCreate($self->{strSpoolPath}, {bIgnoreExists => true, bCreateParent => true});\n"
"\n\n"
"$self->{oArchiveProcess} = new pgBackRest::Protocol::Local::Process(\n"
"CFGOPTVAL_LOCAL_TYPE_BACKUP, cfgOption(CFGOPT_PROTOCOL_TIMEOUT) < 60 ? cfgOption(CFGOPT_PROTOCOL_TIMEOUT) / 2 : 30,\n"
"$self->{strBackRestBin}, false);\n"
"$self->{oArchiveProcess}->hostAdd(1, cfgOption(CFGOPT_PROCESS_MAX));\n"
"\n\n"
"return logDebugReturn($strOperation);\n"
"}\n"
"\n\n\n\n\n\n"
"sub process\n"
"{\n"
"my $self = shift;\n"
"\n\n"
"my ($strOperation) = logDebugParam(__PACKAGE__ . '->process');\n"
"\n\n"
"logFileSet(storageLocal(), cfgOption(CFGOPT_LOG_PATH) . '/' . cfgOption(CFGOPT_STANZA) . '-archive-push-async');\n"
"\n\n\n"
"$self->initServer();\n"
"$self->processQueue();\n"
"\n\n"
"return logDebugReturn($strOperation);\n"
"}\n"
"\n\n\n\n\n\n"
"sub processQueue\n"
"{\n"
"my $self = shift;\n"
"\n\n"
"my ($strOperation) = logDebugParam(__PACKAGE__ . '->processQueue');\n"
"\n\n"
"my $iDropTotal = 0;\n"
"\n"
"if (cfgOptionTest(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX))\n"
"{\n"
"my $stryDropList = $self->dropList($self->readyList());\n"
"\n"
"if (@{$stryDropList} > 0)\n"
"{\n"
"foreach my $strDropFile (@{$stryDropList})\n"
"{\n"
"archiveAsyncStatusWrite(\n"
"WAL_STATUS_OK, $self->{strSpoolPath}, $strDropFile, 0,\n"
"\"dropped WAL file ${strDropFile} because archive queue exceeded \" .\n"
"cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) . ' bytes');\n"
"\n"
"$iDropTotal++;\n"
"}\n"
"}\n"
"}\n"
"\n\n"
"my $stryWalFile = $self->readyList();\n"
"\n\n"
"foreach my $strWalFile (@{$stryWalFile})\n"
"{\n"
"$self->{oArchiveProcess}->queueJob(\n"
"1, 'default', $strWalFile, OP_ARCHIVE_PUSH_FILE,\n"
"[$self->{strWalPath}, $strWalFile, cfgOption(CFGOPT_COMPRESS), cfgOption(CFGOPT_COMPRESS_LEVEL)]);\n"
"}\n"
"\n\n"
"my $iOkTotal = 0;\n"
"my $iErrorTotal = 0;\n"
"\n"
"if ($self->{oArchiveProcess}->jobTotal() > 0)\n"
"{\n"
"&log(INFO,\n"
"'push ' . @{$stryWalFile} . ' WAL file(s) to archive: ' .\n"
"${$stryWalFile}[0] . (@{$stryWalFile} > 1 ? \"...${$stryWalFile}[-1]\" : ''));\n"
"\n"
"eval\n"
"{\n"
"\n"
"lockStopTest();\n"
"\n\n"
"!isRepoLocal() && protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP);\n"
"\n"
"while (my $hyJob = $self->{oArchiveProcess}->process())\n"
"{\n"
"\n"
"protocolKeepAlive();\n"
"\n"
"foreach my $hJob (@{$hyJob})\n"
"{\n"
"my $strWalFile = @{$hJob->{rParam}}[1];\n"
"my $strWarning = @{$hJob->{rResult}}[0];\n"
"\n\n"
"if (defined($hJob->{oException}))\n"
"{\n"
"archiveAsyncStatusWrite(\n"
"WAL_STATUS_ERROR, $self->{strSpoolPath}, $strWalFile, $hJob->{oException}->code(),\n"
"$hJob->{oException}->message());\n"
"\n"
"$iErrorTotal++;\n"
"\n"
"&log(WARN,\n"
"\"could not push WAL file ${strWalFile} to archive (will be retried): [\" .\n"
"$hJob->{oException}->code() . \"] \" . $hJob->{oException}->message());\n"
"}\n"
"\n"
"else\n"
"{\n"
"archiveAsyncStatusWrite(\n"
"WAL_STATUS_OK, $self->{strSpoolPath}, $strWalFile, defined($strWarning) ? 0 : undef,\n"
"defined($strWarning) ? $strWarning : undef);\n"
"\n"
"$iOkTotal++;\n"
"\n"
"&log(DETAIL, \"pushed WAL file ${strWalFile} to archive\", undef, undef, undef, $hJob->{iProcessId});\n"
"}\n"
"}\n"
"}\n"
"\n"
"return 1;\n"
"}\n"
"or do\n"
"{\n"
"\n"
"my $iCode = exceptionCode($EVAL_ERROR);\n"
"my $strMessage = exceptionMessage($EVAL_ERROR);\n"
"\n\n"
"foreach my $strWalFile (@{$stryWalFile})\n"
"{\n"
"archiveAsyncStatusWrite(\n"
"WAL_STATUS_ERROR, $self->{strSpoolPath}, $strWalFile, $iCode, $strMessage, true);\n"
"}\n"
"}\n"
"}\n"
"\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'iNewTotal', value => scalar(@{$stryWalFile})},\n"
"{name => 'iDropTotal', value => $iDropTotal},\n"
"{name => 'iOkTotal', value => $iOkTotal},\n"
"{name => 'iErrorTotal', value => $iErrorTotal}\n"
");\n"
"}\n"
"\n"
"1;\n"
},
{
.name = "pgBackRest/Archive/Push/File.pm",
.data =
"\n\n\n"
"package pgBackRest::Archive::Push::File;\n"
"\n"
"use strict;\n"
"use warnings FATAL => qw(all);\n"
"use Carp qw(confess);\n"
"use English '-no_match_vars';\n"
"\n"
"use Exporter qw(import);\n"
"our @EXPORT = qw();\n"
"use File::Basename qw(basename dirname);\n"
"\n"
"use pgBackRest::Archive::Common;\n"
"use pgBackRest::Archive::Info;\n"
"use pgBackRest::Common::Exception;\n"
"use pgBackRest::Common::Log;\n"
"use pgBackRest::Config::Config;\n"
"use pgBackRest::Protocol::Helper;\n"
"use pgBackRest::Protocol::Storage::Helper;\n"
"use pgBackRest::Storage::Filter::Gzip;\n"
"use pgBackRest::Storage::Filter::Sha;\n"
"use pgBackRest::Storage::Helper;\n"
"\n\n\n\n\n\n\n"
"sub archivePushCheck\n"
"{\n"
"\n"
"my\n"
"(\n"
"$strOperation,\n"
"$strArchiveFile,\n"
"$strDbVersion,\n"
"$ullDbSysId,\n"
"$strWalFile,\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '::archivePushCheck', \\@_,\n"
"{name => 'strArchiveFile'},\n"
"{name => 'strDbVersion', required => false},\n"
"{name => 'ullDbSysId', required => false},\n"
"{name => 'strWalFile', required => false},\n"
");\n"
"\n\n"
"my $oStorageRepo = storageRepo();\n"
"my $strArchiveId;\n"
"my $strChecksum;\n"
"my $strCipherPass = undef;\n"
"\n\n"
"my $bWalSegment = walIsSegment($strArchiveFile);\n"
"\n"
"if (!isRepoLocal())\n"
"{\n"
"\n"
"($strArchiveId, $strChecksum, $strCipherPass) = protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP)->cmdExecute(\n"
"OP_ARCHIVE_PUSH_CHECK, [$strArchiveFile, $strDbVersion, $ullDbSysId], true);\n"
"}\n"
"else\n"
"{\n"
"my $oArchiveInfo = new pgBackRest::Archive::Info($oStorageRepo->pathGet(STORAGE_REPO_ARCHIVE));\n"
"\n\n"
"if ($bWalSegment)\n"
"{\n"
"\n"
"$strArchiveId = $oArchiveInfo->check($strDbVersion, $ullDbSysId);\n"
"\n\n"
"my $strFoundFile = walSegmentFind($oStorageRepo, $strArchiveId, $strArchiveFile);\n"
"\n"
"if (defined($strFoundFile))\n"
"{\n"
"$strChecksum = substr($strFoundFile, length($strArchiveFile) + 1, 40);\n"
"}\n"
"}\n"
"\n"
"else\n"
"{\n"
"$strArchiveId = $oArchiveInfo->archiveId();\n"
"}\n"
"\n\n"
"$strCipherPass = $oArchiveInfo->cipherPassSub();\n"
"}\n"
"\n"
"my $strWarning;\n"
"\n"
"if (defined($strChecksum) && !cfgCommandTest(CFGCMD_REMOTE))\n"
"{\n"
"my ($strChecksumNew) = storageDb()->hashSize($strWalFile);\n"
"\n"
"if ($strChecksumNew ne $strChecksum)\n"
"{\n"
"confess &log(ERROR, \"WAL segment \" . basename($strWalFile) . \" already exists in the archive\", ERROR_ARCHIVE_DUPLICATE);\n"
"}\n"
"\n"
"$strWarning =\n"
"\"WAL segment \" . basename($strWalFile) . \" already exists in the archive with the same checksum\\n\" .\n"
"\"HINT: this is valid in some recovery scenarios but may also indicate a problem.\";\n"
"\n"
"&log(WARN, $strWarning);\n"
"}\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'strArchiveId', value => $strArchiveId},\n"
"{name => 'strChecksum', value => $strChecksum},\n"
"{name => 'strCipherPass', value => $strCipherPass, redact => true},\n"
"{name => 'strWarning', value => $strWarning},\n"
");\n"
"}\n"
"\n"
"push @EXPORT, qw(archivePushCheck);\n"
"\n\n\n\n\n\n"
"sub archivePushFile\n"
"{\n"
"\n"
"my\n"
"(\n"
"$strOperation,\n"
"$strWalPath,\n"
"$strWalFile,\n"
"$bCompress,\n"
"$iCompressLevel,\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '::archivePushFile', \\@_,\n"
"{name => 'strWalPath'},\n"
"{name => 'strWalFile'},\n"
"{name => 'bCompress'},\n"
"{name => 'iCompressLevel'},\n"
");\n"
"\n\n"
"my $oStorageRepo = storageRepo();\n"
"my $strDbVersion;\n"
"my $ullDbSysId;\n"
"\n"
"if (walIsSegment($strWalFile))\n"
"{\n"
"($strDbVersion, $ullDbSysId) = walInfo(\"${strWalPath}/${strWalFile}\");\n"
"}\n"
"\n\n"
"my ($strArchiveId, $strChecksum, $strCipherPass, $strWarning) = archivePushCheck(\n"
"$strWalFile, $strDbVersion, $ullDbSysId, walIsSegment($strWalFile) ? \"${strWalPath}/${strWalFile}\" : undef);\n"
"\n\n\n"
"if (!defined($strChecksum))\n"
"{\n"
"my $strArchiveFile = \"${strArchiveId}/${strWalFile}\";\n"
"\n\n"
"if (walIsSegment($strWalFile))\n"
"{\n"
"\n"
"my ($strSourceHash) = storageDb()->hashSize(\"${strWalPath}/${strWalFile}\");\n"
"\n"
"$strArchiveFile .= \"-${strSourceHash}\";\n"
"\n\n"
"if ($bCompress)\n"
"{\n"
"$strArchiveFile .= qw{.} . COMPRESS_EXT;\n"
"}\n"
"}\n"
"\n\n"
"my $rhyFilter;\n"
"\n"
"if (walIsSegment($strWalFile) && $bCompress)\n"
"{\n"
"push(@{$rhyFilter}, {strClass => STORAGE_FILTER_GZIP, rxyParam => [{iLevel => $iCompressLevel}]});\n"
"}\n"
"\n\n"
"$oStorageRepo->copy(\n"
"storageDb()->openRead(\"${strWalPath}/${strWalFile}\", {rhyFilter => $rhyFilter}),\n"
"$oStorageRepo->openWrite(\n"
"STORAGE_REPO_ARCHIVE . \"/${strArchiveFile}\",\n"
"{bPathCreate => true, bAtomic => true, bProtocolCompress => !walIsSegment($strWalFile) || !$bCompress,\n"
"strCipherPass => $strCipherPass}));\n"
"}\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'strWarning', value => $strWarning}\n"
");\n"
"}\n"
"\n"
"push @EXPORT, qw(archivePushFile);\n"
"\n"
"1;\n"
},
{
.name = "pgBackRest/Archive/Push/Push.pm",
.data =
"\n\n\n"
"package pgBackRest::Archive::Push::Push;\n"
"use parent 'pgBackRest::Archive::Base';\n"
"\n"
"use strict;\n"
"use warnings FATAL => qw(all);\n"
"use Carp qw(confess);\n"
"use English '-no_match_vars';\n"
"\n"
"use Exporter qw(import);\n"
"our @EXPORT = qw();\n"
"use File::Basename qw(basename dirname);\n"
"\n"
"use pgBackRest::Archive::Common;\n"
"use pgBackRest::DbVersion;\n"
"use pgBackRest::Common::Exception;\n"
"use pgBackRest::Common::Lock;\n"
"use pgBackRest::Common::Log;\n"
"use pgBackRest::Common::Wait;\n"
"use pgBackRest::Config::Config;\n"
"use pgBackRest::Protocol::Helper;\n"
"use pgBackRest::Protocol::Storage::Helper;\n"
"use pgBackRest::Storage::Helper;\n"
"\n\n\n\n\n\n"
"sub process\n"
"{\n"
"my $self = shift;\n"
"\n\n"
"my\n"
"(\n"
"$strOperation,\n"
"$strWalPathFile,\n"
"$bAsync,\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '->process', \\@_,\n"
"{name => 'strWalPathFile', required => false},\n"
"{name => 'bAsync', required => true},\n"
");\n"
"\n\n"
"if (!isDbLocal())\n"
"{\n"
"confess &log(ERROR, cfgCommandName(CFGCMD_ARCHIVE_PUSH) . ' operation must run on db host', ERROR_HOST_INVALID);\n"
"}\n"
"\n"
"if (!defined($strWalPathFile))\n"
"{\n"
"confess &log(ERROR, 'WAL file to push required', ERROR_PARAM_REQUIRED);\n"
"}\n"
"\n\n"
"my $strWalPath = dirname(walPath($strWalPathFile, cfgOption(CFGOPT_PG_PATH, false), cfgCommandName(cfgCommandGet())));\n"
"my $strWalFile = basename($strWalPathFile);\n"
"\n\n"
"if ($bAsync)\n"
"{\n"
"\n"
"require pgBackRest::Archive::Push::Async;\n"
"(new pgBackRest::Archive::Push::Async(\n"
"$strWalPath, storageSpool()->pathGet(STORAGE_SPOOL_ARCHIVE_OUT), $self->{strBackRestBin}))->process();\n"
"}\n"
"\n"
"else\n"
"{\n"
"\n"
"lockStopTest();\n"
"\n\n"
"require pgBackRest::Archive::Push::File;\n"
"pgBackRest::Archive::Push::File->import();\n"
"\n\n"
"$self->{strWalPath} = $strWalPath;\n"
"\n"
"if (cfgOptionTest(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) && @{$self->dropList($self->readyList())} > 0)\n"
"{\n"
"&log(WARN,\n"
"\"dropped WAL file ${strWalFile} because archive queue exceeded \" . cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) . ' bytes');\n"
"}\n"
"\n"
"else\n"
"{\n"
"archivePushFile($strWalPath, $strWalFile, cfgOption(CFGOPT_COMPRESS), cfgOption(CFGOPT_COMPRESS_LEVEL));\n"
"&log(INFO, \"pushed WAL segment ${strWalFile}\");\n"
"}\n"
"}\n"
"\n\n"
"return logDebugReturn($strOperation);\n"
"}\n"
"\n\n\n\n\n\n\n"
"sub readyList\n"
"{\n"
"my $self = shift;\n"
"\n\n"
"my ($strOperation) = logDebugParam(__PACKAGE__ . '->readyList');\n"
"\n\n"
"my $hOkFile = {};\n"
"\n"
"if (defined($self->{strSpoolPath}))\n"
"{\n"
"foreach my $strOkFile (storageSpool()->list($self->{strSpoolPath}, {strExpression => '\\.ok$', bIgnoreMissing => true}))\n"
"{\n"
"$strOkFile = substr($strOkFile, 0, length($strOkFile) - length('.ok'));\n"
"$hOkFile->{$strOkFile} = true;\n"
"}\n"
"}\n"
"\n\n"
"my $strWalStatusPath = \"$self->{strWalPath}/archive_status\";\n"
"my @stryReadyFile = storageDb()->list($strWalStatusPath, {strExpression => '\\.ready$'});\n"
"\n\n"
"my @stryNewReadyFile;\n"
"my $hReadyFile = {};\n"
"\n"
"foreach my $strReadyFile (@stryReadyFile)\n"
"{\n"
"\n"
"$strReadyFile = substr($strReadyFile, 0, length($strReadyFile) - length('.ready'));\n"
"\n\n"
"if (!defined($hOkFile->{$strReadyFile}))\n"
"{\n"
"\n"
"push(@stryNewReadyFile, $strReadyFile);\n"
"}\n"
"\n\n"
"$hReadyFile->{$strReadyFile} = true;\n"
"}\n"
"\n\n"
"foreach my $strOkFile (sort(keys(%{$hOkFile})))\n"
"{\n"
"if (!defined($hReadyFile->{$strOkFile}))\n"
"{\n"
"storageSpool()->remove(\"$self->{strSpoolPath}/${strOkFile}.ok\");\n"
"}\n"
"}\n"
"\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'stryWalFile', value => \\@stryNewReadyFile, ref => true}\n"
");\n"
"}\n"
"\n\n\n\n\n\n\n\n"
"sub dropList\n"
"{\n"
"my $self = shift;\n"
"\n\n"
"my\n"
"(\n"
"$strOperation,\n"
"$stryWalFile,\n"
") =\n"
"logDebugParam\n"
"(\n"
"__PACKAGE__ . '->dropList', \\@_,\n"
"{name => 'stryReadyList'},\n"
");\n"
"\n\n"
"my $iTotalSize = 0;\n"
"\n"
"for my $strWalFile (@{$stryWalFile})\n"
"{\n"
"$iTotalSize += (storageDb()->info(\"$self->{strWalPath}/${strWalFile}\"))->size();\n"
"}\n"
"\n\n"
"my $stryDropFile = [];\n"
"\n"
"if ($iTotalSize > cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX))\n"
"{\n"
"$stryDropFile = $stryWalFile;\n"
"}\n"
"\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'stryDropFile', value => $stryDropFile, ref => true}\n"
");\n"
"}\n"
"\n"
"1;\n"
},
{
.name = "pgBackRest/Backup/Backup.pm",
.data =
@ -10527,25 +9715,7 @@ static const EmbeddedModule embeddedModule[] =
"cfgCommandSet(cfgCommandId($strCommand));\n"
"}\n"
"\n\n\n"
"if (cfgCommandTest(CFGCMD_ARCHIVE_PUSH))\n"
"{\n"
"\n"
"require pgBackRest::Archive::Push::Push;\n"
"pgBackRest::Archive::Push::Push->import();\n"
"\n"
"new pgBackRest::Archive::Push::Push()->process($stryCommandArg[0], false);\n"
"}\n"
"\n\n\n"
"elsif (cfgCommandTest(CFGCMD_ARCHIVE_PUSH_ASYNC))\n"
"{\n"
"\n"
"require pgBackRest::Archive::Push::Push;\n"
"pgBackRest::Archive::Push::Push->import();\n"
"\n"
"new pgBackRest::Archive::Push::Push()->process($stryCommandArg[0], true);\n"
"}\n"
"\n\n\n"
"elsif (cfgCommandTest(CFGCMD_REMOTE))\n"
"if (cfgCommandTest(CFGCMD_REMOTE))\n"
"{\n"
"\n"
"cfgOptionSet(CFGOPT_LOG_LEVEL_STDERR, PROTOCOL, true);\n"
@ -12704,11 +11874,6 @@ static const EmbeddedModule embeddedModule[] =
"\n\n"
"use constant OP_ARCHIVE_GET_CHECK => 'archiveCheck';\n"
"push @EXPORT, qw(OP_ARCHIVE_GET_CHECK);\n"
"use constant OP_ARCHIVE_PUSH_CHECK => 'archivePushCheck';\n"
"push @EXPORT, qw(OP_ARCHIVE_PUSH_CHECK);\n"
"\n\n"
"use constant OP_ARCHIVE_PUSH_FILE => 'archivePushFile';\n"
"push @EXPORT, qw(OP_ARCHIVE_PUSH_FILE);\n"
"\n\n"
"use constant OP_CHECK_BACKUP_INFO_CHECK => 'backupInfoCheck';\n"
"push @EXPORT, qw(OP_CHECK_BACKUP_INFO_CHECK);\n"
@ -13174,7 +12339,6 @@ static const EmbeddedModule embeddedModule[] =
"use warnings FATAL => qw(all);\n"
"use Carp qw(confess);\n"
"\n"
"use pgBackRest::Archive::Push::File;\n"
"use pgBackRest::Backup::File;\n"
"use pgBackRest::Common::Log;\n"
"use pgBackRest::Config::Config;\n"
@ -13209,7 +12373,6 @@ static const EmbeddedModule embeddedModule[] =
"\n\n"
"my $hCommandMap =\n"
"{\n"
"&OP_ARCHIVE_PUSH_FILE => sub {archivePushFile(@{shift()})},\n"
"&OP_BACKUP_FILE => sub {backupFile(@{shift()})},\n"
"&OP_RESTORE_FILE => sub {restoreFile(@{shift()})},\n"
"\n\n"
@ -13862,7 +13025,6 @@ static const EmbeddedModule embeddedModule[] =
"use pgBackRest::Common::Io::Buffered;\n"
"use pgBackRest::Common::Wait;\n"
"use pgBackRest::Archive::Get::File;\n"
"use pgBackRest::Archive::Push::File;\n"
"use pgBackRest::Check::Check;\n"
"use pgBackRest::Config::Config;\n"
"use pgBackRest::Db;\n"
@ -13913,8 +13075,6 @@ static const EmbeddedModule embeddedModule[] =
"\n"
"&OP_ARCHIVE_GET_CHECK => sub {archiveGetCheck(@{shift()})},\n"
"\n\n"
"&OP_ARCHIVE_PUSH_CHECK => sub {archivePushCheck(@{shift()})},\n"
"\n\n"
"&OP_CHECK_BACKUP_INFO_CHECK => sub {$oCheck->backupInfoCheck(@{shift()})},\n"
"\n\n"
"&OP_DB_CONNECT => sub {$oDb->connect()},\n"

1
src/postgres/interface.h
View File

@ -14,6 +14,7 @@ Defines for various Postgres paths and files
***********************************************************************************************************************************/
#define PG_FILE_PGCONTROL "pg_control"
#define PG_PATH_ARCHIVE_STATUS "archive_status"
#define PG_PATH_GLOBAL "global"
/***********************************************************************************************************************************

1
src/protocol/parallelJob.c
View File

@ -173,7 +173,6 @@ protocolParallelJobResultSet(ProtocolParallelJob *this, const Variant *result)
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(result != NULL);
ASSERT(this->code == 0);
MEM_CONTEXT_BEGIN(this->memContext)

17
test/define.yaml
View File

@ -629,7 +629,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-common-perl
total: 6
total: 4
coverage:
Archive/Common: partial
@ -653,23 +653,14 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-push
total: 1
total: 4
perlReq: true
coverage:
command/archive/push/file: full
command/archive/push/protocol: full
command/archive/push/push: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-push-perl
total: 7
coverage:
Archive/Base: full
Archive/Push/Async: full
Archive/Push/File: partial
Archive/Push/Push: full
Protocol/Local/Master: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: command
total: 1

43
test/expect/mock-archive-001.log
View File

@ -4,9 +4,11 @@ run 001 - rmt 0, s3 0, enc 1
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/archive/db/archive.info or [TEST_PATH]/db-master/repo/archive/db/archive.info.copy
P00 ERROR: [055]: archive.info does not exist but is required to push/get WAL segments
HINT: is archive_command configured in postgresql.conf?
P00 ERROR: [055]: unable to load info file '[TEST_PATH]/db-master/repo/archive/db/archive.info' or '[TEST_PATH]/db-master/repo/archive/db/archive.info.copy':
FileMissingError: unable to open '[TEST_PATH]/db-master/repo/archive/db/archive.info' for read: [2] No such file or directory
FileMissingError: unable to open '[TEST_PATH]/db-master/repo/archive/db/archive.info.copy' for read: [2] No such file or directory
HINT: archive.info cannot be opened but is required to push/get WAL segments.
HINT: is archive_command configured correctly in postgresql.conf?
HINT: has a stanza-create been performed?
HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
P00 INFO: archive-push command end: aborted with exception [055]
@ -70,7 +72,7 @@ db-version="9.4"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000001
P00 INFO: pushed WAL file '000000010000000100000001' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 700000007000000070000000 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
@ -88,28 +90,15 @@ P00 INFO: archive-get command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000002 asynchronously
P00 INFO: pushed WAL file '000000010000000100000002' to the archive asynchronously
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --archive-async [TEST_PATH]/db-master/db/base/pg_xlog/00000002.history
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/00000002.history] --archive-async --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db
P00 INFO: pushed WAL segment 00000002.history asynchronously
P00 INFO: pushed WAL file '00000002.history' to the archive asynchronously
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 ERROR: [044]: WAL segment version 9.4 does not match archive version 8.0
HINT: are you archiving to the correct stanza?
P00 INFO: archive-push command end: aborted with exception [044]
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: found 000000010000000100000001 in the archive
P00 INFO: archive-get command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
@ -119,7 +108,7 @@ P00 INFO: archive-get command end: aborted with exception [044]
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 ERROR: [044]: WAL segment system-id 1000000000000000094 does not match archive system-id 5000900090001855000
P00 ERROR: [044]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 9.4, system-id 5000900090001855000
HINT: are you archiving to the correct stanza?
P00 INFO: archive-push command end: aborted with exception [044]
@ -156,15 +145,15 @@ P00 INFO: start command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 WARN: WAL segment 000000010000000100000002 already exists in the archive with the same checksum
P00 WARN: WAL file '000000010000000100000002' already exists in the archive with the same checksum
HINT: this is valid in some recovery scenarios but may also indicate a problem.
P00 INFO: pushed WAL segment 000000010000000100000002
P00 INFO: pushed WAL file '000000010000000100000002' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 ERROR: [045]: WAL segment 000000010000000100000002 already exists in the archive
P00 ERROR: [045]: WAL file '000000010000000100000002' already exists in the archive
P00 INFO: archive-push command end: aborted with exception [045]
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get --archive-async --repo-type=cifs --archive-timeout=5 000000010000000100000002 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
@ -188,19 +177,19 @@ P00 INFO: archive-get command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000002.partial
P00 INFO: pushed WAL file '000000010000000100000002.partial' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 WARN: WAL segment 000000010000000100000002.partial already exists in the archive with the same checksum
P00 WARN: WAL file '000000010000000100000002.partial' already exists in the archive with the same checksum
HINT: this is valid in some recovery scenarios but may also indicate a problem.
P00 INFO: pushed WAL segment 000000010000000100000002.partial
P00 INFO: pushed WAL file '000000010000000100000002.partial' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 ERROR: [045]: WAL segment 000000010000000100000002.partial already exists in the archive
P00 ERROR: [045]: WAL file '000000010000000100000002.partial' already exists in the archive
P00 INFO: archive-push command end: aborted with exception [045]

42
test/expect/mock-archive-002.log
View File

@ -4,8 +4,11 @@ run 002 - rmt 1, s3 1, enc 0
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 ERROR: [055]: raised from remote process on 'backup': archive.info does not exist but is required to push/get WAL segments
HINT: is archive_command configured in postgresql.conf?
P00 ERROR: [055]: unable to load info file 'archive/db/archive.info' or 'archive/db/archive.info.copy':
FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info': No such file or directory
FileMissingError: raised from remote-0 protocol on 'backup': unable to open '/archive/db/archive.info.copy': No such file or directory
HINT: archive.info cannot be opened but is required to push/get WAL segments.
HINT: is archive_command configured correctly in postgresql.conf?
HINT: has a stanza-create been performed?
HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
P00 INFO: archive-push command end: aborted with exception [055]
@ -63,7 +66,7 @@ db-version="9.4"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --cmd-ssh=/usr/bin/ssh --compress [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --cmd-ssh=/usr/bin/ssh --compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000001
P00 INFO: pushed WAL file '000000010000000100000001' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 700000007000000070000000 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
@ -81,28 +84,15 @@ P00 INFO: archive-get command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --spool-path=[TEST_PATH]/db-master/spool --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000002 asynchronously
P00 INFO: pushed WAL file '000000010000000100000002' to the archive asynchronously
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --archive-async [TEST_PATH]/db-master/db/base/pg_xlog/00000002.history
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/00000002.history] --archive-async --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --spool-path=[TEST_PATH]/db-master/spool --stanza=db
P00 INFO: pushed WAL segment 00000002.history asynchronously
P00 INFO: pushed WAL file '00000002.history' to the archive asynchronously
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 ERROR: [044]: raised from remote process on 'backup': WAL segment version 9.4 does not match archive version 8.0
HINT: are you archiving to the correct stanza?
P00 INFO: archive-push command end: aborted with exception [044]
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 INFO: found 000000010000000100000001 in the archive
P00 INFO: archive-get command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
@ -112,7 +102,7 @@ P00 INFO: archive-get command end: aborted with exception [044]
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 ERROR: [044]: raised from remote process on 'backup': WAL segment system-id 1000000000000000094 does not match archive system-id 5000900090001855000
P00 ERROR: [044]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 9.4, system-id 5000900090001855000
HINT: are you archiving to the correct stanza?
P00 INFO: archive-push command end: aborted with exception [044]
@ -149,15 +139,15 @@ P00 INFO: start command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 WARN: WAL segment 000000010000000100000002 already exists in the archive with the same checksum
P00 WARN: WAL file '000000010000000100000002' already exists in the archive with the same checksum
HINT: this is valid in some recovery scenarios but may also indicate a problem.
P00 INFO: pushed WAL segment 000000010000000100000002
P00 INFO: pushed WAL file '000000010000000100000002' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 ERROR: [045]: WAL segment 000000010000000100000002 already exists in the archive
P00 ERROR: [045]: WAL file '000000010000000100000002' already exists in the archive
P00 INFO: archive-push command end: aborted with exception [045]
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get --cmd-ssh=/usr/bin/ssh --archive-async --archive-timeout=5 000000010000000100000002 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
@ -181,19 +171,19 @@ P00 INFO: archive-get command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000002.partial
P00 INFO: pushed WAL file '000000010000000100000002.partial' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 WARN: WAL segment 000000010000000100000002.partial already exists in the archive with the same checksum
P00 WARN: WAL file '000000010000000100000002.partial' already exists in the archive with the same checksum
HINT: this is valid in some recovery scenarios but may also indicate a problem.
P00 INFO: pushed WAL segment 000000010000000100000002.partial
P00 INFO: pushed WAL file '000000010000000100000002.partial' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002.partial] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 ERROR: [045]: WAL segment 000000010000000100000002.partial already exists in the archive
P00 ERROR: [045]: WAL file '000000010000000100000002.partial' already exists in the archive
P00 INFO: archive-push command end: aborted with exception [045]

6
test/expect/mock-archive-stop-001.log
View File

@ -50,17 +50,17 @@ db-version="9.4"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 ERROR: [044]: raised from local-1 process: WAL segment version 9.4 does not match archive version 8.0
P00 ERROR: [044]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 8.0, system-id 1000000000000000094
HINT: are you archiving to the correct stanza?
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
------------------------------------------------------------------------------------------------------------------------------------
P00 ERROR: [044]: raised from local-1 process: WAL segment version 9.4 does not match archive version 8.0
P00 ERROR: [044]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 8.0, system-id 1000000000000000094
HINT: are you archiving to the correct stanza?
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000004 --repo1-host=bogus
------------------------------------------------------------------------------------------------------------------------------------
P00 WARN: dropped WAL file 000000010000000100000004 because archive queue exceeded 33554432 bytes
P00 WARN: dropped WAL file '000000010000000100000004' because archive queue exceeded 32MB
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005
------------------------------------------------------------------------------------------------------------------------------------

6
test/expect/mock-archive-stop-002.log
View File

@ -44,15 +44,15 @@ db-version="9.4"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db --repo1-host=bogus archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 ERROR: [042]: remote process on 'bogus' terminated unexpectedly: ssh: Could not resolve hostname bogus: Name or service not known
P00 ERROR: [125]: remote-0 process on 'bogus' terminated unexpectedly [255]: ssh: Could not resolve hostname bogus: Name or service not known
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db --repo1-host=bogus archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000003
------------------------------------------------------------------------------------------------------------------------------------
P00 ERROR: [042]: remote process on 'bogus' terminated unexpectedly: ssh: Could not resolve hostname bogus: Name or service not known
P00 ERROR: [125]: remote-0 process on 'bogus' terminated unexpectedly [255]: ssh: Could not resolve hostname bogus: Name or service not known
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000004 --repo1-host=bogus
------------------------------------------------------------------------------------------------------------------------------------
P00 WARN: dropped WAL file 000000010000000100000004 because archive queue exceeded 33554432 bytes
P00 WARN: dropped WAL file '000000010000000100000004' because archive queue exceeded 32MB
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000005
------------------------------------------------------------------------------------------------------------------------------------

7
test/expect/mock-stanza-001.log
View File

@ -178,7 +178,7 @@ db-version="9.3"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000001
P00 INFO: pushed WAL file '000000010000000100000001' to the archive
P00 INFO: archive-push command end: completed successfully
stanza-create db - fail on archive info file missing from non-empty dir (db-master host)
@ -386,13 +386,12 @@ db-version="9.3"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000002
P00 INFO: pushed WAL file '000000010000000100000002' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
P00 ERROR: [044]: WAL segment version 9.4 does not match archive version 9.3
WAL segment system-id 1000000000000000094 does not match archive system-id 1000000000000000093
P00 ERROR: [044]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 9.3, system-id 1000000000000000093
HINT: are you archiving to the correct stanza?
stanza-upgrade db - successful upgrade creates additional history (db-master host)

7
test/expect/mock-stanza-002.log
View File

@ -202,7 +202,7 @@ db-version="9.3"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-2] --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000001
P00 INFO: pushed WAL file '000000010000000100000001' to the archive
P00 INFO: archive-push command end: completed successfully
stanza-create db - force create archive.info from gz file (backup host)
@ -235,13 +235,12 @@ db-version="9.3"
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-2] --stanza=db
P00 INFO: pushed WAL segment 000000010000000100000002
P00 INFO: pushed WAL file '000000010000000100000002' to the archive
P00 INFO: archive-push command end: completed successfully
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=warn --archive-push-queue-max=33554432 --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
------------------------------------------------------------------------------------------------------------------------------------
P00 ERROR: [044]: raised from remote process on 'backup': WAL segment version 9.4 does not match archive version 9.3
WAL segment system-id 1000000000000000094 does not match archive system-id 1000000000000000093
P00 ERROR: [044]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 9.3, system-id 1000000000000000093
HINT: are you archiving to the correct stanza?
stanza-upgrade db - successful upgrade creates additional history (backup host)

2
test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm
View File

@ -152,7 +152,7 @@ sub archivePush
' --config=' . $self->backrestConfig() .
' --log-level-console=warn --archive-push-queue-max=' . int(2 * PG_WAL_SIZE_TEST) .
' --stanza=' . $self->stanza() .
(defined($iExpectedError) && $iExpectedError == ERROR_FILE_READ ? ' --repo1-host=bogus' : '') .
(defined($iExpectedError) && $iExpectedError == ERROR_UNKNOWN ? ' --repo1-host=bogus' : '') .
($bAsync ? '' : ' --no-archive-async') .
" archive-push" . (defined($strSourceFile) ? " ${strSourceFile}" : '') .
(defined($strOptionalParam) ? " ${strOptionalParam}" : ''),

79
test/lib/pgBackRestTest/Module/Command/CommandArchiveCommonPerlTest.pm
View File

@ -50,36 +50,6 @@ sub run
'(0000000700000FFE, 0000000700000FFF, 0000000800000000, 0000000800000001)', 'get range >= 11/1MB');
}
################################################################################################################################
if ($self->begin("${strModule}::walPath()"))
{
my $strPgPath = '/db';
my $strWalFileRelative = 'pg_wal/000000010000000100000001';
my $strWalFileAbsolute = "${strPgPath}/${strWalFileRelative}";
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {walPath($strWalFileRelative, undef, cfgCommandName(CFGCMD_ARCHIVE_GET))}, ERROR_OPTION_REQUIRED,
"option 'pg1-path' must be specified when relative wal paths are used\n" .
"HINT: Is \%f passed to " . cfgCommandName(CFGCMD_ARCHIVE_GET) . " instead of \%p?\n" .
"HINT: PostgreSQL may pass relative paths even with \%p depending on the environment.");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {walPath($strWalFileRelative, $strPgPath, cfgCommandName(CFGCMD_ARCHIVE_PUSH))}, $strWalFileAbsolute,
'relative path is contructed');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {walPath($strWalFileAbsolute, $strPgPath, cfgCommandName(CFGCMD_ARCHIVE_PUSH))}, $strWalFileAbsolute,
'path is not relative and pg-path is still specified');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {walPath($strWalFileAbsolute, $strPgPath, cfgCommandName(CFGCMD_ARCHIVE_PUSH))}, $strWalFileAbsolute,
'path is not relative and pg-path is undef');
}
################################################################################################################################
if ($self->begin("${strModule}::walIsSegment()"))
{
@ -179,55 +149,6 @@ sub run
$self->testResult(
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
}
################################################################################################################################
if ($self->begin("archiveAsyncStatusWrite()"))
{
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
# Create the spool path
my $strSpoolPath = $self->testPath() . "/spool/out";
$self->storageTest()->pathCreate($strSpoolPath, {bIgnoreExists => true, bCreateParent => true});
#---------------------------------------------------------------------------------------------------------------------------
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
# Generate a normal ok
archiveAsyncStatusWrite(WAL_STATUS_OK, $strSpoolPath, $strSegment);
#---------------------------------------------------------------------------------------------------------------------------
# Generate a valid warning ok
archiveAsyncStatusWrite(WAL_STATUS_OK, $strSpoolPath, $strSegment, 0, 'Test Warning');
# Skip error when an ok file already exists
#---------------------------------------------------------------------------------------------------------------------------
archiveAsyncStatusWrite(
WAL_STATUS_ERROR, $strSpoolPath, $strSegment, ERROR_ARCHIVE_DUPLICATE,
"WAL segment ${strSegment} already exists in the archive", true);
$self->testResult(
$self->storageTest()->exists("${strSpoolPath}/${strSegment}.error"), false, "error file should not exist");
#---------------------------------------------------------------------------------------------------------------------------
# Generate an invalid error
$self->testException(
sub {archiveAsyncStatusWrite(WAL_STATUS_ERROR, $strSpoolPath, $strSegment)}, ERROR_ASSERT,
"error status must have iCode and strMessage set");
#---------------------------------------------------------------------------------------------------------------------------
# Generate an invalid error
$self->testException(
sub {archiveAsyncStatusWrite(WAL_STATUS_ERROR, $strSpoolPath, $strSegment, ERROR_ASSERT)},
ERROR_ASSERT, "strMessage must be set when iCode is set");
#---------------------------------------------------------------------------------------------------------------------------
# Generate a valid error
archiveAsyncStatusWrite(
WAL_STATUS_ERROR, $strSpoolPath, $strSegment, ERROR_ARCHIVE_DUPLICATE,
"WAL segment ${strSegment} already exists in the archive");
}
}
1;

679
test/lib/pgBackRestTest/Module/Command/CommandArchivePushPerlTest.pm
View File

@ -1,679 +0,0 @@
####################################################################################################################################
# Archive Push Tests
####################################################################################################################################
package pgBackRestTest::Module::Command::CommandArchivePushPerlTest;
use parent 'pgBackRestTest::Env::HostEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use English '-no_match_vars';
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Push::Push;
use pgBackRest::Archive::Push::Async;
use pgBackRest::Archive::Push::File;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Env::HostEnvTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Env::Host::HostBackupTest;
use pgBackRestTest::Common::RunTest;
####################################################################################################################################
# Test WAL size
####################################################################################################################################
use constant PG_WAL_SIZE_TEST => 16777216;
####################################################################################################################################
# initModule
####################################################################################################################################
sub initModule
{
my $self = shift;
$self->{strDbPath} = $self->testPath() . '/db';
$self->{strWalPath} = "$self->{strDbPath}/pg_xlog";
$self->{strWalStatusPath} = "$self->{strWalPath}/archive_status";
$self->{strWalHash} = $self->walGenerateContentChecksum(PG_VERSION_94);
$self->{strRepoPath} = $self->testPath() . '/repo';
$self->{strArchivePath} = "$self->{strRepoPath}/archive/" . $self->stanza();
$self->{strSpoolPath} = "$self->{strArchivePath}/out";
}
####################################################################################################################################
# initTest
####################################################################################################################################
sub initTest
{
my $self = shift;
# Create WAL path
storageTest()->pathCreate($self->{strWalStatusPath}, {bIgnoreExists => true, bCreateParent => true});
# Create archive info
storageTest()->pathCreate($self->{strArchivePath}, {bIgnoreExists => true, bCreateParent => true});
$self->initOption();
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
my $oArchiveInfo = new pgBackRest::Archive::Info($self->{strArchivePath}, false, {bIgnoreMissing => true});
$oArchiveInfo->create(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), true);
$self->{strArchiveId} = $oArchiveInfo->archiveId();
}
####################################################################################################################################
# initOption
####################################################################################################################################
sub initOption
{
my $self = shift;
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_PG_PATH, $self->{strDbPath});
$self->optionTestSet(CFGOPT_REPO_PATH, $self->{strRepoPath});
$self->optionTestSet(CFGOPT_LOG_PATH, $self->testPath());
$self->optionTestSetBool(CFGOPT_COMPRESS, false);
$self->optionTestSet(CFGOPT_DB_TIMEOUT, 5);
$self->optionTestSet(CFGOPT_PROTOCOL_TIMEOUT, 6);
$self->optionTestSet(CFGOPT_ARCHIVE_TIMEOUT, 5);
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
################################################################################################################################
if ($self->begin("ArchivePushFile::archivePushCheck"))
{
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
#---------------------------------------------------------------------------------------------------------------------------
my $strWalSegment = '000000010000000100000001';
$self->testResult(sub {archivePushCheck(
$strWalSegment, PG_VERSION_94, $self->dbSysId(PG_VERSION_94), "$self->{strWalPath}/${strWalSegment}")},
'(9.4-1, [undef], [undef], [undef])', "${strWalSegment} WAL not found");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalMajorPath = "$self->{strArchivePath}/9.4-1/" . substr($strWalSegment, 0, 16);
my $strWalSegmentHash = "${strWalSegment}-$self->{strWalHash}";
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strWalSegment);
storageTest()->pathCreate($strWalMajorPath, {bCreateParent => true});
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(sub {archivePushCheck(
$strWalSegment, PG_VERSION_94, $self->dbSysId(PG_VERSION_94), "$self->{strWalPath}/${strWalSegment}")},
"(9.4-1, $self->{strWalHash}, [undef]," .
" WAL segment ${strWalSegment} already exists in the archive with the same checksum\n" .
'HINT: this is valid in some recovery scenarios but may also indicate a problem.)',
"${strWalSegment} WAL found");
storageTest()->remove("${strWalMajorPath}/${strWalSegmentHash}");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegmentHash = "${strWalSegment}-10be15a0ab8e1653dfab18c83180e74f1507cab1";
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testException(sub {archivePushCheck(
$strWalSegment, PG_VERSION_94, $self->dbSysId(PG_VERSION_94), "$self->{strWalPath}/${strWalSegment}")},
ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strWalSegment} already exists in the archive");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegment = "${strWalSegment}.partial";
$strWalSegmentHash = "${strWalSegment}-$self->{strWalHash}";
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strWalSegment);
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(sub {archivePushCheck(
$strWalSegment, PG_VERSION_94, $self->dbSysId(PG_VERSION_94), "$self->{strWalPath}/${strWalSegment}")},
"(9.4-1, $self->{strWalHash}, [undef]," .
" WAL segment ${strWalSegment} already exists in the archive with the same checksum\n" .
'HINT: this is valid in some recovery scenarios but may also indicate a problem.)',
"${strWalSegment} WAL found");
storageTest()->remove("${strWalMajorPath}/${strWalSegmentHash}");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegmentHash = "${strWalSegment}-10be15a0ab8e1653dfab18c83180e74f1507cab1";
storageTest()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testException(sub {archivePushCheck(
$strWalSegment, PG_VERSION_94, $self->dbSysId(PG_VERSION_94), "$self->{strWalPath}/${strWalSegment}")},
ERROR_ARCHIVE_DUPLICATE, "WAL segment ${strWalSegment} already exists in the archive");
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(sub {archivePushCheck(
$strWalSegment, PG_VERSION_94, $self->dbSysId(PG_VERSION_94))},
ERROR_ASSERT, "xFileExp is required in Storage::Local->hashSize");
#---------------------------------------------------------------------------------------------------------------------------
my $strHistoryFile = "00000001.history";
storageTest()->put("$self->{strArchivePath}/9.4-1/${strHistoryFile}");
$self->testResult(sub {archivePushCheck(
$strHistoryFile, PG_VERSION_94, $self->dbSysId(PG_VERSION_94), "$self->{strWalPath}/${strHistoryFile}")},
'(9.4-1, [undef], [undef], [undef])', "history file ${strHistoryFile} found");
}
################################################################################################################################
if ($self->begin("ArchivePushFile::archivePushFile"))
{
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
$self->optionTestSet(CFGOPT_REPO_HOST, 'localhost');
$self->optionTestSet(CFGOPT_REPO_HOST_USER, $self->pgUser());
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
protocolGet(CFGOPTVAL_REMOTE_TYPE_BACKUP, undef, {strBackRestBin => $self->backrestExe()});
# Generate a normal segment
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$self->testResult(
sub {archivePushFile($self->{strWalPath}, $strSegment, false, false)}, '[undef]',
"${strSegment} WAL segment to remote");
$self->testResult(
sub {archivePushFile($self->{strWalPath}, $strSegment, false, false)},
"WAL segment 000000010000000100000001 already exists in the archive with the same checksum\n" .
'HINT: this is valid in some recovery scenarios but may also indicate a problem.',
"${strSegment} WAL duplicate segment to remote");
# Destroy protocol object
protocolDestroy();
$self->optionTestClear(CFGOPT_REPO_HOST);
$self->optionTestClear(CFGOPT_REPO_HOST_USER);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
}
################################################################################################################################
if ($self->begin("ArchivePush->readyList()"))
{
my $oPushAsync = new pgBackRest::Archive::Push::Async($self->{strWalPath}, $self->{strSpoolPath});
$self->optionTestSetBool(CFGOPT_ARCHIVE_ASYNC, true);
$self->optionTestSet(CFGOPT_SPOOL_PATH, $self->{strRepoPath});
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$oPushAsync->initServer();
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put("$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.done');
$self->testResult(
sub {$oPushAsync->readyList()}, '()',
'ignore files without .ready extension');
#---------------------------------------------------------------------------------------------------------------------------
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
$self->testResult(
sub {$oPushAsync->readyList()}, '(000000010000000100000002, 000000010000000100000003)',
'.ready files are found');
storageTest()->put("$self->{strSpoolPath}/000000010000000100000002.ok");
storageTest()->put("$self->{strSpoolPath}/000000010000000100000003.ok");
#---------------------------------------------------------------------------------------------------------------------------
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++));
$self->testResult(
sub {$oPushAsync->readyList()}, '(000000010000000100000004)',
'new .ready files are found and duplicates ignored');
storageTest()->put("$self->{strSpoolPath}/000000010000000100000004.ok");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {$oPushAsync->readyList()}, '()',
'no new .ready files returns empty list');
#---------------------------------------------------------------------------------------------------------------------------
$iWalTimeline++;
$iWalMinor = 1;
storageTest()->put("$self->{strWalStatusPath}/00000002.history.ready");
$self->testResult(
sub {$oPushAsync->readyList()}, '(00000002.history)',
'history .ready file');
storageTest()->put("$self->{strSpoolPath}/00000002.history.ok");
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->put(
"$self->{strWalStatusPath}/" . $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++) . '.00000028.backup.ready');
$self->testResult(
sub {$oPushAsync->readyList()}, '(000000020000000100000001.00000028.backup)',
'backup .ready file');
storageTest()->put("$self->{strSpoolPath}/000000020000000100000001.00000028.backup.ok");
#---------------------------------------------------------------------------------------------------------------------------
storageTest()->remove("$self->{strWalStatusPath}/00000002.history.ready");
$self->testResult(
sub {$oPushAsync->readyList()}, '()', 'remove 00000002.history.ok file');
$self->testResult(
sub {storageTest()->exists("$self->{strWalStatusPath}/00000002.history.ready")}, false,
'00000002.history.ok is removed');
}
################################################################################################################################
if ($self->begin("ArchivePush->dropList()"))
{
my $oPushAsync = new pgBackRest::Archive::Push::Async($self->{strWalPath}, $self->{strSpoolPath});
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 4);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
#---------------------------------------------------------------------------------------------------------------------------
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
storageTest()->put("$self->{strWalStatusPath}/${strSegment}.ready");
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
storageTest()->put("$self->{strWalStatusPath}/${strSegment}.ready");
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
storageTest()->put("$self->{strWalStatusPath}/${strSegment}.ready");
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$self->testResult(
sub {$oPushAsync->dropList($oPushAsync->readyList())}, '()',
'WAL files not dropped');
#---------------------------------------------------------------------------------------------------------------------------
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 2);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testResult(
sub {$oPushAsync->dropList($oPushAsync->readyList())},
'(000000010000000100000001, 000000010000000100000002, 000000010000000100000003)', 'WAL files that exceed queue max');
# Reset queue max
$self->optionTestClear(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
}
################################################################################################################################
if ($self->begin("ArchivePushAsync->process()"))
{
my $oPushAsync = new pgBackRest::Archive::Push::Async(
$self->{strWalPath}, $self->{strSpoolPath}, $self->backrestExe());
$self->optionTestSetBool(CFGOPT_ARCHIVE_ASYNC, true);
$self->optionTestSet(CFGOPT_SPOOL_PATH, $self->{strRepoPath});
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$oPushAsync->initServer();
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
#---------------------------------------------------------------------------------------------------------------------------
# Generate a normal segment
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
# Generate an error (.ready file withough a corresponding WAL file)
my $strSegmentError = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
storageTest()->put("$self->{strWalStatusPath}/$strSegmentError.ready");
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(2, 0, 1, 1)', "process ${strSegment}, ${strSegmentError}");
$self->testResult(
sub {storageSpool->list($self->{strSpoolPath})}, "(${strSegment}.ok, ${strSegmentError}.error)",
"${strSegment} pushed, ${strSegmentError} errored");
$self->testResult(
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
"${strSegment} WAL in archive");
$self->testResult(
sub {${storageSpool()->get("$self->{strSpoolPath}/$strSegmentError.error")}},
ERROR_FILE_OPEN . "\nraised from local-1 process: unable to open $self->{strWalPath}/${strSegmentError}",
"test ${strSegmentError}.error contents");
# Remove pushed WAL file
$self->walRemove($self->{strWalPath}, $strSegment);
#---------------------------------------------------------------------------------------------------------------------------
# Fix errored WAL file by providing a valid segment
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegmentError);
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "process ${strSegment}, ${strSegmentError}");
$self->testResult(
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegmentError)}, "${strSegmentError}-$self->{strWalHash}",
"${strSegmentError} WAL in archive");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "${strSegmentError}.ok", "${strSegmentError} pushed");
#---------------------------------------------------------------------------------------------------------------------------
# Remove previously errored WAL file
$self->walRemove($self->{strWalPath}, $strSegmentError);
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "remove ${strSegmentError}.ready");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "[undef]", "${strSegmentError} removed");
#---------------------------------------------------------------------------------------------------------------------------
# Enable compression
$self->optionTestSetBool(CFGOPT_COMPRESS, true);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Create history file
my $strHistoryFile = "00000001.history";
storageTest()->put("$self->{strWalPath}/${strHistoryFile}");
storageTest()->put("$self->{strWalStatusPath}/$strHistoryFile.ready");
# Create backup file
my $strBackupFile = "${strSegment}.00000028.backup";
storageTest()->put("$self->{strWalPath}/${strBackupFile}");
storageTest()->put("$self->{strWalStatusPath}/$strBackupFile.ready");
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(2, 0, 2, 0)', "end processing ${strHistoryFile}, ${strBackupFile}");
$self->testResult(
sub {storageSpool()->list($self->{strSpoolPath})}, "(${strHistoryFile}.ok, ${strBackupFile}.ok)",
"${strHistoryFile}, ${strBackupFile} pushed");
$self->testResult(
sub {storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/$self->{strArchiveId}/${strHistoryFile}")}, true,
"${strHistoryFile} in archive");
$self->testResult(
sub {storageRepo()->exists(STORAGE_REPO_ARCHIVE . "/$self->{strArchiveId}/${strBackupFile}")}, true,
"${strBackupFile} in archive");
# Remove history and backup files
storageTest()->remove("$self->{strWalPath}/${strHistoryFile}");
storageTest()->remove("$self->{strWalStatusPath}/$strHistoryFile.ready");
storageTest()->remove("$self->{strWalPath}/${strBackupFile}");
storageTest()->remove("$self->{strWalStatusPath}/$strBackupFile.ready");
#---------------------------------------------------------------------------------------------------------------------------
# Generate a normal segment
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "processing ${strSegment}.gz");
$self->testResult(
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
"${strSegment} WAL in archive");
# Remove the WAL and process so the .ok file is removed
$self->walRemove($self->{strWalPath}, $strSegment);
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "remove ${strSegment}.ready");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "[undef]", "${strSegment}.ok removed");
# Generate the same WAL again
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(1, 0, 1, 0)', "processed duplicate ${strSegment}.gz");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "${strSegment}.ok", "${strSegment} pushed");
$self->testResult(
sub {${storageSpool()->get("$self->{strSpoolPath}/${strSegment}.ok")}},
"0\nWAL segment ${strSegment} already exists in the archive with the same checksum\n" .
'HINT: this is valid in some recovery scenarios but may also indicate a problem.',
"${strSegment}.ok warning status");
$self->testResult(
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}.gz",
"${strSegment} WAL in archive");
# Remove the WAL
$self->walRemove($self->{strWalPath}, $strSegment);
# Disable compression
$self->optionTestSetBool(CFGOPT_COMPRESS, false);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
#---------------------------------------------------------------------------------------------------------------------------
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 2);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Generate WAL to test queue limits
my @strySegment =
(
$self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++),
$self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++),
$self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++)
);
foreach my $strSegment (@strySegment)
{
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
}
# Process and check results
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 3, 0, 0)', "process and drop files");
$self->testResult(
sub {storageSpool()->list($self->{strSpoolPath})}, '(' . join('.ok, ', @strySegment) . '.ok)',
join(', ', @strySegment) . " ok drop files written");
foreach my $strSegment (@strySegment)
{
$self->testResult(
sub {${storageSpool()->get("$self->{strSpoolPath}/${strSegment}.ok")}},
"0\ndropped WAL file ${strSegment} because archive queue exceeded " . cfgOption(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX) .
' bytes',
"verify ${strSegment} status");
$self->walRemove($self->{strWalPath}, $strSegment);
}
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oPushAsync->processQueue()}, '(0, 0, 0, 0)', "final process to remove ok files");
$self->testResult(sub {storageSpool()->list($self->{strSpoolPath})}, "[undef]", "ok files removed");
$self->optionTestClear(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
}
################################################################################################################################
if ($self->begin("ArchivePush->process()"))
{
my $oPush = new pgBackRest::Archive::Push::Push($self->backrestExe());
$self->optionTestClear(CFGOPT_ARCHIVE_ASYNC);
$self->optionTestClear(CFGOPT_SPOOL_PATH);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
my $iProcessId = $PID;
#---------------------------------------------------------------------------------------------------------------------------
# Set pg-host to trick archive-push into thinking it is running on the backup server
$self->optionTestSet(CFGOPT_PG_HOST, BOGUS);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testException(sub {$oPush->process(undef, false)}, ERROR_HOST_INVALID, 'archive-push operation must run on db host');
#---------------------------------------------------------------------------------------------------------------------------
# Reset pg-host
$self->optionTestClear(CFGOPT_PG_HOST);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testException(sub {$oPush->process(undef, false)}, ERROR_PARAM_REQUIRED, 'WAL file to push required');
#---------------------------------------------------------------------------------------------------------------------------
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$self->testResult(
sub {$oPush->process("pg_xlog/${strSegment}", false)}, undef, "${strSegment} WAL pushed (with relative path)");
$self->testResult(
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
"${strSegment} WAL in archive");
$self->walRemove($self->{strWalPath}, $strSegment);
#---------------------------------------------------------------------------------------------------------------------------
# Set unrealistic queue max to make synchronous push drop a WAL
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, 0);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}", false)}, undef, "${strSegment} WAL dropped");
$self->testResult(
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, '[undef]',
"${strSegment} WAL in archive");
# Set more realistic queue max and allow segment to push
$self->optionTestSet(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX, PG_WAL_SIZE_TEST * 4);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}", false)}, undef, "${strSegment} WAL pushed");
$self->testResult(
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment)}, "${strSegment}-$self->{strWalHash}",
"${strSegment} WAL in archive");
$self->walRemove($self->{strWalPath}, $strSegment);
# Reset queue max
$self->optionTestClear(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
#---------------------------------------------------------------------------------------------------------------------------
# Enable async archiving
$self->optionTestSetBool(CFGOPT_ARCHIVE_ASYNC, true);
$self->optionTestSet(CFGOPT_SPOOL_PATH, $self->{strRepoPath});
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$self->testResult(
sub {$oPush->process("$self->{strWalPath}/${strSegment}", true)}, undef, "${strSegment} WAL pushed async");
$self->testResult(
sub {walSegmentFind(storageRepo(), $self->{strArchiveId}, $strSegment, 5)}, "${strSegment}-$self->{strWalHash}",
"${strSegment} WAL in archive");
$self->walRemove($self->{strWalPath}, $strSegment);
#---------------------------------------------------------------------------------------------------------------------------
$strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$self->optionTestSet(CFGOPT_REPO_HOST, BOGUS);
$self->optionTestSet(CFGOPT_PROTOCOL_TIMEOUT, 60);
$self->optionTestSet(CFGOPT_ARCHIVE_TIMEOUT, 5);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
$self->testResult(sub {$oPush->process("$self->{strWalPath}/${strSegment}", true)}, undef, 'process connect error');
# Check contents of error file
my $strErrorFile = STORAGE_SPOOL_ARCHIVE_OUT . "/${strSegment}.error";
my $strErrorFileContents = ${storageSpool()->get($strErrorFile)};
$self->testResult(
$strErrorFileContents =~ ("42\nremote process on '" . BOGUS . "' terminated.*"), true, "check error file contents");
# Disable async archiving
$self->optionTestClear(CFGOPT_REPO_HOST);
$self->optionTestClear(CFGOPT_PROTOCOL_TIMEOUT);
$self->optionTestClear(CFGOPT_ARCHIVE_TIMEOUT);
$self->optionTestClear(CFGOPT_ARCHIVE_ASYNC);
$self->optionTestClear(CFGOPT_SPOOL_PATH);
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
}
################################################################################################################################
if ($self->begin("ArchivePushFile::archivePushFile - encryption"))
{
my $iWalTimeline = 1;
my $iWalMajor = 1;
my $iWalMinor = 1;
$self->optionTestSet(CFGOPT_REPO_CIPHER_TYPE, CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC);
$self->optionTestSet(CFGOPT_REPO_CIPHER_PASS, 'x');
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
# Remove any archive info files
executeTest('sudo rm ' . $self->{strArchivePath} . '/archive.info*');
# Clear the repo settings
storageRepoCacheClear($self->stanza());
my $oArchiveInfo = new pgBackRest::Archive::Info(storageRepo()->pathGet(STORAGE_REPO_ARCHIVE), false,
{bLoad => false, bIgnoreMissing => true, strCipherPassSub => 'y'});
$oArchiveInfo->create(PG_VERSION_94, $self->dbSysId(PG_VERSION_94), true);
# Generate a normal segment
my $strSegment = $self->walSegment($iWalTimeline, $iWalMajor, $iWalMinor++);
$self->walGenerate($self->{strWalPath}, PG_VERSION_94, 1, $strSegment);
$self->testResult(
sub {archivePushFile($self->{strWalPath}, $strSegment, false, false)}, '[undef]',
"${strSegment} WAL segment to pushed");
$self->testResult(storageRepo()->encrypted($self->{strArchivePath} . "/" . $self->{strArchiveId} . "/" .
substr($strSegment, 0, 16) . "/$strSegment-" . $self->{strWalHash}), true, ' pushed segment is encrypted');
}
}
1;

16
test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm
View File

@ -90,6 +90,7 @@ sub run
$oHostBackup->stanzaCreate('create required data for stanza', {strOptionalParam => '--no-' . cfgOptionName(CFGOPT_ONLINE)});
# Push a WAL segment
&log(INFO, ' push first WAL');
$oHostDbMaster->archivePush($strWalPath, $strWalTestFile, 1);
# Break the database version of the archive info file
@ -97,17 +98,24 @@ sub run
{
$oHostBackup->infoMunge(
$oStorage->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'},
&INFO_ARCHIVE_SECTION_DB_HISTORY => {1 => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}}});
}
# Push two more segments with errors to exceed archive-push-queue-max
$oHostDbMaster->archivePush(
$strWalPath, $strWalTestFile, 2, $iError ? ERROR_FILE_READ : ERROR_ARCHIVE_MISMATCH);
&log(INFO, ' push second WAL');
$oHostDbMaster->archivePush(
$strWalPath, $strWalTestFile, 3, $iError ? ERROR_FILE_READ : ERROR_ARCHIVE_MISMATCH);
$strWalPath, $strWalTestFile, 2, $iError ? ERROR_UNKNOWN : ERROR_ARCHIVE_MISMATCH);
&log(INFO, ' push third WAL');
$oHostDbMaster->archivePush(
$strWalPath, $strWalTestFile, 3, $iError ? ERROR_UNKNOWN : ERROR_ARCHIVE_MISMATCH);
# Now this segment will get dropped
&log(INFO, ' push fourth WAL');
$oHostDbMaster->archivePush($strWalPath, $strWalTestFile, 4, undef, undef, '--repo1-host=bogus');
# Fix the database version

33
test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm
View File

@ -273,39 +273,6 @@ sub run
storageTest()->remove("${strWalPath}/archive_status/00000002.history.ready");
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' db version mismatch in db section only - archive-push errors but archive-get succeeds');
$oHostBackup->infoMunge(
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . qw{/} . ARCHIVE_INFO_FILE),
{&INFO_ARCHIVE_SECTION_DB => {&INFO_ARCHIVE_KEY_DB_VERSION => '8.0'}});
$oHostDbMaster->executeSimple(
$strCommandPush . " ${strWalPath}/${strSourceFile}",
{iExpectedExitStatus => ERROR_ARCHIVE_MISMATCH, oLogTest => $self->expect()});
# Remove RECOVERYXLOG so it can be recovered
storageTest()->remove("${strWalPath}/RECOVERYXLOG", {bIgnoreMissing => false});
$oHostDbMaster->executeSimple(
$strCommandGet . " ${strSourceFile1} ${strWalPath}/RECOVERYXLOG",
{oLogTest => $self->expect()});
# Check that the destination file exists
if (storageDb()->exists("${strWalPath}/RECOVERYXLOG"))
{
my ($strActualChecksum) = storageDb()->hashSize("${strWalPath}/RECOVERYXLOG");
if ($strActualChecksum ne $strArchiveChecksum)
{
confess "recovered file hash '${strActualChecksum}' does not match expected '${strArchiveChecksum}'";
}
}
else
{
confess "archive file '${strWalPath}/RECOVERYXLOG' is not in destination";
}
#---------------------------------------------------------------------------------------------------------------------------
&log(INFO, ' db version mismatch error - archive-get unable to retrieve archiveId');

1
test/lib/pgBackRestTest/Module/Protocol/ProtocolHelperPerlTest.pm
View File

@ -16,7 +16,6 @@ use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Archive::Common;
use pgBackRest::Archive::Push::Push;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;

728
test/src/module/command/archivePushTest.c
View File

@ -1,9 +1,16 @@
/***********************************************************************************************************************************
Test Archive Push Command
***********************************************************************************************************************************/
#include "common/io/bufferRead.h"
#include "common/io/bufferWrite.h"
#include "common/io/handleRead.h"
#include "common/io/handleWrite.h"
#include "common/time.h"
#include "postgres/version.h"
#include "storage/driver/posix/storage.h"
#include "common/harnessConfig.h"
#include "common/harnessFork.h"
/***********************************************************************************************************************************
Test Run
@ -17,77 +24,702 @@ testRun(void)
Storage *storageTest = storageDriverPosixInterface(
storageDriverPosixNew(strNew(testPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, true, NULL));
// Start a protocol server to test the protocol directly
Buffer *serverWrite = bufNew(8192);
IoWrite *serverWriteIo = ioBufferWriteIo(ioBufferWriteNew(serverWrite));
ioWriteOpen(serverWriteIo);
ProtocolServer *server = protocolServerNew(
strNew("test"), strNew("test"), ioBufferReadIo(ioBufferReadNew(bufNew(0))), serverWriteIo);
bufUsedSet(serverWrite, 0);
// *****************************************************************************************************************************
if (testBegin("cmdArchivePush()"))
if (testBegin("archivePushReadyList(), archivePushProcessList(), and archivePushDrop()"))
{
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest-bogus");
strLstAddZ(argList, "--archive-timeout=1");
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=db");
strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath()));
strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath()));
strLstAddZ(argList, "archive-push-async");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
storagePathCreateNP(storagePgWrite(), strNew("pg_wal/archive_status"));
storagePathCreateNP(storageTest, strNew("spool/archive/db/out"));
// Create ok files to indicate WAL that has already been archived
storagePutNP(
storageNewWriteNP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok")), NULL);
storagePutNP(
storageNewWriteNP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000003.ok")), NULL);
storagePutNP(
storageNewWriteNP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000004.ok")), NULL);
storagePutNP(
storageNewWriteNP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000005.error")), NULL);
storagePutNP(
storageNewWriteNP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000006.error")), NULL);
storagePutNP(
storageNewWriteNP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/global.error")), NULL);
// Create ready files for wal that still needs to be archived
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/archive_status/000000010000000100000002.ready")), NULL);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/archive_status/000000010000000100000003.ready")), NULL);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/archive_status/000000010000000100000005.ready")), NULL);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/archive_status/000000010000000100000006.ready")), NULL);
TEST_RESULT_STR(
strPtr(strLstJoin(archivePushProcessList(strNewFmt("%s/db/pg_wal", testPath())), "|")),
"000000010000000100000002|000000010000000100000005|000000010000000100000006", "ready list");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc), "|")),
"000000010000000100000003.ok", "remaining status list");
// Test drop
// -------------------------------------------------------------------------------------------------------------------------
StringList *argListDrop = strLstDup(argList);
strLstAdd(argListDrop, strNewFmt("--archive-push-queue-max=%zu", (size_t)1024 * 1024 * 1024));
harnessCfgLoad(strLstSize(argListDrop), strLstPtr(argListDrop));
// Write the files that we claim are in pg_wal
Buffer *walBuffer = bufNew((size_t)16 * 1024 * 1024);
bufUsedSet(walBuffer, bufSize(walBuffer));
memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/000000010000000100000002")), walBuffer);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/000000010000000100000003")), walBuffer);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/000000010000000100000005")), walBuffer);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/000000010000000100000006")), walBuffer);
// Queue max is high enough that no WAL will be dropped
TEST_RESULT_BOOL(
archivePushDrop(strNew("pg_wal"), archivePushProcessList(strNewFmt("%s/db/pg_wal", testPath()))), false,
"wal is not dropped");
// Now set queue max low enough that WAL will be dropped
argListDrop = strLstDup(argList);
strLstAdd(argListDrop, strNewFmt("--archive-push-queue-max=%zu", (size_t)16 * 1024 * 1024 * 2));
harnessCfgLoad(strLstSize(argListDrop), strLstPtr(argListDrop));
TEST_RESULT_BOOL(
archivePushDrop(strNew("pg_wal"), archivePushProcessList(strNewFmt("%s/db/pg_wal", testPath()))), true,
"wal is dropped");
}
// *****************************************************************************************************************************
if (testBegin("archivePushCheck()"))
{
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test");
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAddZ(argList, "archive-push");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// Check mismatched pg_control and archive.info
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(storageTest, strNew("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
pgControlTestToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 0xFACEFACEFACEFACE}));
// Create incorrect archive info
storagePutNP(
storageNewWriteNP(storageTest, strNew("repo/archive/test/archive.info")),
bufNewZ(
"[backrest]\n"
"backrest-checksum=\"806471e1481804dc3ddf8dc6f1da7c34939420a8\"\n"
"backrest-format=5\n"
"backrest-version=\"2.06\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n"));
TEST_ERROR(
archivePushCheck(cipherTypeNone, NULL), ArchiveMismatchError,
"PostgreSQL version 9.6, system-id 18072658121562454734 do not match stanza version 9.4, system-id 5555555555555555555"
"\nHINT: are you archiving to the correct stanza?");
// Fix the version
storagePutNP(
storageNewWriteNP(storageTest, strNew("repo/archive/test/archive.info")),
bufNewZ(
"[backrest]\n"
"backrest-checksum=\"4656aad82fa16a5d87b1aed36cb8a20c2707b9f9\"\n"
"backrest-format=5\n"
"backrest-version=\"2.08\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.6\"}\n"));
TEST_ERROR(
archivePushCheck(cipherTypeNone, NULL), ArchiveMismatchError,
"PostgreSQL version 9.6, system-id 18072658121562454734 do not match stanza version 9.6, system-id 5555555555555555555"
"\nHINT: are you archiving to the correct stanza?");
// Fix archive info
storagePutNP(
storageNewWriteNP(storageTest, strNew("repo/archive/test/archive.info")),
bufNewZ(
"[backrest]\n"
"backrest-checksum=\"724b21c26c64f4c25567a11eb222c18766cbec00\"\n"
"backrest-format=5\n"
"backrest-version=\"2.11\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n"));
ArchivePushCheckResult result = {0};
TEST_ASSIGN(result, archivePushCheck(cipherTypeNone, NULL), "get archive check result");
TEST_RESULT_UINT(result.pgVersion, PG_VERSION_96, "check pg version");
TEST_RESULT_UINT(result.pgSystemId, 0xFACEFACEFACEFACE, "check pg system id");
TEST_RESULT_UINT(result.pgWalSegmentSize, 16 * 1024 * 1024, "check wal segment size");
TEST_RESULT_STR(strPtr(result.archiveId), "9.6-1", "check archive id");
TEST_RESULT_STR(strPtr(result.archiveCipherPass), NULL, "check archive cipher pass (not set in this test)");
}
// *****************************************************************************************************************************
if (testBegin("Synchronous cmdArchivePush(), archivePushFile() and archivePushProtocol()"))
{
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test");
strLstAddZ(argList, "archive-push");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_ERROR(cmdArchivePush(), ParamRequiredError, "WAL segment to push required");
// -------------------------------------------------------------------------------------------------------------------------
strLstAddZ(argList, "000000010000000100000001");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_ERROR(cmdArchivePush(), OptionRequiredError , "===PERL-EMBED-ERROR===");
// Make sure the process times out when there is nothing to archive
// -------------------------------------------------------------------------------------------------------------------------
storagePathCreateNP(storageTest, strNewFmt("%s/db/archive_status", testPath()));
strLstAdd(argList, strNewFmt("--spool-path=%s", testPath()));
strLstAddZ(argList, "--archive-async");
strLstAdd(argList, strNewFmt("--log-path=%s", testPath()));
strLstAdd(argList, strNewFmt("--log-level-file=debug"));
strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath()));
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
StringList *argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
TEST_ERROR(
cmdArchivePush(), ArchiveTimeoutError,
"unable to push WAL segment '000000010000000100000001' asynchronously after 1 second(s)");
cmdArchivePush(), OptionRequiredError,
"option 'pg1-path' must be specified when relative wal paths are used"
"\nHINT: Is %f passed to archive-push instead of %p?"
"\nHINT: PostgreSQL may pass relative paths even with %p depending on the environment.");
// Wait for the lock to release
lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 30000, true);
lockRelease(true);
// Write out a bogus .error file to make sure it is ignored on the first loop
// Create pg_control and archive.info
// -------------------------------------------------------------------------------------------------------------------------
// Remove the archive status path so async will error and not overwrite the bogus error file
storagePathRemoveNP(storageTest, strNewFmt("%s/db/archive_status", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
String *errorFile = storagePathNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.error"));
storagePutNP(storageNewWriteNP(storageSpoolWrite(), errorFile), bufNewZ("25\n" BOGUS_STR));
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
TEST_ERROR(cmdArchivePush(), AssertError, BOGUS_STR);
storageRemoveP(storageTest, errorFile, .errorOnMissing = true);
// Write out a valid ok file and test for success
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageNewWriteNP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok")), bufNew(0));
storageNewWriteNP(storageTest, strNew("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
pgControlTestToBuffer((PgControl){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}));
TEST_RESULT_VOID(cmdArchivePush(), "successful push");
harnessLogResult("P00 INFO: pushed WAL segment 000000010000000100000001 asynchronously");
storagePutNP(
storageNewWriteNP(storageTest, strNew("repo/archive/test/archive.info")),
bufNewZ(
"[backrest]\n"
"backrest-checksum=\"f5c9e17258db65e8d22609e23d1e99985d6bb063\"\n"
"backrest-format=5\n"
"backrest-version=\"2.12\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}\n"));
storageRemoveP(
storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok"), .errorOnMissing = true);
// Make sure the process times out when there is nothing to archive and it can't get a lock. This test MUST go last since
// the lock is lost and cannot be closed by the main process.
// Generate WAL with incorrect headers and try to push them
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(
lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 30000, true), "acquire lock");
TEST_RESULT_VOID(lockClear(true), "clear lock");
Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024);
bufUsedSet(walBuffer1, bufSize(walBuffer1));
memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer1);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/000000010000000100000001")), walBuffer1);
TEST_ERROR(
cmdArchivePush(), ArchiveMismatchError,
strPtr(
strNewFmt(
"WAL file '%s/pg/pg_wal/000000010000000100000001' version 10, system-id 18072658121562454734 do not match"
" stanza version 11, system-id 18072658121562454734",
testPath())));
memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAFECAFECAF}, walBuffer1);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/000000010000000100000001")), walBuffer1);
TEST_ERROR(
cmdArchivePush(), ArchiveMismatchError,
strPtr(
strNewFmt(
"WAL file '%s/pg/pg_wal/000000010000000100000001' version 11, system-id 17055110554209741999 do not match"
" stanza version 11, system-id 18072658121562454734",
testPath())));
// Generate valid WAL and push them
// -------------------------------------------------------------------------------------------------------------------------
memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer1);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/000000010000000100000001")), walBuffer1);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
harnessLogResult("P00 INFO: pushed WAL file '000000010000000100000001' to the archive");
TEST_RESULT_BOOL(
storageExistsNP(
storageTest,
strNewFmt(
"repo/archive/test/11-1/0000000100000001/000000010000000100000001-%s.gz",
TEST_64BIT() ? "3e5ecd22712f319b2420d5b901fd29f4f6be2336" : "6903dce7e3cd64ba9a6134056405eaeb8dedcd37")),
true, "check repo for WAL file");
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment again");
harnessLogResult(
"P00 WARN: WAL file '000000010000000100000001' already exists in the archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P00 INFO: pushed WAL file '000000010000000100000001' to the archive");
// Now create a new WAL buffer with a different checksum to test checksum errors
Buffer *walBuffer2 = bufNew((size_t)16 * 1024 * 1024);
bufUsedSet(walBuffer2, bufSize(walBuffer2));
memset(bufPtr(walBuffer2), 0xFF, bufSize(walBuffer2));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer2);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/000000010000000100000001")), walBuffer2);
TEST_ERROR(cmdArchivePush(), ArchiveDuplicateError, "WAL file '000000010000000100000001' already exists in the archive");
// Save it to a new file instead
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/000000010000000100000002")), walBuffer2);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
harnessLogResult("P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
TEST_RESULT_BOOL(
storageExistsNP(
storageTest,
strNewFmt(
"repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz",
TEST_64BIT() ? "edad2f5a9d8a03ee3c09e8ce92c771e0d20232f5" : "e7c81f5513e0c6e3f19b9dbfc450019165994dda")),
true, "check repo for WAL file");
// Push a history file
// -------------------------------------------------------------------------------------------------------------------------
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/00000001.history");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/00000001.history")), bufNewStr(strNew("FAKEHISTORY")));
TEST_RESULT_VOID(cmdArchivePush(), "push a history file");
harnessLogResult("P00 INFO: pushed WAL file '00000001.history' to the archive");
TEST_RESULT_BOOL(
storageExistsNP(storageTest, strNew("repo/archive/test/11-1/00000001.history")), true, "check repo for history file");
// Check drop functionality
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/archive_status/000000010000000100000001.ready")), NULL);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_wal/archive_status/000000010000000100000002.ready")), NULL);
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "--archive-push-queue-max=16m");
strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
TEST_RESULT_VOID(cmdArchivePush(), "drop WAL file");
harnessLogResult("P00 WARN: dropped WAL file '000000010000000100000002' because archive queue exceeded 16MB");
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "--archive-push-queue-max=1GB");
strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
TEST_RESULT_VOID(cmdArchivePush(), "push WAL file again");
harnessLogResult(
"P00 WARN: WAL file '000000010000000100000002' already exists in the archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
// Check protocol function directly
// -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNewFmt("%s/pg/pg_wal/000000010000000100000002", testPath())));
varLstAdd(paramList, varNewStrZ("11-1"));
varLstAdd(paramList, varNewUInt64(PG_VERSION_11));
varLstAdd(paramList, varNewUInt64(0xFACEFACEFACEFACE));
varLstAdd(paramList, varNewStrZ("000000010000000100000002"));
varLstAdd(paramList, varNewUInt64(cipherTypeNone));
varLstAdd(paramList, NULL);
varLstAdd(paramList, varNewBool(false));
varLstAdd(paramList, varNewInt(6));
TEST_RESULT_BOOL(
archivePushProtocol(PROTOCOL_COMMAND_ARCHIVE_PUSH_STR, paramList, server), true, "protocol archive put");
TEST_RESULT_STR(
strPtr(strNewBuf(serverWrite)),
"{\"out\":\"WAL file '000000010000000100000002' already exists in the archive with the same checksum"
"\\nHINT: this is valid in some recovery scenarios but may also indicate a problem.\"}\n",
"check result");
bufUsedSet(serverWrite, 0);
// Check invalid protocol function
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_BOOL(archivePushProtocol(strNew(BOGUS_STR), paramList, server), false, "invalid function");
// Create a new encrypted repo to test encryption
// -------------------------------------------------------------------------------------------------------------------------
storagePathRemoveP(storageTest, strNew("repo"), .errorOnMissing = true, .recurse = true);
StorageFileWrite *infoWrite = storageNewWriteNP(storageTest, strNew("repo/archive/test/archive.info"));
ioWriteFilterGroupSet(
storageFileWriteIo(infoWrite),
ioFilterGroupAdd(
ioFilterGroupNew(),
cipherBlockFilter(
cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, bufNewStr(strNew("badpassphrase")), NULL))));
storagePutNP(
infoWrite,
bufNewZ(
"[backrest]\n"
"backrest-checksum=\"4ce1b0e8f7132e3b5193a9c385f2d039dd75062f\"\n"
"backrest-format=5\n"
"backrest-version=\"2.12\"\n"
"\n"
"[cipher]\n"
"cipher-pass=\"badsubpassphrase\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}"));
// Push encrypted WAL segment
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
strLstAddZ(argListTemp, "--repo1-cipher-type=aes-256-cbc");
strLstAddZ(argListTemp, "--no-compress");
setenv("PGBACKREST_REPO1_CIPHER_PASS", "badpassphrase", true);
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
unsetenv("PGBACKREST_REPO1_CIPHER_PASS");
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
harnessLogResult("P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
TEST_RESULT_BOOL(
storageExistsNP(
storageTest,
strNewFmt(
"repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s",
TEST_64BIT() ? "edad2f5a9d8a03ee3c09e8ce92c771e0d20232f5" : "e7c81f5513e0c6e3f19b9dbfc450019165994dda")),
true, "check repo for WAL file");
}
// *****************************************************************************************************************************
if (testBegin("Asynchronous cmdArchivePush() and cmdArchivePushAsync()"))
{
harnessLogLevelSet(logLevelDetail);
// Call with a bogus exe name so the async process will error out and we can make sure timeouts work
// -------------------------------------------------------------------------------------------------------------------------
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest-bogus");
strLstAddZ(argList, "--stanza=test");
strLstAddZ(argList, "--archive-async");
strLstAddZ(argList, "--archive-timeout=1");
strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAddZ(argList, "archive-push");
strLstAddZ(argList, "pg_wal/bogus");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_ERROR(
cmdArchivePush(), ArchiveTimeoutError,
"unable to push WAL segment '000000010000000100000001' asynchronously after 1 second(s)");
}
"unable to push WAL file 'bogus' to the archive asynchronously after 1 second(s)");
// Create pg_control and archive.info
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test");
strLstAddZ(argList, "--archive-async");
strLstAddZ(argList, "--no-compress");
strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAdd(argList, strNewFmt("--log-path=%s/log", testPath()));
strLstAddZ(argList, "--log-level-file=trace");
strLstAddZ(argList, "--log-subprocess");
strLstAddZ(argList, "archive-push");
storagePathCreateNP(storageTest, strNew("log"));
storagePutNP(
storageNewWriteNP(storageTest, strNew("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
pgControlTestToBuffer((PgControl){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}));
storagePutNP(
storageNewWriteNP(storageTest, strNew("repo/archive/test/archive.info")),
bufNewZ(
"[backrest]\n"
"backrest-checksum=\"d12be58a906afa894c34f49ac9cbb6b968efc5a4\"\n"
"backrest-format=5\n"
"backrest-version=\"2.12\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n"));
// Write out an error file that will be ignored on the first pass, then the async process will write a new one
// -------------------------------------------------------------------------------------------------------------------------
StringList *argListTemp = strLstDup(argList);
strLstAdd(argListTemp, strNewFmt("%s/pg/pg_xlog/000000010000000100000001", testPath()));
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
storagePathCreateNP(storagePgWrite(), strNew("pg_xlog/archive_status"));
storagePutNP(
storageNewWriteNP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.error")),
bufNewStr(strNew("25\nbogus error")));
TEST_ERROR(cmdArchivePush(), AssertError, "no WAL files to process");
storageRemoveP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/global.error"), .errorOnMissing = true);
// Acquire a lock so the async process will not be able to run -- this will result in a timeout
// -------------------------------------------------------------------------------------------------------------------------
argListTemp = strLstDup(argList);
strLstAdd(argListTemp, strNewFmt("%s/pg/pg_xlog/000000010000000100000001", testPath()));
strLstAddZ(argListTemp, "--archive-timeout=1");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
HARNESS_FORK_BEGIN()
{
HARNESS_FORK_CHILD_BEGIN(0, true)
{
IoRead *read = ioHandleReadIo(ioHandleReadNew(strNew("child read"), HARNESS_FORK_CHILD_READ(), 2000));
ioReadOpen(read);
IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(strNew("child write"), HARNESS_FORK_CHILD_WRITE()));
ioWriteOpen(write);
lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 30000, true);
// Let the parent know the lock has been acquired and wait for the parent to allow lock release
ioWriteLine(write, strNew(""));
ioWriteFlush(write);
ioReadLine(read);
lockRelease(true);
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN()
{
IoRead *read = ioHandleReadIo(ioHandleReadNew(strNew("parent read"), HARNESS_FORK_PARENT_READ_PROCESS(0), 2000));
ioReadOpen(read);
IoWrite *write = ioHandleWriteIo(ioHandleWriteNew(strNew("parent write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0)));
ioWriteOpen(write);
// Wait for the child to acquire the lock
ioReadLine(read);
TEST_ERROR(
cmdArchivePush(), ArchiveTimeoutError,
"unable to push WAL file '000000010000000100000001' to the archive asynchronously after 1 second(s)");
// Notify the child to release the lock
ioWriteLine(write, strNew(""));
ioWriteFlush(write);
}
HARNESS_FORK_PARENT_END();
}
HARNESS_FORK_END();
// Actually push a WAL file
// -------------------------------------------------------------------------------------------------------------------------
argListTemp = strLstDup(argList);
strLstAdd(argListTemp, strNewFmt("%s/pg/pg_xlog/000000010000000100000001", testPath()));
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_xlog/archive_status/000000010000000100000001.ready")), NULL);
Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024);
bufUsedSet(walBuffer1, bufSize(walBuffer1));
memset(bufPtr(walBuffer1), 0xFF, bufSize(walBuffer1));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer1);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_xlog/000000010000000100000001")), walBuffer1);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
harnessLogResult("P00 INFO: pushed WAL file '000000010000000100000001' to the archive asynchronously");
TEST_RESULT_BOOL(
storageExistsNP(
storageTest,
strNewFmt(
"repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s",
TEST_64BIT() ? "f81d63dd5e258cd607534f3531bbd71442797e37" : "02d228126281e8e102b35a2737e45a0527946296")),
true, "check repo for WAL file");
// Direct tests of the async function
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test");
strLstAddZ(argList, "--no-compress");
strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath()));
strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAdd(argList, strNewFmt("--log-path=%s/log", testPath()));
strLstAddZ(argList, "--log-level-file=trace");
strLstAddZ(argList, "--log-subprocess");
strLstAddZ(argList, "archive-push-async");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_ERROR(cmdArchivePushAsync(), ParamRequiredError, "WAL path to push required");
// Check that global.error is created
// -------------------------------------------------------------------------------------------------------------------------
// Remove data from prior tests
storagePathRemoveP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT_STR, .recurse = true);
storagePathCreateNP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT_STR);
storagePathRemoveP(storageRepoWrite(), strNew(STORAGE_REPO_ARCHIVE "/9.4-1"), .recurse = true);
storagePathCreateNP(storageRepoWrite(), strNew(STORAGE_REPO_ARCHIVE "/9.4-1"));
storagePathRemoveP(storagePgWrite(), strNew("pg_xlog/archive_status"), .recurse = true);
storagePathCreateNP(storagePgWrite(), strNew("pg_xlog/archive_status"));
strLstAdd(argList, strNewFmt("%s/pg/pg_xlog", testPath()));
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_ERROR(cmdArchivePushAsync(), AssertError, "no WAL files to process");
TEST_RESULT_STR(
strPtr(strNewBuf(storageGetNP(storageNewReadNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/global.error"))))),
"25\nno WAL files to process", "check global.error");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc), "|")),
"global.error", "check status files");
// Push WAL
// -------------------------------------------------------------------------------------------------------------------------
// Recreate ready file for WAL 1
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_xlog/archive_status/000000010000000100000001.ready")), NULL);
// Create a ready file for WAL 2 but don't create the segment yet -- this will test the file error
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_xlog/archive_status/000000010000000100000002.ready")), NULL);
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
harnessLogResult(
strPtr(
strNewFmt(
"P00 INFO: push 2 WAL file(s) to archive: 000000010000000100000001...000000010000000100000002\n"
"P00 DETAIL: pushed WAL file '000000010000000100000001' to the archive\n"
"P00 WARN: could not push WAL file '000000010000000100000002' to the archive (will be retried): "
"[55] raised from local-1 protocol: unable to open '%s/pg/pg_xlog/000000010000000100000002' for read: "
"[2] No such file or directory", testPath())));
TEST_RESULT_BOOL(
storageExistsNP(
storageTest,
strNewFmt(
"repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s",
TEST_64BIT() ? "f81d63dd5e258cd607534f3531bbd71442797e37" : "02d228126281e8e102b35a2737e45a0527946296")),
true, "check repo for WAL 1 file");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc), "|")),
"000000010000000100000001.ok|000000010000000100000002.error", "check status files");
// Create WAL 2 segment
Buffer *walBuffer2 = bufNew((size_t)16 * 1024 * 1024);
bufUsedSet(walBuffer2, bufSize(walBuffer2));
memset(bufPtr(walBuffer2), 0x0C, bufSize(walBuffer2));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer2);
storagePutNP(storageNewWriteNP(storagePgWrite(), strNew("pg_xlog/000000010000000100000002")), walBuffer2);
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "--archive-push-queue-max=1gb");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
harnessLogResult(
"P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000002\n"
"P00 DETAIL: pushed WAL file '000000010000000100000002' to the archive");
TEST_RESULT_BOOL(
storageExistsNP(
storageTest,
strNewFmt(
"repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s",
TEST_64BIT() ? "0aea6fa5d53500ce548b84a86bc3a29ae77fa048" : "408822a89ef44ef6740e785743bf1b870d8024a2")),
true, "check repo for WAL 2 file");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc), "|")),
"000000010000000100000001.ok|000000010000000100000002.ok", "check status files");
// Check that drop functionality works
// -------------------------------------------------------------------------------------------------------------------------
// Remove status files
storagePathRemoveP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT_STR, .recurse = true);
storagePathCreateNP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT_STR);
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "--archive-push-queue-max=16m");
harnessCfgLoad(strLstSize(argListTemp), strLstPtr(argListTemp));
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
harnessLogResult(
"P00 INFO: push 2 WAL file(s) to archive: 000000010000000100000001...000000010000000100000002\n"
"P00 WARN: dropped WAL file '000000010000000100000001' because archive queue exceeded 16MB\n"
"P00 WARN: dropped WAL file '000000010000000100000002' because archive queue exceeded 16MB");
TEST_RESULT_STR(
strPtr(
strNewBuf(
storageGetNP(
storageNewReadNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok"))))),
"0\ndropped WAL file '000000010000000100000001' because archive queue exceeded 16MB", "check WAL 1 warning");
TEST_RESULT_STR(
strPtr(
strNewBuf(
storageGetNP(
storageNewReadNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000002.ok"))))),
"0\ndropped WAL file '000000010000000100000002' because archive queue exceeded 16MB", "check WAL 2 warning");
TEST_RESULT_STR(
strPtr(strLstJoin(strLstSort(storageListNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc), "|")),
"000000010000000100000001.ok|000000010000000100000002.ok", "check status files");
}
FUNCTION_HARNESS_RESULT_VOID();
}

2
test/src/module/perl/execTest.c
View File

@ -52,7 +52,7 @@ testRun(void)
TEST_RESULT_VOID(perlFree(0), "free Perl before it is init'd");
TEST_RESULT_VOID(perlInit(), "init Perl");
TEST_ERROR(perlExec(), ParamRequiredError, PERL_EMBED_ERROR);
TEST_ERROR(perlExec(), PathMissingError, PERL_EMBED_ERROR);
TEST_RESULT_VOID(perlFree(0), "free Perl");
}