db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

The backup command is implemented entirely in C.

Browse Source 
For the most part this is a direct migration of the Perl code into C except as noted below.

A backup can now be initiated from a linked directory.  The link will not be stored in the manifest or recreated on restore.  If a link or directory does not already exist in the restore location then a directory will be created.

The logic for creating backup labels has been improved and it should no longer be possible to get a backup label earlier than the latest backup even with timezone changes or clock skew.  This has never been an issue in the field that we know of, but we found it in testing.

For online backups all times are fetched from the PostgreSQL primary host (before only copy start was).  This doesn't affect backup integrity but it does prevent clock skew between hosts affecting backup duration reporting.

Archive copy now works as expected when the archive and backup have different compression settings, i.e. when one is compressed and the other is not.  This was a long-standing bug in the Perl code.

Resume will now work even if hardlink settings have been changed.

Reviewed by Cynthia Shang.
This commit is contained in:
David Steele 2019-12-13 17:14:26 -05:00
parent e206093beb
commit 1f2ce45e6b
22 changed files with 4061 additions and 4864 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/xml/release.xml
View File

@ -13,6 +13,17 @@
<release-list>
<release date="XXXX-XX-XX" version="2.21dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-improvement-list>
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>The <cmd>backup</cmd> command is implemented entirely in C.</p>
</release-item>
</release-improvement-list>
</release-core-list>
</release>
<release date="2019-12-12" version="2.20" title="Bug Fixes">

4
doc/xml/user-guide.xml
View File

@ -2503,7 +2503,7 @@
<execute-list host="{[host-repo1]}">
<title>Attempt a backup</title>
<execute user="{[br-user]}" err-expect="62" output="y">
<execute user="{[br-user]}" err-expect="56" output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} backup</exe-cmd>
<exe-highlight>\: stop file exists for all stanzas</exe-highlight>
</execute>
@ -2544,7 +2544,7 @@
<execute-list host="{[host-repo1]}">
<title>Attempt a backup</title>
<execute user="{[br-user]}" err-expect="62" output="y">
<execute user="{[br-user]}" err-expect="56" output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} backup</exe-cmd>
<exe-highlight>\: stop file exists for stanza demo</exe-highlight>
</execute>

234
lib/pgBackRest/Archive/Common.pm
View File

@ -43,238 +43,4 @@ use constant PG_WAL_SYSTEM_ID_OFFSET_LT_93 => 12;
use constant PG_WAL_SEGMENT_SIZE => 16777216;
push @EXPORT, qw(PG_WAL_SEGMENT_SIZE);
####################################################################################################################################
# lsnNormalize
#
# Generates a normalized form from an LSN that can be used for comparison.
####################################################################################################################################
sub lsnNormalize
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strLsn,
) =
logDebugParam
(
__PACKAGE__ . '::lsnFile', \@_,
{name => 'strLsn', trace => true},
);
# Split the LSN into major and minor parts
my @stryLsnSplit = split('/', $strLsn);
if (@stryLsnSplit != 2)
{
confess &log(ASSERT, "invalid lsn ${strLsn}");
}
my $strLsnNormal = uc(sprintf("%08s%08s", $stryLsnSplit[0], $stryLsnSplit[1]));
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strLsnNormal', value => $strLsnNormal, trace => true}
);
}
push @EXPORT, qw(lsnNormalize);
####################################################################################################################################
# lsnFileRange
#
# Generates a range of WAL filenames given the start and stop LSN. For pre-9.3 databases, use bSkipFF to exclude the FF that
# prior versions did not generate.
####################################################################################################################################
sub lsnFileRange
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strLsnStart,
$strLsnStop,
$strDbVersion,
$iWalSegmentSize,
) =
logDebugParam
(
__PACKAGE__ . '::lsnFileRange', \@_,
{name => 'strLsnStart'},
{name => 'strLsnStop'},
{name => '$strDbVersion'},
{name => '$iWalSegmentSize'},
);
# Working variables
my @stryArchive;
my $iArchiveIdx = 0;
my $bSkipFF = $strDbVersion < PG_VERSION_93;
# Iterate through all archive logs between start and stop
my @stryArchiveSplit = split('/', $strLsnStart);
my $iStartMajor = hex($stryArchiveSplit[0]);
my $iStartMinor = int(hex($stryArchiveSplit[1]) / $iWalSegmentSize);
@stryArchiveSplit = split('/', $strLsnStop);
my $iStopMajor = hex($stryArchiveSplit[0]);
my $iStopMinor = int(hex($stryArchiveSplit[1]) / $iWalSegmentSize);
$stryArchive[$iArchiveIdx] = uc(sprintf("%08x%08x", $iStartMajor, $iStartMinor));
$iArchiveIdx += 1;
while (!($iStartMajor == $iStopMajor && $iStartMinor == $iStopMinor))
{
$iStartMinor += 1;
if ($bSkipFF && $iStartMinor == 255 || !$bSkipFF && $iStartMinor > int(0xFFFFFFFF / $iWalSegmentSize))
{
$iStartMajor += 1;
$iStartMinor = 0;
}
$stryArchive[$iArchiveIdx] = uc(sprintf("%08x%08x", $iStartMajor, $iStartMinor));
$iArchiveIdx += 1;
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'stryWalFileName', value => \@stryArchive}
);
}
push @EXPORT, qw(lsnFileRange);
####################################################################################################################################
# walSegmentFind
#
# Returns the filename of a WAL segment in the archive. Optionally, a wait time can be specified. In this case an error will be
# thrown when the WAL segment is not found. If the same WAL segment with multiple checksums is found then error.
####################################################################################################################################
sub walSegmentFind
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oStorageRepo,
$strArchiveId,
$strWalSegment,
$iWaitSeconds,
) =
logDebugParam
(
__PACKAGE__ . '::walSegmentFind', \@_,
{name => 'oStorageRepo'},
{name => 'strArchiveId'},
{name => 'strWalSegment'},
{name => 'iWaitSeconds', required => false},
);
# Error if not a segment
if (!walIsSegment($strWalSegment))
{
confess &log(ERROR, "${strWalSegment} is not a WAL segment", ERROR_ASSERT);
}
# Loop and wait for file to appear
my $oWait = waitInit($iWaitSeconds);
my @stryWalFileName;
do
{
# Get the name of the requested WAL segment (may have compression extension)
push(@stryWalFileName, $oStorageRepo->list(
STORAGE_REPO_ARCHIVE . "/${strArchiveId}/" . substr($strWalSegment, 0, 16),
{strExpression =>
'^' . substr($strWalSegment, 0, 24) . (walIsPartial($strWalSegment) ? "\\.partial" : '') .
"-[0-f]{40}(\\." . COMPRESS_EXT . "){0,1}\$",
bIgnoreMissing => true}));
}
while (@stryWalFileName == 0 && waitMore($oWait));
# If there is more than one matching archive file then there is a serious issue - either a bug in the archiver or the user has
# copied files around or removed archive.info.
if (@stryWalFileName > 1)
{
confess &log(ERROR,
"duplicates found in archive for WAL segment ${strWalSegment}: " . join(', ', @stryWalFileName) .
"\nHINT: are multiple primaries archiving to this stanza?",
ERROR_ARCHIVE_DUPLICATE);
}
# If waiting and no WAL segment was found then throw an error
if (@stryWalFileName == 0 && defined($iWaitSeconds))
{
confess &log(
ERROR,
"could not find WAL segment ${strWalSegment} after ${iWaitSeconds} second(s)" .
"\nHINT: is archive_command configured correctly?" .
"\nHINT: use the check command to verify that PostgreSQL is archiving.",
ERROR_ARCHIVE_TIMEOUT);
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strWalFileName', value => $stryWalFileName[0]}
);
}
push @EXPORT, qw(walSegmentFind);
####################################################################################################################################
# walIsSegment
#
# Is the file a segment or some other file (e.g. .history, .backup, etc).
####################################################################################################################################
sub walIsSegment
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strWalFile,
) =
logDebugParam
(
__PACKAGE__ . '::walIsSegment', \@_,
{name => 'strWalFile', trace => true},
);
return $strWalFile =~ /^[0-F]{24}(\.partial){0,1}$/ ? true : false;
}
push @EXPORT, qw(walIsSegment);
####################################################################################################################################
# walIsPartial
#
# Is the file a segment and partial.
####################################################################################################################################
sub walIsPartial
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strWalFile,
) =
logDebugParam
(
__PACKAGE__ . '::walIsPartial', \@_,
{name => 'strWalFile', trace => true},
);
return walIsSegment($strWalFile) && $strWalFile =~ /\.partial$/ ? true : false;
}
push @EXPORT, qw(walIsPartial);
1;

1117
lib/pgBackRest/Backup/Backup.pm
View File

File diff suppressed because it is too large Load Diff

267
lib/pgBackRest/Backup/File.pm
View File

@ -1,267 +0,0 @@
####################################################################################################################################
# BACKUP FILE MODULE
####################################################################################################################################
package pgBackRest::Backup::File;
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Common::Exception;
use pgBackRest::Common::Io::Handle;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Base;
use pgBackRest::Storage::Helper;
####################################################################################################################################
# Result constants
####################################################################################################################################
use constant BACKUP_FILE_CHECKSUM => 0;
push @EXPORT, qw(BACKUP_FILE_CHECKSUM);
use constant BACKUP_FILE_COPY => 1;
push @EXPORT, qw(BACKUP_FILE_COPY);
use constant BACKUP_FILE_RECOPY => 2;
push @EXPORT, qw(BACKUP_FILE_RECOPY);
use constant BACKUP_FILE_SKIP => 3;
push @EXPORT, qw(BACKUP_FILE_SKIP);
use constant BACKUP_FILE_NOOP => 4;
push @EXPORT, qw(BACKUP_FILE_NOOP);
####################################################################################################################################
# backupManifestUpdate
####################################################################################################################################
sub backupManifestUpdate
{
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oManifest,
$strHost,
$iLocalId,
$strDbFile,
$strRepoFile,
$lSize,
$strChecksum,
$bChecksumPage,
$iCopyResult,
$lSizeCopy,
$lSizeRepo,
$strChecksumCopy,
$rExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent
) =
logDebugParam
(
__PACKAGE__ . '::backupManifestUpdate', \@_,
{name => 'oManifest', trace => true},
{name => 'strHost', required => false, trace => true},
{name => 'iLocalId', required => false, trace => true},
# Parameters to backupFile()
{name => 'strDbFile', trace => true},
{name => 'strRepoFile', trace => true},
{name => 'lSize', required => false, trace => true},
{name => 'strChecksum', required => false, trace => true},
{name => 'bChecksumPage', trace => true},
# Results from backupFile()
{name => 'iCopyResult', trace => true},
{name => 'lSizeCopy', required => false, trace => true},
{name => 'lSizeRepo', required => false, trace => true},
{name => 'strChecksumCopy', required => false, trace => true},
{name => 'rExtra', required => false, trace => true},
# Accumulators
{name => 'lSizeTotal', trace => true},
{name => 'lSizeCurrent', trace => true},
{name => 'lManifestSaveSize', trace => true},
{name => 'lManifestSaveCurrent', trace => true}
);
# Increment current backup progress
$lSizeCurrent += $lSize;
# If the file is in a prior backup and nothing changed, then nothing needs to be done
if ($iCopyResult == BACKUP_FILE_NOOP)
{
# File copy was not needed so just restore the size and checksum to the manifest
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_SIZE, $lSizeCopy);
$oManifest->set(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM, $strChecksumCopy);
&log(DETAIL,
'match file from prior backup ' . (defined($strHost) ? "${strHost}:" : '') . "${strDbFile} (" .
fileSizeFormat($lSizeCopy) . ', ' . int($lSizeCurrent * 100 / $lSizeTotal) . '%)' .
($lSizeCopy != 0 ? " checksum ${strChecksumCopy}" : ''),
undef, undef, undef, $iLocalId);
}
# Else process the results
else
{
# Log invalid checksum
if ($iCopyResult == BACKUP_FILE_RECOPY)
{
&log(
WARN,
"resumed backup file ${strRepoFile} does not have expected checksum ${strChecksum}. The file will be recopied and" .
" backup will continue but this may be an issue unless the resumed backup path in the repository is known to be" .
" corrupted.\n" .
"NOTE: this does not indicate a problem with the PostgreSQL page checksums.");
}
# If copy was successful store the checksum and size
if ($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY || $iCopyResult == BACKUP_FILE_CHECKSUM)
{
# Log copy or checksum
&log($iCopyResult == BACKUP_FILE_CHECKSUM ? DETAIL : INFO,
($iCopyResult == BACKUP_FILE_CHECKSUM ?
'checksum resumed file ' : 'backup file ' . (defined($strHost) ? "${strHost}:" : '')) .
"${strDbFile} (" . fileSizeFormat($lSizeCopy) .
', ' . int($lSizeCurrent * 100 / $lSizeTotal) . '%)' .
($lSizeCopy != 0 ? " checksum ${strChecksumCopy}" : ''), undef, undef, undef, $iLocalId);
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_SIZE, $lSizeCopy);
if ($lSizeRepo != $lSizeCopy)
{
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_REPO_SIZE, $lSizeRepo);
}
if ($lSizeCopy > 0)
{
$oManifest->set(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM, $strChecksumCopy);
}
# If the file was copied, then remove any reference to the file's existence in a prior backup.
if ($iCopyResult == BACKUP_FILE_COPY || $iCopyResult == BACKUP_FILE_RECOPY)
{
$oManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_REFERENCE);
}
# If the file had page checksums calculated during the copy
if ($bChecksumPage)
{
# The valid flag should be set
if (defined($rExtra->{valid}))
{
# Store the valid flag
$oManifest->boolSet(
MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rExtra->{valid});
# If the page was not valid
if (!$rExtra->{valid})
{
# Check for a page misalignment
if ($lSizeCopy % PG_PAGE_SIZE != 0)
{
# Make sure the align flag was set, otherwise there is a bug
if (!defined($rExtra->{align}) || $rExtra->{align})
{
confess &log(ASSERT, 'align flag should have been set for misaligned page');
}
# Emit a warning so the user knows something is amiss
&log(WARN,
'page misalignment in file ' . (defined($strHost) ? "${strHost}:" : '') .
"${strDbFile}: file size ${lSizeCopy} is not divisible by page size " . PG_PAGE_SIZE);
}
# Else process the page check errors
else
{
$oManifest->set(
MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR,
dclone($rExtra->{error}));
# Build a pretty list of the page errors
my $strPageError;
my $iPageErrorTotal = 0;
foreach my $iyPage (@{$rExtra->{error}})
{
$strPageError .= (defined($strPageError) ? ', ' : '');
# If a range of pages
if (ref($iyPage))
{
$strPageError .= $$iyPage[0] . '-' . $$iyPage[1];
$iPageErrorTotal += ($$iyPage[1] - $$iyPage[0]) + 1;
}
# Else a single page
else
{
$strPageError .= $iyPage;
$iPageErrorTotal += 1;
}
}
# There should be at least one page in the error list
if ($iPageErrorTotal == 0)
{
confess &log(ASSERT, 'page checksum error list should have at least one entry');
}
# Emit a warning so the user knows something is amiss
&log(WARN,
'invalid page checksum' . ($iPageErrorTotal > 1 ? 's' : '') .
' found in file ' . (defined($strHost) ? "${strHost}:" : '') . "${strDbFile} at page" .
($iPageErrorTotal > 1 ? 's' : '') . " ${strPageError}");
}
}
}
# If it's not set that's a bug in the code
elsif (!$oManifest->test(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE))
{
confess &log(ASSERT, "${strDbFile} should have calculated page checksums");
}
}
}
# Else the file was removed during backup so remove from manifest
elsif ($iCopyResult == BACKUP_FILE_SKIP)
{
&log(DETAIL, 'skip file removed by database ' . (defined($strHost) ? "${strHost}:" : '') . $strDbFile);
$oManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strRepoFile);
}
}
# Determine whether to save the manifest
$lManifestSaveCurrent += $lSize;
if ($lManifestSaveCurrent >= $lManifestSaveSize)
{
$oManifest->saveCopy();
logDebugMisc
(
$strOperation, 'save manifest',
{name => 'lManifestSaveSize', value => $lManifestSaveSize},
{name => 'lManifestSaveCurrent', value => $lManifestSaveCurrent}
);
$lManifestSaveCurrent = 0;
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'lSizeCurrent', value => $lSizeCurrent, trace => true},
{name => 'lManifestSaveCurrent', value => $lManifestSaveCurrent, trace => true},
);
}
push @EXPORT, qw(backupManifestUpdate);
1;

338
lib/pgBackRest/Db.pm
View File

@ -25,13 +25,6 @@ use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Version;
####################################################################################################################################
# PostgreSQL 8.3 WAL size
#
# WAL segment size in 8.3 cannot be determined from pg_control, so use this constant instead.
####################################################################################################################################
use constant PG_WAL_SIZE_83 => 16777216;
####################################################################################################################################
# Backup advisory lock
####################################################################################################################################
@ -330,61 +323,6 @@ sub executeSqlOne
);
}
####################################################################################################################################
# tablespaceMapGet
#
# Get the mapping between oid and tablespace name.
####################################################################################################################################
sub tablespaceMapGet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->tablespaceMapGet');
my $hTablespaceMap = {};
for my $strRow (@{$self->executeSql('select oid, spcname from pg_tablespace')})
{
$hTablespaceMap->{@{$strRow}[0]} = @{$strRow}[1];
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hTablespaceMap', value => $hTablespaceMap}
);
}
####################################################################################################################################
# databaseMapGet
#
# Get the mapping between oid and database name.
####################################################################################################################################
sub databaseMapGet
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->databaseMapGet');
my $hDatabaseMap = {};
for my $strRow (@{$self->executeSql('select datname, oid, datlastsysoid from pg_database')})
{
$hDatabaseMap->{@{$strRow}[0]}{&MANIFEST_KEY_DB_ID} = @{$strRow}[1];
$hDatabaseMap->{@{$strRow}[0]}{&MANIFEST_KEY_DB_LAST_SYSTEM_ID} = @{$strRow}[2];
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hDatabaseMap', value => $hDatabaseMap}
);
}
####################################################################################################################################
# info
####################################################################################################################################
@ -527,157 +465,6 @@ sub versionGet
);
}
####################################################################################################################################
# backupStart
####################################################################################################################################
sub backupStart
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strLabel,
$bStartFast
) =
logDebugParam
(
__PACKAGE__ . '->backupStart', \@_,
{name => 'strLabel'},
{name => 'bStartFast'}
);
# Validate the database configuration
$self->configValidate();
# Only allow start-fast option for version >= 8.4
if ($self->{strDbVersion} < PG_VERSION_84 && $bStartFast)
{
&log(WARN, cfgOptionName(CFGOPT_START_FAST) . ' option is only available in PostgreSQL >= ' . PG_VERSION_84);
$bStartFast = false;
}
# Determine if page checksums can be enabled
my $bChecksumPage =
$self->executeSqlOne("select count(*) = 1 from pg_settings where name = 'data_checksums' and setting = 'on'");
# If checksum page option is not explicitly set then set it to whatever the database says
if (!cfgOptionTest(CFGOPT_CHECKSUM_PAGE))
{
cfgOptionSet(CFGOPT_CHECKSUM_PAGE, $bChecksumPage);
}
# Else if enabled make sure they are in the database as well, else throw a warning
elsif (cfgOption(CFGOPT_CHECKSUM_PAGE) && !$bChecksumPage)
{
&log(WARN, 'unable to enable page checksums since they are not enabled in the database');
cfgOptionSet(CFGOPT_CHECKSUM_PAGE, false);
}
# Acquire the backup advisory lock to make sure that backups are not running from multiple backup servers against the same
# database cluster. This lock helps make the stop-auto option safe.
if (!$self->executeSqlOne('select pg_try_advisory_lock(' . DB_BACKUP_ADVISORY_LOCK . ')'))
{
confess &log(ERROR, 'unable to acquire ' . PROJECT_NAME . " advisory lock\n" .
'HINT: is another ' . PROJECT_NAME . ' backup already running on this cluster?', ERROR_LOCK_ACQUIRE);
}
# If stop-auto is enabled check for a running backup. This feature is not supported for PostgreSQL >= 9.6 since backups are
# run in non-exclusive mode.
if (cfgOption(CFGOPT_STOP_AUTO) && $self->{strDbVersion} < PG_VERSION_96)
{
# Running backups can only be detected in PostgreSQL >= 9.3
if ($self->{strDbVersion} >= PG_VERSION_93)
{
# If a backup is currently in progress emit a warning and then stop it
if ($self->executeSqlOne('select pg_is_in_backup()'))
{
&log(WARN, 'the cluster is already in backup mode but no ' . PROJECT_NAME . ' backup process is running.' .
' pg_stop_backup() will be called so a new backup can be started.');
$self->backupStop();
}
}
# Else emit a warning that the feature is not supported and continue. If a backup is running then an error will be
# generated later on.
else
{
&log(WARN, cfgOptionName(CFGOPT_STOP_AUTO) . ' option is only available in PostgreSQL >= ' . PG_VERSION_93);
}
}
# Start the backup
&log(INFO, 'execute ' . ($self->{strDbVersion} >= PG_VERSION_96 ? 'non-' : '') .
"exclusive pg_start_backup() with label \"${strLabel}\": backup begins after " .
($bStartFast ? "the requested immediate checkpoint" : "the next regular checkpoint") . " completes");
my ($strTimestampDbStart, $strArchiveStart, $strLsnStart, $iWalSegmentSize) = $self->executeSqlRow(
"select to_char(current_timestamp, 'YYYY-MM-DD HH24:MI:SS.US TZ'), pg_" . $self->walId() . "file_name(lsn), lsn::text," .
($self->{strDbVersion} < PG_VERSION_84 ? PG_WAL_SIZE_83 :
" (select setting::int8 from pg_settings where name = 'wal_segment_size')" .
# In Pre-11 versions the wal_segment_sise was expressed in terms of blocks rather than total size
($self->{strDbVersion} < PG_VERSION_11 ?
" * (select setting::int8 from pg_settings where name = 'wal_block_size')" : '')) .
" from pg_start_backup('${strLabel}'" .
($bStartFast ? ', true' : $self->{strDbVersion} >= PG_VERSION_84 ? ', false' : '') .
($self->{strDbVersion} >= PG_VERSION_96 ? ', false' : '') . ') as lsn');
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strArchiveStart', value => $strArchiveStart},
{name => 'strLsnStart', value => $strLsnStart},
{name => 'iWalSegmentSize', value => $iWalSegmentSize},
{name => 'strTimestampDbStart', value => $strTimestampDbStart}
);
}
####################################################################################################################################
# backupStop
####################################################################################################################################
sub backupStop
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->backupStop');
# Stop the backup
&log(INFO, 'execute ' . ($self->{strDbVersion} >= PG_VERSION_96 ? 'non-' : '') .
'exclusive pg_stop_backup() and wait for all WAL segments to archive');
my ($strTimestampDbStop, $strArchiveStop, $strLsnStop, $strLabel, $strTablespaceMap) =
$self->executeSqlRow(
"select to_char(clock_timestamp(), 'YYYY-MM-DD HH24:MI:SS.US TZ'), pg_" .
$self->walId() . "file_name(lsn), lsn::text, " .
($self->{strDbVersion} >= PG_VERSION_96 ?
'labelfile, ' .
'case when length(trim(both \'\t\n \' from spcmapfile)) = 0 then null else spcmapfile end as spcmapfile' :
'null as labelfile, null as spcmapfile') .
' from pg_stop_backup(' .
# Add flag to use non-exclusive backup
($self->{strDbVersion} >= PG_VERSION_96 ? 'false' : '') .
# Add flag to exit immediately after backup stop rather than waiting for WAL to archive (this is checked later)
($self->{strDbVersion} >= PG_VERSION_10 ? ', false' : '') . ') as lsn');
# Build a hash of the files that need to be written to the backup
my $oFileHash =
{
&MANIFEST_FILE_BACKUPLABEL => $strLabel,
&MANIFEST_FILE_TABLESPACEMAP => $strTablespaceMap
};
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strArchiveStop', value => $strArchiveStop},
{name => 'strLsnStop', value => $strLsnStop},
{name => 'strTimestampDbStop', value => $strTimestampDbStop},
{name => 'oFileHash', value => $oFileHash}
);
}
####################################################################################################################################
# configValidate
#
@ -761,18 +548,6 @@ sub walId
return $self->{strDbVersion} >= PG_VERSION_10 ? 'wal' : 'xlog';
}
####################################################################################################################################
# lsnId
#
# Returns 'lsn' or 'location' depending on the version of PostgreSQL.
####################################################################################################################################
sub lsnId
{
my $self = shift;
return $self->{strDbVersion} >= PG_VERSION_10 ? 'lsn' : 'location';
}
####################################################################################################################################
# isStandby
#
@ -807,119 +582,6 @@ sub isStandby
);
}
####################################################################################################################################
# replayWait
#
# Waits for replay on the standby to equal specified LSN
####################################################################################################################################
sub replayWait
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strTargetLSN,
) =
logDebugParam
(
__PACKAGE__ . '->replayWait', \@_,
{name => 'strTargetLSN'}
);
# Load ArchiveCommon Module
require pgBackRest::Archive::Common;
pgBackRest::Archive::Common->import();
# Initialize working variables
my $oWait = waitInit(cfgOption(CFGOPT_ARCHIVE_TIMEOUT));
my $bTimeout = true;
my $strReplayedLSN = undef;
# Monitor the replay location
do
{
my $strLastWalReplayLsnFunction =
'pg_last_' . $self->walId() . '_replay_' . $self->lsnId() . '()';
# Get the replay location
my $strLastReplayedLSN = $self->executeSqlOne(
"select coalesce(${strLastWalReplayLsnFunction}::text, '<NONE>')");
# Error if the replay location could not be retrieved
if ($strLastReplayedLSN eq '<NONE>')
{
confess &log(
ERROR,
"unable to query replay lsn on the standby using ${strLastWalReplayLsnFunction}\n" .
"Hint: Is this a standby?",
ERROR_ARCHIVE_TIMEOUT);
}
# Is the replay lsn > target lsn? It needs to be greater because the checkpoint record is directly after the LSN returned
# by pg_start_backup().
if (lsnNormalize($strLastReplayedLSN) ge lsnNormalize($strTargetLSN))
{
$bTimeout = false;
}
else
{
# Reset the timer if the LSN is advancing
if (defined($strReplayedLSN) &&
lsnNormalize($strLastReplayedLSN) gt lsnNormalize($strReplayedLSN) &&
!waitMore($oWait))
{
$oWait = waitInit(cfgOption(CFGOPT_ARCHIVE_TIMEOUT));
}
}
# Assigned last replayed to replayed
$strReplayedLSN = $strLastReplayedLSN;
} while ($bTimeout && waitMore($oWait));
# Error if a timeout occurred before the target lsn was reached
if ($bTimeout == true)
{
confess &log(
ERROR, "timeout before standby replayed ${strTargetLSN} - only reached ${strReplayedLSN}", ERROR_ARCHIVE_TIMEOUT);
}
# Perform a checkpoint
$self->executeSql('checkpoint', undef, false);
# On PostgreSQL >= 9.6 the checkpoint location can be verified
#
# ??? We have seen one instance where this check failed. Is there any chance that the replayed position could be ahead of the
# checkpoint recorded in pg_control? It seems possible, so in the C version of this add a loop to keep checking pg_control
# until the checkpoint has been recorded.
my $strCheckpointLSN = undef;
if ($self->{strDbVersion} >= PG_VERSION_96)
{
$strCheckpointLSN = $self->executeSqlOne('select checkpoint_' . $self->lsnId() .'::text from pg_control_checkpoint()');
if (lsnNormalize($strCheckpointLSN) le lsnNormalize($strTargetLSN))
{
confess &log(
ERROR,
"the checkpoint location ${strCheckpointLSN} is less than the target location ${strTargetLSN} even though the" .
" replay location is ${strReplayedLSN}\n" .
"Hint: This should not be possible and may indicate a bug in PostgreSQL.",
ERROR_ARCHIVE_TIMEOUT);
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strReplayedLSN', value => $strReplayedLSN},
{name => 'strCheckpointLSN', value => $strCheckpointLSN},
);
}
####################################################################################################################################
# dbObjectGet
#

22
lib/pgBackRest/Main.pm
View File

@ -13,7 +13,6 @@ $SIG{__DIE__} = sub {Carp::confess @_};
use File::Basename qw(dirname);
use pgBackRest::Backup::Info;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Lock;
use pgBackRest::Common::Log;
@ -115,27 +114,6 @@ sub main
logFileSet(
storageLocal(),
cfgOption(CFGOPT_LOG_PATH) . '/' . cfgOption(CFGOPT_STANZA) . '-' . lc(cfgCommandName(cfgCommandGet())));
# Check if processes have been stopped
lockStopTest();
# Check locality
if (!isRepoLocal())
{
confess &log(ERROR,
cfgCommandName(cfgCommandGet()) . ' command must be run on the repository host', ERROR_HOST_INVALID);
}
# Process backup command
# ----------------------------------------------------------------------------------------------------------------------
if (cfgCommandTest(CFGCMD_BACKUP))
{
# Load module dynamically
require pgBackRest::Backup::Backup;
pgBackRest::Backup::Backup->import();
new pgBackRest::Backup::Backup()->process();
}
}
return 1;

6
src/Makefile.in
View File

@ -51,6 +51,7 @@ SRCS = \
command/archive/push/file.c \
command/archive/push/protocol.c \
command/archive/push/push.c \
command/backup/backup.c \
command/backup/common.c \
command/backup/file.c \
command/backup/pageChecksum.c \
@ -234,6 +235,9 @@ command/archive/push/protocol.o: command/archive/push/protocol.c build.auto.h co
command/archive/push/push.o: command/archive/push/push.c build.auto.h command/archive/common.h command/archive/push/file.h command/archive/push/protocol.h command/command.h command/control/common.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/fork.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/exec.h info/info.h info/infoArchive.h info/infoPg.h postgres/interface.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/archive/push/push.c -o command/archive/push/push.o
command/backup/backup.o: command/backup/backup.c build.auto.h command/archive/common.h command/backup/backup.h command/backup/common.h command/backup/file.h command/backup/protocol.h command/check/common.h command/control/common.h command/stanza/common.h common/assert.h common/compress/gzip/common.h common/compress/gzip/compress.h common/compress/gzip/decompress.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/filter/size.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h db/helper.h info/info.h info/infoArchive.h info/infoBackup.h info/infoPg.h info/manifest.h postgres/client.h postgres/interface.h postgres/version.h protocol/client.h protocol/command.h protocol/helper.h protocol/parallel.h protocol/parallelJob.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/backup.c -o command/backup/backup.o
command/backup/common.o: command/backup/common.c build.auto.h command/backup/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/string.h common/type/stringz.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/common.c -o command/backup/common.o
@ -498,7 +502,7 @@ info/infoPg.o: info/infoPg.c build.auto.h common/assert.h common/crypto/common.h
info/manifest.o: info/manifest.c build.auto.h command/backup/common.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/crypto/hash.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/mcv.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h info/info.h info/manifest.h postgres/interface.h postgres/version.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/manifest.c -o info/manifest.o
main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/check/check.h command/command.h command/control/start.h command/control/stop.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/restore/restore.h command/stanza/create.h command/stanza/delete.h command/stanza/upgrade.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h
main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/backup/backup.h command/check/check.h command/command.h command/control/start.h command/control/stop.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/restore/restore.h command/stanza/create.h command/stanza/delete.h command/stanza/upgrade.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c main.c -o main.o
perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/param.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h

1986
src/command/backup/backup.c Normal file
View File

File diff suppressed because it is too large Load Diff

12
src/command/backup/backup.h Normal file
View File

@ -0,0 +1,12 @@
/***********************************************************************************************************************************
Backup Command
***********************************************************************************************************************************/
#ifndef COMMAND_BACKUP_BACKUP_H
#define COMMAND_BACKUP_BACKUP_H
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void cmdBackup(void);
#endif

13
src/command/backup/protocol.c
View File

@ -40,13 +40,12 @@ backupProtocol(const String *command, const VariantList *paramList, ProtocolServ
{
// Backup the file
BackupFileResult result = backupFile(
varStr(varLstGet(paramList, 0)), varBoolForce(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)),
varStr(varLstGet(paramList, 3)), varBoolForce(varLstGet(paramList, 4)),
varUInt64(varLstGet(paramList, 5)) << 32 | varUInt64(varLstGet(paramList, 6)), varStr(varLstGet(paramList, 7)),
varBoolForce(varLstGet(paramList, 8)), varBoolForce(varLstGet(paramList, 9)),
varUIntForce(varLstGet(paramList, 10)), varStr(varLstGet(paramList, 11)), varBoolForce(varLstGet(paramList, 12)),
varLstSize(paramList) == 14 ? cipherTypeAes256Cbc : cipherTypeNone,
varLstSize(paramList) == 14 ? varStr(varLstGet(paramList, 13)) : NULL);
varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)),
varStr(varLstGet(paramList, 3)), varBool(varLstGet(paramList, 4)),
varUInt64(varLstGet(paramList, 5)), varStr(varLstGet(paramList, 6)),
varBool(varLstGet(paramList, 7)), varBool(varLstGet(paramList, 8)), varUIntForce(varLstGet(paramList, 9)),
varStr(varLstGet(paramList, 10)), varBool(varLstGet(paramList, 11)),
varStr(varLstGet(paramList, 12)) == NULL ? cipherTypeNone : cipherTypeAes256Cbc, varStr(varLstGet(paramList, 12)));
// Return backup result
VariantList *resultList = varLstNew();

11
src/main.c
View File

@ -9,6 +9,7 @@ Main
#include "command/archive/get/get.h"
#include "command/archive/push/push.h"
#include "command/backup/backup.h"
#include "command/check/check.h"
#include "command/command.h"
#include "command/control/start.h"
@ -106,16 +107,8 @@ main(int argListSize, const char *argList[])
// -----------------------------------------------------------------------------------------------------------------
case cfgCmdBackup:
{
#ifdef DEBUG
// Check pg_control during testing so errors are more obvious. Otherwise errors only happen in
// archive-get/archive-push and end up in the PostgreSQL log which is not output in CI. This can be removed
// once backup is written in C.
if (cfgOptionBool(cfgOptOnline) && !cfgOptionBool(cfgOptBackupStandby) && !cfgOptionTest(cfgOptPgHost))
pgControlFromFile(storagePg());
#endif
// Run backup
perlExec();
cmdBackup();
// Switch to expire command
cmdEnd(0, NULL);

1694
src/perl/embed.auto.c
View File

File diff suppressed because it is too large Load Diff

21
test/define.yaml
View File

@ -592,10 +592,6 @@ unit:
coverage:
command/archive/common: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-common-perl
total: 4
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-get
total: 5
@ -626,13 +622,16 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: backup
total: 3
total: 10
perlReq: true
coverage:
command/backup/backup: full
command/backup/file: full
command/backup/protocol: full
include:
- info/manifest
- storage/storage
# ----------------------------------------------------------------------------------------------------------------------------
@ -728,18 +727,6 @@ unit:
coverage:
command/storage/list: full
# ********************************************************************************************************************************
- name: backup
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: unit-perl
total: 4
# ----------------------------------------------------------------------------------------------------------------------------
- name: file-unit-perl
total: 2
# ********************************************************************************************************************************
- name: manifest

232
test/expect/mock-all-001.log
View File

@ -61,7 +61,7 @@ full backup - error on identical link destinations (db-master host)
P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [070]: link [TEST_PATH]/db-master/db/base/pg_hba.conf (../pg_config) references a subdirectory of or the same directory as link [TEST_PATH]/db-master/db/base/pg_config_bad (../../db/pg_config)
P00 ERROR: [070]: link 'pg_config_bad/pg_hba.conf.link' destination '[TEST_PATH]/db-master/db/base/pg_config_bad' is in PGDATA
P00 INFO: backup command end: aborted with exception [070]
full backup - error on link to a link (db-master host)
@ -70,7 +70,7 @@ full backup - error on link to a link (db-master host)
P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [070]: link '[TEST_PATH]/db-master/db/base/postgresql.conf.bad' -> '../pg_config/postgresql.conf.link' cannot reference another link
P00 ERROR: [070]: link '[TEST_PATH]/db-master/db/base/postgresql.conf.bad' cannot reference another link '[TEST_PATH]/db-master/db/pg_config/postgresql.conf.link'
P00 INFO: backup command end: aborted with exception [070]
full backup - create pg_stat link, pg_clog dir (db-master host)
@ -318,13 +318,14 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: backup '[BACKUP-FULL-1]' missing manifest removed from backup.info
P00 WARN: --no-online passed and postmaster.pid exists but --force was passed so backup will continue though it looks like the postmaster is running and the backup will probably not be consistent
P00 INFO: exclude apipe from backup using 'apipe' exclusion
P00 INFO: exclude pg_log/logfile from backup using 'pg_log/' exclusion
P00 INFO: exclude pg_log2 from backup using 'pg_log2' exclusion
P00 INFO: exclude pg_log2/logfile from backup using 'pg_log2' exclusion
P00 INFO: exclude postgresql.auto.conf from backup using 'postgresql.auto.conf' exclusion
P00 WARN: aborted backup [BACKUP-FULL-2] of same type exists, will be cleaned to remove invalid files and resumed
P00 DETAIL: clean resumed backup path: [TEST_PATH]/db-master/repo/backup/db/[BACKUP-FULL-2]
P00 INFO: exclude '[TEST_PATH]/db-master/db/base/apipe' from backup using 'apipe' exclusion
P00 INFO: exclude contents of '[TEST_PATH]/db-master/db/base/pg_log' from backup using 'pg_log/' exclusion
P00 INFO: exclude '[TEST_PATH]/db-master/db/base/pg_log2' from backup using 'pg_log2' exclusion
P00 INFO: exclude '[TEST_PATH]/db-master/db/base/postgresql.auto.conf' from backup using 'postgresql.auto.conf' exclusion
P00 WARN: resumable backup [BACKUP-FULL-2] of same type exists -- remove invalid files and resume
P00 DETAIL: remove file '[TEST_PATH]/db-master/repo/backup/db/[BACKUP-FULL-2]/file.tmp' from resumed backup (missing in manifest)
P00 DETAIL: remove file '[TEST_PATH]/db-master/repo/backup/db/[BACKUP-FULL-2]/pg_data/PG_VERSION' from resumed backup (no checksum in resumed manifest)
P00 DETAIL: remove file '[TEST_PATH]/db-master/repo/backup/db/[BACKUP-FULL-2]/pg_data/special-!_.*'()&!@;:+,?' from resumed backup (zero size)
P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/32768/33001 (64KB, 33%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b
P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/32768/44000_init (32KB, 49%) checksum 7a16d165e4775f7c92e8cdf60c0af57313f0bf90
P01 DETAIL: checksum resumed file [TEST_PATH]/db-master/db/base/base/32768/33000.32767 (32KB, 66%) checksum 6e99b589e550e68e934fd235ccba59fe5b592a9e
@ -508,8 +509,12 @@ full backup - invalid repo (db-master host)
P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --exclude=postgresql.auto.conf --exclude=pg_log/ --exclude=pg_log2 --exclude=apipe --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=/bogus_path --stanza=db --start-fast --type=full
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [073]: repo1-path '/bogus_path' does not exist
P00 INFO: backup command end: aborted with exception [073]
P00 ERROR: [055]: unable to load info file '/bogus_path/backup/db/backup.info' or '/bogus_path/backup/db/backup.info.copy':
FileMissingError: unable to open missing file '/bogus_path/backup/db/backup.info' for read
FileMissingError: unable to open missing file '/bogus_path/backup/db/backup.info.copy' for read
HINT: backup.info cannot be opened and is required to perform a backup.
HINT: has a stanza-create been performed?
P00 INFO: backup command end: aborted with exception [055]
restore delta, backup '[BACKUP-FULL-2]' - add and delete files (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --delta --set=[BACKUP-FULL-2] --link-all --stanza=db restore
@ -971,7 +976,7 @@ incr backup - invalid database version (db-master host)
P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [051]: database version = 9.4, system-id 1000000000000000094 does not match backup version = 8.0, system-id = 1000000000000000094
P00 ERROR: [051]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 8.0, system-id 1000000000000000094
HINT: is this the correct stanza?
P00 INFO: backup command end: aborted with exception [051]
@ -981,7 +986,7 @@ incr backup - invalid system id (db-master host)
P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [051]: database version = 9.4, system-id 1000000000000000094 does not match backup version = 9.4, system-id = 6999999999999999999
P00 ERROR: [051]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 9.4, system-id 6999999999999999999
HINT: is this the correct stanza?
P00 INFO: backup command end: aborted with exception [051]
@ -993,7 +998,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [069]: pg_tblspc/path is not a symlink - pg_tblspc should contain only symlinks
P00 ERROR: [069]: 'pg_data/pg_tblspc/path' is not a symlink - pg_tblspc should contain only symlinks
P00 INFO: backup command end: aborted with exception [069]
incr backup - invalid relative tablespace is ../ (db-master host)
@ -1004,7 +1009,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [070]: tablespace symlink ../ destination must not be in $PGDATA
P00 ERROR: [070]: link 'pg_tblspc/99999' destination '[TEST_PATH]/db-master/db/base' is in PGDATA
P00 INFO: backup command end: aborted with exception [070]
incr backup - invalid relative tablespace is .. (db-master host)
@ -1015,7 +1020,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [070]: tablespace symlink .. destination must not be in $PGDATA
P00 ERROR: [070]: link 'pg_tblspc/99999' destination '[TEST_PATH]/db-master/db/base' is in PGDATA
P00 INFO: backup command end: aborted with exception [070]
incr backup - invalid relative tablespace is ../../$PGDATA (db-master host)
@ -1026,7 +1031,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [070]: tablespace symlink ../../base/ destination must not be in $PGDATA
P00 ERROR: [070]: link 'pg_tblspc/99999' destination '[TEST_PATH]/db-master/db/base' is in PGDATA
P00 INFO: backup command end: aborted with exception [070]
incr backup - invalid relative tablespace is ../../$PGDATA (db-master host)
@ -1037,7 +1042,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [070]: tablespace symlink ../../base destination must not be in $PGDATA
P00 ERROR: [070]: link 'pg_tblspc/99999' destination '[TEST_PATH]/db-master/db/base' is in PGDATA
P00 INFO: backup command end: aborted with exception [070]
incr backup - tablespace link references a link (db-master host)
@ -1048,7 +1053,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [070]: link '[TEST_PATH]/db-master/db/base/pg_tblspc/99999' -> '[TEST_PATH]/db-master/db/intermediate_link' cannot reference another link
P00 ERROR: [070]: link '[TEST_PATH]/db-master/db/base/pg_tblspc/99999' cannot reference another link '[TEST_PATH]/db-master/db/intermediate_link'
P00 INFO: backup command end: aborted with exception [070]
incr backup - invalid relative tablespace in $PGDATA (db-master host)
@ -1059,7 +1064,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [070]: tablespace symlink ../invalid_tblspc destination must not be in $PGDATA
P00 ERROR: [070]: link 'pg_tblspc/99999' destination '[TEST_PATH]/db-master/db/base/invalid_tblspc' is in PGDATA
P00 INFO: backup command end: aborted with exception [070]
incr backup - $PGDATA is a substring of valid tblspc excluding / (file missing err expected) (db-master host)
@ -1070,8 +1075,8 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [073]: unable to list file info for missing path '[TEST_PATH]/db-master/db/base_tbs'
P00 INFO: backup command end: aborted with exception [073]
P00 ERROR: [041]: unable to get info for missing path/file '[TEST_PATH]/db-master/db/base/pg_tblspc/99999/[TS_PATH-1]': [2] No such file or directory
P00 INFO: backup command end: aborted with exception [041]
incr backup - invalid tablespace in $PGDATA (db-master host)
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --stanza=db backup
@ -1081,7 +1086,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [070]: tablespace symlink [TEST_PATH]/db-master/db/base/invalid_tblspc destination must not be in $PGDATA
P00 ERROR: [070]: link 'pg_tblspc/99999' destination '[TEST_PATH]/db-master/db/base/invalid_tblspc' is in PGDATA
P00 INFO: backup command end: aborted with exception [070]
incr backup - add tablespace 1 (db-master host)
@ -1097,6 +1102,23 @@ P01 INFO: backup file [TEST_PATH]/db-master/db/base/changesize.txt (4B, 68%) c
P01 INFO: backup file [TEST_PATH]/db-master/db/base/zerosize.txt (0B, 68%)
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 100%) checksum 14c44cef6287269b08d41de489fd492bb9fc795d
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192
P00 DETAIL: reference pg_data/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/12000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/17000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33001 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/44000_init to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/changecontent.txt to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/changetime.txt to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/global/pg_control to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/postgresql.conf to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2]
P00 INFO: incr backup size = 22B
P00 INFO: new backup label = [BACKUP-INCR-1]
P00 INFO: backup command end: completed successfully
@ -1266,8 +1288,9 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
P00 WARN: backup '[BACKUP-INCR-1]' missing manifest removed from backup.info
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 WARN: aborted backup [BACKUP-INCR-2] of same type exists, will be cleaned to remove invalid files and resumed
P00 DETAIL: clean resumed backup path: [TEST_PATH]/db-master/repo/backup/db/[BACKUP-INCR-2]
P00 WARN: resumable backup [BACKUP-INCR-2] of same type exists -- remove invalid files and resume
P00 DETAIL: remove file '[TEST_PATH]/db-master/repo/backup/db/[BACKUP-INCR-2]/pg_data/changesize.txt' from resumed backup (mismatched size)
P00 DETAIL: remove file '[TEST_PATH]/db-master/repo/backup/db/[BACKUP-INCR-2]/pg_data/zerosize.txt' from resumed backup (zero size)
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/32768/33001 (64KB, 33%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/32768/44000_init (32KB, 49%) checksum 7a16d165e4775f7c92e8cdf60c0af57313f0bf90
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/32768/33000.32767 (32KB, 66%) checksum 6e99b589e550e68e934fd235ccba59fe5b592a9e
@ -1288,10 +1311,26 @@ P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/1638
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/1/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29
P01 INFO: backup file [TEST_PATH]/db-master/db/base/zerosize.txt (0B, 99%)
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 99%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 100%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 99%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 100%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P00 DETAIL: reference pg_data/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/12000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/17000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33001 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/44000_init to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/changetime.txt to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/global/pg_control to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/postgresql.conf to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2]
P00 INFO: incr backup size = 192KB
P00 INFO: new backup label = [BACKUP-INCR-2]
P00 INFO: backup command end: completed successfully
@ -1470,8 +1509,8 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: backup '[BACKUP-INCR-2]' missing manifest removed from backup.info
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: aborted backup [BACKUP-INCR-2] cannot be resumed: new backup-type 'diff' does not match aborted backup-type 'incr'
P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 WARN: backup '[BACKUP-INCR-2]' cannot be resumed: new backup type 'diff' does not match resumable backup type 'incr'
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/32768/33001 (64KB, 33%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/32768/44000_init (32KB, 49%) checksum 7a16d165e4775f7c92e8cdf60c0af57313f0bf90
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/32768/33000.32767 (32KB, 66%) checksum 6e99b589e550e68e934fd235ccba59fe5b592a9e
@ -1489,10 +1528,26 @@ P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/1638
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/1/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29
P01 INFO: backup file [TEST_PATH]/db-master/db/base/zerosize.txt (0B, 99%)
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 99%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 100%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 99%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 100%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P00 DETAIL: reference pg_data/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/12000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/17000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33001 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/44000_init to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/changetime.txt to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/global/pg_control to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/postgresql.conf to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2]
P00 INFO: diff backup size = 192KB
P00 INFO: new backup label = [BACKUP-DIFF-1]
P00 INFO: backup command end: completed successfully
@ -1666,8 +1721,8 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: backup '[BACKUP-DIFF-1]' missing manifest removed from backup.info
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: aborted backup [BACKUP-DIFF-1] cannot be resumed: resume is disabled
P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 WARN: backup '[BACKUP-DIFF-1]' cannot be resumed: resume is disabled
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/32768/33001 (64KB, 33%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/32768/44000_init (32KB, 49%) checksum 7a16d165e4775f7c92e8cdf60c0af57313f0bf90
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/32768/33000.32767 (32KB, 66%) checksum 6e99b589e550e68e934fd235ccba59fe5b592a9e
@ -1685,10 +1740,26 @@ P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/1638
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/base/1/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29
P01 INFO: backup file [TEST_PATH]/db-master/db/base/zerosize.txt (0B, 99%)
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 99%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 100%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt (7B, 99%) checksum d85de07d6421d90aa9191c11c889bfde43680f0f
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 100%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P00 DETAIL: reference pg_data/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/12000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/17000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33001 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/44000_init to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/changetime.txt to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/global/pg_control to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/postgresql.conf to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2]
P00 INFO: diff backup size = 192KB
P00 INFO: new backup label = [BACKUP-DIFF-2]
P00 INFO: backup command end: completed successfully
@ -2024,11 +2095,31 @@ incr backup - add files and remove tablespace 2 (db-master host)
P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --process-max=1 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-DIFF-2], version = [VERSION-1]
P00 INFO: last backup label = [BACKUP-DIFF-2], version = 0.00
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-DIFF-2]
P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt (8B, 61%) checksum e324463005236d83e6e54795dbddd20a74533bf3
P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/base/base2.txt (5B, 38%) checksum 09b5e31766be1dba1ec27de82f975c1b6eea2a92
P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt (8B, 100%) checksum e324463005236d83e6e54795dbddd20a74533bf3
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt: file size 8 is not divisible by page size 8192
P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/base/base2.txt (5B, 100%) checksum 09b5e31766be1dba1ec27de82f975c1b6eea2a92
P00 DETAIL: reference pg_data/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/badchecksum.txt to [BACKUP-DIFF-2]
P00 DETAIL: reference pg_data/base/1/12000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/17000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33001 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/44000_init to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/changecontent.txt to [BACKUP-DIFF-2]
P00 DETAIL: reference pg_data/changetime.txt to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/global/pg_control to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/postgresql.conf to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zerosize.txt to [BACKUP-DIFF-2]
P00 DETAIL: reference pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt to [BACKUP-DIFF-2]
P00 INFO: incr backup size = 13B
P00 INFO: new backup label = [BACKUP-INCR-3]
P00 INFO: backup command end: completed successfully
@ -2201,8 +2292,10 @@ incr backup - update files - fail on missing backup.info (db-master host)
P00 INFO: backup command begin [BACKREST-VERSION]: --no-compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [055]: unable to open [TEST_PATH]/db-master/repo/backup/db/backup.info or [TEST_PATH]/db-master/repo/backup/db/backup.info.copy
P00 ERROR: [055]: [TEST_PATH]/db-master/repo/backup/db/backup.info does not exist and is required to perform a backup.
P00 ERROR: [055]: unable to load info file '[TEST_PATH]/db-master/repo/backup/db/backup.info' or '[TEST_PATH]/db-master/repo/backup/db/backup.info.copy':
FileMissingError: unable to open missing file '[TEST_PATH]/db-master/repo/backup/db/backup.info' for read
FileMissingError: unable to open missing file '[TEST_PATH]/db-master/repo/backup/db/backup.info.copy' for read
HINT: backup.info cannot be opened and is required to perform a backup.
HINT: has a stanza-create been performed?
P00 INFO: backup command end: aborted with exception [055]
@ -2239,7 +2332,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 INFO: last backup label = [BACKUP-INCR-3], version = [VERSION-1]
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-INCR-3]
P00 WARN: file pg_data/base/16384/17000 timestamp in the past or size changed but timestamp did not, enabling delta checksum
P00 WARN: file 'base/16384/17000' has same timestamp as prior but different size, enabling delta checksum
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 36%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/base/32768/44000_init (32KB, 54%) checksum 7a16d165e4775f7c92e8cdf60c0af57313f0bf90
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/base/32768/33000.32767 (32KB, 72%) checksum 6e99b589e550e68e934fd235ccba59fe5b592a9e
@ -2260,6 +2353,27 @@ P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/base/1/
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/PG_VERSION (3B, 99%) checksum 184473f470864e067ee3a22e64b47b0a1c356f29
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt (8B, 99%) checksum e324463005236d83e6e54795dbddd20a74533bf3
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 100%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578
P00 DETAIL: reference pg_data/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/badchecksum.txt to [BACKUP-DIFF-2]
P00 DETAIL: reference pg_data/base/1/12000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33001 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/44000_init to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/base2.txt to [BACKUP-INCR-3]
P00 DETAIL: reference pg_data/changecontent.txt to [BACKUP-DIFF-2]
P00 DETAIL: reference pg_data/changetime.txt to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/global/pg_control to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/postgresql.conf to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zerosize.txt to [BACKUP-DIFF-2]
P00 DETAIL: reference pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt to [BACKUP-DIFF-2]
P00 DETAIL: reference pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt to [BACKUP-INCR-3]
P00 INFO: incr backup size = 176KB
P00 INFO: new backup label = [BACKUP-INCR-4]
P00 INFO: backup command end: completed successfully
@ -2458,6 +2572,21 @@ P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt: file size 8 is not divisible by page size 8192
P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 100%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P00 DETAIL: reference pg_data/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/12000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33001 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/44000_init to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/changetime.txt to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/global/pg_control to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/postgresql.conf to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2]
P00 INFO: diff backup size = 176KB
P00 INFO: new backup label = [BACKUP-DIFF-3]
P00 INFO: backup command end: completed successfully
@ -2647,8 +2776,8 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
P00 WARN: diff backup cannot alter compress option to 'true', reset to value in [BACKUP-FULL-2]
P00 WARN: diff backup cannot alter hardlink option to 'true', reset to value in [BACKUP-FULL-2]
P00 WARN: aborted backup [BACKUP-INCR-5] cannot be resumed: new backup-type 'diff' does not match aborted backup-type 'incr'
P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 WARN: backup '[BACKUP-INCR-5]' cannot be resumed: new backup type 'diff' does not match resumable backup type 'incr'
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 36%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/base/32768/44000_init (32KB, 54%) checksum 7a16d165e4775f7c92e8cdf60c0af57313f0bf90
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/base/32768/33000.32767 (32KB, 72%) checksum 6e99b589e550e68e934fd235ccba59fe5b592a9e
@ -2669,6 +2798,21 @@ P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt: file size 12 is not divisible by page size 8192
P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt (7B, 100%) checksum dc7f76e43c46101b47acc55ae4d593a9e6983578
P00 WARN: page misalignment in file [TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P00 DETAIL: reference pg_data/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/12000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/1/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/16384/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33000.32767 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/33001 to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/44000_init to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/base/32768/PG_VERSION to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/changetime.txt to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/global/pg_control to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/pg_stat/global.stat to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/postgresql.conf to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/special-!_.*'()&!@;:+,? to [BACKUP-FULL-2]
P00 DETAIL: reference pg_data/zero_from_start to [BACKUP-FULL-2]
P00 INFO: diff backup size = 176KB
P00 INFO: new backup label = [BACKUP-DIFF-4]
P00 INFO: backup command end: completed successfully
@ -3658,7 +3802,7 @@ diff backup - option backup-standby reset - backup performed from master (db-mas
P00 INFO: backup command begin [BACKREST-VERSION]: --backup-standby --compress --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=info --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2/base --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: option backup-standby is enabled but standby is not properly configured - backups will be performed from the master
P00 WARN: option backup-standby is enabled but backup is offline - backups will be performed from the primary
P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1]
P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/base/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0
P00 INFO: diff backup size = 9B

50
test/expect/mock-all-002.log
View File

@ -52,14 +52,14 @@ full backup - error on identical link destinations (backup host)
------------------------------------------------------------------------------------------------------------------------------------
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [070]: link [TEST_PATH]/db-master/db/base/pg_hba.conf (../pg_config) references a subdirectory of or the same directory as link [TEST_PATH]/db-master/db/base/pg_config_bad (../../db/pg_config)
P00 ERROR: [070]: link 'pg_config_bad/pg_hba.conf.link' destination '[TEST_PATH]/db-master/db/base/pg_config_bad' is in PGDATA
full backup - error on link to a link (backup host)
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --type=full --stanza=db backup
------------------------------------------------------------------------------------------------------------------------------------
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [070]: link '[TEST_PATH]/db-master/db/base/postgresql.conf.bad' -> '../pg_config/postgresql.conf.link' cannot reference another link
P00 ERROR: [070]: link '[TEST_PATH]/db-master/db/base/postgresql.conf.bad' cannot reference another link '[TEST_PATH]/db-master/db/pg_config/postgresql.conf.link'
full backup - create pg_stat link, pg_clog dir (backup host)
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --manifest-save-threshold=3 --cmd-ssh=/usr/bin/ssh --pg1-port=9999 --pg1-socket-path=/test_socket_path --buffer-size=16384 --checksum-page --process-max=1 --type=full --stanza=db backup
@ -257,7 +257,7 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: backup '[BACKUP-FULL-1]' missing manifest removed from backup.info
P00 WARN: --no-online passed and postmaster.pid exists but --force was passed so backup will continue though it looks like the postmaster is running and the backup will probably not be consistent
P00 WARN: aborted backup [BACKUP-FULL-2] of same type exists, will be cleaned to remove invalid files and resumed
P00 WARN: resumable backup [BACKUP-FULL-2] of same type exists -- remove invalid files and resume
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
----------------------------------------------------------
@ -453,8 +453,10 @@ full backup - invalid repo (backup host)
------------------------------------------------------------------------------------------------------------------------------------
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [055]: unable to open /bogus_path/backup/db/backup.info or /bogus_path/backup/db/backup.info.copy
P00 ERROR: [055]: /bogus_path/backup/db/backup.info does not exist and is required to perform a backup.
P00 ERROR: [055]: unable to load info file '/bogus_path/backup/db/backup.info' or '/bogus_path/backup/db/backup.info.copy':
FileMissingError: unable to open '/bogus_path/backup/db/backup.info': No such file or directory
FileMissingError: unable to open '/bogus_path/backup/db/backup.info.copy': No such file or directory
HINT: backup.info cannot be opened and is required to perform a backup.
HINT: has a stanza-create been performed?
restore delta, backup '[BACKUP-FULL-2]' - add and delete files (db-master host)
@ -504,7 +506,7 @@ incr backup - invalid database version (backup host)
------------------------------------------------------------------------------------------------------------------------------------
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [051]: database version = 9.4, system-id 1000000000000000094 does not match backup version = 8.0, system-id = 1000000000000000094
P00 ERROR: [051]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 8.0, system-id 1000000000000000094
HINT: is this the correct stanza?
incr backup - invalid system id (backup host)
@ -512,7 +514,7 @@ incr backup - invalid system id (backup host)
------------------------------------------------------------------------------------------------------------------------------------
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 ERROR: [051]: database version = 9.4, system-id 1000000000000000094 does not match backup version = 9.4, system-id = 6999999999999999999
P00 ERROR: [051]: PostgreSQL version 9.4, system-id 1000000000000000094 do not match stanza version 9.4, system-id 6999999999999999999
HINT: is this the correct stanza?
incr backup - invalid path in pg_tblspc (backup host)
@ -521,7 +523,7 @@ incr backup - invalid path in pg_tblspc (backup host)
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [069]: pg_tblspc/path is not a symlink - pg_tblspc should contain only symlinks
P00 ERROR: [069]: 'pg_data/pg_tblspc/path' is not a symlink - pg_tblspc should contain only symlinks
incr backup - invalid relative tablespace in $PGDATA (backup host)
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --stanza=db backup
@ -529,7 +531,7 @@ incr backup - invalid relative tablespace in $PGDATA (backup host)
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [070]: tablespace symlink ../invalid_tblspc destination must not be in $PGDATA
P00 ERROR: [070]: link 'pg_tblspc/99999' destination '[TEST_PATH]/db-master/db/base/invalid_tblspc' is in PGDATA
incr backup - invalid tablespace in $PGDATA (backup host)
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --stanza=db backup
@ -537,7 +539,7 @@ incr backup - invalid tablespace in $PGDATA (backup host)
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 ERROR: [070]: tablespace symlink [TEST_PATH]/db-master/db/base/invalid_tblspc destination must not be in $PGDATA
P00 ERROR: [070]: link 'pg_tblspc/99999' destination '[TEST_PATH]/db-master/db/base/invalid_tblspc' is in PGDATA
incr backup - add tablespace 1 (backup host)
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --stanza=db backup
@ -744,10 +746,10 @@ P00 WARN: option repo1-retention-full is not set, the repository may run out o
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: backup '[BACKUP-INCR-1]' missing manifest removed from backup.info
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 WARN: file pg_data/changetime.txt timestamp in the past or size changed but timestamp did not, enabling delta checksum
P00 WARN: aborted backup [BACKUP-INCR-2] of same type exists, will be cleaned to remove invalid files and resumed
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P00 WARN: file 'changetime.txt' has timestamp earlier than prior backup, enabling delta checksum
P00 WARN: resumable backup [BACKUP-INCR-2] of same type exists -- remove invalid files and resume
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
----------------------------------------------------------
@ -955,11 +957,11 @@ diff backup - cannot resume - new diff (backup host)
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: backup '[BACKUP-INCR-2]' missing manifest removed from backup.info
P00 WARN: aborted backup [BACKUP-INCR-2] cannot be resumed: new backup-type 'diff' does not match aborted backup-type 'incr'
P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 WARN: file pg_data/changetime.txt timestamp in the past or size changed but timestamp did not, enabling delta checksum
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P00 WARN: file 'changetime.txt' has timestamp earlier than prior backup, enabling delta checksum
P00 WARN: backup '[BACKUP-INCR-2]' cannot be resumed: new backup type 'diff' does not match resumable backup type 'incr'
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
----------------------------------------------------------
@ -1163,11 +1165,11 @@ diff backup - cannot resume - disabled / no repo link (backup host)
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: backup '[BACKUP-DIFF-1]' missing manifest removed from backup.info
P00 WARN: aborted backup [BACKUP-DIFF-1] cannot be resumed: resume is disabled
P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 WARN: file pg_data/changetime.txt timestamp in the past or size changed but timestamp did not, enabling delta checksum
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
P00 WARN: file 'changetime.txt' has timestamp earlier than prior backup, enabling delta checksum
P00 WARN: backup '[BACKUP-DIFF-1]' cannot be resumed: resume is disabled
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/1/[TS_PATH-1]/16384/tablespace1.txt: file size 7 is not divisible by page size 8192
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
----------------------------------------------------------
@ -1600,7 +1602,7 @@ incr backup - update files (backup host)
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: incr backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-INCR-3]
P00 WARN: file pg_data/base/16384/17000 timestamp in the past or size changed but timestamp did not, enabling delta checksum
P00 WARN: file 'base/16384/17000' has same timestamp as prior but different size, enabling delta checksum
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/base/16384/17000: file size 8 is not divisible by page size 8192
+ supplemental file: [TEST_PATH]/db-master/pgbackrest.conf
@ -1807,7 +1809,7 @@ diff backup - updates since last full (backup host)
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 WARN: file pg_data/base/16384/17000 timestamp in the past or size changed but timestamp did not, enabling delta checksum
P00 WARN: file 'base/16384/17000' has same timestamp as prior but different size, enabling delta checksum
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/base/16384/17000: file size 8 is not divisible by page size 8192
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2b.txt: file size 8 is not divisible by page size 8192
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
@ -2026,9 +2028,9 @@ diff backup - remove files during backup (backup host)
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: diff backup cannot alter compress option to 'true', reset to value in [BACKUP-FULL-2]
P00 WARN: aborted backup [BACKUP-INCR-5] cannot be resumed: new backup-type 'diff' does not match aborted backup-type 'incr'
P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
P00 WARN: file pg_data/changetime.txt timestamp in the past or size changed but timestamp did not, enabling delta checksum
P00 WARN: file 'changetime.txt' has timestamp earlier than prior backup, enabling delta checksum
P00 WARN: backup '[BACKUP-INCR-5]' cannot be resumed: new backup type 'diff' does not match resumable backup type 'incr'
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt: file size 12 is not divisible by page size 8192
P00 WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2.txt: file size 7 is not divisible by page size 8192
@ -2897,7 +2899,7 @@ diff backup - option backup-standby reset - backup performed from master (backup
P00 INFO: backup command begin [BACKREST-VERSION]: --backup-standby --compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=info --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/backup/log --log-subprocess --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base-2/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db --start-fast --type=diff
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
P00 WARN: option backup-standby is enabled but standby is not properly configured - backups will be performed from the master
P00 WARN: option backup-standby is enabled but backup is offline - backups will be performed from the primary
P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1]
P01 INFO: backup file db-master:[TEST_PATH]/db-master/db/base-2/base/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0
P00 INFO: diff backup size = 9B

511
test/lib/pgBackRestTest/Module/Backup/BackupFileUnitPerlTest.pm
View File

@ -1,511 +0,0 @@
####################################################################################################################################
# Tests for Backup File module
####################################################################################################################################
package pgBackRestTest::Module::Backup::BackupFileUnitPerlTest;
use parent 'pgBackRestTest::Env::HostEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Backup::File;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Ini;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Env::Host::HostBackupTest;
####################################################################################################################################
# initModule
####################################################################################################################################
sub initModule
{
my $self = shift;
$self->{strDbPath} = $self->testPath() . '/db';
$self->{strRepoPath} = $self->testPath() . '/repo';
$self->{strBackupPath} = "$self->{strRepoPath}/backup/" . $self->stanza();
$self->{strPgControl} = $self->{strDbPath} . '/' . DB_FILE_PGCONTROL;
# Create backup path
storageTest()->pathCreate($self->{strBackupPath}, {bIgnoreExists => true, bCreateParent => true});
# Generate pg_control file
storageTest()->pathCreate($self->{strDbPath} . '/' . DB_PATH_GLOBAL, {bCreateParent => true});
$self->controlGenerate($self->{strDbPath}, PG_VERSION_94);
}
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_PG_PATH, $self->{strDbPath});
$self->optionTestSet(CFGOPT_REPO_PATH, $self->{strRepoPath});
$self->optionTestSet(CFGOPT_LOG_PATH, $self->testPath());
$self->optionTestSetBool(CFGOPT_ONLINE, false);
$self->optionTestSet(CFGOPT_DB_TIMEOUT, 5);
$self->optionTestSet(CFGOPT_PROTOCOL_TIMEOUT, 6);
$self->optionTestSet(CFGOPT_COMPRESS_LEVEL, 3);
$self->configTestLoad(CFGCMD_BACKUP);
# Repo
my $strRepoBackupPath = storageRepo()->pathGet(STORAGE_REPO_BACKUP);
my $strBackupLabel = "20180724-182750F";
# File
my $strFileName = "12345";
my $strFileDb = $self->{strDbPath} . "/$strFileName";
my $strFileHash = '1c7e00fd09b9dd11fc2966590b3e3274645dd031';
my $strFileRepo = storageRepo()->pathGet(
STORAGE_REPO_BACKUP . "/$strBackupLabel/" . MANIFEST_TARGET_PGDATA . "/$strFileName");
my $strRepoFile = MANIFEST_TARGET_PGDATA . "/$strFileName";
my $strRepoPgControl = MANIFEST_FILE_PGCONTROL;
my $strPgControlRepo = storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/$strBackupLabel/$strRepoPgControl");
my $strPgControlHash =
$self->archBits() == 32 ? '8107e546c59c72a8c1818fc3610d7cc1e5623660' : '4c77c900f7af0d9ab13fa9982051a42e0b637f6c';
# Copy file to db path
executeTest('cp ' . $self->dataPath() . "/filecopy.archive2.bin ${strFileDb}");
# Get size and data info for the files in the db path
my $hManifest = storageDb()->manifest($self->{strDbPath});
my $lFileSize = $hManifest->{$strFileName}{size} + 0;
my $lFileTime = $hManifest->{$strFileName}{modification_time} + 0;
my $lPgControlSize = $hManifest->{&DB_FILE_PGCONTROL}{size} + 0;
my $lPgControlTime = $hManifest->{&DB_FILE_PGCONTROL}{modification_time} + 0;
my $lRepoFileCompressSize = 3646899;
my $strBackupPath = $self->{strBackupPath} . "/$strBackupLabel";
my $strHost = "host";
my $iLocalId = 1;
# Initialize the manifest
my $oBackupManifest = new pgBackRest::Manifest("$strBackupPath/" . FILE_MANIFEST,
{bLoad => false, strDbVersion => PG_VERSION_94, iDbCatalogVersion => 201409291});
$oBackupManifest->build(storageDb(), $self->{strDbPath}, undef, true, false);
# Set the initial size values for backupManifestUpdate - running size and size for when to save the file
my $lSizeCurrent = 0;
my $lSizeTotal = 16785408;
my $lManifestSaveCurrent = 0;
my $lManifestSaveSize = int($lSizeTotal / 100);
# Result variables
my $iResultCopyResult;
my $lResultCopySize;
my $lResultRepoSize;
my $strResultCopyChecksum;
my $rResultExtra;
################################################################################################################################
if ($self->begin('backupFile(), backupManifestUpdate()'))
{
#---------------------------------------------------------------------------------------------------------------------------
# Create backup path so manifest can be saved
storageRepo->pathCreate(storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/$strBackupLabel"));
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
$self->{strPgControl},
$strRepoPgControl,
$lPgControlSize,
undef,
false,
BACKUP_FILE_COPY,
8192,
8192,
$strPgControlHash,
undef,
16785408,
0,
167854,
0);
# Accumulators should be same size as pg_control
$self->testResult(($lSizeCurrent == $lPgControlSize && $lManifestSaveCurrent == $lPgControlSize), true,
"file size in repo and repo size equal pg_control size");
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, MANIFEST_FILE_PGCONTROL,
MANIFEST_SUBKEY_CHECKSUM, $strPgControlHash)}, true, "manifest updated for pg_control");
# Neither backup.manifest nor backup.manifest.copy written because size threshold not met
$self->testResult(sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST)}, false, "backup.manifest missing");
$self->testResult(
sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT)}, false, "backup.manifest.copy missing");
#---------------------------------------------------------------------------------------------------------------------------
# No prior checksum, no compression, no page checksum, no extra, no delta, no hasReference
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
false,
BACKUP_FILE_COPY,
16777216,
16777216,
'1c7e00fd09b9dd11fc2966590b3e3274645dd031',
undef,
16785408,
8192,
167854,
8192);
# Accumulator includes size of pg_control and file. Manifest saved so ManifestSaveCurrent returns to 0
$self->testResult(($lSizeCurrent == ($lPgControlSize + $lFileSize) && $lManifestSaveCurrent == 0), true,
"repo size increased and ManifestSaveCurrent returns to 0");
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, $strRepoFile,
MANIFEST_SUBKEY_CHECKSUM, $strFileHash)}, true, "manifest updated for $strRepoFile");
# Backup.manifest not written but backup.manifest.copy written because size threshold met
$self->testResult(sub {storageTest()->exists("$strBackupPath/" . FILE_MANIFEST . INI_COPY_EXT)}, true,
'backup.manifest.copy exists in repo');
$self->testResult(
sub {storageRepo()->exists("$strBackupPath/" . FILE_MANIFEST)}, false, 'backup.manifest.copy missing in repo');
#---------------------------------------------------------------------------------------------------------------------------
# Set up page checksum result
$rResultExtra = {'valid' => true,'align' => true};
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
true,
BACKUP_FILE_COPY,
16777216,
3646899,
'1c7e00fd09b9dd11fc2966590b3e3274645dd031',
$rResultExtra,
16785408,
16785408,
167854,
0);
# File is compressed in repo so make sure repo-size added to manifest
$self->testResult(sub {$oBackupManifest->test(
MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_REPO_SIZE, $lResultRepoSize)},
true, "repo-size set");
$self->testResult(sub {$oBackupManifest->test(
MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM_PAGE, $rResultExtra->{bValid})},
true, "checksum page set");
# Set a section in the manifest to ensure it is removed in the next test
$oBackupManifest->set(
MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1", MANIFEST_SUBKEY_CHECKSUM, $strResultCopyChecksum);
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, MANIFEST_TARGET_PGDATA . "/$strFileName.1")},
true, MANIFEST_TARGET_PGDATA . "/$strFileName.1 section exists in manifest - skip file");
#---------------------------------------------------------------------------------------------------------------------------
# Removed db file is removed from manifest
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
"$strFileDb.1",
"$strRepoFile.1",
$lFileSize,
$strFileHash,
false,
BACKUP_FILE_SKIP,
undef,
undef,
undef,
undef,
16785408,
33562624,
167854,
0);
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1")},
false, " $strRepoFile.1 section removed from manifest");
# Add back the section
$oBackupManifest->set(MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1");
# Code coverage for code path when host not defined for logged message of skipped file
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
undef,
$iLocalId,
"$strFileDb.1",
MANIFEST_TARGET_PGDATA . "/$strFileName.1",
$lFileSize,
$strFileHash,
false,
BACKUP_FILE_SKIP,
undef,
undef,
undef,
undef,
16785408,
50339840,
167854,
0);
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, "$strRepoFile.1")},
false, " $strRepoFile.1 section removed from manifest on undef host");
#---------------------------------------------------------------------------------------------------------------------------
# Has reference - Code path to ensure reference is removed
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
false,
BACKUP_FILE_COPY,
16777216,
16777216,
'1c7e00fd09b9dd11fc2966590b3e3274645dd031',
undef,
16785408,
67117056,
167854,
0);
# Confirm reference to prior backup removed
$self->testResult(sub {$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, MANIFEST_TARGET_PGDATA . "/$strFileName.",
MANIFEST_SUBKEY_REFERENCE)},
false, "reference to prior backup in manifest removed");
#---------------------------------------------------------------------------------------------------------------------------
# BACKUP_FILE_NOOP
# Calculate running counts
my $lSizeCurrentAfter = $lSizeCurrent + $lFileSize;
my $lManifestSaveCurrentAfter = $lManifestSaveCurrent + $lFileSize;
# Increase manifest save size, so manifest will not be saved so counts can be tested
$lManifestSaveSize = $lFileSize * 2;
($lSizeCurrent, $lManifestSaveCurrent) = backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
false,
BACKUP_FILE_NOOP,
16777216,
undef,
'1c7e00fd09b9dd11fc2966590b3e3274645dd031',
undef,
16785408,
83894272,
$lManifestSaveSize,
0);
$self->testResult(($lSizeCurrent ==$lSizeCurrentAfter && $lManifestSaveCurrent == $lManifestSaveCurrentAfter),
true, ' running counts updated');
}
################################################################################################################################
# This section for for code coverage that is not covered in the above tests
if ($self->begin('backupManifestUpdate()'))
{
$oBackupManifest = new pgBackRest::Manifest("$strBackupPath/" . FILE_MANIFEST,
{bLoad => false, strDbVersion => PG_VERSION_94, iDbCatalogVersion => 201409291});
#---------------------------------------------------------------------------------------------------------------------------
# Check BACKUP_FILE_RECOPY warning
$iResultCopyResult = BACKUP_FILE_RECOPY;
$lResultCopySize = 0;
$lResultRepoSize = $lResultCopySize + 1;
$strResultCopyChecksum = $strFileHash;
$lSizeCurrent = 0;
$lManifestSaveSize = $lFileSize * 2;
$lManifestSaveCurrent = 0;
$rResultExtra = undef;
$self->testResult(sub {backupManifestUpdate(
$oBackupManifest,
undef,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
false,
$iResultCopyResult,
$lResultCopySize,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent)}, "($lFileSize, $lFileSize)",
'backup file recopy warning', {strLogExpect =>
"WARN: resumed backup file $strRepoFile does not have expected checksum $strFileHash. The file will be recopied and" .
" backup will continue but this may be an issue unless the resumed backup path in the repository is known to be" .
" corrupted.\n" .
"NOTE: this does not indicate a problem with the PostgreSQL page checksums."});
# Check size code paths
$self->testResult(
$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_SIZE, $lResultCopySize),
true, " copy size set");
$self->testResult(
$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_REPO_SIZE, $lResultRepoSize),
true, " repo size set");
$self->testResult(
$oBackupManifest->test(MANIFEST_SECTION_TARGET_FILE, $strRepoFile, MANIFEST_SUBKEY_CHECKSUM, $strResultCopyChecksum),
false, " checksum not set since copy size 0");
#---------------------------------------------------------------------------------------------------------------------------
# Checksum page exception
$iResultCopyResult = BACKUP_FILE_COPY;
$self->testException(sub {backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
true,
$iResultCopyResult,
$lResultCopySize,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent)},
ERROR_ASSERT, "$strFileDb should have calculated page checksums");
$rResultExtra->{valid} = false;
$self->testException(sub {backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
true,
$iResultCopyResult,
$lResultCopySize + 1,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent)},
ERROR_ASSERT, "align flag should have been set for misaligned page");
$rResultExtra->{align} = true;
$self->testException(sub {backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
true,
$iResultCopyResult,
$lResultCopySize + 1,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent)},
ERROR_ASSERT, "align flag should have been set for misaligned page");
$rResultExtra->{align} = false;
$self->testResult(sub {backupManifestUpdate(
$oBackupManifest,
$strHost,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
true,
$iResultCopyResult,
$lResultCopySize + 1,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent)},
"($lFileSize, $lFileSize)",
'page misalignment warning - host defined', {strLogExpect =>
"WARN: page misalignment in file $strHost:$strFileDb: file size " . ($lResultCopySize + 1) .
" is not divisible by page size " . PG_PAGE_SIZE});
$self->testResult(sub {backupManifestUpdate(
$oBackupManifest,
undef,
$iLocalId,
$strFileDb,
$strRepoFile,
$lFileSize,
$strFileHash,
true,
$iResultCopyResult,
$lResultCopySize + 1,
$lResultRepoSize,
$strResultCopyChecksum,
$rResultExtra,
$lSizeTotal,
$lSizeCurrent,
$lManifestSaveSize,
$lManifestSaveCurrent)},
"($lFileSize, $lFileSize)",
'page misalignment warning - host not defined', {strLogExpect =>
"WARN: page misalignment in file $strFileDb: file size " . ($lResultCopySize + 1) .
" is not divisible by page size " . PG_PAGE_SIZE});
}
}
1;

409
test/lib/pgBackRestTest/Module/Backup/BackupUnitPerlTest.pm
View File

@ -1,409 +0,0 @@
####################################################################################################################################
# Tests for Backup module
####################################################################################################################################
package pgBackRestTest::Module::Backup::BackupUnitPerlTest;
use parent 'pgBackRestTest::Env::HostEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use File::Basename qw(dirname);
use Storable qw(dclone);
use pgBackRest::Backup::Backup;
use pgBackRest::Backup::Common;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Common::Wait;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Manifest;
use pgBackRest::Protocol::Helper;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRest::Storage::Helper;
use pgBackRestTest::Common::ContainerTest;
use pgBackRestTest::Common::ExecuteTest;
use pgBackRestTest::Common::FileTest;
use pgBackRestTest::Common::RunTest;
use pgBackRestTest::Env::Host::HostBackupTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
################################################################################################################################
if ($self->begin('backupRegExpGet()'))
{
# Expected results matrix
my $hExpected = {};
$hExpected->{&false}{&false}{&true}{&false} = '[0-9]{8}\-[0-9]{6}F\_[0-9]{8}\-[0-9]{6}I';
$hExpected->{&false}{&false}{&true}{&true} = '^[0-9]{8}\-[0-9]{6}F\_[0-9]{8}\-[0-9]{6}I$';
$hExpected->{&false}{&true}{&false}{&false} = '[0-9]{8}\-[0-9]{6}F\_[0-9]{8}\-[0-9]{6}D';
$hExpected->{&false}{&true}{&false}{&true} = '^[0-9]{8}\-[0-9]{6}F\_[0-9]{8}\-[0-9]{6}D$';
$hExpected->{&false}{&true}{&true}{&false} = '[0-9]{8}\-[0-9]{6}F\_[0-9]{8}\-[0-9]{6}(D|I)';
$hExpected->{&false}{&true}{&true}{&true} = '^[0-9]{8}\-[0-9]{6}F\_[0-9]{8}\-[0-9]{6}(D|I)$';
$hExpected->{&true}{&false}{&false}{&false} = '[0-9]{8}\-[0-9]{6}F';
$hExpected->{&true}{&false}{&false}{&true} = '^[0-9]{8}\-[0-9]{6}F$';
$hExpected->{&true}{&false}{&true}{&false} = '[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}I){0,1}';
$hExpected->{&true}{&false}{&true}{&true} = '^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}I){0,1}$';
$hExpected->{&true}{&true}{&false}{&false} = '[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}D){0,1}';
$hExpected->{&true}{&true}{&false}{&true} = '^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}D){0,1}$';
$hExpected->{&true}{&true}{&true}{&false} = '[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}';
$hExpected->{&true}{&true}{&true}{&true} = '^[0-9]{8}\-[0-9]{6}F(\_[0-9]{8}\-[0-9]{6}(D|I)){0,1}$';
# Iterate though all possible combinations
for (my $bFull = false; $bFull <= true; $bFull++)
{
for (my $bDiff = false; $bDiff <= true; $bDiff++)
{
for (my $bIncr = false; $bIncr <= true; $bIncr++)
{
for (my $bAnchor = false; $bAnchor <= true; $bAnchor++)
{
# Make sure that an assertion is thrown if no types are requested
if (!($bFull || $bDiff || $bIncr))
{
$self->testException(
sub {backupRegExpGet($bFull, $bDiff, $bIncr, $bAnchor)},
ERROR_ASSERT, 'at least one backup type must be selected');
}
# Else make sure the returned value is correct
else
{
$self->testResult(
sub {backupRegExpGet($bFull, $bDiff, $bIncr, $bAnchor)}, $hExpected->{$bFull}{$bDiff}{$bIncr}{$bAnchor},
"expression full $bFull, diff $bDiff, incr $bIncr, anchor $bAnchor = " .
$hExpected->{$bFull}{$bDiff}{$bIncr}{$bAnchor});
}
}
}
}
}
}
################################################################################################################################
if ($self->begin('backupLabelFormat()'))
{
my $strBackupLabelFull = timestampFileFormat(undef, 1482000000) . 'F';
$self->testResult(sub {backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_FULL, undef, 1482000000)}, $strBackupLabelFull,
'full backup label');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_FULL, $strBackupLabelFull, 1482000000)},
ERROR_ASSERT, "strBackupLabelLast must not be defined when strType = 'full'");
#---------------------------------------------------------------------------------------------------------------------------
my $strBackupLabelDiff = "${strBackupLabelFull}_" . timestampFileFormat(undef, 1482000400) . 'D';
$self->testResult(
sub {backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_DIFF, $strBackupLabelFull, 1482000400)}, $strBackupLabelDiff,
'diff backup label');
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_DIFF, undef, 1482000400)},
ERROR_ASSERT, "strBackupLabelLast must be defined when strType = 'diff'");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_INCR, $strBackupLabelDiff, 1482000800)},
"${strBackupLabelFull}_" . timestampFileFormat(undef, 1482000800) . 'I',
'incremental backup label');
}
################################################################################################################################
if ($self->begin('backupLabel()'))
{
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath() . '/repo');
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
#---------------------------------------------------------------------------------------------------------------------------
my $lTime = time();
my $strFullLabel = backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_FULL, undef, $lTime);
storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/${strFullLabel}", {bCreateParent => true});
my $strNewFullLabel = backupLabel(storageRepo(), CFGOPTVAL_BACKUP_TYPE_FULL, undef, $lTime);
$self->testResult(sub {$strFullLabel ne $strNewFullLabel}, true, 'new full label <> existing full backup dir');
#---------------------------------------------------------------------------------------------------------------------------
executeTest('rmdir ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strFullLabel}"));
storageRepo()->pathCreate(
STORAGE_REPO_BACKUP . qw(/) . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime), {bCreateParent => true});
storageRepo()->put(
STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime) .
"/${strFullLabel}.manifest." . COMPRESS_EXT);
$strNewFullLabel = backupLabel(storageRepo(), CFGOPTVAL_BACKUP_TYPE_FULL, undef, $lTime);
$self->testResult(sub {$strFullLabel ne $strNewFullLabel}, true, 'new full label <> existing full history file');
#---------------------------------------------------------------------------------------------------------------------------
$lTime = time() + 1000;
$strFullLabel = backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_FULL, undef, $lTime);
$strNewFullLabel = backupLabel(storageRepo(), CFGOPTVAL_BACKUP_TYPE_FULL, undef, $lTime);
$self->testResult(sub {$strFullLabel eq $strNewFullLabel}, true, 'new full label in future');
#---------------------------------------------------------------------------------------------------------------------------
$lTime = time();
$strFullLabel = backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_FULL, undef, $lTime);
my $strDiffLabel = backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/${strDiffLabel}", {bCreateParent => true});
my $strNewDiffLabel = backupLabel(storageRepo(), CFGOPTVAL_BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$self->testResult(sub {$strDiffLabel ne $strNewDiffLabel}, true, 'new diff label <> existing diff backup dir');
#---------------------------------------------------------------------------------------------------------------------------
executeTest('rmdir ' . storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strDiffLabel}"));
storageRepo()->pathCreate(
STORAGE_REPO_BACKUP . qw(/) . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime),
{bIgnoreExists => true, bCreateParent => true});
storageRepo()->put(
STORAGE_REPO_BACKUP . qw{/} . PATH_BACKUP_HISTORY . '/' . timestampFormat('%4d', $lTime) .
"/${strDiffLabel}.manifest." . COMPRESS_EXT);
$strNewDiffLabel = backupLabel(storageRepo(), CFGOPTVAL_BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$self->testResult(sub {$strDiffLabel ne $strNewDiffLabel}, true, 'new full label <> existing diff history file');
#---------------------------------------------------------------------------------------------------------------------------
$lTime = time() + 1000;
$strDiffLabel = backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$strNewDiffLabel = backupLabel(storageRepo(), CFGOPTVAL_BACKUP_TYPE_DIFF, $strFullLabel, $lTime);
$self->testResult(sub {$strDiffLabel eq $strNewDiffLabel}, true, 'new diff label in future');
}
################################################################################################################################
if ($self->begin('resumeClean()'))
{
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath() . '/repo');
$self->optionTestSet(CFGOPT_PG_PATH, $self->testPath() . '/db');
$self->configTestLoad(CFGCMD_BACKUP);
my $lTime = time();
my $strFullLabel = backupLabelFormat(CFGOPTVAL_BACKUP_TYPE_FULL, undef, $lTime);
storageRepo()->pathCreate(STORAGE_REPO_BACKUP . "/${strFullLabel}", {bCreateParent => true});
my $strBackupPath = storageRepo()->pathGet(STORAGE_REPO_BACKUP . "/${strFullLabel}");
my $strBackupManifestFile = "$strBackupPath/" . FILE_MANIFEST;
my $strPath = "path";
my $strSubPath = "$strBackupPath/$strPath";
my $strInManifestNoChecksum = 'in_manifest_no_checksum';
my $strInManifestWithChecksum = 'in_manifest_with_checksum';
my $strInManifestWithReference = 'in_manifest_with_reference';
my $strExpectedManifest = $self->testPath() . '/expected.manifest';
my $strAbortedManifest = $self->testPath() . '/aborted.manifest';
my $oManifest = new pgBackRest::Manifest(
$strBackupManifestFile,
{bLoad => false, strDbVersion => PG_VERSION_94, iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94)});
my $oAbortedManifest = new pgBackRest::Manifest(
$strBackupManifestFile,
{bLoad => false, strDbVersion => PG_VERSION_94, iDbCatalogVersion => $self->dbCatalogVersion(PG_VERSION_94)});
my $oBackup = new pgBackRest::Backup::Backup();
$oAbortedManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, false);
# Compression prior enabled, gzip file exists and not in manifest, dir exists and is in manifest, delta not enabled
#---------------------------------------------------------------------------------------------------------------------------
$oAbortedManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, true);
storageRepo()->put(storageRepo()->openWrite($strBackupPath . '/' . BOGUS . '.gz',
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}));
storageRepo()->pathCreate($strSubPath, {bIgnoreExists => true});
my $hDefault = {};
$oManifest->set(MANIFEST_SECTION_TARGET_PATH, $strPath, undef, $hDefault);
$self->testResult(sub {$oBackup->resumeClean(storageRepo(), $strFullLabel, $oManifest, $oAbortedManifest, false, false,
undef, undef)}, false, 'resumeClean, prior compression enabled, delta not enabled');
$self->testResult(sub {!storageRepo()->exists($strBackupPath . '/' . BOGUS . '.gz')}, true, ' gzip file removed');
$self->testResult(sub {storageRepo()->pathExists($strSubPath)}, true, ' path not removed');
# Disable compression
$oAbortedManifest->boolSet(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_COMPRESS, undef, false);
$oManifest->remove(MANIFEST_SECTION_TARGET_PATH, $strPath);
# Path and files to be removed (not in oManifest)
#---------------------------------------------------------------------------------------------------------------------------
storageRepo()->put(storageRepo()->openWrite($strBackupPath . "/" . FILE_MANIFEST_COPY,
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}));
storageRepo()->put(storageRepo()->openWrite($strSubPath . "/" . BOGUS,
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}));
storageRepo()->put(storageRepo()->openWrite($strBackupPath . "/" . BOGUS,
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}));
$self->testResult(sub {storageRepo()->pathExists($strSubPath) && storageRepo()->exists($strSubPath . "/" . BOGUS) &&
storageRepo()->exists($strBackupPath . "/" . BOGUS)},
true, 'dir and files to be removed exist');
$self->testResult(sub {$oBackup->resumeClean(storageRepo(), $strFullLabel, $oManifest, $oAbortedManifest, false, true,
undef, undef)}, true, 'resumeClean, delta enabled, path and files to remove, manifest copy to remain');
$self->testResult(sub {!storageRepo()->pathExists($strSubPath) && !storageRepo()->exists($strSubPath . "/" . BOGUS)},
true, ' dir removed');
$self->testResult(sub {!storageRepo()->exists($strBackupPath . "/" . BOGUS) &&
storageRepo()->exists($strBackupPath . "/" . FILE_MANIFEST_COPY)}, true,
' file removed, manifest copy remains');
# Online changed, delta enabled
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {$oBackup->resumeClean(storageRepo(), $strFullLabel, $oManifest, $oAbortedManifest, true, false,
undef, undef)}, true, 'resumeClean, online changed, delta enabled');
# Online does not change, only backup.manifest.copy exists, delta not enabled
#---------------------------------------------------------------------------------------------------------------------------
storageRepo()->put(storageRepo()->openWrite($strBackupPath . '/' . BOGUS,
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}));
storageRepo()->put(storageRepo()->openWrite($strBackupPath . "/$strInManifestWithReference",
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}));
storageRepo()->put(storageRepo()->openWrite($strBackupPath . "/$strInManifestNoChecksum",
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}));
storageRepo()->put(storageRepo()->openWrite($strBackupPath . "/$strInManifestWithChecksum",
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}), 'test');
my ($strHash, $iSize) = storageRepo()->hashSize(storageRepo()->openRead($strBackupPath . "/$strInManifestWithChecksum"));
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestNoChecksum,
MANIFEST_SUBKEY_SIZE, 0);
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestNoChecksum,
MANIFEST_SUBKEY_TIMESTAMP, $lTime);
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_SIZE, $iSize);
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_TIMESTAMP, $lTime);
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithReference,
MANIFEST_SUBKEY_SIZE, 0);
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithReference,
MANIFEST_SUBKEY_TIMESTAMP, $lTime);
$oManifest->set(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithReference,
MANIFEST_SUBKEY_REFERENCE, BOGUS);
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestNoChecksum,
MANIFEST_SUBKEY_SIZE, 0);
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestNoChecksum,
MANIFEST_SUBKEY_TIMESTAMP, $lTime);
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_SIZE, $iSize);
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_TIMESTAMP, $lTime);
$oAbortedManifest->set(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_CHECKSUM, $strHash);
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithReference,
MANIFEST_SUBKEY_SIZE, 0);
$oManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithReference,
MANIFEST_SUBKEY_TIMESTAMP, $lTime);
$oManifest->set(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithReference,
MANIFEST_SUBKEY_REFERENCE, BOGUS);
$self->testResult(sub {$oBackup->resumeClean(storageRepo(), $strFullLabel, $oManifest, $oAbortedManifest, false, false,
undef, undef)}, false, 'resumeClean, online not changed, delta not enabled');
$self->testResult(sub {!storageRepo()->exists($strBackupPath . "/" . BOGUS) &&
!storageRepo()->exists($strBackupPath . "/$strInManifestNoChecksum") &&
!storageRepo()->exists($strBackupPath . "/$strInManifestWithReference") &&
storageRepo()->exists($strBackupPath . "/$strInManifestWithChecksum") &&
storageRepo()->exists($strBackupPath . "/" . FILE_MANIFEST_COPY)}, true,
' file not in manifest or in manifest but no-checksum removed, file in manifest and manifest.copy remains');
$self->testResult(sub {$oManifest->test(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum, MANIFEST_SUBKEY_CHECKSUM,
$strHash)}, true, ' checksum copied to manifest');
# Timestamp in the past for same-sized file with checksum.
#---------------------------------------------------------------------------------------------------------------------------
$oManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum, MANIFEST_SUBKEY_CHECKSUM);
$self->testResult(sub {$oManifest->test(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_CHECKSUM)}, false, 'manifest checksum does not exist');
# Set the timestamp so that the new manifest appears to have a time in the past. This should enable delta.
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_TIMESTAMP, $lTime + 100);
# Set checksum page for code coverage
$oAbortedManifest->set(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum, MANIFEST_SUBKEY_CHECKSUM_PAGE, false);
$self->testResult(sub {$oBackup->resumeClean(storageRepo(), $strFullLabel, $oManifest, $oAbortedManifest, false, false,
undef, undef)}, true, ' resumeClean, timestamp in past, delta enabled');
$self->testResult(sub {$oManifest->test(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum, MANIFEST_SUBKEY_CHECKSUM,
$strHash) && $oManifest->boolTest(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_CHECKSUM_PAGE, false)}, true, ' checksum copied to manifest');
# Timestamp different for same-sized file with checksum.
#---------------------------------------------------------------------------------------------------------------------------
$oManifest->remove(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum, MANIFEST_SUBKEY_CHECKSUM);
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_TIMESTAMP, $lTime - 100);
$self->testResult(sub {$oBackup->resumeClean(storageRepo(), $strFullLabel, $oManifest, $oAbortedManifest, false, false,
undef, undef)}, false, 'resumeClean, timestamp different but size the same, delta not enabled');
$self->testResult(sub {$oManifest->test(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_CHECKSUM) && !storageRepo()->exists($strBackupPath . "/$strInManifestWithChecksum")},
false, ' checksum not copied to manifest, file removed');
# Size different, timestamp same for file with checksum.
#---------------------------------------------------------------------------------------------------------------------------
storageRepo()->put(storageRepo()->openWrite($strBackupPath . "/$strInManifestWithChecksum",
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}), 'test');
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_SIZE, $iSize - 1);
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_TIMESTAMP, $lTime);
$self->testResult(sub {$oBackup->resumeClean(storageRepo(), $strFullLabel, $oManifest, $oAbortedManifest, false, false,
undef, undef)}, true, 'resumeClean, size different, timestamp same, delta enabled');
$self->testResult(sub {$oManifest->test(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_CHECKSUM) && !storageRepo()->exists($strBackupPath . "/$strInManifestWithChecksum")},
false, ' checksum not copied to manifest, file removed');
# Checksum page error and link to file.
#---------------------------------------------------------------------------------------------------------------------------
storageRepo()->put(storageRepo()->openWrite($strBackupPath . "/$strInManifestWithChecksum",
{strMode => '0750', strUser => TEST_USER, strGroup => TEST_GROUP, lTimestamp => $lTime}), 'test');
testLinkCreate($strBackupPath . "/testlink", $strBackupPath . "/$strInManifestWithChecksum");
$self->testResult(sub {storageRepo()->exists($strBackupPath . "/testlink")}, true, 'link exists');
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_SIZE, $iSize);
$oAbortedManifest->numericSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_TIMESTAMP, $lTime);
# Set checksum page for code coverage
$oAbortedManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum, MANIFEST_SUBKEY_CHECKSUM_PAGE, false);
$oAbortedManifest->set(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, 'E');
$self->testResult(sub {$oBackup->resumeClean(storageRepo(), $strFullLabel, $oManifest, $oAbortedManifest, false, false,
undef, undef)}, false, ' resumeClean, delta not enabled');
$self->testResult(sub {$oManifest->test(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR, 'E') && !storageRepo()->exists($strBackupPath . "/testlink")},
true, ' checksum page error copied to manifest, link removed');
# Checksum page=true
#---------------------------------------------------------------------------------------------------------------------------
$oAbortedManifest->boolSet(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum, MANIFEST_SUBKEY_CHECKSUM_PAGE, true);
$self->testResult(sub {$oBackup->resumeClean(storageRepo(), $strFullLabel, $oManifest, $oAbortedManifest, false, false,
undef, undef)}, false, ' resumeClean, checksum page = true');
$self->testResult(sub {$oManifest->boolTest(MANIFEST_SECTION_TARGET_FILE, $strInManifestWithChecksum,
MANIFEST_SUBKEY_CHECKSUM_PAGE, true)}, true, ' checksum page set true in manifest');
}
}
1;

154
test/lib/pgBackRestTest/Module/Command/CommandArchiveCommonPerlTest.pm
View File

@ -1,154 +0,0 @@
####################################################################################################################################
# Archive Common Tests
####################################################################################################################################
package pgBackRestTest::Module::Command::CommandArchiveCommonPerlTest;
use parent 'pgBackRestTest::Env::HostEnvTest';
####################################################################################################################################
# Perl includes
####################################################################################################################################
use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);
use Storable qw(dclone);
use pgBackRest::Archive::Common;
use pgBackRest::Common::Exception;
use pgBackRest::Common::Log;
use pgBackRest::Config::Config;
use pgBackRest::DbVersion;
use pgBackRest::Protocol::Storage::Helper;
use pgBackRestTest::Env::Host::HostBackupTest;
####################################################################################################################################
# run
####################################################################################################################################
sub run
{
my $self = shift;
my $strModule = 'ArchiveCommon';
################################################################################################################################
if ($self->begin("${strModule}::lsnFileRange()"))
{
$self->testResult(sub {lsnFileRange("1/60", "1/60", PG_VERSION_92, 16 * 1024 * 1024)}, "0000000100000000", 'get single');
$self->testResult(
sub {lsnFileRange("1/FD000000", "2/1000000", PG_VERSION_92, 16 * 1024 * 1024)},
"(00000001000000FD, 00000001000000FE, 0000000200000000, 0000000200000001)", 'get range < 9.3');
$self->testResult(
sub {lsnFileRange("1/FD000000", "2/60", PG_VERSION_93, 16 * 1024 * 1024)},
"(00000001000000FD, 00000001000000FE, 00000001000000FF, 0000000200000000)", 'get range >= 9.3');
$self->testResult(
sub {lsnFileRange("A/800", "B/C0000000", PG_VERSION_11, 1024 * 1024 * 1024)},
'(0000000A00000000, 0000000A00000001, 0000000A00000002, 0000000A00000003, 0000000B00000000, 0000000B00000001, ' .
'0000000B00000002, 0000000B00000003)',
'get range >= 11/1GB');
$self->testResult(
sub {lsnFileRange("7/FFEFFFFF", "8/001AAAAA", PG_VERSION_11, 1024 * 1024)},
'(0000000700000FFE, 0000000700000FFF, 0000000800000000, 0000000800000001)', 'get range >= 11/1MB');
}
################################################################################################################################
if ($self->begin("${strModule}::walIsSegment()"))
{
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {walIsSegment('0000000200ABCDEF0000001')}, false, 'invalid segment');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {walIsSegment('0000000200ABCDEF00000001')}, true, 'valid segment');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {walIsSegment('000000010000000100000001.partial')}, true, 'valid partial segment');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {walIsSegment('00000001.history')}, false, 'valid history file');
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(sub {walIsSegment('000000020000000100000001.00000028.backup')}, false, 'valid backup file');
}
################################################################################################################################
if ($self->begin("${strModule}::walIsPartial()"))
{
#---------------------------------------------------------------------------------------------------------------------------
my $strWalSegment = '0000000200ABCDEF00000001';
$self->testResult(sub {walIsPartial($strWalSegment)}, false, "${strWalSegment} WAL is not partial");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegment = $strWalSegment . '.partial';
$self->testResult(sub {walIsPartial($strWalSegment)}, true, "${strWalSegment} WAL is partial");
}
################################################################################################################################
if ($self->begin("${strModule}::walSegmentFind()"))
{
$self->optionTestSet(CFGOPT_STANZA, $self->stanza());
$self->optionTestSet(CFGOPT_REPO_PATH, $self->testPath());
$self->configTestLoad(CFGCMD_ARCHIVE_PUSH);
my $strArchiveId = '9.4-1';
my $strArchivePath = storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/${strArchiveId}");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalSegment = '000000010000000100000001ZZ';
$self->testException(
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, ERROR_ASSERT,
"${strWalSegment} is not a WAL segment");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegment = '000000010000000100000001';
$self->testResult(
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, undef, "${strWalSegment} WAL not found");
#---------------------------------------------------------------------------------------------------------------------------
$self->testException(
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment, .1)}, ERROR_ARCHIVE_TIMEOUT,
"could not find WAL segment ${strWalSegment} after 0.1 second(s)" .
"\nHINT: is archive_command configured correctly?" .
"\nHINT: use the check command to verify that PostgreSQL is archiving.");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalMajorPath = "${strArchivePath}/" . substr($strWalSegment, 0, 16);
my $strWalSegmentHash = "${strWalSegment}-53aa5d59515aa7288ae02ba414c009aed1ca73ad";
storageRepo()->pathCreate($strWalMajorPath, {bCreateParent => true});
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
#---------------------------------------------------------------------------------------------------------------------------
my $strWalSegmentHash2 = "${strWalSegment}-a0b0d38b8aa263e25b8ff52a0a4ba85b6be97f9b.gz";
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash2}");
$self->testException(
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, ERROR_ARCHIVE_DUPLICATE,
"duplicates found in archive for WAL segment ${strWalSegment}: ${strWalSegmentHash}, ${strWalSegmentHash2}");
storageRepo()->remove("${strWalMajorPath}/${strWalSegmentHash}");
#---------------------------------------------------------------------------------------------------------------------------
$self->testResult(
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash2,
"${strWalSegment} WAL found with compressed extension");
storageRepo()->remove("${strWalMajorPath}/${strWalSegmentHash2}");
#---------------------------------------------------------------------------------------------------------------------------
$strWalSegment = $strWalSegment . '.partial';
$strWalSegmentHash = "${strWalSegment}-996195c807713ef9262170043e7222cb150aef70";
storageRepo()->put("${strWalMajorPath}/${strWalSegmentHash}");
$self->testResult(
sub {walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment)}, $strWalSegmentHash, "${strWalSegment} WAL found");
}
}
1;

4
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
View File

@ -516,7 +516,7 @@ sub run
$oHostBackup->backup(
$strType, 'invalid repo',
{oExpectedManifest => \%oManifest, strOptionalParam => '--' . cfgOptionName(CFGOPT_REPO_PATH) . '=/bogus_path',
iExpectedExitStatus => $bS3 ? ERROR_FILE_MISSING : ERROR_PATH_MISSING});
iExpectedExitStatus => ERROR_FILE_MISSING});
# Restore - tests various mode, extra files/paths, missing files/paths
#---------------------------------------------------------------------------------------------------------------------------
@ -822,7 +822,7 @@ sub run
$oHostBackup->backup(
$strType, '$PGDATA is a substring of valid tblspc excluding / (file missing err expected)',
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_PATH_MISSING});
{oExpectedManifest => \%oManifest, iExpectedExitStatus => ERROR_FILE_OPEN});
testFileRemove("${strTblSpcPath}/99999");
}

6
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
View File

@ -475,7 +475,7 @@ sub run
my $strStandbyBackup = $oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_FULL, 'backup from standby, failure to access at least one standby',
{bStandby => true,
iExpectedExitStatus => ERROR_HOST_CONNECT,
iExpectedExitStatus => ERROR_DB_CONNECT,
strOptionalParam => '--' .
cfgOptionName(cfgOptionIdFromIndex(CFGOPT_PG_HOST, cfgOptionIndexTotal(CFGOPT_PG_PATH))) . '=' . BOGUS});
}
@ -535,7 +535,9 @@ sub run
$oHostDbMaster->stop();
$oHostBackup->backup(CFGOPTVAL_BACKUP_TYPE_INCR, 'attempt backup when stopped', {iExpectedExitStatus => ERROR_STOP});
$oHostBackup->backup(
CFGOPTVAL_BACKUP_TYPE_INCR, 'attempt backup when stopped',
{iExpectedExitStatus => $oHostBackup == $oHostDbMaster ? ERROR_STOP : ERROR_DB_CONNECT});
$oHostDbMaster->start();
}

1823
test/src/module/command/backupTest.c
View File

File diff suppressed because it is too large Load Diff