db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-17 20:58:34 +02:00
Code Issues Releases Activity

Implement issue #90: Move archive-max-mb archive-stop checks to end of archive-push.

Browse Source
This commit is contained in:
David Steele 2015-05-29 11:41:19 -04:00
parent 639c27e113
commit 13e4eec629
11 changed files with 260 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -747,7 +747,7 @@ example: db-path=/data/db
* Various command behavior and help fixes. Reported by Michael Renner.
* Fixed an issue in async archiving where archive-push was not properly returning 0 when archive-max-mb was reached. Also added unit tests for this case and improved error messages to make it clearer to the user what went wrong. Reported by Michael Renner.
* Fixed an issue in async archiving where archive-push was not properly returning 0 when archive-max-mb was reached and moved the async check after transfer to avoid having to remove the stop file twice. Also added unit tests for this case and improved error messages to make it clearer to the user what went wrong. Reported by Michael Renner.
* Replaced JSON module with JSON::PP which ships with core Perl.

2
bin/pg_backrest.pl
View File

@ -300,5 +300,5 @@ if ($@)
}
safe_exit();
confess $@;
confess $oMessage;
}

2
doc/doc.xml
View File

@ -705,7 +705,7 @@ Run a <id>full</id> backup on the <id>db</id> stanza. <param>--type</param> can
<text>Various command behavior and help fixes. Reported by Michael Renner.</text>
</release-feature>
<release-feature>
<text>Fixed an issue in async archiving where archive-push was not properly returning 0 when archive-max-mb was reached. Also added unit tests for this case and improved error messages to make it clearer to the user what went wrong. Reported by Michael Renner.</text>
<text>Fixed an issue in async archiving where archive-push was not properly returning 0 when archive-max-mb was reached and moved the async check after transfer to avoid having to remove the stop file twice. Also added unit tests for this case and improved error messages to make it clearer to the user what went wrong. Reported by Michael Renner.</text>
</release-feature>
<release-feature>
<text>Replaced JSON module with JSON::PP which ships with core Perl.</text>

175
lib/BackRest/Archive.pm
View File

@ -618,25 +618,11 @@ sub xfer
{
CORE::push(@stryFile, $strFile);
$lFileSize += $oManifestHash{name}{"${strFile}"}{size};
$lFileSize += $oManifestHash{name}{$strFile}{size};
$lFileTotal++;
}
}
if (optionTest(OPTION_ARCHIVE_MAX_MB))
{
my $iArchiveMaxMB = optionGet(OPTION_ARCHIVE_MAX_MB);
if ($iArchiveMaxMB < int($lFileSize / 1024 / 1024))
{
&log(ERROR, "local archive store max size has exceeded limit of ${iArchiveMaxMB}MB, archive logs will be discarded");
my $hStopFile;
open($hStopFile, '>', $strStopFile) or confess &log(ERROR, "unable to create stop file file ${strStopFile}");
close($hStopFile);
}
}
if ($lFileTotal == 0)
{
&log(DEBUG, 'no archive logs to be copied to backup');
@ -644,74 +630,105 @@ sub xfer
return 0;
}
# If the archive repo is remote create a new file object to do the copies
if (!optionRemoteTypeTest(NONE))
eval
{
$oFile = new BackRest::File
(
optionGet(OPTION_STANZA),
optionGet(OPTION_REPO_REMOTE_PATH),
optionRemoteType(),
optionRemote()
);
# If the archive repo is remote create a new file object to do the copies
if (!optionRemoteTypeTest(NONE))
{
$oFile = new BackRest::File
(
optionGet(OPTION_STANZA),
optionGet(OPTION_REPO_REMOTE_PATH),
optionRemoteType(),
optionRemote()
);
}
# Modify process name to indicate async archiving
$0 = $^X . ' ' . $0 . " --stanza=" . optionGet(OPTION_STANZA) .
"archive-push-async " . $stryFile[0] . '-' . $stryFile[scalar @stryFile - 1];
# Output files to be moved to backup
&log(INFO, "archive to be copied to backup total ${lFileTotal}, size " . file_size_format($lFileSize));
# Transfer each file
foreach my $strFile (sort @stryFile)
{
# Construct the archive filename to backup
my $strArchiveFile = "${strArchivePath}/${strFile}";
# Determine if the source file is already compressed
my $bSourceCompressed = $strArchiveFile =~ "^.*\.$oFile->{strCompressExtension}\$" ? true : false;
# Determine if this is an archive file (don't want to do compression or checksum on .backup files)
my $bArchiveFile = basename($strFile) =~
"^[0-F]{24}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$" ? true : false;
# Figure out whether the compression extension needs to be added or removed
my $bDestinationCompress = $bArchiveFile && optionGet(OPTION_COMPRESS);
my $strDestinationFile = basename($strFile);
if (!$bSourceCompressed && $bDestinationCompress)
{
$strDestinationFile .= ".$oFile->{strCompressExtension}";
}
elsif ($bSourceCompressed && !$bDestinationCompress)
{
$strDestinationFile = substr($strDestinationFile, 0, length($strDestinationFile) - 3);
}
&log(DEBUG, "archive ${strFile}, is WAL ${bArchiveFile}, source_compressed = ${bSourceCompressed}, " .
"destination_compress ${bDestinationCompress}, default_compress = " . optionGet(OPTION_COMPRESS));
# Check that there are no issues with pushing this WAL segment
if ($bArchiveFile)
{
my ($strDbVersion, $ullDbSysId) = $self->walInfo($strArchiveFile);
$self->pushCheck($oFile, substr(basename($strArchiveFile), 0, 24), $strArchiveFile, $strDbVersion, $ullDbSysId);
}
# Copy the archive file
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveFile, # Source file
PATH_BACKUP_ARCHIVE, $strDestinationFile, # Destination file
$bSourceCompressed, # Source is not compressed
$bDestinationCompress, # Destination compress is configurable
undef, undef, undef, # Unused params
true); # Create path if it does not exist
# Remove the source archive file
unlink($strArchiveFile)
or confess &log(ERROR, "copied ${strArchiveFile} to archive successfully but unable to remove it locally. " .
'This file will need to be cleaned up manually. If the problem persists, check if ' .
OP_ARCHIVE_PUSH . ' is being run with different permissions in different contexts.');
# Remove the copied segment from the total size
$lFileSize -= $oManifestHash{name}{$strFile}{size};
}
};
my $oException = $@;
# Create a stop file if the archive store exceeds the max even after xfer
if (optionTest(OPTION_ARCHIVE_MAX_MB))
{
my $iArchiveMaxMB = optionGet(OPTION_ARCHIVE_MAX_MB);
if ($iArchiveMaxMB < int($lFileSize / 1024 / 1024))
{
&log(ERROR, "local archive store max size has exceeded limit of ${iArchiveMaxMB}MB" .
" - WAL segments will be discarded until the stop file (${strStopFile}) is removed");
my $hStopFile;
open($hStopFile, '>', $strStopFile)
or confess &log(ERROR, "unable to create stop file file ${strStopFile}");
close($hStopFile);
}
}
# Modify process name to indicate async archiving
$0 = $^X . ' ' . $0 . " --stanza=" . optionGet(OPTION_STANZA) .
"archive-push-async " . $stryFile[0] . '-' . $stryFile[scalar @stryFile - 1];
# Output files to be moved to backup
&log(INFO, "archive to be copied to backup total ${lFileTotal}, size " . file_size_format($lFileSize));
# Transfer each file
foreach my $strFile (sort @stryFile)
# If there was an exception before throw it now
if ($oException)
{
# Construct the archive filename to backup
my $strArchiveFile = "${strArchivePath}/${strFile}";
# Determine if the source file is already compressed
my $bSourceCompressed = $strArchiveFile =~ "^.*\.$oFile->{strCompressExtension}\$" ? true : false;
# Determine if this is an archive file (don't want to do compression or checksum on .backup files)
my $bArchiveFile = basename($strFile) =~
"^[0-F]{24}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$" ? true : false;
# Figure out whether the compression extension needs to be added or removed
my $bDestinationCompress = $bArchiveFile && optionGet(OPTION_COMPRESS);
my $strDestinationFile = basename($strFile);
if (!$bSourceCompressed && $bDestinationCompress)
{
$strDestinationFile .= ".$oFile->{strCompressExtension}";
}
elsif ($bSourceCompressed && !$bDestinationCompress)
{
$strDestinationFile = substr($strDestinationFile, 0, length($strDestinationFile) - 3);
}
&log(DEBUG, "archive ${strFile}, is WAL ${bArchiveFile}, source_compressed = ${bSourceCompressed}, " .
"destination_compress ${bDestinationCompress}, default_compress = " . optionGet(OPTION_COMPRESS));
# Check that there are no issues with pushing this WAL segment
if ($bArchiveFile)
{
my ($strDbVersion, $ullDbSysId) = $self->walInfo($strArchiveFile);
$self->pushCheck($oFile, substr(basename($strArchiveFile), 0, 24), $strArchiveFile, $strDbVersion, $ullDbSysId);
}
# Copy the archive file
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveFile, # Source file
PATH_BACKUP_ARCHIVE, $strDestinationFile, # Destination file
$bSourceCompressed, # Source is not compressed
$bDestinationCompress, # Destination compress is configurable
undef, undef, undef, # Unused params
true); # Create path if it does not exist
# Remove the source archive file
unlink($strArchiveFile)
or confess &log(ERROR, "copied ${strArchiveFile} to archive successfully but unable to remove it locally. " .
'This file will need to be cleaned up manually. If the problem persists, check if ' .
OP_ARCHIVE_PUSH . ' is being run with different permissions in different contexts.');
confess $oException;
}
# Return number of files indicating that processing should continue

102
test/lib/BackRestTest/BackupTest.pm
View File

@ -1757,23 +1757,35 @@ sub BackRestTestBackup_Test
true, # archive-async
undef);
my $strCommand = BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_DbPathGet() .
'/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push';
# Helper function to push archive logs
sub archivePush
{
my $oFile = shift;
my $strXlogPath = shift;
my $strArchiveTestFile = shift;
my $iArchiveNo = shift;
my $iExpectedError = shift;
# &log(INFO, ' backup ' . sprintf('%02d', $iBackup) .
# ', archive ' .sprintf('%02x', $iArchive) .
# " - ${strArchiveFile}");
my $strSourceFile = "${strXlogPath}/" . uc(sprintf('0000000100000001%08x', $iArchiveNo));
my $strSourceFile = "${strXlogPath}/000000010000000100000001";
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false, # Destination is not compressed
undef, undef, undef, # Unused params
true); # Create path if it does not exist
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false, # Destination is not compressed
undef, undef, undef, # Unused params
true); # Create path if it does not exist
my $strCommand = BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_DbPathGet() .
'/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push' .
(defined($iExpectedError) && $iExpectedError == ERROR_HOST_CONNECT ?
" --backup-host=bogus" : '');
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}");
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}", undef, undef, undef,
$iExpectedError);
}
# Push a WAL segment
archivePush($oFile, $strXlogPath, $strArchiveTestFile, 1);
# load the archive info file so it can be munged for testing
my $strInfoFile = $oFile->path_get(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE);
@ -1788,74 +1800,34 @@ sub BackRestTestBackup_Test
$oInfo{database}{version} = '8.0';
BackRestTestCommon_iniSave($strInfoFile, \%oInfo, $bRemote);
}
else
{
$strCommand .= " --backup-host=bogus";
}
$strSourceFile = "${strXlogPath}/000000010000000100000002";
# Push two more segments with errors to exceed archive-max-mb
archivePush($oFile, $strXlogPath, $strArchiveTestFile, 2,
$iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false, # Destination is not compressed
undef, undef, undef, # Unused params
true); # Create path if it does not exist
archivePush($oFile, $strXlogPath, $strArchiveTestFile, 3,
$iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}", undef, undef, undef,
$iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
# Now this segment will get dropped
archivePush($oFile, $strXlogPath, $strArchiveTestFile, 4);
$strSourceFile = "${strXlogPath}/000000010000000100000003";
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false, # Destination is not compressed
undef, undef, undef, # Unused params
true); # Create path if it does not exist
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}", undef, undef, undef,
$iError ? ERROR_HOST_CONNECT : ERROR_ARCHIVE_MISMATCH);
$strSourceFile = "${strXlogPath}/000000010000000100000004";
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false, # Destination is not compressed
undef, undef, undef, # Unused params
true); # Create path if it does not exist
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}");
# Fix the database version and remove stop file
# Fix the database version
if ($iError == 0)
{
$oInfo{database}{version} = '9.3';
BackRestTestCommon_iniSave($strInfoFile, \%oInfo, $bRemote);
}
else
{
$strCommand = BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_DbPathGet() .
'/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push';
}
# Remove the stop file
my $strStopFile = ($bRemote ? BackRestTestCommon_LocalPathGet() : BackRestTestCommon_RepoPathGet()) .
'/lock/db-archive.stop';
unlink($strStopFile)
or die "unable to remove stop file ${strStopFile}";
$strSourceFile = "${strXlogPath}/000000010000000100000005";
$oFile->copy(PATH_DB_ABSOLUTE, $strArchiveTestFile, # Source file
PATH_DB_ABSOLUTE, $strSourceFile, # Destination file
false, # Source is not compressed
false, # Destination is not compressed
undef, undef, undef, # Unused params
true); # Create path if it does not exist
BackRestTestCommon_Execute($strCommand . " ${strSourceFile}");
# Push two more segments - only #4 should be missing from the archive at the end
archivePush($oFile, $strXlogPath, $strArchiveTestFile, 5);
archivePush($oFile, $strXlogPath, $strArchiveTestFile, 6);
}
}

25
test/log/backup-archive-stop-001.log
View File

@ -52,13 +52,13 @@ run 001 - rmt 0, cmp 0, error version
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/backrest/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 2, size 32MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 0, default_compress = 0
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
DEBUG: File->exists: backup:archive:[TEST_PATH]/backrest/archive/db
DEBUG: File->exists: backup:archive:[TEST_PATH]/backrest/archive/db/archive.info
ERROR: [119]: WAL segment version 9.3 does not match archive version 8.0
ERROR: local archive store max size has exceeded limit of 24MB - WAL segments will be discarded until the stop file ([TEST_PATH]/backrest/lock/db-archive.stop) is removed
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000004
@ -74,7 +74,6 @@ run 001 - rmt 0, cmp 0, error version
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/backrest/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 3, size 48MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 0, default_compress = 0
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
@ -102,3 +101,25 @@ run 001 - rmt 0, cmp 0, error version
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000006
------------------------------------------------------------------------------------------------------------------------------------
INFO: pushing WAL segment [TEST_PATH]/db/common/pg_xlog/000000010000000100000006 asynchronously
DEBUG: File->copy: local db:absolute:[TEST_PATH]/db/common/pg_xlog/000000010000000100000006 to local backup:archive:out:[TEST_PATH]/backrest/archive/db/out/000000010000000100000006, source_compressed = false, destination_compress = false, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: File->move: absolute:[TEST_PATH]/backrest/archive/db/out/000000010000000100000006.tmp to absolute:[TEST_PATH]/backrest/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, destination_path_create = true
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/backrest/archive/db/out
INFO: archive to be copied to backup total 1, size 16MB
DEBUG: archive 000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 0, default_compress = 0
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000006
DEBUG: File->exists: backup:archive:[TEST_PATH]/backrest/archive/db
DEBUG: File->exists: backup:archive:[TEST_PATH]/backrest/archive/db/archive.info
DEBUG: File->list: backup:absolute:[TEST_PATH]/backrest/archive/db/0000000100000001, expression ^000000010000000100000006(-[0-f]+){0,1}(\.gz){0,1}$, sort forward
DEBUG: File->copy: local db:absolute:[TEST_PATH]/backrest/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031 to local backup:archive:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, source_compressed = false, destination_compress = false, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: File->move: absolute:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031.tmp to absolute:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, destination_path_create = true
DEBUG: 1 WAL segments were transferred, calling Archive->xfer() again
DEBUG: File->manifest: db:absolute:[TEST_PATH]/backrest/archive/db/out
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads

25
test/log/backup-archive-stop-002.log
View File

@ -52,13 +52,13 @@ run 002 - rmt 0, cmp 1, error version
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/backrest/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 2, size 32MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 1, default_compress = 1
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
DEBUG: File->exists: backup:archive:[TEST_PATH]/backrest/archive/db
DEBUG: File->exists: backup:archive:[TEST_PATH]/backrest/archive/db/archive.info
ERROR: [119]: WAL segment version 9.3 does not match archive version 8.0
ERROR: local archive store max size has exceeded limit of 24MB - WAL segments will be discarded until the stop file ([TEST_PATH]/backrest/lock/db-archive.stop) is removed
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000004
@ -74,7 +74,6 @@ run 002 - rmt 0, cmp 1, error version
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/backrest/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 3, size 48MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 1, default_compress = 1
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
@ -102,3 +101,25 @@ run 002 - rmt 0, cmp 1, error version
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000006
------------------------------------------------------------------------------------------------------------------------------------
INFO: pushing WAL segment [TEST_PATH]/db/common/pg_xlog/000000010000000100000006 asynchronously
DEBUG: File->copy: local db:absolute:[TEST_PATH]/db/common/pg_xlog/000000010000000100000006 to local backup:archive:out:[TEST_PATH]/backrest/archive/db/out/000000010000000100000006, source_compressed = false, destination_compress = false, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: File->move: absolute:[TEST_PATH]/backrest/archive/db/out/000000010000000100000006.tmp to absolute:[TEST_PATH]/backrest/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, destination_path_create = true
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/backrest/archive/db/out
INFO: archive to be copied to backup total 1, size 16MB
DEBUG: archive 000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 1, default_compress = 1
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000006
DEBUG: File->exists: backup:archive:[TEST_PATH]/backrest/archive/db
DEBUG: File->exists: backup:archive:[TEST_PATH]/backrest/archive/db/archive.info
DEBUG: File->list: backup:absolute:[TEST_PATH]/backrest/archive/db/0000000100000001, expression ^000000010000000100000006(-[0-f]+){0,1}(\.gz){0,1}$, sort forward
DEBUG: File->copy: local db:absolute:[TEST_PATH]/backrest/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031 to local backup:archive:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031.gz, source_compressed = false, destination_compress = true, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: File->move: absolute:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031.gz.tmp to absolute:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031.gz, destination_path_create = true
DEBUG: 1 WAL segments were transferred, calling Archive->xfer() again
DEBUG: File->manifest: db:absolute:[TEST_PATH]/backrest/archive/db/out
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads

21
test/log/backup-archive-stop-003.log
View File

@ -43,11 +43,11 @@ run 003 - rmt 1, cmp 0, error version
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 2, size 32MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 0, default_compress = 0
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
ERROR: [119]: : WAL segment version 9.3 does not match archive version 8.0
ERROR: local archive store max size has exceeded limit of 24MB - WAL segments will be discarded until the stop file ([TEST_PATH]/local/lock/db-archive.stop) is removed
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000004
@ -63,7 +63,6 @@ run 003 - rmt 1, cmp 0, error version
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 3, size 48MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 0, default_compress = 0
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
@ -79,3 +78,21 @@ run 003 - rmt 1, cmp 0, error version
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000006
------------------------------------------------------------------------------------------------------------------------------------
INFO: pushing WAL segment [TEST_PATH]/db/common/pg_xlog/000000010000000100000006 asynchronously
DEBUG: File->copy: local db:absolute:[TEST_PATH]/db/common/pg_xlog/000000010000000100000006 to local backup:archive:out:[TEST_PATH]/local/archive/db/out/000000010000000100000006, source_compressed = false, destination_compress = false, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: File->move: absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006.tmp to absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, destination_path_create = true
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
INFO: archive to be copied to backup total 1, size 16MB
DEBUG: archive 000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 0, default_compress = 0
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000006
DEBUG: File->copy: local db:absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031 to remote backup:archive:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, source_compressed = false, destination_compress = false, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: 1 WAL segments were transferred, calling Archive->xfer() again
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads

23
test/log/backup-archive-stop-004.log
View File

@ -41,12 +41,12 @@ resolve hostname bogus: nodename nor servname provided, or not known
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
resolve hostname bogus: nodename nor servname provided, or not known
ERROR: [124]: unable to connect to bogus: unable to establish master SSH connection: master process exited unexpectedly
ERROR: local archive store max size has exceeded limit of 24MB - WAL segments will be discarded until the stop file ([TEST_PATH]/local/lock/db-archive.stop) is removed
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push --backup-host=bogus [TEST_PATH]/db/common/pg_xlog/000000010000000100000004
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000004
------------------------------------------------------------------------------------------------------------------------------------
ERROR: discarding 000000010000000100000004 due to the archive store max size exceeded - remove the archive stop file ([TEST_PATH]/local/lock/db-archive.stop) to resume archiving and be sure to take a new backup as soon as possible
DEBUG: safe exit called, terminating threads
@ -59,7 +59,6 @@ resolve hostname bogus: nodename nor servname provided, or not known
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 3, size 48MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 0, default_compress = 0
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
@ -75,3 +74,21 @@ resolve hostname bogus: nodename nor servname provided, or not known
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000006
------------------------------------------------------------------------------------------------------------------------------------
INFO: pushing WAL segment [TEST_PATH]/db/common/pg_xlog/000000010000000100000006 asynchronously
DEBUG: File->copy: local db:absolute:[TEST_PATH]/db/common/pg_xlog/000000010000000100000006 to local backup:archive:out:[TEST_PATH]/local/archive/db/out/000000010000000100000006, source_compressed = false, destination_compress = false, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: File->move: absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006.tmp to absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, destination_path_create = true
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
INFO: archive to be copied to backup total 1, size 16MB
DEBUG: archive 000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 0, default_compress = 0
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000006
DEBUG: File->copy: local db:absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031 to remote backup:archive:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, source_compressed = false, destination_compress = false, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: 1 WAL segments were transferred, calling Archive->xfer() again
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads

21
test/log/backup-archive-stop-005.log
View File

@ -43,11 +43,11 @@ run 005 - rmt 1, cmp 1, error version
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 2, size 32MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 1, default_compress = 1
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
ERROR: [119]: : WAL segment version 9.3 does not match archive version 8.0
ERROR: local archive store max size has exceeded limit of 24MB - WAL segments will be discarded until the stop file ([TEST_PATH]/local/lock/db-archive.stop) is removed
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000004
@ -63,7 +63,6 @@ run 005 - rmt 1, cmp 1, error version
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 3, size 48MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 1, default_compress = 1
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
@ -79,3 +78,21 @@ run 005 - rmt 1, cmp 1, error version
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000006
------------------------------------------------------------------------------------------------------------------------------------
INFO: pushing WAL segment [TEST_PATH]/db/common/pg_xlog/000000010000000100000006 asynchronously
DEBUG: File->copy: local db:absolute:[TEST_PATH]/db/common/pg_xlog/000000010000000100000006 to local backup:archive:out:[TEST_PATH]/local/archive/db/out/000000010000000100000006, source_compressed = false, destination_compress = false, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: File->move: absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006.tmp to absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, destination_path_create = true
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
INFO: archive to be copied to backup total 1, size 16MB
DEBUG: archive 000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 1, default_compress = 1
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000006
DEBUG: File->copy: local db:absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031 to remote backup:archive:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031.gz, source_compressed = false, destination_compress = true, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: 1 WAL segments were transferred, calling Archive->xfer() again
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads

23
test/log/backup-archive-stop-006.log
View File

@ -41,12 +41,12 @@ resolve hostname bogus: nodename nor servname provided, or not known
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
resolve hostname bogus: nodename nor servname provided, or not known
ERROR: [124]: unable to connect to bogus: unable to establish master SSH connection: master process exited unexpectedly
ERROR: local archive store max size has exceeded limit of 24MB - WAL segments will be discarded until the stop file ([TEST_PATH]/local/lock/db-archive.stop) is removed
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push --backup-host=bogus [TEST_PATH]/db/common/pg_xlog/000000010000000100000004
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000004
------------------------------------------------------------------------------------------------------------------------------------
ERROR: discarding 000000010000000100000004 due to the archive store max size exceeded - remove the archive stop file ([TEST_PATH]/local/lock/db-archive.stop) to resume archiving and be sure to take a new backup as soon as possible
DEBUG: safe exit called, terminating threads
@ -59,7 +59,6 @@ resolve hostname bogus: nodename nor servname provided, or not known
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
ERROR: local archive store max size has exceeded limit of 24MB, archive logs will be discarded
INFO: archive to be copied to backup total 3, size 48MB
DEBUG: archive 000000010000000100000002-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 1, default_compress = 1
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000002
@ -75,3 +74,21 @@ resolve hostname bogus: nodename nor servname provided, or not known
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads
> ../bin/pg_backrest.pl --config=[TEST_PATH]/db/pg_backrest.conf --archive-max-mb=24 --no-fork --stanza=db archive-push [TEST_PATH]/db/common/pg_xlog/000000010000000100000006
------------------------------------------------------------------------------------------------------------------------------------
INFO: pushing WAL segment [TEST_PATH]/db/common/pg_xlog/000000010000000100000006 asynchronously
DEBUG: File->copy: local db:absolute:[TEST_PATH]/db/common/pg_xlog/000000010000000100000006 to local backup:archive:out:[TEST_PATH]/local/archive/db/out/000000010000000100000006, source_compressed = false, destination_compress = false, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: File->move: absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006.tmp to absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, destination_path_create = true
INFO: No fork on archive local for TESTING
INFO: starting async archive-push
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
INFO: archive to be copied to backup total 1, size 16MB
DEBUG: archive 000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031, is WAL 1, source_compressed = 0, destination_compress 1, default_compress = 1
DEBUG: Archive->pushCheck: backup:archive:000000010000000100000006
DEBUG: File->copy: local db:absolute:[TEST_PATH]/local/archive/db/out/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031 to remote backup:archive:[TEST_PATH]/backrest/archive/db/0000000100000001/000000010000000100000006-1c7e00fd09b9dd11fc2966590b3e3274645dd031.gz, source_compressed = false, destination_compress = true, ignore_missing_source = false, destination_path_create = true, modification_time = [undef], mode = [undef], user = [undef], group = [undef]
DEBUG: 1 WAL segments were transferred, calling Archive->xfer() again
DEBUG: File->manifest: db:absolute:[TEST_PATH]/local/archive/db/out
DEBUG: no archive logs to be copied to backup
DEBUG: no more WAL segments to transfer - exiting
DEBUG: safe exit called, terminating threads