db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-20 10:10:46 +02:00
Code Issues Releases Activity

Improved archive logging and put in max limit

Browse Source
This commit is contained in:
David Steele
2014-02-16 18:20:55 -05:00
parent 13086e9626
commit 5c695a6522
3 changed files with 100 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pg_backrest.conf
+1
View File
@@ -21,6 +21,7 @@ thread-max=2
[global:archive]
path=/Users/dsteele/test
compress-async=y
archive-max-mb=500
[global:retention]
full_retention=2
pg_backrest.pl
+76 -43
View File
@@ -40,6 +40,7 @@ use constant
CONFIG_KEY_THREAD_MAX => "thread-max",
CONFIG_KEY_HARDLINK => "hardlink",
CONFIG_KEY_ARCHIVE_REQUIRED => "archive-required",
CONFIG_KEY_ARCHIVE_MAX_MB => "archive-max-mb",
CONFIG_KEY_LEVEL_FILE => "level-file",
CONFIG_KEY_LEVEL_CONSOLE => "level-console",
@@ -194,10 +195,37 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
# Get the async compress flag. If compress_async=y then compression is off for the initial push
my $bCompressAsync = config_load($strSection, CONFIG_KEY_COMPRESS_ASYNC, true, "n") eq "n" ? false : true;
# Get the async compress flag. If compress_async=y then compression is off for the initial push
my $strStopFile;
my $strArchivePath;
# If logging locally then create the stop archiving file name
if ($strSection eq CONFIG_SECTION_ARCHIVE)
{
$strArchivePath = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH);
$strStopFile = "${strArchivePath}/lock/${strStanza}-archive.stop";
}
# Perform the archive-push
if ($strOperation eq OP_ARCHIVE_PUSH)
{
# Call the archive_push function
if (!defined($ARGV[1]))
{
confess &log(ERROR, "source archive file not provided - show usage");
}
# If the stop file exists then discard the archive log
if (defined($strStopFile))
{
if (-e $strStopFile)
{
&log(ERROR, "archive stop file exists ($strStopFile), discarding " . basename($ARGV[1]));
exit 0;
}
}
# Make sure that archive-push is running locally
if (defined(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST)))
{
@@ -232,12 +260,6 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
!$bChecksum
);
# Call the archive_push function
if (!defined($ARGV[1]))
{
confess &log(ERROR, "source archive file not provided - show usage");
}
&log(INFO, "pushing archive log " . $ARGV[1] . ($bCompressAsync ? " asynchronously" : ""));
archive_push($ARGV[1]);
@@ -268,7 +290,6 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
}
# Create a lock file to make sure archive-pull does not run more than once
my $strArchivePath = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH);
my $strLockFile = "${strArchivePath}/lock/${strStanza}-archive.lock";
if (!lock_file_create($strLockFile))
@@ -283,15 +304,17 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
# Get the new operational flags
my $bCompress = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
my $bChecksum = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
my $iArchiveMaxMB = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_ARCHIVE_MAX_MB);
# Do async compression
if ($bCompressAsync)
eval
{
# Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file
# Run file_init_archive - this is the minimal config needed to run archive pulling
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => false,
bNoCompression => !$bCompress,
strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER),
strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
@@ -309,43 +332,53 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX)
);
archive_compress($strArchivePath . "/archive/${strStanza}", $strCommand);
}
# Call the archive_pull function Continue to loop as long as there are files to process.
while (archive_pull($strArchivePath . "/archive/${strStanza}", $strStopFile, $strCommand, $iArchiveMaxMB))
{
}
};
# Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER),
strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST)
);
backup_init
(
undef,
$oFile,
undef,
undef,
!$bChecksum,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX)
);
# Call the archive_pull function Continue to loop as long as there are files to process.
while (archive_pull($strArchivePath . "/archive/${strStanza}", $strCommand))
# If there were errors above then start compressing
if ($@)
{
sleep(5);
archive_compress($strArchivePath . "/archive/${strStanza}", $strCommand);
if ($bCompressAsync)
{
&log(ERROR, "error during transfer: $@");
&log(WARN, "errors during transter, starting compression");
# Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => false,
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST)
);
backup_init
(
undef,
$oFile,
undef,
undef,
!$bChecksum,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX)
);
archive_compress($strArchivePath . "/archive/${strStanza}", $strCommand, 256);
}
else
{
confess $@;
}
}
lock_file_remove();
}
exit 0;
}
pg_backrest_backup.pm
+23 -3
View File
@@ -229,7 +229,9 @@ sub archive_push
sub archive_pull
{
my $strArchivePath = shift;
my $strStopFile = shift;
my $strCommand = shift;
my $iArchiveMaxMB = shift;
# Load the archive manifest - all the files that need to be pushed
my %oManifestHash = $oFile[0]->manifest_get(PATH_DB_ABSOLUTE, $strArchivePath);
@@ -250,6 +252,18 @@ sub archive_pull
}
}
if (defined($iArchiveMaxMB))
{
if ($iArchiveMaxMB < int($lFileSize / 1024 / 1024))
{
&log(ERROR, "local archive store has exceeded limit of ${iArchiveMaxMB}MB, archive logs will be discarded");
my $hStopFile;
open($hStopFile, ">", $strStopFile) or confess &log(ERROR, "unable to create stop file file ${strStopFile}");
close($hStopFile);
}
}
if ($lFileTotal == 0)
{
&log(DEBUG, "no archive logs to be copied to backup");
@@ -299,7 +313,7 @@ sub archive_pull
}
}
# Return 1 indicating that processing should continue
# Return number of files indicating that processing should continue
return $lFileTotal;
}
@@ -331,6 +345,7 @@ sub archive_compress
{
my $strArchivePath = shift;
my $strCommand = shift;
my $iFileCompressMax = shift;
# Load the archive manifest - all the files that need to be pushed
my %oManifestHash = $oFile[0]->manifest_get(PATH_DB_ABSOLUTE, $strArchivePath);
@@ -348,6 +363,11 @@ sub archive_compress
$lFileSize += $oManifestHash{name}{"$strFile"}{size};
$lFileTotal++;
if ($lFileTotal >= $iFileCompressMax)
{
last;
}
}
}
@@ -382,8 +402,8 @@ sub archive_compress
$oThread[$iThreadIdx] = threads->create(\&archive_pull_compress_thread, $iThreadIdx, $strArchivePath);
}
# Don't die on an error because we'd still like to try transferring
backup_thread_complete(false);
# Complete the threads
backup_thread_complete();
}
sub archive_pull_compress_thread