You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-11-06 08:49:29 +02:00
New implementation for archive logging
This commit is contained in:
David Steele
204
pg_backrest.pl
204
pg_backrest.pl
@@ -105,7 +105,6 @@ sub backup_type_find
|
||||
my @stryFile = sort {$b cmp $a} grep(/^[0-F]{8}\-[0-F]{6}F\_[0-F]{8}\-[0-F]{6}(I|D)$/i, readdir $hDir);
|
||||
close $hDir;
|
||||
|
||||
#print "incremental: @stryFile\n";
|
||||
$strDirectory = $stryFile[0];
|
||||
}
|
||||
|
||||
@@ -115,7 +114,6 @@ sub backup_type_find
|
||||
my @stryFile = sort {$b cmp $a} grep(/^[0-F]{8}\-[0-F]{6}F$/i, readdir $hDir);
|
||||
close $hDir;
|
||||
|
||||
#print "next: @stryFile\n";
|
||||
$strDirectory = $stryFile[0];
|
||||
}
|
||||
|
||||
@@ -576,97 +574,6 @@ my $strCommandCompress = config_load(\%oConfig, "command", "compress", !$bNoComp
|
||||
my $strCommandDecompress = config_load(\%oConfig, "command", "decompress", !$bNoCompression);
|
||||
my $strCommandCopy = config_load(\%oConfig, "command", "copy", $bNoCompression);
|
||||
|
||||
####################################################################################################################################
|
||||
# ARCHIVE-PUSH Command
|
||||
####################################################################################################################################
|
||||
if ($strOperation eq "archive-push")
|
||||
{
|
||||
# archive-push command must have three arguments
|
||||
if (@ARGV != 3)
|
||||
{
|
||||
die "not enough arguments - show usage";
|
||||
}
|
||||
|
||||
# Get the source dir/file
|
||||
my $strSourceFile = $ARGV[1];
|
||||
|
||||
unless (-e $strSourceFile)
|
||||
{
|
||||
die "source file does not exist - show usage";
|
||||
}
|
||||
|
||||
# Get the destination dir/file
|
||||
my $strDestinationFile = $ARGV[2];
|
||||
|
||||
# Make sure the destination directory exists
|
||||
unless (-e dirname($strDestinationFile))
|
||||
{
|
||||
die "destination dir does not exist - show usage";
|
||||
}
|
||||
|
||||
# Make sure the destination file does NOT exist - ignore checksum and extension in case they (or options) have changed
|
||||
if (glob("$strDestinationFile*"))
|
||||
{
|
||||
die "destination file already exists";
|
||||
}
|
||||
|
||||
# Setup the copy command
|
||||
my $strCommand = "";
|
||||
|
||||
# !!! Modify this to skip compression and checksum for any file that is not a log file
|
||||
if ($strSourceFile =~ /\.backup$/)
|
||||
{
|
||||
$strCommand = $strCommandCopy;
|
||||
$strCommand =~ s/\%source\%/$strSourceFile/g;
|
||||
$strCommand =~ s/\%destination\%/$strDestinationFile/g;
|
||||
}
|
||||
else
|
||||
{
|
||||
# Calculate sha1 hash for the file (unless disabled)
|
||||
if (!$bNoChecksum)
|
||||
{
|
||||
$strDestinationFile .= "-" . file_hash_get($strCommandChecksum, $strSourceFile);
|
||||
}
|
||||
|
||||
if ($bNoCompression)
|
||||
{
|
||||
$strCommand = $strCommandCopy;
|
||||
$strCommand =~ s/\%source\%/$strSourceFile/g;
|
||||
$strCommand =~ s/\%destination\%/$strDestinationFile/g;
|
||||
}
|
||||
else
|
||||
{
|
||||
$strCommand = $strCommandCompress;
|
||||
$strCommand =~ s/\%file\%/$strSourceFile/g;
|
||||
$strCommand .= " > $strDestinationFile.gz";
|
||||
}
|
||||
}
|
||||
|
||||
# Execute the copy
|
||||
execute($strCommand);
|
||||
|
||||
exit 0;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# GET MORE CONFIG INFO
|
||||
####################################################################################################################################
|
||||
# Check the backup type
|
||||
if ($strType eq "diff")
|
||||
{
|
||||
$strType = "differential";
|
||||
}
|
||||
|
||||
if ($strType eq "incr")
|
||||
{
|
||||
$strType = "incremental";
|
||||
}
|
||||
|
||||
if ($strType ne "full" && $strType ne "differential" && $strType ne "incremental")
|
||||
{
|
||||
die &log(ERROR, "backup type must be full, differential (diff), incremental (incr)");
|
||||
}
|
||||
|
||||
# Load and check the base backup path
|
||||
my $strBasePath = $oConfig{common}{backup_path};
|
||||
|
||||
@@ -694,6 +601,117 @@ unless (-e $strBackupClusterPath)
|
||||
mkdir $strBackupClusterPath or die &log(ERROR, "cluster backup path '${strBackupClusterPath}' create failed");
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# ARCHIVE-PUSH Command
|
||||
####################################################################################################################################
|
||||
if ($strOperation eq "archive-push")
|
||||
{
|
||||
# archive-push command must have three arguments
|
||||
if (@ARGV != 2)
|
||||
{
|
||||
die "not enough arguments - show usage";
|
||||
}
|
||||
|
||||
my $strBackupClusterArchivePath = "${strBackupClusterPath}/archive";
|
||||
|
||||
unless (-e $strBackupClusterArchivePath)
|
||||
{
|
||||
&log (INFO, "creating cluster archive path ${strBackupClusterArchivePath}");
|
||||
mkdir $strBackupClusterArchivePath or die &log(ERROR, "cluster backup archive path '${strBackupClusterArchivePath}' create failed");
|
||||
}
|
||||
|
||||
# Get the source dir/file
|
||||
my $strSourceFile = $ARGV[1];
|
||||
|
||||
unless (-e $strSourceFile)
|
||||
{
|
||||
die "source file does not exist - show usage";
|
||||
}
|
||||
|
||||
# Get the destination dir/file
|
||||
my $strDestinationFile = basename($strSourceFile);
|
||||
my $strDestinationTmpFile = "${strBackupClusterArchivePath}/archive.tmp";
|
||||
my $strBackupClusterArchiveSubPath = "${strBackupClusterArchivePath}";
|
||||
|
||||
if (-e $strDestinationTmpFile)
|
||||
{
|
||||
unlink($strDestinationTmpFile);
|
||||
}
|
||||
|
||||
# Setup the copy command
|
||||
my $strCommand;
|
||||
|
||||
# !!! Modify this to skip compression and checksum for any file that is not a log file
|
||||
if ($strDestinationFile =~ /^([0-9]|[A-F]){24}$/)
|
||||
{
|
||||
$strBackupClusterArchiveSubPath = "${strBackupClusterArchivePath}/" . substr($strDestinationFile, 0, 16);
|
||||
|
||||
unless (-e $strBackupClusterArchiveSubPath)
|
||||
{
|
||||
&log (INFO, "creating cluster archive sub path ${strBackupClusterArchiveSubPath}");
|
||||
mkdir $strBackupClusterArchiveSubPath or die &log(ERROR, "cluster backup archive sub path '${strBackupClusterArchiveSubPath}' create failed");
|
||||
}
|
||||
|
||||
# Make sure the destination file does NOT exist - ignore checksum and extension in case they (or options) have changed
|
||||
if (glob("${strBackupClusterArchiveSubPath}/${strDestinationFile}*"))
|
||||
{
|
||||
die "destination file already exists";
|
||||
}
|
||||
|
||||
# Calculate sha1 hash for the file (unless disabled)
|
||||
if (!$bNoChecksum)
|
||||
{
|
||||
$strDestinationFile .= "-" . file_hash_get($strCommandChecksum, $strSourceFile);
|
||||
}
|
||||
|
||||
if ($bNoCompression)
|
||||
{
|
||||
$strCommand = $strCommandCopy;
|
||||
$strCommand =~ s/\%source\%/${strSourceFile}/g;
|
||||
$strCommand =~ s/\%destination\%/${strDestinationTmpFile}/g;
|
||||
}
|
||||
else
|
||||
{
|
||||
$strCommand = $strCommandCompress;
|
||||
$strCommand =~ s/\%file\%/${strSourceFile}/g;
|
||||
$strCommand .= " > ${strDestinationTmpFile}";
|
||||
$strDestinationFile .= ".gz";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
$strCommand = $strCommandCopy;
|
||||
$strCommand =~ s/\%source\%/$strSourceFile/g;
|
||||
$strCommand =~ s/\%destination\%/${strDestinationTmpFile}/g;
|
||||
}
|
||||
|
||||
# Execute the copy
|
||||
execute($strCommand);
|
||||
rename($strDestinationTmpFile, "${strBackupClusterArchiveSubPath}/${strDestinationFile}")
|
||||
or die &log(ERROR, "unable to rename '${strBackupClusterArchiveSubPath}/${strDestinationFile}'");
|
||||
|
||||
exit 0;
|
||||
}
|
||||
|
||||
####################################################################################################################################
|
||||
# GET MORE CONFIG INFO
|
||||
####################################################################################################################################
|
||||
# Check the backup type
|
||||
if ($strType eq "diff")
|
||||
{
|
||||
$strType = "differential";
|
||||
}
|
||||
|
||||
if ($strType eq "incr")
|
||||
{
|
||||
$strType = "incremental";
|
||||
}
|
||||
|
||||
if ($strType ne "full" && $strType ne "differential" && $strType ne "incremental")
|
||||
{
|
||||
die &log(ERROR, "backup type must be full, differential (diff), incremental (incr)");
|
||||
}
|
||||
|
||||
# Load commands required for backup
|
||||
my $strCommandManifest = config_load(\%oConfig, "command", "manifest");
|
||||
my $strCommandPsql = config_load(\%oConfig, "command", "psql");
|
||||
|
||||
16
test/test.pl
16
test/test.pl
@@ -44,8 +44,8 @@ sub pg_create
|
||||
execute("mkdir $strTestPath/$strTestDir/ts1");
|
||||
execute("mkdir $strTestPath/$strTestDir/ts2");
|
||||
execute($strPgBinPath . "/initdb -D $strTestPath/$strTestDir/common -A trust -k");
|
||||
execute("mkdir $strTestPath/$strArchiveDir");
|
||||
execute("mkdir $strTestPath/$strBackupDir");
|
||||
# execute("mkdir -p $strTestPath/$strArchiveDir");
|
||||
}
|
||||
|
||||
sub pg_start
|
||||
@@ -99,7 +99,7 @@ sub archive_command_build
|
||||
my $bCompression = shift;
|
||||
my $bChecksum = shift;
|
||||
|
||||
my $strCommand = "$strBackRestBinPath/pg_backrest.pl archive-push --config=$strBackRestBinPath/pg_backrest.conf";
|
||||
my $strCommand = "$strBackRestBinPath/pg_backrest.pl --cluster=db --config=$strBackRestBinPath/pg_backrest.conf";
|
||||
|
||||
if (!$bCompression)
|
||||
{
|
||||
@@ -111,7 +111,7 @@ sub archive_command_build
|
||||
$strCommand .= " --no-checksum"
|
||||
}
|
||||
|
||||
return $strCommand . " %p $strDestinationPath/%f";
|
||||
return $strCommand . " archive-push %p";
|
||||
}
|
||||
|
||||
sub wait_for_file
|
||||
@@ -158,7 +158,7 @@ my $strUser = execute('whoami');
|
||||
|
||||
my $strTestPath = "/Users/dsteele/test";
|
||||
my $strDbDir = "db";
|
||||
my $strArchiveDir = "archive";
|
||||
my $strArchiveDir = "backup/db/archive";
|
||||
my $strBackupDir = "backup";
|
||||
|
||||
my $strPgBinPath = "/Library/PostgreSQL/9.3/bin";
|
||||
@@ -198,8 +198,10 @@ pg_execute($dbh, "create table test_ts2 (id int) tablespace ts1");
|
||||
pg_execute($dbh, "insert into test values (1)");
|
||||
pg_execute($dbh, "select pg_switch_xlog()");
|
||||
|
||||
execute("mkdir -p $strTestPath/$strArchiveDir/0000000100000000");
|
||||
|
||||
# Test for archive log file 000000010000000000000001
|
||||
wait_for_file("$strTestPath/$strArchiveDir", "^000000010000000000000001\$", 5);
|
||||
wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000001\$", 5);
|
||||
|
||||
# Turn on log checksum for the next test
|
||||
$dbh->disconnect();
|
||||
@@ -214,7 +216,7 @@ pg_execute($dbh, "insert into test values (2)");
|
||||
pg_execute($dbh, "select pg_switch_xlog()");
|
||||
|
||||
# Test for archive log file 000000010000000000000002
|
||||
wait_for_file("$strTestPath/$strArchiveDir", "^000000010000000000000002-([a-f]|[0-9]){40}\$", 5);
|
||||
wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000002-([a-f]|[0-9]){40}\$", 5);
|
||||
|
||||
# Turn on log compression and checksum for the next test
|
||||
$dbh->disconnect();
|
||||
@@ -229,7 +231,7 @@ pg_execute($dbh, "insert into test values (3)");
|
||||
pg_execute($dbh, "select pg_switch_xlog()");
|
||||
|
||||
# Test for archive log file 000000010000000000000003
|
||||
wait_for_file("$strTestPath/$strArchiveDir", "^000000010000000000000003-([a-f]|[0-9]){40}\\.gz\$", 5);
|
||||
wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000003-([a-f]|[0-9]){40}\\.gz\$", 5);
|
||||
|
||||
$dbh->disconnect();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user