db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-09-16 09:06:18 +02:00
Code Issues Releases Activity

Lots of archive-pull plumbing

Browse Source
This commit is contained in:
David Steele
2014-02-12 20:28:27 -05:00
parent 16df430b73
commit 8e60a3111f
2 changed files with 126 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
pg_backrest.pl
View File

@@ -7,6 +7,7 @@ use File::Basename;
use Getopt::Long;
use Config::IniFiles;
use Carp;
use Fcntl qw(:DEFAULT :flock);
use lib dirname($0);
use pg_backrest_utility;
@@ -25,11 +26,22 @@ use constant
use constant
{
CONFIG_SECTION_COMMAND => "command",
CONFIG_SECTION_COMMAND => "command",
CONFIG_SECTION_COMMAND_OPTION => "command:option",
CONFIG_SECTION_BACKUP => "backup",
CONFIG_SECTION_ARCHIVE => "archive",
CONFIG_SECTION_RETENTION => "retention",
CONFIG_SECTION_BACKUP => "backup",
CONFIG_SECTION_ARCHIVE => "archive",
CONFIG_SECTION_RETENTION => "retention",
CONFIG_SECTION_STANZA => "stanza",
CONFIG_KEY_USER => "user",
CONFIG_KEY_HOST => "host",
CONFIG_KEY_PATH => "path",
CONFIG_KEY_COMPRESS => "compress",
CONFIG_KEY_DECOMPRESS => "decompress",
CONFIG_KEY_CHECKSUM => "checksum",
CONFIG_KEY_MANIFEST => "manifest",
CONFIG_KEY_PSQL => "psql"
};
# Command line parameters
@@ -64,7 +76,7 @@ sub config_load
my $strValue;
# Look in the default stanza section
if ($strSection eq "stanza")
if ($strSection eq CONFIG_SECTION_STANZA)
{
$strValue = $oConfig{"${strStanza}"}{"${strKey}"};
}
@@ -154,28 +166,28 @@ if (!defined($strStanza))
}
####################################################################################################################################
# ARCHIVE-PUSH and ARCHIVE-PULL Command
# ARCHIVE-PUSH Command
####################################################################################################################################
if ($strOperation eq OP_ARCHIVE_PUSH)
{
# If an archive section has been defined, use that instead of the backup section when operation is OP_ARCHIVE_PUSH
my $strSection = defined(config_load(CONFIG_SECTION_ARCHIVE, "path")) ? CONFIG_SECTION_ARCHIVE : CONFIG_SECTION_BACKUP;
my $strSection = defined(config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH)) ? CONFIG_SECTION_ARCHIVE : CONFIG_SECTION_BACKUP;
# Get the operational flags
my $bCompress = config_load($strSection, "compress", true, "y") eq "y" ? true : false;
my $bChecksum = config_load($strSection, "checksum", true, "y") eq "y" ? true : false;
my $bCompress = config_load($strSection, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
my $bChecksum = config_load($strSection, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
# Run file_init_archive - this is the minimal config needed to run archiving
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load($strSection, "user"),
strBackupHost => config_load($strSection, "host"),
strBackupPath => config_load($strSection, "path", true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, "checksum", $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, "compress", $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, "decompress", $bCompress)
strBackupUser => config_load($strSection, CONFIG_KEY_USER),
strBackupHost => config_load($strSection, CONFIG_KEY_HOST),
strBackupPath => config_load($strSection, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress)
);
backup_init
@@ -187,28 +199,62 @@ if ($strOperation eq OP_ARCHIVE_PUSH)
!$bChecksum
);
# Call the archive function
if ($strOperation eq OP_ARCHIVE_PUSH)
# Call the archive_push function
if (!defined($ARGV[1]))
{
if (!defined($ARGV[1]))
{
confess &log(ERROR, "source archive file not provided - show usage");
}
archive_push($ARGV[1]);
if ($strSection eq CONFIG_SECTION_ARCHIVE && defined(config_load($strSection, "host")))
{
if (fork())
{
exit 0;
}
}
sleep(5);
&log(INFO, "GOT HERE");
confess &log(ERROR, "source archive file not provided - show usage");
}
archive_push($ARGV[1]);
# Is backup section defined? If so fork, exit parent, and push archive logs to the backup server asynchronously afterwards
if ($strSection eq CONFIG_SECTION_ARCHIVE && defined(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST)) && fork())
{
exit 0;
}
# Create a lock file to make sure archive-pull does not run more than once
my $strArchivePath = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH);
my $fLockFile;
sysopen($fLockFile, "${strArchivePath}/${strStanza}/archive.lock", O_WRONLY | O_CREAT) or die;
if (!flock($fLockFile, LOCK_EX | LOCK_NB))
{
&log(INFO, "archive-pull process is already running - exiting");
exit 0
}
# Get the new operational flags
$bCompress = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
$bChecksum = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
# Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file
$oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER),
strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST)
);
backup_init
(
undef,
$oFile,
undef,
undef,
!$bChecksum
);
# Call the archive_pull function
archive_pull($strArchivePath);
exit 0;
}
@@ -234,31 +280,31 @@ elsif ($strType ne "full" && $strType ne "differential" && $strType ne "incremen
}
# Get the operational flags
my $bCompress = config_load(CONFIG_SECTION_BACKUP, "compress", true, "y") eq "y" ? true : false;
my $bChecksum = config_load(CONFIG_SECTION_BACKUP, "checksum", true, "y") eq "y" ? true : false;
my $bCompress = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
my $bChecksum = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
# Run file_init_archive - the rest of the file config required for backup and restore
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load(CONFIG_SECTION_BACKUP, "user"),
strBackupHost => config_load(CONFIG_SECTION_BACKUP, "host"),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, "path", true),
strDbUser => config_load("stanza", "user"),
strDbHost => config_load("stanza", "host"),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, "checksum", $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, "compress", $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, "decompress", $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, "manifest"),
strCommandPsql => config_load(CONFIG_SECTION_COMMAND, "psql")
strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER),
strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER),
strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST),
strCommandPsql => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL)
);
my $oDb = pg_backrest_db->new
(
strDbUser => config_load("stanza", "user"),
strDbHost => config_load("stanza", "host"),
strCommandPsql => config_load(CONFIG_SECTION_COMMAND, "psql")
strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER),
strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST),
strCommandPsql => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL)
);
# Run backup_init - parameters required for backup and restore operations
@@ -278,7 +324,7 @@ backup_init
####################################################################################################################################
if ($strOperation eq OP_BACKUP)
{
backup(config_load("stanza", "path"));
backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH));
$strOperation = OP_EXPIRE;
}

33
pg_backrest_backup.pm
View File

@@ -25,7 +25,7 @@ use pg_backrest_db;
use Exporter qw(import);
our @EXPORT = qw(backup_init archive_push backup backup_expire archive_list_get);
our @EXPORT = qw(backup_init archive_push archive_pull backup backup_expire archive_list_get);
my $oDb;
my $oFile;
@@ -91,6 +91,25 @@ sub archive_push
$bArchiveFile ? undef : true);
}
####################################################################################################################################
# ARCHIVE_PULL
####################################################################################################################################
sub archive_pull
{
my $strArchivePath = shift;
# Load the archive manifest - all the files that need to be pushed
my %oManifestHash = $oFile->manifest_get(PATH_DB_ABSOLUTE, $strArchivePath . "/archive/" . ${oFile}->{strStanza});
foreach my $strFile (sort(keys $oManifestHash{name}))
{
if ($strFile =~ /^[0-F]{16}\/[0-F]{24}.*/)
{
&log(DEBUG, "SHOULD BE LOGGING ${strFile}");
}
}
}
####################################################################################################################################
# BACKUP_REGEXP_GET - Generate a regexp depending on the backups that need to be found
####################################################################################################################################
@@ -667,6 +686,9 @@ sub backup
# Delete files leftover from a partial backup
# !!! do it
# Save the backup conf file first time - so we can see what is happening in the backup
backup_manifest_save($strBackupConfFile, \%oBackupManifest);
# Perform the backup
backup_file($strBackupPath, $strDbClusterPath, \%oBackupManifest);
@@ -677,9 +699,14 @@ sub backup
&log(INFO, 'Backup archive stop: ' . $strArchiveStop);
# If archive logs are required to complete the backup, then fetch them (this is the default)
# If archive logs are required to complete the backup, then fetch them. This is the default, but can be overridden if the
# archive logs are going to a different server. Be careful here because there is no way to verify that the backup will be
# consistent - at least not in this routine.
if ($bArchiveRequired)
{
# Save the backup conf file second time - before getting archive logs in case that fails
backup_manifest_save($strBackupConfFile, \%oBackupManifest);
# After the backup has been stopped, need to make a copy of the archive logs need to make the db consistent
my @stryArchive = archive_list_get($strArchiveStart, $strArchiveStop, $oDb->version_get() < 9.3);
@@ -700,7 +727,7 @@ sub backup
}
}
# Save the backup conf file
# Save the backup conf file final time
backup_manifest_save($strBackupConfFile, \%oBackupManifest);
# Rename the backup tmp path to complete the backup