db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

Working on restore options and integration with unit tests.

Browse Source
This commit is contained in:
David Steele 2014-12-23 18:52:38 -05:00
parent a96b83beb8
commit 9f8f33f957
7 changed files with 157 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
bin/pg_backrest.pl
View File

@ -17,6 +17,7 @@ use Pod::Usage;
use lib dirname($0) . '/../lib';
use BackRest::Utility;
use BackRest::Config;
use BackRest::Remote;
use BackRest::File;
use BackRest::Backup;
use BackRest::Restore;
@ -38,6 +39,7 @@ pg_backrest.pl [options] [operation]
archive-get retrieve an archive file from backup
archive-push push an archive file to backup
backup backup a cluster
restore restore a cluster
expire expire old backups (automatically run after backup)
General Options:
@ -50,7 +52,14 @@ pg_backrest.pl [options] [operation]
--type type of backup to perform (full, diff, incr)
--no-start-stop do not call pg_start/stop_backup(). Postmaster should not be running.
--force force backup when --no-start-stop passed and postmaster.pid exists.
Use with extreme caution as this will produce an inconsistent backup!
Use with extreme caution as this will probably produce an inconsistent backup!
Restore Options:
--set backup set to restore (defaults to latest set).
--remap remaps the base or a tablespace to another path.
--thread # of threads to use for restore (defaults to 1).
--force force restore when destination paths are not empty.
Use with extreme caution as this will delete data in those paths!
=cut
####################################################################################################################################
@ -64,12 +73,12 @@ my $strRemote; # Defines which side is remote, DB or BACKUP
####################################################################################################################################
sub remote_get
{
if (!defined($oRemote) && $strRemote ne REMOTE_NONE)
if (!defined($oRemote) && $strRemote ne NONE)
{
$oRemote = new BackRest::Remote
(
config_key_load($strRemote eq REMOTE_DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST, true),
config_key_load($strRemote eq REMOTE_DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_USER, true),
config_key_load($strRemote eq DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST, true),
config_key_load($strRemote eq DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_USER, true),
config_key_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_REMOTE, true)
);
}
@ -145,7 +154,7 @@ if (param_get(PARAM_HELP))
# First check if backup is remote
if (defined(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST)))
{
$strRemote = REMOTE_BACKUP;
$strRemote = BACKUP;
}
# Else check if db is remote
elsif (defined(config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST)))
@ -156,11 +165,11 @@ elsif (defined(config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST)))
confess &log(ERROR, 'db and backup cannot both be configured as remote');
}
$strRemote = REMOTE_DB;
$strRemote = DB;
}
else
{
$strRemote = REMOTE_NONE;
$strRemote = NONE;
}
####################################################################################################################################
@ -169,7 +178,7 @@ else
if (operation_get() eq OP_ARCHIVE_PUSH)
{
# Make sure the archive push operation happens on the db side
if ($strRemote eq REMOTE_DB)
if ($strRemote eq DB)
{
confess &log(ERROR, 'archive-push operation must run on the db host');
}
@ -219,7 +228,7 @@ if (operation_get() eq OP_ARCHIVE_PUSH)
(
param_get(PARAM_STANZA),
config_key_load($strSection, CONFIG_KEY_PATH, true),
$bArchiveLocal ? REMOTE_NONE : $strRemote,
$bArchiveLocal ? NONE : $strRemote,
$bArchiveLocal ? undef : remote_get()
);
@ -428,22 +437,26 @@ my $oFile = new BackRest::File
####################################################################################################################################
if (operation_get() eq OP_RESTORE)
{
if ($strRemote eq REMOTE_DB)
if ($strRemote eq DB)
{
confess &log(ASSERT, 'restore operation must be performed locally on the db server');
}
# Open the log file
log_file_set(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . '/log/' . param_get(PARAM_STANZA) . '-restore');
log_file_set(config_key_load(CONFIG_SECTION_RESTORE, CONFIG_KEY_PATH, true) . '/log/' . param_get(PARAM_STANZA) . '-restore');
# Set the lock path
my $strLockPath = config_key_load(CONFIG_SECTION_RESTORE, CONFIG_KEY_PATH, true) . '/lock/' .
param_get(PARAM_STANZA) . '-' . operation_get() . '.lock';
# Do the restore
new BackRest::Restore
(
config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH),
undef,
param_get(PARAM_SET),
param_get(PARAM_REMAP),
$oFile,
4,
param_get(PARAM_THREAD),
param_get(PARAM_FORCE)
)->restore;
@ -457,7 +470,7 @@ if (operation_get() eq OP_RESTORE)
log_file_set(config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . '/log/' . param_get(PARAM_STANZA));
# Make sure backup and expire operations happen on the backup side
if ($strRemote eq REMOTE_BACKUP)
if ($strRemote eq BACKUP)
{
confess &log(ERROR, 'backup and expire operations must run on the backup host');
}

31
lib/BackRest/Config.pm
View File

@ -25,11 +25,11 @@ our @EXPORT = qw(config_load config_key_load operation_get operation_set param_g
BACKUP_TYPE_FULL BACKUP_TYPE_DIFF BACKUP_TYPE_INCR
PARAM_CONFIG PARAM_STANZA PARAM_TYPE PARAM_REMAP PARAM_NO_START_STOP PARAM_FORCE PARAM_VERSION PARAM_HELP
PARAM_TEST PARAM_TEST_DELAY PARAM_TEST_NO_FORK
PARAM_CONFIG PARAM_STANZA PARAM_TYPE PARAM_REMAP PARAM_SET PARAM_NO_START_STOP PARAM_THREAD PARAM_FORCE
PARAM_VERSION PARAM_HELP PARAM_TEST PARAM_TEST_DELAY PARAM_TEST_NO_FORK
CONFIG_SECTION_COMMAND CONFIG_SECTION_COMMAND_OPTION CONFIG_SECTION_LOG CONFIG_SECTION_BACKUP
CONFIG_SECTION_ARCHIVE CONFIG_SECTION_RETENTION CONFIG_SECTION_STANZA
CONFIG_SECTION_RESTORE CONFIG_SECTION_ARCHIVE CONFIG_SECTION_RETENTION CONFIG_SECTION_STANZA
CONFIG_KEY_USER CONFIG_KEY_HOST CONFIG_KEY_PATH
@ -87,6 +87,8 @@ use constant
PARAM_TYPE => 'type',
PARAM_NO_START_STOP => 'no-start-stop',
PARAM_REMAP => 'remap',
PARAM_SET => 'set',
PARAM_THREAD => 'thread',
PARAM_FORCE => 'force',
PARAM_VERSION => 'version',
PARAM_HELP => 'help',
@ -105,6 +107,7 @@ use constant
CONFIG_SECTION_COMMAND_OPTION => 'command:option',
CONFIG_SECTION_LOG => 'log',
CONFIG_SECTION_BACKUP => 'backup',
CONFIG_SECTION_RESTORE => 'restore',
CONFIG_SECTION_ARCHIVE => 'archive',
CONFIG_SECTION_RETENTION => 'retention',
CONFIG_SECTION_STANZA => 'stanza',
@ -163,8 +166,8 @@ sub config_load
param_set(PARAM_TEST_DELAY, 5); # Seconds to delay after a test point (default is not enough for manual tests)
# Get command line parameters
GetOptions (\%oParam, PARAM_CONFIG . '=s', PARAM_STANZA . '=s', PARAM_TYPE . '=s', PARAM_REMAP . '=s%', PARAM_NO_START_STOP,
PARAM_FORCE, PARAM_VERSION, PARAM_HELP,
GetOptions (\%oParam, PARAM_CONFIG . '=s', PARAM_STANZA . '=s', PARAM_TYPE . '=s', PARAM_REMAP . '=s%', PARAM_SET . '=s',
PARAM_THREAD . '=s', PARAM_NO_START_STOP, PARAM_FORCE, PARAM_VERSION, PARAM_HELP,
PARAM_TEST, PARAM_TEST_DELAY . '=s', PARAM_TEST_NO_FORK)
or pod2usage(2);
@ -205,6 +208,13 @@ sub config_load
}
}
# Validate thread parameter
if (defined(param_get(PARAM_THREAD)) && !(param_get(PARAM_THREAD) >= 1))
{
confess &log(ERROR, 'thread parameter should be >= 1');
}
# Get configuration parameter and load it
if (!defined(param_get(PARAM_CONFIG)))
{
param_set(PARAM_CONFIG, '/etc/pg_backrest.conf');
@ -218,6 +228,17 @@ sub config_load
confess 'a backup stanza must be specified';
}
# If this is a restore, then try to default config
if (!defined(config_key_load(CONFIG_SECTION_RESTORE, CONFIG_KEY_PATH)))
{
$oConfig{'global:restore'}{path} = config_key_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH);
if (!defined(config_key_load(CONFIG_SECTION_RESTORE, CONFIG_KEY_PATH)))
{
$oConfig{'global:restore'}{path} = config_key_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH);
}
}
# Set the log levels
log_level_set(uc(config_key_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_FILE, true, INFO)),
uc(config_key_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_CONSOLE, true, ERROR)));

18
lib/BackRest/File.pm
View File

@ -35,8 +35,6 @@ our @EXPORT = qw(PATH_ABSOLUTE PATH_DB PATH_DB_ABSOLUTE PATH_BACKUP PATH_BACKUP_
PIPE_STDIN PIPE_STDOUT PIPE_STDERR
REMOTE_DB REMOTE_BACKUP REMOTE_NONE
OP_FILE_LIST OP_FILE_EXISTS OP_FILE_HASH OP_FILE_REMOVE OP_FILE_MANIFEST OP_FILE_COMPRESS
OP_FILE_MOVE OP_FILE_COPY OP_FILE_COPY_OUT OP_FILE_COPY_IN OP_FILE_PATH_CREATE);
@ -80,16 +78,6 @@ use constant
PIPE_STDERR => '<STDERR>'
};
####################################################################################################################################
# Remote Types
####################################################################################################################################
use constant
{
REMOTE_DB => PATH_DB,
REMOTE_BACKUP => PATH_BACKUP,
REMOTE_NONE => 'none'
};
####################################################################################################################################
# Operation constants
####################################################################################################################################
@ -142,12 +130,12 @@ sub new
$self->{iThreadIdx} = $iThreadIdx;
# If remote is defined check parameters and open session
if (defined($self->{strRemote}) && $self->{strRemote} ne REMOTE_NONE)
if (defined($self->{strRemote}) && $self->{strRemote} ne NONE)
{
# Make sure remote is valid
if ($self->{strRemote} ne REMOTE_DB && $self->{strRemote} ne REMOTE_BACKUP)
if ($self->{strRemote} ne DB && $self->{strRemote} ne BACKUP)
{
confess &log(ASSERT, 'strRemote must be "' . REMOTE_DB . '" or "' . REMOTE_BACKUP .
confess &log(ASSERT, 'strRemote must be "' . DB . '" or "' . BACKUP .
"\", $self->{strRemote} was passed");
}

13
lib/BackRest/Remote.pm
View File

@ -19,6 +19,19 @@ use BackRest::Exception;
use BackRest::Utility;
use BackRest::ProcessAsync;
use Exporter qw(import);
our @EXPORT = qw(DB BACKUP NONE);
####################################################################################################################################
# DB/BACKUP Constants
####################################################################################################################################
use constant
{
DB => 'db',
BACKUP => 'backup',
NONE => 'none'
};
####################################################################################################################################
# Remote xfer default block size constant
####################################################################################################################################

79
lib/BackRest/Restore.pm
View File

@ -38,7 +38,7 @@ sub new
# Initialize variables
$self->{strDbClusterPath} = $strDbClusterPath;
$self->{oFile} = $oFile;
$self->{thread_total} = $iThreadTotal;
$self->{thread_total} = defined($iThreadTotal) ? $iThreadTotal : 1;
$self->{bForce} = $bForce;
$self->{oRemapRef} = $oRemapRef;
@ -71,50 +71,50 @@ sub manifest_load
$self->{oFile}->copy(PATH_BACKUP_CLUSTER, $self->{strBackupPath} . '/' . FILE_MANIFEST,
PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST);
# Load the manifest into a hash
ini_load($self->{oFile}->path_get(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST), $oManifestRef);
# Load the manifest into a hash
ini_load($self->{oFile}->path_get(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST), $oManifestRef);
# Remove the manifest now that it is in memory
$self->{oFile}->remove(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST);
# Remove the manifest now that it is in memory
$self->{oFile}->remove(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST);
# If tablespaces have been remapped, update the manifest
if (defined($self->{oRemapRef}))
{
foreach my $strPathKey (sort(keys $self->{oRemapRef}))
{
my $strRemapPath = ${$self->{oRemapRef}}{$strPathKey};
# If tablespaces have been remapped, update the manifest
if (defined($self->{oRemapRef}))
{
foreach my $strPathKey (sort(keys $self->{oRemapRef}))
{
my $strRemapPath = ${$self->{oRemapRef}}{$strPathKey};
if ($strPathKey eq 'base')
{
&log(INFO, "remapping base to ${strRemapPath}");
${$oManifestRef}{'backup:path'}{$strPathKey} = $strRemapPath;
}
else
{
# If the tablespace beigns with prefix 'tablespace:' then strip the prefix. This only needs to be used in
# the case that there is a tablespace called 'base'
if (index($strPathKey, 'tablespace:') == 0)
{
$strPathKey = substr($strPathKey, length('tablespace:'));
}
if ($strPathKey eq 'base')
{
&log(INFO, "remapping base to ${strRemapPath}");
${$oManifestRef}{'backup:path'}{$strPathKey} = $strRemapPath;
}
else
{
# If the tablespace beigns with prefix 'tablespace:' then strip the prefix. This only needs to be used in
# the case that there is a tablespace called 'base'
if (index($strPathKey, 'tablespace:') == 0)
{
$strPathKey = substr($strPathKey, length('tablespace:'));
}
# Make sure that the tablespace exists in the manifest
if (!defined(${$oManifestRef}{'backup:tablespace'}{$strPathKey}))
{
confess &log(ERROR, "cannot remap invalid tablespace ${strPathKey} to ${strRemapPath}");
}
# Make sure that the tablespace exists in the manifest
if (!defined(${$oManifestRef}{'backup:tablespace'}{$strPathKey}))
{
confess &log(ERROR, "cannot remap invalid tablespace ${strPathKey} to ${strRemapPath}");
}
# Remap the tablespace in the manifest
&log(INFO, "remapping tablespace to ${strRemapPath}");
# Remap the tablespace in the manifest
&log(INFO, "remapping tablespace to ${strRemapPath}");
my $strTablespaceLink = ${$oManifestRef}{'backup:tablespace'}{$strPathKey}{link};
my $strTablespaceLink = ${$oManifestRef}{'backup:tablespace'}{$strPathKey}{link};
${$oManifestRef}{'backup:path'}{"tablespace:${strPathKey}"} = $strRemapPath;
${$oManifestRef}{'backup:tablespace'}{$strPathKey}{path} = $strRemapPath;
${$oManifestRef}{'base:link'}{"pg_tblspc/${strTablespaceLink}"}{link_destination} = $strRemapPath;
}
}
}
${$oManifestRef}{'backup:path'}{"tablespace:${strPathKey}"} = $strRemapPath;
${$oManifestRef}{'backup:tablespace'}{$strPathKey}{path} = $strRemapPath;
${$oManifestRef}{'base:link'}{"pg_tblspc/${strTablespaceLink}"}{link_destination} = $strRemapPath;
}
}
}
}
else
{
@ -292,6 +292,9 @@ sub restore
confess &log(ERROR, 'unable to restore while Postgres is running');
}
# Log the backup set to restore
&log(INFO, "Restoring backup set " . $self->{strBackupPath});
# Make sure the backup path is valid and load the manifest
my %oManifest;
$self->manifest_load(\%oManifest);

56
test/lib/BackRestTest/BackupTest.pm
View File

@ -809,7 +809,7 @@ sub BackRestTestBackup_Test
}
BackRestTestCommon_ConfigCreate('db',
($bRemote ? REMOTE_BACKUP : undef),
($bRemote ? BACKUP : undef),
$bCompress,
$bChecksum, # checksum
undef, # hardlink
@ -937,14 +937,14 @@ sub BackRestTestBackup_Test
$bCreate = false;
}
BackRestTestCommon_ConfigCreate('db', # local
($bRemote ? REMOTE_BACKUP : undef), # remote
$bCompress, # compress
$bChecksum, # checksum
undef, # hardlink
undef, # thread-max
undef, # archive-async
undef); # compress-async
BackRestTestCommon_ConfigCreate('db', # local
($bRemote ? BACKUP : undef), # remote
$bCompress, # compress
$bChecksum, # checksum
undef, # hardlink
undef, # thread-max
undef, # archive-async
undef); # compress-async
my $strCommand = BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_DbPathGet() .
'/pg_backrest.conf --stanza=db archive-get';
@ -1099,26 +1099,26 @@ sub BackRestTestBackup_Test
}
# Create db config
BackRestTestCommon_ConfigCreate('db', # local
$bRemote ? REMOTE_BACKUP : undef, # remote
$bCompress, # compress
$bChecksum, # checksum
$bRemote ? undef : $bHardlink, # hardlink
$bRemote ? undef : $iThreadMax, # thread-max
undef, # archive-async
undef); # compress-async
BackRestTestCommon_ConfigCreate('db', # local
$bRemote ? BACKUP : undef, # remote
$bCompress, # compress
$bChecksum, # checksum
$bRemote ? undef : $bHardlink, # hardlink
$iThreadMax, # thread-max
undef, # archive-async
undef); # compress-async
# Create backup config
if ($bRemote)
{
BackRestTestCommon_ConfigCreate('backup', # local
$bRemote ? REMOTE_DB : undef, # remote
$bCompress, # compress
$bChecksum, # checksum
$bHardlink, # hardlink
$iThreadMax, # thread-max
undef, # archive-async
undef); # compress-async
BackRestTestCommon_ConfigCreate('backup', # local
$bRemote ? DB : undef, # remote
$bCompress, # compress
$bChecksum, # checksum
$bHardlink, # hardlink
$iThreadMax, # thread-max
undef, # archive-async
undef); # compress-async
}
# Create the backup command
@ -1390,11 +1390,11 @@ sub BackRestTestBackup_Test
# Create db config
BackRestTestCommon_ConfigCreate('db', # local
$bRemote ? REMOTE_BACKUP : undef, # remote
$bRemote ? BACKUP : undef, # remote
false, # compress
false, # checksum
$bRemote ? undef : true, # hardlink
$bRemote ? undef : $iThreadMax, # thread-max
$iThreadMax, # thread-max
$bArchiveAsync, # archive-async
undef); # compress-async
@ -1402,7 +1402,7 @@ sub BackRestTestBackup_Test
if ($bRemote)
{
BackRestTestCommon_ConfigCreate('backup', # local
$bRemote ? REMOTE_DB : undef, # remote
$bRemote ? DB : undef, # remote
false, # compress
false, # checksum
true, # hardlink

34
test/lib/BackRestTest/CommonTest.pm
View File

@ -20,6 +20,7 @@ use IO::Select;
use lib dirname($0) . '/../lib';
use BackRest::Utility;
use BackRest::Remote;
use BackRest::File;
use Exporter qw(import);
@ -387,12 +388,12 @@ sub BackRestTestCommon_ConfigCreate
$oParamHash{'global:command'}{'psql'} = $strCommonCommandPsql;
if (defined($strRemote) && $strRemote eq REMOTE_BACKUP)
if (defined($strRemote) && $strRemote eq BACKUP)
{
$oParamHash{'global:backup'}{'host'} = $strCommonHost;
$oParamHash{'global:backup'}{'user'} = $strCommonUserBackRest;
}
elsif (defined($strRemote) && $strRemote eq REMOTE_DB)
elsif (defined($strRemote) && $strRemote eq DB)
{
$oParamHash{$strCommonStanza}{'host'} = $strCommonHost;
$oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
@ -401,19 +402,21 @@ sub BackRestTestCommon_ConfigCreate
$oParamHash{'global:log'}{'level-console'} = 'error';
$oParamHash{'global:log'}{'level-file'} = 'trace';
if (defined($bHardlink) && !$bHardlink)
{
$oParamHash{'global:backup'}{'hardlink'} = 'n';
}
if ($strLocal eq REMOTE_BACKUP)
if ($strLocal eq BACKUP)
{
}
elsif ($strLocal eq REMOTE_DB)
elsif ($strLocal eq DB)
{
if (defined($strRemote))
{
$oParamHash{'global:log'}{'level-console'} = 'trace';
if (!$bArchiveLocal)
{
$oParamHash{'global:restore'}{path} = BackRestTestCommon_ArchivePathGet();
}
$oParamHash{'global:restore'}{'thread-max'} = $iThreadMax;
}
if ($bArchiveLocal)
@ -431,9 +434,15 @@ sub BackRestTestCommon_ConfigCreate
confess "invalid local type ${strLocal}";
}
if (($strLocal eq REMOTE_BACKUP) || ($strLocal eq REMOTE_DB && !defined($strRemote)))
if (($strLocal eq BACKUP) || ($strLocal eq DB && !defined($strRemote)))
{
$oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}";
$oParamHash{'global:backup'}{'thread-max'} = $iThreadMax;
if (defined($bHardlink) && !$bHardlink)
{
$oParamHash{'global:backup'}{'hardlink'} = 'n';
}
}
if (defined($bCompress) && !$bCompress)
@ -449,11 +458,6 @@ sub BackRestTestCommon_ConfigCreate
$oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath;
$oParamHash{'global:backup'}{'path'} = $strCommonBackupPath;
if (defined($iThreadMax))
{
$oParamHash{'global:backup'}{'thread-max'} = $iThreadMax;
}
# Write out the configuration file
my $strFile = BackRestTestCommon_TestPathGet() . '/pg_backrest.conf';
ini_save($strFile, \%oParamHash);