db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

Restore can now remap base/tablespace locations.

Browse Source 
Made Restore more modular.
This commit is contained in:
David Steele 2014-12-23 15:03:06 -05:00
parent 653ab486af
commit a96b83beb8
3 changed files with 139 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
bin/pg_backrest.pl
View File

@ -441,6 +441,7 @@ if (operation_get() eq OP_RESTORE)
( (
config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH), config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH),
undef, undef,
param_get(PARAM_REMAP),
$oFile, $oFile,
4, 4,
param_get(PARAM_FORCE) param_get(PARAM_FORCE)

7
lib/BackRest/Config.pm
View File

@ -25,7 +25,7 @@ our @EXPORT = qw(config_load config_key_load operation_get operation_set param_g
BACKUP_TYPE_FULL BACKUP_TYPE_DIFF BACKUP_TYPE_INCR BACKUP_TYPE_FULL BACKUP_TYPE_DIFF BACKUP_TYPE_INCR
PARAM_CONFIG PARAM_STANZA PARAM_TYPE PARAM_NO_START_STOP PARAM_FORCE PARAM_VERSION PARAM_HELP PARAM_CONFIG PARAM_STANZA PARAM_TYPE PARAM_REMAP PARAM_NO_START_STOP PARAM_FORCE PARAM_VERSION PARAM_HELP
PARAM_TEST PARAM_TEST_DELAY PARAM_TEST_NO_FORK PARAM_TEST PARAM_TEST_DELAY PARAM_TEST_NO_FORK
CONFIG_SECTION_COMMAND CONFIG_SECTION_COMMAND_OPTION CONFIG_SECTION_LOG CONFIG_SECTION_BACKUP CONFIG_SECTION_COMMAND CONFIG_SECTION_COMMAND_OPTION CONFIG_SECTION_LOG CONFIG_SECTION_BACKUP
@ -86,6 +86,7 @@ use constant
PARAM_STANZA => 'stanza', PARAM_STANZA => 'stanza',
PARAM_TYPE => 'type', PARAM_TYPE => 'type',
PARAM_NO_START_STOP => 'no-start-stop', PARAM_NO_START_STOP => 'no-start-stop',
PARAM_REMAP => 'remap',
PARAM_FORCE => 'force', PARAM_FORCE => 'force',
PARAM_VERSION => 'version', PARAM_VERSION => 'version',
PARAM_HELP => 'help', PARAM_HELP => 'help',
@ -162,8 +163,8 @@ sub config_load
param_set(PARAM_TEST_DELAY, 5); # Seconds to delay after a test point (default is not enough for manual tests) param_set(PARAM_TEST_DELAY, 5); # Seconds to delay after a test point (default is not enough for manual tests)
# Get command line parameters # Get command line parameters
GetOptions (\%oParam, PARAM_CONFIG . '=s', PARAM_STANZA . '=s', PARAM_TYPE . '=s', PARAM_NO_START_STOP, PARAM_FORCE, GetOptions (\%oParam, PARAM_CONFIG . '=s', PARAM_STANZA . '=s', PARAM_TYPE . '=s', PARAM_REMAP . '=s%', PARAM_NO_START_STOP,
PARAM_VERSION, PARAM_HELP, PARAM_FORCE, PARAM_VERSION, PARAM_HELP,
PARAM_TEST, PARAM_TEST_DELAY . '=s', PARAM_TEST_NO_FORK) PARAM_TEST, PARAM_TEST_DELAY . '=s', PARAM_TEST_NO_FORK)
or pod2usage(2); or pod2usage(2);

180
lib/BackRest/Restore.pm
View File

@ -26,6 +26,7 @@ sub new
my $class = shift; # Class name my $class = shift; # Class name
my $strDbClusterPath = shift; # Database cluster path my $strDbClusterPath = shift; # Database cluster path
my $strBackupPath = shift; # Backup to restore my $strBackupPath = shift; # Backup to restore
my $oRemapRef = shift; # Tablespace remaps
my $oFile = shift; # Default file object my $oFile = shift; # Default file object
my $iThreadTotal = shift; # Total threads to run for restore my $iThreadTotal = shift; # Total threads to run for restore
my $bForce = shift; # Force the restore even if files are present my $bForce = shift; # Force the restore even if files are present
@ -39,6 +40,7 @@ sub new
$self->{oFile} = $oFile; $self->{oFile} = $oFile;
$self->{thread_total} = $iThreadTotal; $self->{thread_total} = $iThreadTotal;
$self->{bForce} = $bForce; $self->{bForce} = $bForce;
$self->{oRemapRef} = $oRemapRef;
# If backup path is not specified then default to latest # If backup path is not specified then default to latest
if (defined($strBackupPath)) if (defined($strBackupPath))
@ -54,22 +56,14 @@ sub new
} }
#################################################################################################################################### ####################################################################################################################################
# RESTORE # MANIFEST_LOAD
# #
# Takes a backup and restores it back to the original or a remapped location. # Loads the backup manifest and performs requested tablespace remaps.
#################################################################################################################################### ####################################################################################################################################
sub restore sub manifest_load
{ {
my $self = shift; # Class hash my $self = shift; # Class hash
my $oManifestRef = shift; # Backup manifest
# Make sure that Postgres is not running
if ($self->{oFile}->exists(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_POSTMASTER_PID))
{
confess &log(ERROR, 'unable to restore while Postgres is running');
}
# Make sure the backup path is valid and load the manifest
my %oManifest;
if ($self->{oFile}->exists(PATH_BACKUP_CLUSTER, $self->{strBackupPath})) if ($self->{oFile}->exists(PATH_BACKUP_CLUSTER, $self->{strBackupPath}))
{ {
@ -78,39 +72,81 @@ sub restore
PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST); PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST);
# Load the manifest into a hash # Load the manifest into a hash
ini_load($self->{oFile}->path_get(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST), \%oManifest); ini_load($self->{oFile}->path_get(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST), $oManifestRef);
# Remove the manifest now that it is in memory # Remove the manifest now that it is in memory
$self->{oFile}->remove(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST); $self->{oFile}->remove(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_MANIFEST);
# Set source compression # If tablespaces have been remapped, update the manifest
if ($oManifest{'backup:option'}{compress} eq 'y') if (defined($self->{oRemapRef}))
{ {
$self->{bSourceCompressed} = true; foreach my $strPathKey (sort(keys $self->{oRemapRef}))
{
my $strRemapPath = ${$self->{oRemapRef}}{$strPathKey};
if ($strPathKey eq 'base')
{
&log(INFO, "remapping base to ${strRemapPath}");
${$oManifestRef}{'backup:path'}{$strPathKey} = $strRemapPath;
}
else
{
# If the tablespace beigns with prefix 'tablespace:' then strip the prefix. This only needs to be used in
# the case that there is a tablespace called 'base'
if (index($strPathKey, 'tablespace:') == 0)
{
$strPathKey = substr($strPathKey, length('tablespace:'));
}
# Make sure that the tablespace exists in the manifest
if (!defined(${$oManifestRef}{'backup:tablespace'}{$strPathKey}))
{
confess &log(ERROR, "cannot remap invalid tablespace ${strPathKey} to ${strRemapPath}");
}
# Remap the tablespace in the manifest
&log(INFO, "remapping tablespace to ${strRemapPath}");
my $strTablespaceLink = ${$oManifestRef}{'backup:tablespace'}{$strPathKey}{link};
${$oManifestRef}{'backup:path'}{"tablespace:${strPathKey}"} = $strRemapPath;
${$oManifestRef}{'backup:tablespace'}{$strPathKey}{path} = $strRemapPath;
${$oManifestRef}{'base:link'}{"pg_tblspc/${strTablespaceLink}"}{link_destination} = $strRemapPath;
}
}
} }
} }
else else
{ {
confess &log(ERROR, 'backup ' . $self->{strBackupPath} . ' does not exist'); confess &log(ERROR, 'backup ' . $self->{strBackupPath} . ' does not exist');
} }
}
# Declare thread queues that will be used to process files ####################################################################################################################################
my @oyRestoreQueue; # CLEAN
#
# Checks that the restore paths are empty, or if --force was used then it cleans files/paths/links from the restore directories that
# are not present in the manifest.
####################################################################################################################################
sub clean
{
my $self = shift; # Class hash
my $oManifestRef = shift; # Backup manifest
# Check each restore directory in the manifest and make sure that it exists and is empty. # Check each restore directory in the manifest and make sure that it exists and is empty.
# The --force option can be used to override the empty requirement. # The --force option can be used to override the empty requirement.
foreach my $strPathKey (sort(keys $oManifest{'backup:path'})) foreach my $strPathKey (sort(keys ${$oManifestRef}{'backup:path'}))
{ {
my $strPath = $oManifest{'backup:path'}{$strPathKey}; my $strPath = ${$oManifestRef}{'backup:path'}{$strPathKey};
&log(INFO, "processing db path ${strPath}"); &log(INFO, "checking/cleaning db path ${strPath}");
if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, $strPath)) if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, $strPath))
{ {
confess &log(ERROR, "required db path '${strPath}' does not exist"); confess &log(ERROR, "required db path '${strPath}' does not exist");
} }
# Load path manifest so it can be compared to delete files/paths/links that are not in the backup # Load path manifest so it can be compared to deleted files/paths/links that are not in the backup
my %oPathManifest; my %oPathManifest;
$self->{oFile}->manifest(PATH_DB_ABSOLUTE, $strPath, \%oPathManifest); $self->{oFile}->manifest(PATH_DB_ABSOLUTE, $strPath, \%oPathManifest);
@ -143,9 +179,9 @@ sub restore
} }
# Check to see if the file/path/link exists in the manifest # Check to see if the file/path/link exists in the manifest
if (defined($oManifest{"${strPathKey}:${strType}"}{$strName})) if (defined(${$oManifestRef}{"${strPathKey}:${strType}"}{$strName}))
{ {
my $strMode = $oManifest{"${strPathKey}:${strType}"}{$strName}{permission}; my $strMode = ${$oManifestRef}{"${strPathKey}:${strType}"}{$strName}{permission};
# If file/path mode does not match, fix it # If file/path mode does not match, fix it
if ($strType ne 'link' && $strMode ne $oPathManifest{name}{$strName}{permission}) if ($strType ne 'link' && $strMode ne $oPathManifest{name}{$strName}{permission})
@ -156,8 +192,8 @@ sub restore
or confess 'unable to set mode ${strMode} for ${strFile}'; or confess 'unable to set mode ${strMode} for ${strFile}';
} }
my $strUser = $oManifest{"${strPathKey}:${strType}"}{$strName}{user}; my $strUser = ${$oManifestRef}{"${strPathKey}:${strType}"}{$strName}{user};
my $strGroup = $oManifest{"${strPathKey}:${strType}"}{$strName}{group}; my $strGroup = ${$oManifestRef}{"${strPathKey}:${strType}"}{$strName}{group};
# If ownership does not match, fix it # If ownership does not match, fix it
if ($strUser ne $oPathManifest{name}{$strName}{user} || if ($strUser ne $oPathManifest{name}{$strName}{user} ||
@ -169,7 +205,7 @@ sub restore
} }
# If a link does not have the same destination, then delete it (it will be recreated later) # If a link does not have the same destination, then delete it (it will be recreated later)
if ($strType eq 'link' && $oManifest{"${strPathKey}:${strType}"}{$strName}{link_destination} ne if ($strType eq 'link' && ${$oManifestRef}{"${strPathKey}:${strType}"}{$strName}{link_destination} ne
$oPathManifest{name}{$strName}{link_destination}) $oPathManifest{name}{$strName}{link_destination})
{ {
&log(DEBUG, "removing link ${strFile} - destination changed"); &log(DEBUG, "removing link ${strFile} - destination changed");
@ -193,9 +229,26 @@ sub restore
} }
} }
} }
}
}
####################################################################################################################################
# BUILD
#
# Creates missing paths and links and corrects ownership/mode on existing paths and links.
####################################################################################################################################
sub build
{
my $self = shift; # Class hash
my $oManifestRef = shift; # Backup manifest
# Build paths/links in each restore path
foreach my $strPathKey (sort(keys ${$oManifestRef}{'backup:path'}))
{
my $strPath = ${$oManifestRef}{'backup:path'}{$strPathKey};
# Create all paths in the manifest that do not already exist # Create all paths in the manifest that do not already exist
foreach my $strName (sort (keys $oManifest{"${strPathKey}:path"})) foreach my $strName (sort (keys ${$oManifestRef}{"${strPathKey}:path"}))
{ {
# Skip the root path # Skip the root path
if ($strName eq '.') if ($strName eq '.')
@ -205,23 +258,55 @@ sub restore
# Create the Path # Create the Path
$self->{oFile}->path_create(PATH_DB_ABSOLUTE, "${strPath}/${strName}", $self->{oFile}->path_create(PATH_DB_ABSOLUTE, "${strPath}/${strName}",
$oManifest{"${strPathKey}:path"}{$strName}{permission}); ${$oManifestRef}{"${strPathKey}:path"}{$strName}{permission});
} }
# Create all links in the manifest that do not already exist # Create all links in the manifest that do not already exist
if (defined($oManifest{"${strPathKey}:link"})) if (defined(${$oManifestRef}{"${strPathKey}:link"}))
{ {
foreach my $strName (sort (keys $oManifest{"${strPathKey}:link"})) foreach my $strName (sort (keys ${$oManifestRef}{"${strPathKey}:link"}))
{ {
if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, "${strPath}/${strName}")) if (!$self->{oFile}->exists(PATH_DB_ABSOLUTE, "${strPath}/${strName}"))
{ {
$self->{oFile}->link_create(PATH_DB_ABSOLUTE, $oManifest{"${strPathKey}:link"}{$strName}{link_destination}, $self->{oFile}->link_create(PATH_DB_ABSOLUTE,
${$oManifestRef}{"${strPathKey}:link"}{$strName}{link_destination},
PATH_DB_ABSOLUTE, "${strPath}/${strName}"); PATH_DB_ABSOLUTE, "${strPath}/${strName}");
} }
} }
} }
}
}
# Assign the files in each path to a thread queue ####################################################################################################################################
# RESTORE
#
# Takes a backup and restores it back to the original or a remapped location.
####################################################################################################################################
sub restore
{
my $self = shift; # Class hash
# Make sure that Postgres is not running
if ($self->{oFile}->exists(PATH_DB_ABSOLUTE, $self->{strDbClusterPath} . '/' . FILE_POSTMASTER_PID))
{
confess &log(ERROR, 'unable to restore while Postgres is running');
}
# Make sure the backup path is valid and load the manifest
my %oManifest;
$self->manifest_load(\%oManifest);
# Clean the restore paths
$self->clean(\%oManifest);
# Build paths/links in the restore paths
$self->build(\%oManifest);
# Assign the files in each path to a thread queue
my @oyRestoreQueue;
foreach my $strPathKey (sort(keys $oManifest{'backup:path'}))
{
if (defined($oManifest{"${strPathKey}:file"})) if (defined($oManifest{"${strPathKey}:file"}))
{ {
$oyRestoreQueue[@oyRestoreQueue] = Thread::Queue->new(); $oyRestoreQueue[@oyRestoreQueue] = Thread::Queue->new();
@ -255,27 +340,30 @@ sub restore_thread
{ {
my $self = shift; # Class hash my $self = shift; # Class hash
my $iThreadIdx = shift; # Defines the index of this thread my $iThreadIdx = shift; # Defines the index of this thread
my @oyRestoreQueue = @{+shift}; # Restore queues my $oyRestoreQueueRef = shift; # Restore queues
my %oManifest = %{+shift}; # Backup manifest my $oManifestRef = shift; # Backup manifest
my $iDirection = $iThreadIdx % 2 == 0 ? 1 : -1; # Size of files currently copied by this thread my $iDirection = $iThreadIdx % 2 == 0 ? 1 : -1; # Size of files currently copied by this thread
my $oFileThread = $self->{oFile}->clone($iThreadIdx); # Thread local file object my $oFileThread = $self->{oFile}->clone($iThreadIdx); # Thread local file object
# Initialize the starting and current queue index based in the total number of threads in relation to this thread # Initialize the starting and current queue index based in the total number of threads in relation to this thread
my $iQueueStartIdx = int((@oyRestoreQueue / $self->{thread_total}) * $iThreadIdx); my $iQueueStartIdx = int((@{$oyRestoreQueueRef} / $self->{thread_total}) * $iThreadIdx);
my $iQueueIdx = $iQueueStartIdx; my $iQueueIdx = $iQueueStartIdx;
# Set source compression
my $bSourceCompression = ${$oManifestRef}{'backup:option'}{compress} eq 'y';
# When a KILL signal is received, immediately abort # When a KILL signal is received, immediately abort
$SIG{'KILL'} = sub {threads->exit();}; $SIG{'KILL'} = sub {threads->exit();};
# Loop through all the queues to restore files (exit when the original queue is reached # Loop through all the queues to restore files (exit when the original queue is reached
do do
{ {
while (my $strMessage = $oyRestoreQueue[$iQueueIdx]->dequeue()) while (my $strMessage = ${$oyRestoreQueueRef}[$iQueueIdx]->dequeue())
{ {
my $strSourcePath = (split(/\|/, $strMessage))[0]; # Source path from backup my $strSourcePath = (split(/\|/, $strMessage))[0]; # Source path from backup
my $strSection = "${strSourcePath}:file"; # Backup section with file info my $strSection = "${strSourcePath}:file"; # Backup section with file info
my $strDestinationPath = $oManifest{'backup:path'}{$strSourcePath}; # Destination path stored in manifest my $strDestinationPath = ${$oManifestRef}{'backup:path'}{$strSourcePath}; # Destination path stored in manifest
$strSourcePath =~ s/\:/\//g; # Replace : with / in source path $strSourcePath =~ s/\:/\//g; # Replace : with / in source path
my $strName = (split(/\|/, $strMessage))[1]; # Name of file to be restored my $strName = (split(/\|/, $strMessage))[1]; # Name of file to be restored
@ -283,7 +371,7 @@ sub restore_thread
my $strDestinationFile = $oFileThread->path_get(PATH_DB_ABSOLUTE, "${strDestinationPath}/${strName}"); my $strDestinationFile = $oFileThread->path_get(PATH_DB_ABSOLUTE, "${strDestinationPath}/${strName}");
# If checksum is set the destination file already exists, try a checksum before copying # If checksum is set the destination file already exists, try a checksum before copying
my $strChecksum = $oManifest{$strSection}{$strName}{checksum}; my $strChecksum = ${$oManifestRef}{$strSection}{$strName}{checksum};
if ($oFileThread->exists(PATH_DB_ABSOLUTE, $strDestinationFile)) if ($oFileThread->exists(PATH_DB_ABSOLUTE, $strDestinationFile))
{ {
@ -298,12 +386,12 @@ sub restore_thread
# Copy the file from the backup to the database # Copy the file from the backup to the database
$oFileThread->copy(PATH_BACKUP_CLUSTER, $self->{strBackupPath} . "/${strSourcePath}/${strName}" . $oFileThread->copy(PATH_BACKUP_CLUSTER, $self->{strBackupPath} . "/${strSourcePath}/${strName}" .
($self->{bSourceCompressed} ? '.' . $oFileThread->{strCompressExtension} : ''), ($bSourceCompression ? '.' . $oFileThread->{strCompressExtension} : ''),
PATH_DB_ABSOLUTE, $strDestinationFile, PATH_DB_ABSOLUTE, $strDestinationFile,
$self->{bSourceCompressed}, # Source is compressed based on backup settings $bSourceCompression, # Source is compressed based on backup settings
undef, undef, undef, undef,
$oManifest{$strSection}{$strName}{modification_time}, ${$oManifestRef}{$strSection}{$strName}{modification_time},
$oManifest{$strSection}{$strName}{permission}); ${$oManifestRef}{$strSection}{$strName}{permission});
} }
# Even number threads move up when they have finished a queue, odd numbered threads move down # Even number threads move up when they have finished a queue, odd numbered threads move down
@ -312,9 +400,9 @@ sub restore_thread
# Reset the queue index when it goes over or under the number of queues # Reset the queue index when it goes over or under the number of queues
if ($iQueueIdx < 0) if ($iQueueIdx < 0)
{ {
$iQueueIdx = @oyRestoreQueue - 1; $iQueueIdx = @{$oyRestoreQueueRef} - 1;
} }
elsif ($iQueueIdx >= @oyRestoreQueue) elsif ($iQueueIdx >= @{$oyRestoreQueueRef})
{ {
$iQueueIdx = 0; $iQueueIdx = 0;
} }