db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-20 10:10:46 +02:00
Code Issues Releases Activity

Backup handles files that were removed by the db

Browse Source
This commit is contained in:
David Steele
2014-02-18 15:42:51 -05:00
parent 187fdaf49b
commit 27b820cfb4
3 changed files with 135 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pg_backrest_backup.pm
+32 -15
View File
@@ -1045,13 +1045,28 @@ sub backup_file_thread
file_size_format($oFileCopyMap{$strFile}{size}) .
($lSizeTotal > 0 ? ", " . int($lSize * 100 / $lSizeTotal) . "%" : "") . ")");
$oFile[$iThreadIdx]->file_copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file},
undef, $oFileCopyMap{$strFile}{modification_time},
undef, $bPathCreate);
$lSize += $oFileCopyMap{$strFile}{size};
# Copy the file. If the copy fails see if the file exists. During normal operation the database wil remove files.
# Don't error out in that case.
unless($oFile[$iThreadIdx]->file_copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file},
undef, $oFileCopyMap{$strFile}{modification_time},
undef, $bPathCreate, false))
{
if (!$oFile[$iThreadIdx]->file_exists(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}))
{
&log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file});
# Remove the file and the temp file just in case
$oFile[$iThreadIdx]->file_remove(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, true);
$oFile[$iThreadIdx]->file_remove(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file});
}
next;
}
# # Write the hash into the backup manifest (if not suppressed)
# if (!$bNoChecksum)
# {
@@ -1165,6 +1180,10 @@ sub backup
# Save the backup conf file first time - so we can see what is happening in the backup
backup_manifest_save($strBackupConfFile, \%oBackupManifest);
&log(INFO, "sleeping");
sleep(10);
&log(INFO, "waking");
# Perform the backup
backup_file($strBackupPath, $strDbClusterPath, \%oBackupManifest);
@@ -1183,6 +1202,7 @@ sub backup
backup_manifest_save($strBackupConfFile, \%oBackupManifest);
# After the backup has been stopped, need to make a copy of the archive logs need to make the db consistent
&log(DEBUG, "retrieving archive logs ${strArchiveStart}:${strArchiveStop}");
my @stryArchive = archive_list_get($strArchiveStart, $strArchiveStop, $oDb->version_get() < 9.3);
foreach my $strArchive (@stryArchive)
@@ -1260,22 +1280,19 @@ sub archive_list_get
while (!($iStartMajor == $iStopMajor && $iStartMinor == $iStopMinor))
{
if ($strArchiveStart ne $strArchiveStop)
{
$iArchiveIdx += 1;
$iStartMinor += 1;
$iStartMinor += 1;
if ($bSkipFF && $iStartMinor == 255 || !$bSkipFF && $iStartMinor == 256)
{
$iStartMajor += 1;
$iStartMinor = 0;
}
if ($bSkipFF && $iStartMinor == 255 || !$bSkipFF && $iStartMinor == 256)
{
$iStartMajor += 1;
$iStartMinor = 0;
}
$stryArchive[$iArchiveIdx] = uc(sprintf("${strTimeline}%08x%08x", $iStartMajor, $iStartMinor));
$iArchiveIdx += 1;
}
&log(TRACE, " archive_list_get: $strArchiveStart-$strArchiveStop (@stryArchive)");
&log(TRACE, " archive_list_get: $strArchiveStart:$strArchiveStop (@stryArchive)");
return @stryArchive;
}
pg_backrest_file.pm
+102 -9
View File
@@ -503,9 +503,11 @@ sub file_copy
my $lModificationTime = shift;
my $strPermission = shift;
my $bPathCreate = shift;
my $bConfessCopyError = shift;
# if bPathCreate is not defined, default to true
$bPathCreate = defined($bPathCreate) ? $bPathCreate : true;
$bConfessCopyError = defined($bConfessCopyError) ? $bConfessCopyError : false;
# Modification time and permissions cannot be set remotely
if ((defined($lModificationTime) || defined($strPermission)) && $self->is_remote($strDestinationPathType))
@@ -539,10 +541,6 @@ sub file_copy
# Generate the command string depending on compression/decompression/cat
my $strCommand = $self->{strCommandCat};
# if ($bAlreadyCompressed && $bCompress)
# {
# $strDestination .= $strDestination =~ "^.*\.$self->{strCompressExtension}\$" ? ".gz" : "";
# }
if (!$bAlreadyCompressed && $bCompress)
{
$strCommand = $self->{strCommandCompress};
@@ -571,10 +569,17 @@ sub file_copy
# Execute the command through ssh
my $oSSH = $self->remote_get($strSourcePathType);
$oSSH->system({stdout_fh => $hFile}, $strCommand) or confess &log(ERROR, "unable to execute ssh '$strCommand'");
unless ($oSSH->system({stdout_fh => $hFile, stderr_discard => true}, $strCommand))
{
close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}");
my $strResult = "unable to execute ssh '${strCommand}'";
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
# Close the destination file handle
close($hFile) or confess &log(ERROR, "cannot close file");
close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}");
}
# Else if the destination is remote
elsif ($self->is_remote($strDestinationPathType))
@@ -595,7 +600,8 @@ sub file_copy
if ($iExitStatus != 0)
{
confess &log(ERROR, "command '${strCommand}' returned", $iExitStatus);
my $strResult = "command '${strCommand}' returned " . $iExitStatus;
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
}
}
@@ -617,12 +623,22 @@ sub file_copy
&log(TRACE, "file_copy: remote ${strSourcePathType} '${strCommand}'");
my $oSSH = $self->remote_get($strSourcePathType);
$oSSH->system($strCommand) or confess &log(ERROR, "unable to execute remote command ${strCommand}:" . oSSH->error);
unless($oSSH->system({stderr_discard => true}, $strCommand))
{
my $strResult = "unable to execute remote command ${strCommand}:" . oSSH->error;
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
}
else
{
&log(TRACE, "file_copy: local '${strCommand}'");
system($strCommand) == 0 or confess &log(ERROR, "unable to copy local $strSource to local $strDestinationTmp");
unless(system($strCommand) == 0)
{
my $strResult = "unable to copy local ${strSource} to local ${strDestinationTmp}";
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
}
}
@@ -647,6 +663,8 @@ sub file_copy
# Move the file from tmp to final destination
$self->file_move($self->path_type_get($strSourcePathType) . ":absolute", $strDestinationTmp,
$self->path_type_get($strDestinationPathType) . ":absolute", $strDestination, $bPathCreate);
return true;
}
####################################################################################################################################
@@ -767,6 +785,81 @@ sub file_list_get
return @stryFile;
}
####################################################################################################################################
# FILE_EXISTS
####################################################################################################################################
sub file_exists
{
my $self = shift;
my $strPathType = shift;
my $strPath = shift;
# Get the root path for the manifest
my $strPathExists = $self->path_get($strPathType, $strPath);
# Builds the exists command
my $strCommand = "ls ${strPathExists}";
# Run the file exists command
my $strExists = "";
# Run remotely
if ($self->is_remote($strPathType))
{
&log(TRACE, "file_exists: remote ${strPathType}:${strPathExists}");
my $oSSH = $self->remote_get($strPathType);
$strExists = $oSSH->capture({stderr_discard => true}, $strCommand);
}
# Run locally
else
{
&log(TRACE, "file_exists: local ${strPathType}:${strPathExists}");
$strExists = capture($strCommand);
}
# If the return from ls eq strPathExists then true
return ($strExists eq $strPathExists);
}
####################################################################################################################################
# FILE_REMOVE
####################################################################################################################################
sub file_remove
{
my $self = shift;
my $strPathType = shift;
my $strPath = shift;
my $bTemp = shift;
my $bErrorIfNotExists = shift;
if (!defined($bErrorIfNotExists))
{
$bErrorIfNotExists = false;
}
# Get the root path for the manifest
my $strPathRemove = $self->path_get($strPathType, $strPath, $bTemp);
# Builds the exists command
my $strCommand = "rm -f ${strPathRemove}";
# Run remotely
if ($self->is_remote($strPathType))
{
&log(TRACE, "file_remove: remote ${strPathType}:${strPathRemove}");
my $oSSH = $self->remote_get($strPathType);
$oSSH->system({stderr_discard => true}, $strCommand) or $bErrorIfNotExists ? confess &log(ERROR, "unable to remove remote ${strPathType}:${strPathRemove}") : true;
}
# Run locally
else
{
&log(TRACE, "file_exists: local ${strPathType}:${strPathRemove}");
system($strCommand) == 0 or $bErrorIfNotExists ? confess &log(ERROR, "unable to remove local ${strPathType}:${strPathRemove}") : true;
}
}
####################################################################################################################################
# MANIFEST_GET
#
test/test.pl
+1 -1
View File
@@ -53,7 +53,7 @@ sub pg_create
sub pg_start
{
local($strPgBinPath, $strDbPath, $strPort, $strAchiveCommand) = @_;
my $strCommand = "$strPgBinPath/pg_ctl start -o \"-c port=$strPort -c wal_level=archive -c archive_mode=on -c archive_command=\'$strAchiveCommand\'\" -D $strDbPath -l $strDbPath/postgresql.log -w -s";
my $strCommand = "$strPgBinPath/pg_ctl start -o \"-c port=$strPort -c checkpoint_segments=1 -c wal_level=archive -c archive_mode=on -c archive_command=\'$strAchiveCommand\'\" -D $strDbPath -l $strDbPath/postgresql.log -w -s";
execute($strCommand);
}