From 27b820cfb4c5ac0ff5695202cdab98d471e0b95f Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 18 Feb 2014 15:42:51 -0500 Subject: [PATCH] Backup handles files that were removed by the db --- pg_backrest_backup.pm | 47 ++++++++++++------ pg_backrest_file.pm | 111 ++++++++++++++++++++++++++++++++++++++---- test/test.pl | 2 +- 3 files changed, 135 insertions(+), 25 deletions(-) diff --git a/pg_backrest_backup.pm b/pg_backrest_backup.pm index 65deae1fb..6b418b189 100644 --- a/pg_backrest_backup.pm +++ b/pg_backrest_backup.pm @@ -1045,13 +1045,28 @@ sub backup_file_thread file_size_format($oFileCopyMap{$strFile}{size}) . ($lSizeTotal > 0 ? ", " . int($lSize * 100 / $lSizeTotal) . "%" : "") . ")"); - $oFile[$iThreadIdx]->file_copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}, - PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, - undef, $oFileCopyMap{$strFile}{modification_time}, - undef, $bPathCreate); - $lSize += $oFileCopyMap{$strFile}{size}; + # Copy the file. If the copy fails see if the file exists. During normal operation the database wil remove files. + # Don't error out in that case. + unless($oFile[$iThreadIdx]->file_copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}, + PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, + undef, $oFileCopyMap{$strFile}{modification_time}, + undef, $bPathCreate, false)) + { + if (!$oFile[$iThreadIdx]->file_exists(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file})) + { + &log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file}); + + # Remove the file and the temp file just in case + $oFile[$iThreadIdx]->file_remove(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, true); + $oFile[$iThreadIdx]->file_remove(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}); + } + + next; + } + + # # Write the hash into the backup manifest (if not suppressed) # if (!$bNoChecksum) # { @@ -1165,6 +1180,10 @@ sub backup # Save the backup conf file first time - so we can see what is happening in the backup backup_manifest_save($strBackupConfFile, \%oBackupManifest); + &log(INFO, "sleeping"); + sleep(10); + &log(INFO, "waking"); + # Perform the backup backup_file($strBackupPath, $strDbClusterPath, \%oBackupManifest); @@ -1183,6 +1202,7 @@ sub backup backup_manifest_save($strBackupConfFile, \%oBackupManifest); # After the backup has been stopped, need to make a copy of the archive logs need to make the db consistent + &log(DEBUG, "retrieving archive logs ${strArchiveStart}:${strArchiveStop}"); my @stryArchive = archive_list_get($strArchiveStart, $strArchiveStop, $oDb->version_get() < 9.3); foreach my $strArchive (@stryArchive) @@ -1260,22 +1280,19 @@ sub archive_list_get while (!($iStartMajor == $iStopMajor && $iStartMinor == $iStopMinor)) { - if ($strArchiveStart ne $strArchiveStop) - { - $iArchiveIdx += 1; - $iStartMinor += 1; + $iStartMinor += 1; - if ($bSkipFF && $iStartMinor == 255 || !$bSkipFF && $iStartMinor == 256) - { - $iStartMajor += 1; - $iStartMinor = 0; - } + if ($bSkipFF && $iStartMinor == 255 || !$bSkipFF && $iStartMinor == 256) + { + $iStartMajor += 1; + $iStartMinor = 0; } $stryArchive[$iArchiveIdx] = uc(sprintf("${strTimeline}%08x%08x", $iStartMajor, $iStartMinor)); + $iArchiveIdx += 1; } - &log(TRACE, " archive_list_get: $strArchiveStart-$strArchiveStop (@stryArchive)"); + &log(TRACE, " archive_list_get: $strArchiveStart:$strArchiveStop (@stryArchive)"); return @stryArchive; } diff --git a/pg_backrest_file.pm b/pg_backrest_file.pm index 73f8d98e3..ca69107ec 100644 --- a/pg_backrest_file.pm +++ b/pg_backrest_file.pm @@ -503,9 +503,11 @@ sub file_copy my $lModificationTime = shift; my $strPermission = shift; my $bPathCreate = shift; + my $bConfessCopyError = shift; # if bPathCreate is not defined, default to true $bPathCreate = defined($bPathCreate) ? $bPathCreate : true; + $bConfessCopyError = defined($bConfessCopyError) ? $bConfessCopyError : false; # Modification time and permissions cannot be set remotely if ((defined($lModificationTime) || defined($strPermission)) && $self->is_remote($strDestinationPathType)) @@ -539,10 +541,6 @@ sub file_copy # Generate the command string depending on compression/decompression/cat my $strCommand = $self->{strCommandCat}; -# if ($bAlreadyCompressed && $bCompress) -# { -# $strDestination .= $strDestination =~ "^.*\.$self->{strCompressExtension}\$" ? ".gz" : ""; -# } if (!$bAlreadyCompressed && $bCompress) { $strCommand = $self->{strCommandCompress}; @@ -571,10 +569,17 @@ sub file_copy # Execute the command through ssh my $oSSH = $self->remote_get($strSourcePathType); - $oSSH->system({stdout_fh => $hFile}, $strCommand) or confess &log(ERROR, "unable to execute ssh '$strCommand'"); + + unless ($oSSH->system({stdout_fh => $hFile, stderr_discard => true}, $strCommand)) + { + close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}"); + + my $strResult = "unable to execute ssh '${strCommand}'"; + $bConfessCopyError ? confess &log(ERROR, $strResult) : return false; + } # Close the destination file handle - close($hFile) or confess &log(ERROR, "cannot close file"); + close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}"); } # Else if the destination is remote elsif ($self->is_remote($strDestinationPathType)) @@ -595,7 +600,8 @@ sub file_copy if ($iExitStatus != 0) { - confess &log(ERROR, "command '${strCommand}' returned", $iExitStatus); + my $strResult = "command '${strCommand}' returned " . $iExitStatus; + $bConfessCopyError ? confess &log(ERROR, $strResult) : return false; } } } @@ -617,12 +623,22 @@ sub file_copy &log(TRACE, "file_copy: remote ${strSourcePathType} '${strCommand}'"); my $oSSH = $self->remote_get($strSourcePathType); - $oSSH->system($strCommand) or confess &log(ERROR, "unable to execute remote command ${strCommand}:" . oSSH->error); + + unless($oSSH->system({stderr_discard => true}, $strCommand)) + { + my $strResult = "unable to execute remote command ${strCommand}:" . oSSH->error; + $bConfessCopyError ? confess &log(ERROR, $strResult) : return false; + } } else { &log(TRACE, "file_copy: local '${strCommand}'"); - system($strCommand) == 0 or confess &log(ERROR, "unable to copy local $strSource to local $strDestinationTmp"); + + unless(system($strCommand) == 0) + { + my $strResult = "unable to copy local ${strSource} to local ${strDestinationTmp}"; + $bConfessCopyError ? confess &log(ERROR, $strResult) : return false; + } } } @@ -647,6 +663,8 @@ sub file_copy # Move the file from tmp to final destination $self->file_move($self->path_type_get($strSourcePathType) . ":absolute", $strDestinationTmp, $self->path_type_get($strDestinationPathType) . ":absolute", $strDestination, $bPathCreate); + + return true; } #################################################################################################################################### @@ -767,6 +785,81 @@ sub file_list_get return @stryFile; } +#################################################################################################################################### +# FILE_EXISTS +#################################################################################################################################### +sub file_exists +{ + my $self = shift; + my $strPathType = shift; + my $strPath = shift; + + # Get the root path for the manifest + my $strPathExists = $self->path_get($strPathType, $strPath); + + # Builds the exists command + my $strCommand = "ls ${strPathExists}"; + + # Run the file exists command + my $strExists = ""; + + # Run remotely + if ($self->is_remote($strPathType)) + { + &log(TRACE, "file_exists: remote ${strPathType}:${strPathExists}"); + + my $oSSH = $self->remote_get($strPathType); + $strExists = $oSSH->capture({stderr_discard => true}, $strCommand); + } + # Run locally + else + { + &log(TRACE, "file_exists: local ${strPathType}:${strPathExists}"); + $strExists = capture($strCommand); + } + + # If the return from ls eq strPathExists then true + return ($strExists eq $strPathExists); +} + +#################################################################################################################################### +# FILE_REMOVE +#################################################################################################################################### +sub file_remove +{ + my $self = shift; + my $strPathType = shift; + my $strPath = shift; + my $bTemp = shift; + my $bErrorIfNotExists = shift; + + if (!defined($bErrorIfNotExists)) + { + $bErrorIfNotExists = false; + } + + # Get the root path for the manifest + my $strPathRemove = $self->path_get($strPathType, $strPath, $bTemp); + + # Builds the exists command + my $strCommand = "rm -f ${strPathRemove}"; + + # Run remotely + if ($self->is_remote($strPathType)) + { + &log(TRACE, "file_remove: remote ${strPathType}:${strPathRemove}"); + + my $oSSH = $self->remote_get($strPathType); + $oSSH->system({stderr_discard => true}, $strCommand) or $bErrorIfNotExists ? confess &log(ERROR, "unable to remove remote ${strPathType}:${strPathRemove}") : true; + } + # Run locally + else + { + &log(TRACE, "file_exists: local ${strPathType}:${strPathRemove}"); + system($strCommand) == 0 or $bErrorIfNotExists ? confess &log(ERROR, "unable to remove local ${strPathType}:${strPathRemove}") : true; + } +} + #################################################################################################################################### # MANIFEST_GET # diff --git a/test/test.pl b/test/test.pl index 9d79199fe..f0558d5ca 100755 --- a/test/test.pl +++ b/test/test.pl @@ -53,7 +53,7 @@ sub pg_create sub pg_start { local($strPgBinPath, $strDbPath, $strPort, $strAchiveCommand) = @_; - my $strCommand = "$strPgBinPath/pg_ctl start -o \"-c port=$strPort -c wal_level=archive -c archive_mode=on -c archive_command=\'$strAchiveCommand\'\" -D $strDbPath -l $strDbPath/postgresql.log -w -s"; + my $strCommand = "$strPgBinPath/pg_ctl start -o \"-c port=$strPort -c checkpoint_segments=1 -c wal_level=archive -c archive_mode=on -c archive_command=\'$strAchiveCommand\'\" -D $strDbPath -l $strDbPath/postgresql.log -w -s"; execute($strCommand); }