db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

Backup/restore copy will be run in the main process when thread-max=1. I've resisted this change because it adds complexity, but I have to accept that threads are not stable on all platforms. Or maybe any platform.

Browse Source
This commit is contained in:
David Steele 2015-02-28 10:23:33 -05:00
parent 5d10a18b25
commit f93c6caec2
2 changed files with 135 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
lib/BackRest/Backup.pm
View File

@ -1047,6 +1047,8 @@ sub backup_file
}
}
if ($iThreadLocalMax > 1)
{
# End each thread queue and start the backup_file threads
for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++)
{
@ -1057,8 +1059,8 @@ sub backup_file
"size $oyThreadData[$iThreadIdx]{small_size}");
# Start the thread
$oThread[$iThreadIdx] = threads->create(\&backup_file_thread, $iThreadIdx, !$bNoChecksum, !$bPathCreate,
$oyThreadData[$iThreadIdx]{size});
$oThread[$iThreadIdx] = threads->create(\&backup_file_thread, true, $iThreadIdx, !$bNoChecksum, !$bPathCreate,
$oyThreadData[$iThreadIdx]{size}, $oBackupManifest);
}
# Wait for the threads to complete
@ -1113,20 +1115,38 @@ sub backup_file
}
}
}
}
else
{
&log(DEBUG, "starting backup in main process");
backup_file_thread(false, 0, !$bNoChecksum, !$bPathCreate, $oyThreadData[0]{size}, $oBackupManifest);
}
}
sub backup_file_thread
{
my @args = @_;
my $iThreadIdx = $args[0]; # Defines the index of this thread
my $bChecksum = $args[1]; # Should checksums be generated on files after they have been backed up?
my $bPathCreate = $args[2]; # Should paths be created automatically?
my $lSizeTotal = $args[3]; # Total size of the files to be copied by this thread
my $bMulti = $args[0]; # Is this thread one of many?
my $iThreadIdx = $args[1]; # Defines the index of this thread
my $bChecksum = $args[2]; # Should checksums be generated on files after they have been backed up?
my $bPathCreate = $args[3]; # Should paths be created automatically?
my $lSizeTotal = $args[4]; # Total size of the files to be copied by this thread
my $oBackupManifest = $args[5]; # Backup manifest object (only used when single-threaded)
my $lSize = 0; # Size of files currently copied by this thread
my $strLog; # Store the log message
my $oFileThread = $oFile->clone($iThreadIdx); # Thread local file object
my $oFileThread; # Thread local file object
# If multi-threaded, then clone the file object
if ($bMulti)
{
$oFileThread = $oFile->clone($iThreadIdx);
}
else
{
$oFileThread = $oFile;
}
# When a KILL signal is received, immediately abort
$SIG{'KILL'} = sub {threads->exit();};
@ -1158,8 +1178,18 @@ sub backup_file_thread
# If file is missing assume the database removed it (else corruption and nothing we can do!)
&log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file});
# Remove file from the manifest
if ($bMulti)
{
# Write a message into the master queue to have the file removed from the manifest
$oMasterQueue[$iThreadIdx]->enqueue("remove|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}");
$oMasterQueue[$iThreadIdx]->enqueue("remove|$oFileCopyMap{$strFile}{file_section}|".
"$oFileCopyMap{$strFile}{file}");
}
else
{
# remove it directly
$oBackupManifest->remove($oFileCopyMap{$strFile}{file_section}, $oFileCopyMap{$strFile}{file});
}
# Move on to the next file
next;
@ -1173,8 +1203,19 @@ sub backup_file_thread
my $strChecksum = $oFileThread->hash(PATH_BACKUP_TMP,
$oFileCopyMap{$strFile}{backup_file} . ($bCompress ? '.gz' : ''), $bCompress);
# Store checksum in the manifest
if ($bMulti)
{
# Write the checksum message into the master queue
$oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}");
$oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|" .
"$oFileCopyMap{$strFile}{file}|${strChecksum}");
}
else
{
# Write it directly
$oBackupManifest->set($oFileCopyMap{$strFile}{file_section}, $oFileCopyMap{$strFile}{file},
MANIFEST_SUBKEY_CHECKSUM, $strChecksum);
}
# Output information about the file to be checksummed
if (!defined($strLog))

27
lib/BackRest/Restore.pm
View File

@ -587,17 +587,28 @@ sub restore
}
}
# If multi-threaded then create threads to copy files
if ($self->{iThreadTotal} > 1)
{
# Create threads to process the thread queues
my $oThreadGroup = thread_group_create();
for (my $iThreadIdx = 0; $iThreadIdx < $self->{iThreadTotal}; $iThreadIdx++)
{
&log(DEBUG, "starting restore thread ${iThreadIdx}");
thread_group_add($oThreadGroup, threads->create(\&restore_thread, $self, $iThreadIdx, \@oyRestoreQueue, $oManifest));
thread_group_add($oThreadGroup, threads->create(\&restore_thread, true, $self,
$iThreadIdx, \@oyRestoreQueue, $oManifest));
}
# Complete thread queues
thread_group_complete($oThreadGroup);
}
# Else copy in the main process
else
{
&log(DEBUG, "starting restore in main process");
$self->restore_thread(false, 0, \@oyRestoreQueue, $oManifest);
}
# Create recovery.conf file
$self->recovery();
@ -611,12 +622,24 @@ sub restore
sub restore_thread
{
my $self = shift; # Class hash
my $bMulti = shift; # Is this thread one of many?
my $iThreadIdx = shift; # Defines the index of this thread
my $oyRestoreQueueRef = shift; # Restore queues
my $oManifest = shift; # Backup manifest
my $iDirection = $iThreadIdx % 2 == 0 ? 1 : -1; # Size of files currently copied by this thread
my $oFileThread = $self->{oFile}->clone($iThreadIdx); # Thread local file object
my $oFileThread = $self->{oFile}; # Thread local file object
# If multi-threaded, then clone the file object
if ($bMulti)
{
$oFileThread = $self->{oFile}->clone($iThreadIdx);
}
# Else use the master file object
else
{
$oFileThread = $self->{oFile};
}
# Initialize the starting and current queue index based in the total number of threads in relation to this thread
my $iQueueStartIdx = int((@{$oyRestoreQueueRef} / $self->{iThreadTotal}) * $iThreadIdx);