You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-13 01:00:23 +02:00
Backup/restore copy will be run in the main process when thread-max=1. I've resisted this change because it adds complexity, but I have to accept that threads are not stable on all platforms. Or maybe any platform.
This commit is contained in:
@ -1047,6 +1047,8 @@ sub backup_file
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ($iThreadLocalMax > 1)
|
||||||
|
{
|
||||||
# End each thread queue and start the backup_file threads
|
# End each thread queue and start the backup_file threads
|
||||||
for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++)
|
for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++)
|
||||||
{
|
{
|
||||||
@ -1057,8 +1059,8 @@ sub backup_file
|
|||||||
"size $oyThreadData[$iThreadIdx]{small_size}");
|
"size $oyThreadData[$iThreadIdx]{small_size}");
|
||||||
|
|
||||||
# Start the thread
|
# Start the thread
|
||||||
$oThread[$iThreadIdx] = threads->create(\&backup_file_thread, $iThreadIdx, !$bNoChecksum, !$bPathCreate,
|
$oThread[$iThreadIdx] = threads->create(\&backup_file_thread, true, $iThreadIdx, !$bNoChecksum, !$bPathCreate,
|
||||||
$oyThreadData[$iThreadIdx]{size});
|
$oyThreadData[$iThreadIdx]{size}, $oBackupManifest);
|
||||||
}
|
}
|
||||||
|
|
||||||
# Wait for the threads to complete
|
# Wait for the threads to complete
|
||||||
@ -1113,20 +1115,38 @@ sub backup_file
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
&log(DEBUG, "starting backup in main process");
|
||||||
|
backup_file_thread(false, 0, !$bNoChecksum, !$bPathCreate, $oyThreadData[0]{size}, $oBackupManifest);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
sub backup_file_thread
|
sub backup_file_thread
|
||||||
{
|
{
|
||||||
my @args = @_;
|
my @args = @_;
|
||||||
|
|
||||||
my $iThreadIdx = $args[0]; # Defines the index of this thread
|
my $bMulti = $args[0]; # Is this thread one of many?
|
||||||
my $bChecksum = $args[1]; # Should checksums be generated on files after they have been backed up?
|
my $iThreadIdx = $args[1]; # Defines the index of this thread
|
||||||
my $bPathCreate = $args[2]; # Should paths be created automatically?
|
my $bChecksum = $args[2]; # Should checksums be generated on files after they have been backed up?
|
||||||
my $lSizeTotal = $args[3]; # Total size of the files to be copied by this thread
|
my $bPathCreate = $args[3]; # Should paths be created automatically?
|
||||||
|
my $lSizeTotal = $args[4]; # Total size of the files to be copied by this thread
|
||||||
|
my $oBackupManifest = $args[5]; # Backup manifest object (only used when single-threaded)
|
||||||
|
|
||||||
my $lSize = 0; # Size of files currently copied by this thread
|
my $lSize = 0; # Size of files currently copied by this thread
|
||||||
my $strLog; # Store the log message
|
my $strLog; # Store the log message
|
||||||
my $oFileThread = $oFile->clone($iThreadIdx); # Thread local file object
|
my $oFileThread; # Thread local file object
|
||||||
|
|
||||||
|
# If multi-threaded, then clone the file object
|
||||||
|
if ($bMulti)
|
||||||
|
{
|
||||||
|
$oFileThread = $oFile->clone($iThreadIdx);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
$oFileThread = $oFile;
|
||||||
|
}
|
||||||
|
|
||||||
# When a KILL signal is received, immediately abort
|
# When a KILL signal is received, immediately abort
|
||||||
$SIG{'KILL'} = sub {threads->exit();};
|
$SIG{'KILL'} = sub {threads->exit();};
|
||||||
@ -1158,8 +1178,18 @@ sub backup_file_thread
|
|||||||
# If file is missing assume the database removed it (else corruption and nothing we can do!)
|
# If file is missing assume the database removed it (else corruption and nothing we can do!)
|
||||||
&log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file});
|
&log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file});
|
||||||
|
|
||||||
|
# Remove file from the manifest
|
||||||
|
if ($bMulti)
|
||||||
|
{
|
||||||
# Write a message into the master queue to have the file removed from the manifest
|
# Write a message into the master queue to have the file removed from the manifest
|
||||||
$oMasterQueue[$iThreadIdx]->enqueue("remove|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}");
|
$oMasterQueue[$iThreadIdx]->enqueue("remove|$oFileCopyMap{$strFile}{file_section}|".
|
||||||
|
"$oFileCopyMap{$strFile}{file}");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
# remove it directly
|
||||||
|
$oBackupManifest->remove($oFileCopyMap{$strFile}{file_section}, $oFileCopyMap{$strFile}{file});
|
||||||
|
}
|
||||||
|
|
||||||
# Move on to the next file
|
# Move on to the next file
|
||||||
next;
|
next;
|
||||||
@ -1173,8 +1203,19 @@ sub backup_file_thread
|
|||||||
my $strChecksum = $oFileThread->hash(PATH_BACKUP_TMP,
|
my $strChecksum = $oFileThread->hash(PATH_BACKUP_TMP,
|
||||||
$oFileCopyMap{$strFile}{backup_file} . ($bCompress ? '.gz' : ''), $bCompress);
|
$oFileCopyMap{$strFile}{backup_file} . ($bCompress ? '.gz' : ''), $bCompress);
|
||||||
|
|
||||||
|
# Store checksum in the manifest
|
||||||
|
if ($bMulti)
|
||||||
|
{
|
||||||
# Write the checksum message into the master queue
|
# Write the checksum message into the master queue
|
||||||
$oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}");
|
$oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|" .
|
||||||
|
"$oFileCopyMap{$strFile}{file}|${strChecksum}");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
# Write it directly
|
||||||
|
$oBackupManifest->set($oFileCopyMap{$strFile}{file_section}, $oFileCopyMap{$strFile}{file},
|
||||||
|
MANIFEST_SUBKEY_CHECKSUM, $strChecksum);
|
||||||
|
}
|
||||||
|
|
||||||
# Output information about the file to be checksummed
|
# Output information about the file to be checksummed
|
||||||
if (!defined($strLog))
|
if (!defined($strLog))
|
||||||
|
@ -587,17 +587,28 @@ sub restore
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# If multi-threaded then create threads to copy files
|
||||||
|
if ($self->{iThreadTotal} > 1)
|
||||||
|
{
|
||||||
# Create threads to process the thread queues
|
# Create threads to process the thread queues
|
||||||
my $oThreadGroup = thread_group_create();
|
my $oThreadGroup = thread_group_create();
|
||||||
|
|
||||||
for (my $iThreadIdx = 0; $iThreadIdx < $self->{iThreadTotal}; $iThreadIdx++)
|
for (my $iThreadIdx = 0; $iThreadIdx < $self->{iThreadTotal}; $iThreadIdx++)
|
||||||
{
|
{
|
||||||
&log(DEBUG, "starting restore thread ${iThreadIdx}");
|
&log(DEBUG, "starting restore thread ${iThreadIdx}");
|
||||||
thread_group_add($oThreadGroup, threads->create(\&restore_thread, $self, $iThreadIdx, \@oyRestoreQueue, $oManifest));
|
thread_group_add($oThreadGroup, threads->create(\&restore_thread, true, $self,
|
||||||
|
$iThreadIdx, \@oyRestoreQueue, $oManifest));
|
||||||
}
|
}
|
||||||
|
|
||||||
# Complete thread queues
|
# Complete thread queues
|
||||||
thread_group_complete($oThreadGroup);
|
thread_group_complete($oThreadGroup);
|
||||||
|
}
|
||||||
|
# Else copy in the main process
|
||||||
|
else
|
||||||
|
{
|
||||||
|
&log(DEBUG, "starting restore in main process");
|
||||||
|
$self->restore_thread(false, 0, \@oyRestoreQueue, $oManifest);
|
||||||
|
}
|
||||||
|
|
||||||
# Create recovery.conf file
|
# Create recovery.conf file
|
||||||
$self->recovery();
|
$self->recovery();
|
||||||
@ -611,12 +622,24 @@ sub restore
|
|||||||
sub restore_thread
|
sub restore_thread
|
||||||
{
|
{
|
||||||
my $self = shift; # Class hash
|
my $self = shift; # Class hash
|
||||||
|
my $bMulti = shift; # Is this thread one of many?
|
||||||
my $iThreadIdx = shift; # Defines the index of this thread
|
my $iThreadIdx = shift; # Defines the index of this thread
|
||||||
my $oyRestoreQueueRef = shift; # Restore queues
|
my $oyRestoreQueueRef = shift; # Restore queues
|
||||||
my $oManifest = shift; # Backup manifest
|
my $oManifest = shift; # Backup manifest
|
||||||
|
|
||||||
my $iDirection = $iThreadIdx % 2 == 0 ? 1 : -1; # Size of files currently copied by this thread
|
my $iDirection = $iThreadIdx % 2 == 0 ? 1 : -1; # Size of files currently copied by this thread
|
||||||
my $oFileThread = $self->{oFile}->clone($iThreadIdx); # Thread local file object
|
my $oFileThread = $self->{oFile}; # Thread local file object
|
||||||
|
|
||||||
|
# If multi-threaded, then clone the file object
|
||||||
|
if ($bMulti)
|
||||||
|
{
|
||||||
|
$oFileThread = $self->{oFile}->clone($iThreadIdx);
|
||||||
|
}
|
||||||
|
# Else use the master file object
|
||||||
|
else
|
||||||
|
{
|
||||||
|
$oFileThread = $self->{oFile};
|
||||||
|
}
|
||||||
|
|
||||||
# Initialize the starting and current queue index based in the total number of threads in relation to this thread
|
# Initialize the starting and current queue index based in the total number of threads in relation to this thread
|
||||||
my $iQueueStartIdx = int((@{$oyRestoreQueueRef} / $self->{iThreadTotal}) * $iThreadIdx);
|
my $iQueueStartIdx = int((@{$oyRestoreQueueRef} / $self->{iThreadTotal}) * $iThreadIdx);
|
||||||
|
Reference in New Issue
Block a user