db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-01-30 05:39:12 +02:00
Code Issues Releases Activity

Using separate master queues

Browse Source
This commit is contained in:
David Steele 2014-02-19 15:13:07 -05:00
parent 800cd89831
commit 857e177429
1 changed files with 43 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
pg_backrest_backup.pm
View File

@ -42,8 +42,7 @@ my $bArchiveRequired;
# Thread variables
my @oThread;
my @oThreadQueue;
my @oThreadFile;
my $oMasterQueue;
my @oMasterQueue;
my %oFileCopyMap;
####################################################################################################################################
@ -59,11 +58,6 @@ sub backup_init
my $iThreadMaxParam = shift;
my $bArchiveRequiredParam = shift;
# for (my $iFileIdx = 0; $iFileIdx < scalar @oFile; $iFileIdx++)
# {
# undef($oFile[$iFileIdx]);
# }
$oDb = $oDbParam;
$oFile = $oFileParam;
$strType = $strTypeParam;
@ -109,15 +103,9 @@ sub thread_init
for (my $iThreadIdx = 0; $iThreadIdx < $iThreadActualTotal; $iThreadIdx++)
{
$oThreadQueue[$iThreadIdx] = Thread::Queue->new();
# if (!defined($oFile[$iThreadIdx + 1]))
# {
# $oFileThread[$iThreadIdx] = $oFile->clone($iThreadIdx + 1);
# }
$oMasterQueue[$iThreadIdx] = Thread::Queue->new();
}
$oMasterQueue = Thread::Queue->new();
return $iThreadActualTotal;
}
@ -1037,7 +1025,9 @@ sub backup_file
# Read the messages that we passed back from the threads. These should be two types:
# 1) remove - files that were skipped because they were removed from the database during backup
# 2) checksum - file checksums calculated by the threads
while (my $strMessage = $oMasterQueue->dequeue_nb())
for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++)
{
while (my $strMessage = $oMasterQueue[$iThreadIdx]->dequeue_nb())
{
&log (DEBUG, "message received in master queue: ${strMessage}");
@ -1081,6 +1071,7 @@ sub backup_file
}
}
}
}
sub backup_file_thread
{
@ -1115,6 +1106,8 @@ sub backup_file_thread
undef, $oFileCopyMap{$strFile}{modification_time},
undef, $bPathCreate, false))
{
&log(DEBUG, "thread ${iThreadIdx} unable to copy file: " . $oFileCopyMap{$strFile}{db_file});
# If the copy fails then see if the file still exists on the database
if (!$oFileThread->file_exists(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}))
{
@ -1127,7 +1120,7 @@ sub backup_file_thread
}
# Write a message into the master queue to have the file removed from the manifest
$oMasterQueue->enqueue("remove|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}");
$oMasterQueue[$iThreadIdx]->enqueue("remove|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}");
# Move on to the next file
next;
@ -1140,7 +1133,7 @@ sub backup_file_thread
my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file});
# Write the checksum message into the master queue
$oMasterQueue->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}");
$oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}");
&log(INFO, $strLog . " checksum ${strChecksum}");
}