db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

Changed file sizes are now detected and stored in the manifest.

Browse Source 
Remove thread file minimums - they are unrealistic for a real db and hinder unit tests.
This commit is contained in:
David Steele 2015-03-01 22:43:32 -05:00
parent 23102f19e5
commit 0df1b7c473
2 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
lib/BackRest/Backup.pm
View File

@ -33,7 +33,7 @@ my $bCompress;
my $bHardLink; my $bHardLink;
my $iThreadMax; my $iThreadMax;
my $iThreadLocalMax; my $iThreadLocalMax;
my $iThreadThreshold = 10; #my $iThreadThreshold = 10;
my $iSmallFileThreshold = 65536; my $iSmallFileThreshold = 65536;
my $bArchiveRequired; my $bArchiveRequired;
my $bNoStartStop; my $bNoStartStop;
@ -546,7 +546,7 @@ sub archive_compress
&log(INFO, "archive to be compressed total ${lFileTotal}, size " . file_size_format($lFileSize)); &log(INFO, "archive to be compressed total ${lFileTotal}, size " . file_size_format($lFileSize));
# Init the thread variables # Init the thread variables
$iThreadLocalMax = thread_init(int($lFileTotal / $iThreadThreshold) + 1); $iThreadLocalMax = thread_init($iThreadMax);
my $iThreadIdx = 0; my $iThreadIdx = 0;
# Distribute files among the threads # Distribute files among the threads
@ -977,7 +977,7 @@ sub backup_file
} }
# Build the thread queues # Build the thread queues
$iThreadLocalMax = thread_init(int($lFileTotal / $iThreadThreshold) + 1); $iThreadLocalMax = thread_init($iThreadMax);
&log(DEBUG, "actual threads ${iThreadLocalMax}/${iThreadMax}"); &log(DEBUG, "actual threads ${iThreadLocalMax}/${iThreadMax}");
# Initialize the thread size array # Initialize the thread size array
@ -1094,6 +1094,7 @@ sub backup_file
elsif ($strCommand eq 'checksum') elsif ($strCommand eq 'checksum')
{ {
my $strChecksum = $strSplit[3]; # File checksum calculated by the thread my $strChecksum = $strSplit[3]; # File checksum calculated by the thread
my $lFileSize = $strSplit[4]; # File size calculated by the thread
# Checksum must be defined # Checksum must be defined
if (!defined($strChecksum)) if (!defined($strChecksum))
@ -1101,10 +1102,17 @@ sub backup_file
confess &log(ASSERT, 'thread checksum messages must have strChecksum defined'); confess &log(ASSERT, 'thread checksum messages must have strChecksum defined');
} }
# Checksum must be defined
if (!defined($lFileSize))
{
confess &log(ASSERT, 'thread checksum messages must have lFileSize defined');
}
$oBackupManifest->set($strFileSection, $strFile, MANIFEST_SUBKEY_CHECKSUM, $strChecksum); $oBackupManifest->set($strFileSection, $strFile, MANIFEST_SUBKEY_CHECKSUM, $strChecksum);
$oBackupManifest->set($strFileSection, $strFile, MANIFEST_SUBKEY_SIZE, $lFileSize + 0);
# Log the checksum # Log the checksum
&log (DEBUG, "write checksum ${strFileSection}:${strFile} into manifest: ${strChecksum}"); &log (DEBUG, "write checksum ${strFileSection}:${strFile} into manifest: ${strChecksum} (${lFileSize})");
} }
} }
} }
@ -1202,13 +1210,15 @@ sub backup_file_thread
{ {
# Write the checksum message into the master queue # Write the checksum message into the master queue
$oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|" . $oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|" .
"$oFileCopyMap{$strFile}{file}|${strCopyChecksum}"); "$oFileCopyMap{$strFile}{file}|${strCopyChecksum}|${lCopySize}");
} }
else else
{ {
# Write it directly # Write it directly
$oBackupManifest->set($oFileCopyMap{$strFile}{file_section}, $oFileCopyMap{$strFile}{file}, $oBackupManifest->set($oFileCopyMap{$strFile}{file_section}, $oFileCopyMap{$strFile}{file},
MANIFEST_SUBKEY_CHECKSUM, $strCopyChecksum); MANIFEST_SUBKEY_CHECKSUM, $strCopyChecksum);
$oBackupManifest->set($oFileCopyMap{$strFile}{file_section}, $oFileCopyMap{$strFile}{file},
MANIFEST_SUBKEY_SIZE, $lCopySize + 0);
} }
# Output information about the file to be checksummed # Output information about the file to be checksummed

3
test/lib/BackRestTest/BackupTest.pm
View File

@ -1723,6 +1723,9 @@ sub BackRestTestBackup_Test
BackRestTestBackup_BackupBegin($strType, $strStanza, $bRemote, "remove files during backup", true, true, 1); BackRestTestBackup_BackupBegin($strType, $strStanza, $bRemote, "remove files during backup", true, true, 1);
BackRestTestCommon_ExecuteEnd(TEST_MANIFEST_BUILD); BackRestTestCommon_ExecuteEnd(TEST_MANIFEST_BUILD);
BackRestTestBackup_ManifestFileCreate(\%oManifest, "tablespace:2", 'tablespace2c.txt', 'TBLSPCBIGGER',
'dfcb8679956b734706cf87259d50c88f83e80e66', $lTime);
BackRestTestBackup_ManifestFileRemove(\%oManifest, 'base', 'base/base2.txt', true); BackRestTestBackup_ManifestFileRemove(\%oManifest, 'base', 'base/base2.txt', true);
$strBackup = BackRestTestBackup_BackupEnd($strType, $oFile, $bRemote, undef, \%oManifest, true); $strBackup = BackRestTestBackup_BackupEnd($strType, $oFile, $bRemote, undef, \%oManifest, true);