From c85413ec6837a36ccf3911f07d63f9eb8648e770 Mon Sep 17 00:00:00 2001 From: David Steele Date: Sun, 29 Jun 2014 10:53:39 -0400 Subject: [PATCH] Lots of improvements to unit tests. A few bug fixes. --- bin/pg_backrest_remote.pl | 2 +- lib/BackRest/Backup.pm | 3 +- lib/BackRest/File.pm | 22 ++--- test/lib/BackRestTest/BackupTest.pm | 129 +++++++++++++++++----------- test/lib/BackRestTest/CommonTest.pm | 58 +++++++++---- 5 files changed, 134 insertions(+), 80 deletions(-) diff --git a/bin/pg_backrest_remote.pl b/bin/pg_backrest_remote.pl index 94d1ccced..38292e4ea 100755 --- a/bin/pg_backrest_remote.pl +++ b/bin/pg_backrest_remote.pl @@ -154,7 +154,7 @@ while ($strCommand ne OP_EXIT) PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'), param_get(\%oParamHash, 'source_compressed'), param_get(\%oParamHash, 'destination_compress'), - param_get(\%oParamHash, 'ignore_missing_source', false). + param_get(\%oParamHash, 'ignore_missing_source', false), undef, param_get(\%oParamHash, 'permission', false), param_get(\%oParamHash, 'destination_path_create')) ? 'Y' : 'N'); diff --git a/lib/BackRest/Backup.pm b/lib/BackRest/Backup.pm index 6c17ef07c..2f45118dd 100644 --- a/lib/BackRest/Backup.pm +++ b/lib/BackRest/Backup.pm @@ -1204,7 +1204,8 @@ sub backup_file_thread # Copy the file from the database to the backup (will return false if the source file is missing) unless($oFileThread->copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}, - PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, + PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file} . + ($bCompress ? '.' . $oFile->{strCompressExtension} : ''), false, # Source is not compressed since it is the db directory $bCompress, # Destination should be compressed based on backup settings true, # Ignore missing files diff --git a/lib/BackRest/File.pm b/lib/BackRest/File.pm index 00562af1c..30eaa0fb7 100644 --- a/lib/BackRest/File.pm +++ b/lib/BackRest/File.pm @@ -499,6 +499,7 @@ sub compress # Run locally else { + # Compress the file if (!gzip($strPathOp => "${strPathOp}.gz")) { my $strError = "${strPathOp} could not be compressed:" . $!; @@ -518,6 +519,7 @@ sub compress confess &log(ERROR, "${strDebug}: " . $strError); } + # Remove the old file unlink($strPathOp) or die &log(ERROR, "${strDebug}: unable to remove ${strPathOp}"); } @@ -1373,18 +1375,18 @@ sub copy if (!$bDestinationRemote) { # Set the file permission if required - # if (defined($strPermission)) - # { - # system("chmod ${strPermission} ${strDestinationTmpOp}") == 0 - # or confess &log(ERROR, "unable to set permissions for local ${strDestinationTmpOp}"); - # } + if (defined($strPermission)) + { + chmod(oct($strPermission), $strDestinationTmpOp) + or confess &log(ERROR, "unable to set permissions for local ${strDestinationTmpOp}"); + } # Set the file modification time if required - # if (defined($lModificationTime)) - # { - # utime($lModificationTime, $lModificationTime, $strDestinationTmpOp) - # or confess &log(ERROR, "unable to set time for local ${strDestinationTmpOp}"); - # } + if (defined($lModificationTime)) + { + utime($lModificationTime, $lModificationTime, $strDestinationTmpOp) + or confess &log(ERROR, "unable to set time for local ${strDestinationTmpOp}"); + } # Move the file from tmp to final destination $self->move(PATH_ABSOLUTE, $strDestinationTmpOp, PATH_ABSOLUTE, $strDestinationOp, true); diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm index 570c99ab9..ee0df22ce 100755 --- a/test/lib/BackRestTest/BackupTest.pm +++ b/test/lib/BackRestTest/BackupTest.pm @@ -33,9 +33,9 @@ my $strHost; my $strUserBackRest; #################################################################################################################################### -# BackRestTestBackup_ClusterDrop +# BackRestTestBackup_ClusterStop #################################################################################################################################### -sub BackRestTestBackup_ClusterDrop +sub BackRestTestBackup_ClusterStop { my $strPath = shift; @@ -78,41 +78,66 @@ sub BackRestTestBackup_ClusterCreate } #################################################################################################################################### -# BackRestTestBackup_Setup +# BackRestTestBackup_Drop #################################################################################################################################### -sub BackRestTestBackup_Setup +sub BackRestTestBackup_Drop { - my $strRemote; - my $bDropOnly = shift; - - BackRestTestBackup_ClusterDrop($strTestPath . "/db/common"); + # Stop the cluster if one is running + BackRestTestBackup_ClusterStop(BackRestTestCommon_DbCommonPathGet()); # Remove the backrest private directory - if (-e "${strTestPath}/backrest") + if (-e BackRestTestCommon_BackupPathGet()) { - BackRestTestCommon_ExecuteBackRest("rm -rf ${strTestPath}/backrest", true); + BackRestTestCommon_Execute('rm -rf ' . BackRestTestCommon_BackupPathGet(), true, true); } # Remove the test directory - system("rm -rf ${strTestPath}") == 0 or die 'unable to remove ${strTestPath} path'; + system('rm -rf ' . BackRestTestCommon_TestPathGet()) == 0 + or die 'unable to remove ' . BackRestTestCommon_TestPathGet() . 'path'; +} - if (!defined($bDropOnly) || !$bDropOnly) +#################################################################################################################################### +# BackRestTestBackup_Create +#################################################################################################################################### +sub BackRestTestBackup_Create +{ + my $bRemote = shift; + + # Set defaults + $bRemote = defined($bRemote) ? $bRemote : false; + + # Drop the old test directory + BackRestTestBackup_Drop(); + + # Create the test directory + mkdir(BackRestTestCommon_TestPathGet(), oct('0770')) + or confess 'Unable to create ' . BackRestTestCommon_TestPathGet() . ' path'; + + # Create the db directory + mkdir(BackRestTestCommon_DbPathGet(), oct('0700')) + or confess 'Unable to create ' . BackRestTestCommon_DbPathGet() . ' path'; + + # Create the db/common directory + mkdir(BackRestTestCommon_DbCommonPathGet()) + or confess 'Unable to create ' . BackRestTestCommon_DbCommonPathGet() . ' path'; + + # Create the archive directory + mkdir(BackRestTestCommon_ArchivePathGet(), oct('0700')) + or confess 'Unable to create ' . BackRestTestCommon_ArchivePathGet() . ' path'; + + # Create the backup directory + if ($bRemote) { - # Create the test directory - mkdir($strTestPath, oct("0770")) or confess "Unable to create ${strTestPath} path"; - - # Create the db directory - mkdir($strTestPath . "/db", oct("0700")) or confess "Unable to create ${strTestPath}/db path"; - - # Create the db/common directory - mkdir($strTestPath . "/db/common") or confess "Unable to create ${strTestPath}/db/common path"; - - # Create the cluster - BackRestTestBackup_ClusterCreate($strTestPath . "/db/common", BackRestTestCommon_DbPortGet); - - # Create the backrest directory - BackRestTestCommon_ExecuteBackRest("mkdir -m 770 ${strTestPath}/backrest") + BackRestTestCommon_Execute("mkdir -m 700 " . BackRestTestCommon_BackupPathGet(), true); } + else + { + mkdir(BackRestTestCommon_BackupPathGet(), oct('0700')) + or confess 'Unable to create ' . BackRestTestCommon_BackupPathGet() . ' path'; + } + + # Create the cluster + BackRestTestBackup_ClusterCreate(BackRestTestCommon_DbCommonPathGet(), BackRestTestCommon_DbPortGet()); } #################################################################################################################################### @@ -147,14 +172,12 @@ sub BackRestTestBackup_Test &log(INFO, "Test Full Backup\n"); - for (my $bRemote = 0; $bRemote <= 1; $bRemote++) + for (my $bRemote = false; $bRemote <= true; $bRemote++) { - BackRestTestBackup_Setup(); + BackRestTestBackup_Create($bRemote); - for (my $bHardlink = 0; $bHardlink <= 1; $bHardlink++) + for (my $bHardlink = false; $bHardlink <= true; $bHardlink++) { - # BackRestTestBackup_ClusterRestart(); - my %oDbConfigHash; my %oBackupConfigHash; @@ -164,33 +187,39 @@ sub BackRestTestBackup_Test $oBackupConfigHash{'global:backup'}{hardlink} = 'y'; } - BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', 'db', - ($bRemote ? REMOTE_BACKUP : undef), \%oDbConfigHash); - BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', 'backup', - ($bRemote ? REMOTE_DB : undef), \%oBackupConfigHash); + # for (my $bArchiveLocal = false; $bArchiveLocal <= true; $bArchiveLocal++) + # { + BackRestTestCommon_ConfigCreate('db', + ($bRemote ? REMOTE_BACKUP : undef), \%oDbConfigHash); + BackRestTestCommon_ConfigCreate('backup', + ($bRemote ? REMOTE_DB : undef), \%oBackupConfigHash); - for (my $iFull = 1; $iFull <= 1; $iFull++) - { - $iRun++; - - &log(INFO, "run ${iRun} - " . - "remote ${bRemote}, full ${iFull}"); - - BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() . - "/pg_backrest.conf --type=full --stanza=${strStanza} backup"); - - for (my $iIncr = 1; $iIncr <= 1; $iIncr++) + for (my $iFull = 1; $iFull <= 1; $iFull++) { $iRun++; &log(INFO, "run ${iRun} - " . - "remote ${bRemote}, full ${iFull}, hardlink ${bHardlink}, incr ${iIncr}"); + "remote ${bRemote}, full ${iFull}"); - BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() . - "/pg_backrest.conf --type=incr --stanza=${strStanza} backup"); + my $strCommand = BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() . + "/pg_backrest.conf --type=incr --stanza=${strStanza} backup"; + + BackRestTestCommon_Execute($strCommand, $bRemote); + + for (my $iIncr = 1; $iIncr <= 1; $iIncr++) + { + $iRun++; + + &log(INFO, "run ${iRun} - " . + "remote ${bRemote}, full ${iFull}, hardlink ${bHardlink}, incr ${iIncr}"); + + BackRestTestCommon_Execute($strCommand, $bRemote); + } } - } + # } } + + BackRestTestBackup_Drop(); } } diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm index 048851644..39651f308 100755 --- a/test/lib/BackRestTest/CommonTest.pm +++ b/test/lib/BackRestTest/CommonTest.pm @@ -26,7 +26,8 @@ our @EXPORT = qw(BackRestTestCommon_Setup BackRestTestCommon_Execute BackRestTes BackRestTestCommon_StanzaGet BackRestTestCommon_CommandMainGet BackRestTestCommon_CommandRemoteGet BackRestTestCommon_HostGet BackRestTestCommon_UserGet BackRestTestCommon_GroupGet BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_BackupPathGet - BackRestTestCommon_DbPathGet BackRestTestCommon_DbCommonPathGet BackRestTestCommon_DbPortGet); + BackRestTestCommon_ArchivePathGet BackRestTestCommon_DbPathGet BackRestTestCommon_DbCommonPathGet + BackRestTestCommon_DbPortGet); my $strCommonStanza; my $strCommonCommandMain; @@ -38,6 +39,7 @@ my $strCommonGroup; my $strCommonUserBackRest; my $strCommonTestPath; my $strCommonBackupPath; +my $strCommonArchivePath; my $strCommonDbPath; my $strCommonDbCommonPath; my $iCommonDbPort; @@ -48,30 +50,27 @@ my $iCommonDbPort; sub BackRestTestCommon_Execute { my $strCommand = shift; + my $bRemote = shift; my $bSuppressError = shift; + # Set defaults + $bRemote = defined($bRemote) ? $bRemote : false; + $bSuppressError = defined($bSuppressError) ? $bSuppressError : false; + + if ($bRemote) + { + $strCommand = "ssh ${strCommonUserBackRest}\@${strCommonHost} '${strCommand}'"; + } + if (system($strCommand) != 0) { - if (!defined($bSuppressError) || !$bSuppressError) + if (!$bSuppressError) { confess &log(ERROR, "unable to execute command: ${strCommand}"); } } } -#################################################################################################################################### -# BackRestTestBackup_ExecuteBackRest -#################################################################################################################################### -sub BackRestTestCommon_ExecuteBackRest -{ - my $strCommand = shift; - my $bSuppressError = shift; - - $strCommand = "ssh ${strCommonUserBackRest}\@${strCommonHost} '${strCommand}'"; - - BackRestTestCommon_Execute($strCommand, $bSuppressError); -} - #################################################################################################################################### # BackRestTestCommon_Setup #################################################################################################################################### @@ -87,6 +86,7 @@ sub BackRestTestCommon_Setup $strCommonUserBackRest = 'backrest'; $strCommonTestPath = dirname(abs_path($0)) . '/test'; $strCommonBackupPath = "${strCommonTestPath}/backrest"; + $strCommonArchivePath = "${strCommonTestPath}/archive"; $strCommonDbPath = "${strCommonTestPath}/db"; $strCommonDbCommonPath = "${strCommonTestPath}/db/common"; $iCommonDbPort = 6543; @@ -97,7 +97,6 @@ sub BackRestTestCommon_Setup #################################################################################################################################### sub BackRestTestCommon_ConfigCreate { - my $strFile = shift; my $strLocal = shift; my $strRemote = shift; my $oParamHashRef = shift; @@ -146,9 +145,27 @@ sub BackRestTestCommon_ConfigCreate } } - tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}"; + # Write out the configuration file + my $strFile = BackRestTestCommon_TestPathGet() . '/pg_backrest.conf'; - chmod(0770, $strFile) or die "unable to set permissions for ${strFile}"; + tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}"; + chmod(0660, $strFile) or die "unable to set permissions for ${strFile}"; + + # Move the configuration file based on local + if ($strLocal eq 'db') + { + rename($strFile, BackRestTestCommon_DbPathGet() . '/pg_backrest.conf') + or die "unable to move ${strFile} to " . BackRestTestCommon_DbPathGet() . '/pg_backrest.conf path'; + } + elsif ($strLocal eq 'backup' && !defined($strRemote)) + { + rename($strFile, BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf') + or die "unable to move ${strFile} to " . BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf path'; + } + else + { + BackRestTestCommon_Execute("mv $strFile " . BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', true); + } } #################################################################################################################################### @@ -199,6 +216,11 @@ sub BackRestTestCommon_BackupPathGet return $strCommonBackupPath; } +sub BackRestTestCommon_ArchivePathGet +{ + return $strCommonArchivePath; +} + sub BackRestTestCommon_DbPathGet { return $strCommonDbPath;