From f91dce259f30b20608e882052c91c8466c95c20d Mon Sep 17 00:00:00 2001 From: David Steele Date: Sun, 22 Jun 2014 14:51:28 -0400 Subject: [PATCH] Working on backup unit tests. --- bin/pg_backrest.pl | 16 +++--- lib/BackRest/Backup.pm | 41 ++++++++------- lib/BackRest/Db.pm | 3 +- lib/BackRest/File.pm | 46 +++++++++-------- lib/BackRest/Utility.pm | 5 ++ test/lib/BackRestTest/BackupTest.pm | 21 ++++---- test/lib/BackRestTest/CommonTest.pm | 80 +++++++++++++++++++++++++++-- 7 files changed, 146 insertions(+), 66 deletions(-) diff --git a/bin/pg_backrest.pl b/bin/pg_backrest.pl index 58284a0ad..bbf4a7101 100755 --- a/bin/pg_backrest.pl +++ b/bin/pg_backrest.pl @@ -235,7 +235,7 @@ my $oRemote; if ($strRemote ne REMOTE_NONE) { - my $oRemote = BackRest::Remote->new + $oRemote = BackRest::Remote->new ( strHost => config_load($strRemote eq REMOTE_DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST, true), strUser => config_load($strRemote eq REMOTE_DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_USER, true), @@ -261,7 +261,7 @@ if ($strOperation eq OP_ARCHIVE_GET) } # Init the file object - my $oFile = pg_backrest_file->new + my $oFile = BackRest::File->new ( strStanza => $strStanza, strRemote => $strRemote, @@ -337,7 +337,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL) my $bChecksum = config_load($strSection, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false; # Run file_init_archive - this is the minimal config needed to run archiving - my $oFile = pg_backrest_file->new + my $oFile = BackRest::File->new ( # strStanza => $strStanza, # bNoCompression => !$bCompress, @@ -408,7 +408,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL) eval { # Run file_init_archive - this is the minimal config needed to run archive pulling - my $oFile = pg_backrest_file->new + my $oFile = BackRest::File->new ( # strStanza => $strStanza, # bNoCompression => !$bCompress, @@ -449,7 +449,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL) &log(WARN, "errors during transfer, starting compression"); # Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file - my $oFile = pg_backrest_file->new + my $oFile = BackRest::File->new ( # strStanza => $strStanza, # bNoCompression => false, @@ -499,7 +499,7 @@ log_file_set(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . "/log/$ # GET MORE CONFIG INFO #################################################################################################################################### # Make sure backup and expire operations happen on the db side -if ($strRemote eq backup) +if ($strRemote eq REMOTE_BACKUP) { confess &log(ERROR, 'backup and expire operations must run on the backup host'); } @@ -536,7 +536,7 @@ if (!lock_file_create($strLockPath)) } # Run file_init_archive - the rest of the file config required for backup and restore -my $oFile = pg_backrest_file->new +my $oFile = BackRest::File->new ( strStanza => $strStanza, strRemote => $strRemote, @@ -544,7 +544,7 @@ my $oFile = pg_backrest_file->new strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) ); -my $oDb = pg_backrest_db->new +my $oDb = BackRest::Db->new ( strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER), strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST), diff --git a/lib/BackRest/Backup.pm b/lib/BackRest/Backup.pm index 9209d5bfd..3364595f4 100644 --- a/lib/BackRest/Backup.pm +++ b/lib/BackRest/Backup.pm @@ -224,7 +224,7 @@ sub archive_get my $strArchivePath = dirname($oFile->path_get(PATH_BACKUP_ARCHIVE, $strSourceArchive)); # Get the name of the requested archive file (may have hash info and compression extension) - my @stryArchiveFile = $oFile->file_list_get(PATH_BACKUP_ABSOLUTE, $strArchivePath, + my @stryArchiveFile = $oFile->list(PATH_BACKUP_ABSOLUTE, $strArchivePath, "^${strSourceArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$"); # If there is more than one matching archive file then there is a serious issue - likely a bug in the archiver @@ -372,7 +372,7 @@ sub archive_pull &log(DEBUG, "local archive path max = ${strPathMax}"); - foreach my $strPath ($oFile->file_list_get(PATH_DB_ABSOLUTE, $strArchivePath, "^[0-F]{16}\$")) + foreach my $strPath ($oFile->list(PATH_DB_ABSOLUTE, $strArchivePath, "^[0-F]{16}\$")) { if ($strPath lt $strPathMax) { @@ -572,12 +572,12 @@ sub backup_type_find if ($strType eq 'incremental') { - $strDirectory = ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"))[0]; + $strDirectory = ($oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"))[0]; } if (!defined($strDirectory) && $strType ne "full") { - $strDirectory = ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"))[0]; + $strDirectory = ($oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"))[0]; } return $strDirectory; @@ -755,10 +755,11 @@ sub backup_manifest_build $strLevel = "base"; } - my %oManifestHash = $oFile->manifest_get(PATH_DB_ABSOLUTE, $strDbClusterPath); - my $strName; + my %oManifestHash; - foreach $strName (sort(keys $oManifestHash{name})) + $oFile->manifest(PATH_DB_ABSOLUTE, $strDbClusterPath, \%oManifestHash); + + foreach my $strName (sort(keys $oManifestHash{name})) { # Skip certain files during backup if ($strName =~ /^pg\_xlog\/.*/ || # pg_xlog/ - this will be reconstructed @@ -1258,7 +1259,8 @@ sub backup &log(DEBUG, "cluster path is $strDbClusterPath"); # Create the cluster backup path - $oFile->path_create(PATH_BACKUP_CLUSTER); + $oFile->path_create(PATH_BACKUP, "backup", undef, true); + $oFile->path_create(PATH_BACKUP_CLUSTER, undef, undef, true); # Find the previous backup based on the type my $strBackupLastPath = backup_type_find($strType, $oFile->path_get(PATH_BACKUP_CLUSTER)); @@ -1317,7 +1319,8 @@ sub backup &log(INFO, 'archive start: ' . ${oBackupManifest}{backup}{"archive-start"}); # Build the backup manifest - my %oTablespaceMap = $oDb->tablespace_map_get(); + my %oTablespaceMap; + $oDb->tablespace_map_get(\%oTablespaceMap); backup_manifest_build($oFile->{strCommandManifest}, $strDbClusterPath, \%oBackupManifest, \%oLastManifest, \%oTablespaceMap); @@ -1369,7 +1372,7 @@ sub backup wait_for_file($strArchivePath, "^${strArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$", 600); - my @stryArchiveFile = $oFile->file_list_get(PATH_BACKUP_ABSOLUTE, $strArchivePath, + my @stryArchiveFile = $oFile->list(PATH_BACKUP_ABSOLUTE, $strArchivePath, "^${strArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$"); if (scalar @stryArchiveFile != 1) @@ -1487,13 +1490,13 @@ sub backup_expire } my $iIndex = $iFullRetention; - @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"); + @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"); while (defined($stryPath[$iIndex])) { # Delete all backups that depend on the full backup. Done in reverse order so that remaining backups will still # be consistent if the process dies - foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse")) + foreach $strPath ($oFile->list(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse")) { system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}"); } @@ -1513,12 +1516,12 @@ sub backup_expire confess &log(ERROR, "differential_rentention must be a number >= 1"); } - @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 0), "reverse"); + @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 0), "reverse"); if (defined($stryPath[$iDifferentialRetention])) { # Get a list of all differential and incremental backups - foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 1), "reverse")) + foreach $strPath ($oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 1), "reverse")) { # Remove all differential and incremental backups before the oldest valid differential if (substr($strPath, 0, length($strPath) - 1) lt $stryPath[$iDifferentialRetention]) @@ -1540,15 +1543,15 @@ sub backup_expire # Determine which backup type to use for archive retention (full, differential, incremental) if ($strArchiveRetentionType eq "full") { - @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"); + @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"); } elsif ($strArchiveRetentionType eq "differential" || $strArchiveRetentionType eq "diff") { - @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 0), "reverse"); + @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 0), "reverse"); } elsif ($strArchiveRetentionType eq "incremental" || $strArchiveRetentionType eq "incr") { - @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"); + @stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"); } else { @@ -1607,7 +1610,7 @@ sub backup_expire &log(INFO, "archive retention starts at " . $strArchiveLast); # Remove any archive directories or files that are out of date - foreach $strPath ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, undef, "^[0-F]{16}\$")) + foreach $strPath ($oFile->list(PATH_BACKUP_ARCHIVE, undef, "^[0-F]{16}\$")) { &log(DEBUG, "found major archive path " . $strPath); @@ -1626,7 +1629,7 @@ sub backup_expire my $strSubPath; # Look for archive files in the archive directory - foreach $strSubPath ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, $strPath, "^[0-F]{24}.*\$")) + foreach $strSubPath ($oFile->list(PATH_BACKUP_ARCHIVE, $strPath, "^[0-F]{24}.*\$")) { # Delete if the first 24 characters less than the current archive file if ($strSubPath lt substr($strArchiveLast, 0, 24)) diff --git a/lib/BackRest/Db.pm b/lib/BackRest/Db.pm index 5b3e4425f..db009e123 100644 --- a/lib/BackRest/Db.pm +++ b/lib/BackRest/Db.pm @@ -97,8 +97,9 @@ sub psql_execute sub tablespace_map_get { my $self = shift; + my $oHashRef = shift; - return data_hash_build("oid\tname\n" . $self->psql_execute( + return data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute( "copy (select oid, spcname from pg_tablespace) to stdout"), "\t"); } diff --git a/lib/BackRest/File.pm b/lib/BackRest/File.pm index c278a146f..833372867 100644 --- a/lib/BackRest/File.pm +++ b/lib/BackRest/File.pm @@ -134,7 +134,7 @@ sub BUILD my $self = shift; # If remote is defined check parameters and open session - if (defined($self->{strRemote})) + if (defined($self->{strRemote}) && $self->{strRemote} ne REMOTE_NONE) { # Make sure remote is valid if ($self->{strRemote} ne REMOTE_DB && $self->{strRemote} ne REMOTE_BACKUP) @@ -544,6 +544,7 @@ sub path_create my $strPathType = shift; my $strPath = shift; my $strPermission = shift; + my $bIgnoreExists = shift; # Set operation variables my $strPathOp = $self->path_get($strPathType, $strPath); @@ -575,31 +576,34 @@ sub path_create } else { - # Attempt the create the directory - my $bResult; - - if (defined($strPermission)) + if (!($bIgnoreExists && $self->exists($strPathType, $strPath))) { - $bResult = mkdir($strPathOp, oct($strPermission)); - } - else - { - $bResult = mkdir($strPathOp); - } + # Attempt the create the directory + my $bResult; - if (!$bResult) - { - # Capture the error - my $strError = "${strPathOp} could not be created: " . $!; - - # If running on command line the return directly - if ($strPathType eq PATH_ABSOLUTE) + if (defined($strPermission)) { - confess &log(ERROR, $strError, COMMAND_ERR_PATH_CREATE); + $bResult = mkdir($strPathOp, oct($strPermission)); + } + else + { + $bResult = mkdir($strPathOp); } - # Error the normal way - confess &log(ERROR, "${strDebug}: " . $strError, COMMAND_ERR_PATH_CREATE); + if (!$bResult) + { + # Capture the error + my $strError = "${strPathOp} could not be created: " . $!; + + # If running on command line the return directly + if ($strPathType eq PATH_ABSOLUTE) + { + confess &log(ERROR, $strError, COMMAND_ERR_PATH_CREATE); + } + + # Error the normal way + confess &log(ERROR, "${strDebug}: " . $strError, COMMAND_ERR_PATH_CREATE); + } } } } diff --git a/lib/BackRest/Utility.pm b/lib/BackRest/Utility.pm index e9c6e4853..552f01955 100644 --- a/lib/BackRest/Utility.pm +++ b/lib/BackRest/Utility.pm @@ -293,6 +293,11 @@ sub date_string_get sub log_file_set { my $strFile = shift; + + unless (-e dirname($strFile)) + { + mkdir(dirname($strFile)) or die "unable to create directory for log file ${strFile}"; + } $strFile .= "-" . date_string_get("%4d%02d%02d") . ".log"; my $bExists = false; diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm index 2043ddb6f..21e0d96d9 100755 --- a/test/lib/BackRestTest/BackupTest.pm +++ b/test/lib/BackRestTest/BackupTest.pm @@ -55,7 +55,7 @@ sub BackRestTestBackup_ClusterCreate my $iPort = shift; BackRestTestCommon_Execute("initdb -D $strPath -A trust"); - BackRestTestCommon_Execute("pg_ctl start -o \"-c port=$iPort\" -D $strPath -l $strPath/postgresql.log -w -s"); + BackRestTestCommon_Execute("pg_ctl start -o \"-c port=$iPort -c checkpoint_segments=1 -c wal_level=archive -c archive_mode=on -c archive_command=true\" -D $strPath -l $strPath/postgresql.log -w -s"); } #################################################################################################################################### @@ -63,6 +63,7 @@ sub BackRestTestBackup_ClusterCreate #################################################################################################################################### sub BackRestTestBackup_Setup { + my $strRemote; my $bDropOnly = shift; BackRestTestBackup_ClusterDrop($strTestPath . "/db/common"); @@ -91,7 +92,7 @@ sub BackRestTestBackup_Setup BackRestTestBackup_ClusterCreate($strTestPath . "/db/common", BackRestTestCommon_DbPortGet); # Create the backrest directory - BackRestTestCommon_ExecuteBackRest("mkdir -m 700 ${strTestPath}/backrest") + BackRestTestCommon_ExecuteBackRest("mkdir -m 770 ${strTestPath}/backrest") } } @@ -123,15 +124,11 @@ sub BackRestTestBackup_Test BackRestTestBackup_Setup(); - #------------------------------------------------------------------------------------------------------------------------------- - # Create remote - #------------------------------------------------------------------------------------------------------------------------------- - # my $oRemote = BackRest::Remote->new - # ( - # strHost => $strHost, - # strUser => $strUser, - # strCommand => BackRestTestCommon_CommandRemoteGet(), - # ); + BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_BACKUP); + BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_DB); + + BackRestTestCommon_ExecuteBackRest(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() . + "/pg_backrest.conf --type=incr --stanza=${strStanza} backup"); #------------------------------------------------------------------------------------------------------------------------------- # Test path_create() @@ -255,7 +252,7 @@ sub BackRestTestBackup_Test # } # } - BackRestTestBackup_Setup(true); +# BackRestTestBackup_Setup(true); } 1; diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm index 5531bee2b..43ebd03c2 100755 --- a/test/lib/BackRestTest/CommonTest.pm +++ b/test/lib/BackRestTest/CommonTest.pm @@ -14,23 +14,32 @@ use Carp; use File::Basename; use Cwd 'abs_path'; +use Config::IniFiles; use lib dirname($0) . "/../lib"; use BackRest::Utility; +use BackRest::File; use Exporter qw(import); our @EXPORT = qw(BackRestTestCommon_Setup BackRestTestCommon_Execute BackRestTestCommon_ExecuteBackRest - BackRestTestCommon_StanzaGet BackRestTestCommon_CommandRemoteGet + BackRestTestCommon_ConfigCreate + BackRestTestCommon_StanzaGet BackRestTestCommon_CommandMainGet BackRestTestCommon_CommandRemoteGet BackRestTestCommon_HostGet BackRestTestCommon_UserGet BackRestTestCommon_GroupGet - BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_DbPortGet); + BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_BackupPathGet + BackRestTestCommon_DbPathGet BackRestTestCommon_DbCommonPathGet BackRestTestCommon_DbPortGet); my $strCommonStanza; +my $strCommonCommandMain; my $strCommonCommandRemote; +my $strCommonCommandPsql; my $strCommonHost; my $strCommonUser; my $strCommonGroup; my $strCommonUserBackRest; my $strCommonTestPath; +my $strCommonBackupPath; +my $strCommonDbPath; +my $strCommonDbCommonPath; my $iCommonDbPort; #################################################################################################################################### @@ -69,15 +78,56 @@ sub BackRestTestCommon_ExecuteBackRest sub BackRestTestCommon_Setup { $strCommonStanza = "db"; - $strCommonCommandRemote = "/Users/dsteele/pg_backrest/bin/pg_backrest_remote.pl"; - $strCommonHost = "127.0.0.1"; + $strCommonCommandMain = '/Users/dsteele/pg_backrest/bin/pg_backrest.pl'; + $strCommonCommandRemote = '/Users/dsteele/pg_backrest/bin/pg_backrest_remote.pl'; + $strCommonCommandPsql = '/Library/PostgreSQL/9.3/bin/psql -X %option%'; + $strCommonHost = '127.0.0.1'; $strCommonUser = getpwuid($<); $strCommonGroup = getgrgid($(); $strCommonUserBackRest = 'backrest'; - $strCommonTestPath = dirname(abs_path($0)) . "/test"; + $strCommonTestPath = dirname(abs_path($0)) . '/test'; + $strCommonBackupPath = "${strCommonTestPath}/backrest"; + $strCommonDbPath = "${strCommonTestPath}/db"; + $strCommonDbCommonPath = "${strCommonTestPath}/db/common"; $iCommonDbPort = 6543; } +#################################################################################################################################### +# BackRestTestCommon_ConfigCreate +#################################################################################################################################### +sub BackRestTestCommon_ConfigCreate +{ + my $strFile = shift; + my $strRemote = shift; + my $oParamHashRef = shift; + + my %oParamHash; + tie %oParamHash, 'Config::IniFiles'; + + $oParamHash{'global:command'}{'remote'} = $strCommonCommandRemote; + $oParamHash{'global:command'}{'psql'} = $strCommonCommandPsql; + + if (defined($strRemote) && $strRemote eq REMOTE_BACKUP) + { + $oParamHash{'global:backup'}{'host'} = $strCommonHost; + $oParamHash{'global:backup'}{'user'} = $strCommonUserBackRest; + } + elsif (defined($strRemote) && $strRemote eq REMOTE_DB) + { + $oParamHash{$strCommonStanza}{'host'} = $strCommonHost; + $oParamHash{$strCommonStanza}{'user'} = $strCommonUser; + $oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath; + + $oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}"; + } + + $oParamHash{'global:backup'}{'path'} = $strCommonBackupPath; + + tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}"; + + chmod(0770, $strFile) or die "unable to set permissions for ${strFile}"; +} + #################################################################################################################################### # Get Methods #################################################################################################################################### @@ -86,6 +136,11 @@ sub BackRestTestCommon_StanzaGet return $strCommonStanza; } +sub BackRestTestCommon_CommandMainGet +{ + return $strCommonCommandMain; +} + sub BackRestTestCommon_CommandRemoteGet { return $strCommonCommandRemote; @@ -116,6 +171,21 @@ sub BackRestTestCommon_TestPathGet return $strCommonTestPath; } +sub BackRestTestCommon_BackupPathGet +{ + return $strCommonBackupPath; +} + +sub BackRestTestCommon_DbPathGet +{ + return $strCommonDbPath; +} + +sub BackRestTestCommon_DbCommonPathGet +{ + return $strCommonDbCommonPath; +} + sub BackRestTestCommon_DbPortGet { return $iCommonDbPort;