db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-15 01:04:37 +02:00
Code Issues Releases Activity

Working on backup unit tests.

Browse Source
This commit is contained in:
David Steele
2014-06-22 14:51:28 -04:00
parent 8fff74ccfb
commit f91dce259f
7 changed files with 146 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
bin/pg_backrest.pl
View File

@ -235,7 +235,7 @@ my $oRemote;
if ($strRemote ne REMOTE_NONE)
{
my $oRemote = BackRest::Remote->new
$oRemote = BackRest::Remote->new
(
strHost => config_load($strRemote eq REMOTE_DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST, true),
strUser => config_load($strRemote eq REMOTE_DB ? CONFIG_SECTION_STANZA : CONFIG_SECTION_BACKUP, CONFIG_KEY_USER, true),
@ -261,7 +261,7 @@ if ($strOperation eq OP_ARCHIVE_GET)
}
# Init the file object
my $oFile = pg_backrest_file->new
my $oFile = BackRest::File->new
(
strStanza => $strStanza,
strRemote => $strRemote,
@ -337,7 +337,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
my $bChecksum = config_load($strSection, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
# Run file_init_archive - this is the minimal config needed to run archiving
my $oFile = pg_backrest_file->new
my $oFile = BackRest::File->new
(
# strStanza => $strStanza,
# bNoCompression => !$bCompress,
@ -408,7 +408,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
eval
{
# Run file_init_archive - this is the minimal config needed to run archive pulling
my $oFile = pg_backrest_file->new
my $oFile = BackRest::File->new
(
# strStanza => $strStanza,
# bNoCompression => !$bCompress,
@ -449,7 +449,7 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
&log(WARN, "errors during transfer, starting compression");
# Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file
my $oFile = pg_backrest_file->new
my $oFile = BackRest::File->new
(
# strStanza => $strStanza,
# bNoCompression => false,
@ -499,7 +499,7 @@ log_file_set(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . "/log/$
# GET MORE CONFIG INFO
####################################################################################################################################
# Make sure backup and expire operations happen on the db side
if ($strRemote eq backup)
if ($strRemote eq REMOTE_BACKUP)
{
confess &log(ERROR, 'backup and expire operations must run on the backup host');
}
@ -536,7 +536,7 @@ if (!lock_file_create($strLockPath))
}
# Run file_init_archive - the rest of the file config required for backup and restore
my $oFile = pg_backrest_file->new
my $oFile = BackRest::File->new
(
strStanza => $strStanza,
strRemote => $strRemote,
@ -544,7 +544,7 @@ my $oFile = pg_backrest_file->new
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true)
);
my $oDb = pg_backrest_db->new
my $oDb = BackRest::Db->new
(
strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER),
strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST),

41
lib/BackRest/Backup.pm
View File

@ -224,7 +224,7 @@ sub archive_get
my $strArchivePath = dirname($oFile->path_get(PATH_BACKUP_ARCHIVE, $strSourceArchive));
# Get the name of the requested archive file (may have hash info and compression extension)
my @stryArchiveFile = $oFile->file_list_get(PATH_BACKUP_ABSOLUTE, $strArchivePath,
my @stryArchiveFile = $oFile->list(PATH_BACKUP_ABSOLUTE, $strArchivePath,
"^${strSourceArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$");
# If there is more than one matching archive file then there is a serious issue - likely a bug in the archiver
@ -372,7 +372,7 @@ sub archive_pull
&log(DEBUG, "local archive path max = ${strPathMax}");
foreach my $strPath ($oFile->file_list_get(PATH_DB_ABSOLUTE, $strArchivePath, "^[0-F]{16}\$"))
foreach my $strPath ($oFile->list(PATH_DB_ABSOLUTE, $strArchivePath, "^[0-F]{16}\$"))
{
if ($strPath lt $strPathMax)
{
@ -572,12 +572,12 @@ sub backup_type_find
if ($strType eq 'incremental')
{
$strDirectory = ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"))[0];
$strDirectory = ($oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"))[0];
}
if (!defined($strDirectory) && $strType ne "full")
{
$strDirectory = ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"))[0];
$strDirectory = ($oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"))[0];
}
return $strDirectory;
@ -755,10 +755,11 @@ sub backup_manifest_build
$strLevel = "base";
}
my %oManifestHash = $oFile->manifest_get(PATH_DB_ABSOLUTE, $strDbClusterPath);
my $strName;
my %oManifestHash;
foreach $strName (sort(keys $oManifestHash{name}))
$oFile->manifest(PATH_DB_ABSOLUTE, $strDbClusterPath, \%oManifestHash);
foreach my $strName (sort(keys $oManifestHash{name}))
{
# Skip certain files during backup
if ($strName =~ /^pg\_xlog\/.*/ || # pg_xlog/ - this will be reconstructed
@ -1258,7 +1259,8 @@ sub backup
&log(DEBUG, "cluster path is $strDbClusterPath");
# Create the cluster backup path
$oFile->path_create(PATH_BACKUP_CLUSTER);
$oFile->path_create(PATH_BACKUP, "backup", undef, true);
$oFile->path_create(PATH_BACKUP_CLUSTER, undef, undef, true);
# Find the previous backup based on the type
my $strBackupLastPath = backup_type_find($strType, $oFile->path_get(PATH_BACKUP_CLUSTER));
@ -1317,7 +1319,8 @@ sub backup
&log(INFO, 'archive start: ' . ${oBackupManifest}{backup}{"archive-start"});
# Build the backup manifest
my %oTablespaceMap = $oDb->tablespace_map_get();
my %oTablespaceMap;
$oDb->tablespace_map_get(\%oTablespaceMap);
backup_manifest_build($oFile->{strCommandManifest}, $strDbClusterPath, \%oBackupManifest, \%oLastManifest, \%oTablespaceMap);
@ -1369,7 +1372,7 @@ sub backup
wait_for_file($strArchivePath, "^${strArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$", 600);
my @stryArchiveFile = $oFile->file_list_get(PATH_BACKUP_ABSOLUTE, $strArchivePath,
my @stryArchiveFile = $oFile->list(PATH_BACKUP_ABSOLUTE, $strArchivePath,
"^${strArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$");
if (scalar @stryArchiveFile != 1)
@ -1487,13 +1490,13 @@ sub backup_expire
}
my $iIndex = $iFullRetention;
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
@stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
while (defined($stryPath[$iIndex]))
{
# Delete all backups that depend on the full backup. Done in reverse order so that remaining backups will still
# be consistent if the process dies
foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse"))
foreach $strPath ($oFile->list(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse"))
{
system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}");
}
@ -1513,12 +1516,12 @@ sub backup_expire
confess &log(ERROR, "differential_rentention must be a number >= 1");
}
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 0), "reverse");
@stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 0), "reverse");
if (defined($stryPath[$iDifferentialRetention]))
{
# Get a list of all differential and incremental backups
foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 1), "reverse"))
foreach $strPath ($oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 1), "reverse"))
{
# Remove all differential and incremental backups before the oldest valid differential
if (substr($strPath, 0, length($strPath) - 1) lt $stryPath[$iDifferentialRetention])
@ -1540,15 +1543,15 @@ sub backup_expire
# Determine which backup type to use for archive retention (full, differential, incremental)
if ($strArchiveRetentionType eq "full")
{
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
@stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse");
}
elsif ($strArchiveRetentionType eq "differential" || $strArchiveRetentionType eq "diff")
{
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 0), "reverse");
@stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 0), "reverse");
}
elsif ($strArchiveRetentionType eq "incremental" || $strArchiveRetentionType eq "incr")
{
@stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse");
@stryPath = $oFile->list(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse");
}
else
{
@ -1607,7 +1610,7 @@ sub backup_expire
&log(INFO, "archive retention starts at " . $strArchiveLast);
# Remove any archive directories or files that are out of date
foreach $strPath ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, undef, "^[0-F]{16}\$"))
foreach $strPath ($oFile->list(PATH_BACKUP_ARCHIVE, undef, "^[0-F]{16}\$"))
{
&log(DEBUG, "found major archive path " . $strPath);
@ -1626,7 +1629,7 @@ sub backup_expire
my $strSubPath;
# Look for archive files in the archive directory
foreach $strSubPath ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, $strPath, "^[0-F]{24}.*\$"))
foreach $strSubPath ($oFile->list(PATH_BACKUP_ARCHIVE, $strPath, "^[0-F]{24}.*\$"))
{
# Delete if the first 24 characters less than the current archive file
if ($strSubPath lt substr($strArchiveLast, 0, 24))

3
lib/BackRest/Db.pm
View File

@ -97,8 +97,9 @@ sub psql_execute
sub tablespace_map_get
{
my $self = shift;
my $oHashRef = shift;
return data_hash_build("oid\tname\n" . $self->psql_execute(
return data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute(
"copy (select oid, spcname from pg_tablespace) to stdout"), "\t");
}

6
lib/BackRest/File.pm
View File

@ -134,7 +134,7 @@ sub BUILD
my $self = shift;
# If remote is defined check parameters and open session
if (defined($self->{strRemote}))
if (defined($self->{strRemote}) && $self->{strRemote} ne REMOTE_NONE)
{
# Make sure remote is valid
if ($self->{strRemote} ne REMOTE_DB && $self->{strRemote} ne REMOTE_BACKUP)
@ -544,6 +544,7 @@ sub path_create
my $strPathType = shift;
my $strPath = shift;
my $strPermission = shift;
my $bIgnoreExists = shift;
# Set operation variables
my $strPathOp = $self->path_get($strPathType, $strPath);
@ -574,6 +575,8 @@ sub path_create
$self->{oRemote}->command_execute($strOperation, \%oParamHash, false, $strDebug);
}
else
{
if (!($bIgnoreExists && $self->exists($strPathType, $strPath)))
{
# Attempt the create the directory
my $bResult;
@ -602,6 +605,7 @@ sub path_create
confess &log(ERROR, "${strDebug}: " . $strError, COMMAND_ERR_PATH_CREATE);
}
}
}
}
####################################################################################################################################

5
lib/BackRest/Utility.pm
View File

@ -294,6 +294,11 @@ sub log_file_set
{
my $strFile = shift;
unless (-e dirname($strFile))
{
mkdir(dirname($strFile)) or die "unable to create directory for log file ${strFile}";
}
$strFile .= "-" . date_string_get("%4d%02d%02d") . ".log";
my $bExists = false;

21
test/lib/BackRestTest/BackupTest.pm
View File

@ -55,7 +55,7 @@ sub BackRestTestBackup_ClusterCreate
my $iPort = shift;
BackRestTestCommon_Execute("initdb -D $strPath -A trust");
BackRestTestCommon_Execute("pg_ctl start -o \"-c port=$iPort\" -D $strPath -l $strPath/postgresql.log -w -s");
BackRestTestCommon_Execute("pg_ctl start -o \"-c port=$iPort -c checkpoint_segments=1 -c wal_level=archive -c archive_mode=on -c archive_command=true\" -D $strPath -l $strPath/postgresql.log -w -s");
}
####################################################################################################################################
@ -63,6 +63,7 @@ sub BackRestTestBackup_ClusterCreate
####################################################################################################################################
sub BackRestTestBackup_Setup
{
my $strRemote;
my $bDropOnly = shift;
BackRestTestBackup_ClusterDrop($strTestPath . "/db/common");
@ -91,7 +92,7 @@ sub BackRestTestBackup_Setup
BackRestTestBackup_ClusterCreate($strTestPath . "/db/common", BackRestTestCommon_DbPortGet);
# Create the backrest directory
BackRestTestCommon_ExecuteBackRest("mkdir -m 700 ${strTestPath}/backrest")
BackRestTestCommon_ExecuteBackRest("mkdir -m 770 ${strTestPath}/backrest")
}
}
@ -123,15 +124,11 @@ sub BackRestTestBackup_Test
BackRestTestBackup_Setup();
#-------------------------------------------------------------------------------------------------------------------------------
# Create remote
#-------------------------------------------------------------------------------------------------------------------------------
# my $oRemote = BackRest::Remote->new
# (
# strHost => $strHost,
# strUser => $strUser,
# strCommand => BackRestTestCommon_CommandRemoteGet(),
# );
BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);
BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_DB);
BackRestTestCommon_ExecuteBackRest(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
"/pg_backrest.conf --type=incr --stanza=${strStanza} backup");
#-------------------------------------------------------------------------------------------------------------------------------
# Test path_create()
@ -255,7 +252,7 @@ sub BackRestTestBackup_Test
# }
# }
BackRestTestBackup_Setup(true);
# BackRestTestBackup_Setup(true);
}
1;

80
test/lib/BackRestTest/CommonTest.pm
View File

@ -14,23 +14,32 @@ use Carp;
use File::Basename;
use Cwd 'abs_path';
use Config::IniFiles;
use lib dirname($0) . "/../lib";
use BackRest::Utility;
use BackRest::File;
use Exporter qw(import);
our @EXPORT = qw(BackRestTestCommon_Setup BackRestTestCommon_Execute BackRestTestCommon_ExecuteBackRest
BackRestTestCommon_StanzaGet BackRestTestCommon_CommandRemoteGet
BackRestTestCommon_ConfigCreate
BackRestTestCommon_StanzaGet BackRestTestCommon_CommandMainGet BackRestTestCommon_CommandRemoteGet
BackRestTestCommon_HostGet BackRestTestCommon_UserGet BackRestTestCommon_GroupGet
BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_DbPortGet);
BackRestTestCommon_UserBackRestGet BackRestTestCommon_TestPathGet BackRestTestCommon_BackupPathGet
BackRestTestCommon_DbPathGet BackRestTestCommon_DbCommonPathGet BackRestTestCommon_DbPortGet);
my $strCommonStanza;
my $strCommonCommandMain;
my $strCommonCommandRemote;
my $strCommonCommandPsql;
my $strCommonHost;
my $strCommonUser;
my $strCommonGroup;
my $strCommonUserBackRest;
my $strCommonTestPath;
my $strCommonBackupPath;
my $strCommonDbPath;
my $strCommonDbCommonPath;
my $iCommonDbPort;
####################################################################################################################################
@ -69,15 +78,56 @@ sub BackRestTestCommon_ExecuteBackRest
sub BackRestTestCommon_Setup
{
$strCommonStanza = "db";
$strCommonCommandRemote = "/Users/dsteele/pg_backrest/bin/pg_backrest_remote.pl";
$strCommonHost = "127.0.0.1";
$strCommonCommandMain = '/Users/dsteele/pg_backrest/bin/pg_backrest.pl';
$strCommonCommandRemote = '/Users/dsteele/pg_backrest/bin/pg_backrest_remote.pl';
$strCommonCommandPsql = '/Library/PostgreSQL/9.3/bin/psql -X %option%';
$strCommonHost = '127.0.0.1';
$strCommonUser = getpwuid($<);
$strCommonGroup = getgrgid($();
$strCommonUserBackRest = 'backrest';
$strCommonTestPath = dirname(abs_path($0)) . "/test";
$strCommonTestPath = dirname(abs_path($0)) . '/test';
$strCommonBackupPath = "${strCommonTestPath}/backrest";
$strCommonDbPath = "${strCommonTestPath}/db";
$strCommonDbCommonPath = "${strCommonTestPath}/db/common";
$iCommonDbPort = 6543;
}
####################################################################################################################################
# BackRestTestCommon_ConfigCreate
####################################################################################################################################
sub BackRestTestCommon_ConfigCreate
{
my $strFile = shift;
my $strRemote = shift;
my $oParamHashRef = shift;
my %oParamHash;
tie %oParamHash, 'Config::IniFiles';
$oParamHash{'global:command'}{'remote'} = $strCommonCommandRemote;
$oParamHash{'global:command'}{'psql'} = $strCommonCommandPsql;
if (defined($strRemote) && $strRemote eq REMOTE_BACKUP)
{
$oParamHash{'global:backup'}{'host'} = $strCommonHost;
$oParamHash{'global:backup'}{'user'} = $strCommonUserBackRest;
}
elsif (defined($strRemote) && $strRemote eq REMOTE_DB)
{
$oParamHash{$strCommonStanza}{'host'} = $strCommonHost;
$oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
$oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath;
$oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}";
}
$oParamHash{'global:backup'}{'path'} = $strCommonBackupPath;
tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}";
chmod(0770, $strFile) or die "unable to set permissions for ${strFile}";
}
####################################################################################################################################
# Get Methods
####################################################################################################################################
@ -86,6 +136,11 @@ sub BackRestTestCommon_StanzaGet
return $strCommonStanza;
}
sub BackRestTestCommon_CommandMainGet
{
return $strCommonCommandMain;
}
sub BackRestTestCommon_CommandRemoteGet
{
return $strCommonCommandRemote;
@ -116,6 +171,21 @@ sub BackRestTestCommon_TestPathGet
return $strCommonTestPath;
}
sub BackRestTestCommon_BackupPathGet
{
return $strCommonBackupPath;
}
sub BackRestTestCommon_DbPathGet
{
return $strCommonDbPath;
}
sub BackRestTestCommon_DbCommonPathGet
{
return $strCommonDbCommonPath;
}
sub BackRestTestCommon_DbPortGet
{
return $iCommonDbPort;