db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

Changes for testing - finished changing config to local implementation.

Browse Source
This commit is contained in:
David Steele 2014-07-27 14:03:21 -04:00
parent 50dd95c005
commit 6f4310db42
5 changed files with 236 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
bin/pg_backrest.pl
View File

@ -75,15 +75,25 @@ my $strType; # Type of backup: full, differential (diff), incremental
# Test parameters - not for general use
my $bNoFork = false; # Prevents the archive process from forking when local archiving is enabled
my $bTest = false; # Enters test mode - not harmful in anyway, but adds special logging and pauses for unit testing
my $iTestDelay = 5; # Amount of time to delay after hitting a test point (the default would not be enough for manual tests)
GetOptions ("config=s" => \$strConfigFile,
"stanza=s" => \$strStanza,
"type=s" => \$strType,
"type=s" => \$strType,
# Test parameters - not for general use
"no-fork" => \$bNoFork)
"no-fork" => \$bNoFork,
"test" => \$bTest,
"test-delay=s" => \$iTestDelay)
or confess("Error in command line arguments\n");
# Test delay should be between 1 and 600 seconds
if (!($iTestDelay >= 1 && $iTestDelay <= 600))
{
confess &log(ERROR, 'test-delay must be between 1 and 600 seconds');
}
####################################################################################################################################
# Global variables
####################################################################################################################################
@ -599,7 +609,9 @@ backup_init
!$bChecksum,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX),
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_ARCHIVE_REQUIRED, true, "y") eq "y" ? true : false,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT)
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT),
$bTest,
$iTestDelay
);
####################################################################################################################################

92
lib/BackRest/Backup.pm
View File

@ -10,9 +10,7 @@ use warnings;
use Carp;
use File::Basename;
use File::Path qw(remove_tree);
use JSON;
use Scalar::Util qw(looks_like_number);
use Storable;
use Thread::Queue;
use lib dirname($0);
@ -37,6 +35,8 @@ my $iThreadThreshold = 10;
my $iSmallFileThreshold = 65536;
my $bArchiveRequired;
my $iThreadTimeout;
my $bTest;
my $iTestDelay;
# Thread variables
my @oThread;
@ -58,6 +58,8 @@ sub backup_init
my $iThreadMaxParam = shift;
my $bArchiveRequiredParam = shift;
my $iThreadTimeoutParam = shift;
my $bTestParam = shift;
my $iTestDelayParam = shift;
$oDb = $oDbParam;
$oFile = $oFileParam;
@ -68,6 +70,14 @@ sub backup_init
$iThreadMax = $iThreadMaxParam;
$bArchiveRequired = $bArchiveRequiredParam;
$iThreadTimeout = $iThreadTimeoutParam;
$bTest = defined($bTestParam) ? $bTestParam : false;
$iTestDelay = defined($bTestParam) ? $iTestDelayParam : undef;
# Make sure that a delay is specified in test mode
if ($bTest && !defined($iTestDelay))
{
confess &log(ASSERT, "iTestDelay must be provided when bTest is true");
}
if (!defined($iThreadMax))
{
@ -641,66 +651,6 @@ sub backup_type_find
return $strDirectory;
}
####################################################################################################################################
# BACKUP_MANIFEST_LOAD - Load the backup manifest
####################################################################################################################################
sub backup_manifest_load
{
my $strBackupManifestFile = shift;
return %{retrieve($strBackupManifestFile) or confess &log(ERROR, "unable to read ${strBackupManifestFile}")};
}
####################################################################################################################################
# BACKUP_MANIFEST_SAVE - Save the backup manifest
####################################################################################################################################
sub backup_manifest_save
{
my $strBackupManifestFile = shift;
my $oConfig = shift;
my $hFile;
my $bFirst = true;
open($hFile, '>', $strBackupManifestFile)
or confess "unable to open ${$strBackupManifestFile}";
foreach my $strSection (sort(keys $oConfig))
{
if (!$bFirst)
{
syswrite($hFile, "\n")
or confess "unable to write lf: $!";
}
syswrite($hFile, "[${strSection}]\n")
or confess "unable to write section ${strSection}: $!";
foreach my $strKey (sort(keys ${$oConfig}{"${strSection}"}))
{
my $strValue = ${$oConfig}{"${strSection}"}{"${strKey}"};
if (defined($strValue))
{
if (ref($strValue) eq "HASH")
{
syswrite($hFile, "${strKey}=" . encode_json($strValue) . "\n")
or confess "unable to write key ${strKey}: $!";
}
else
{
syswrite($hFile, "${strKey}=${strValue}\n")
or confess "unable to write key ${strKey}: $!";
}
}
}
$bFirst = false;
}
# store($oBackupManifestRef, $strBackupManifestFile) or confess &log(ERROR, "unable to open ${strBackupManifestFile}");
}
####################################################################################################################################
# BACKUP_FILE_NOT_IN_MANIFEST - Find all files in a backup path that are not in the supplied manifest
####################################################################################################################################
@ -1359,7 +1309,8 @@ sub backup
if (defined($strBackupLastPath))
{
%oLastManifest = backup_manifest_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/$strBackupLastPath/backup.manifest");
my %oLastManifest;
config_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/$strBackupLastPath/backup.manifest", \%oLastManifest);
if (!defined($oLastManifest{backup}{label}))
{
@ -1430,7 +1381,7 @@ sub backup
}
# Save the backup conf file first time - so we can see what is happening in the backup
backup_manifest_save($strBackupConfFile, \%oBackupManifest);
config_save($strBackupConfFile, \%oBackupManifest);
# Perform the backup
backup_file($strBackupPath, $strDbClusterPath, \%oBackupManifest);
@ -1450,7 +1401,7 @@ sub backup
if ($bArchiveRequired)
{
# Save the backup conf file second time - before getting archive logs in case that fails
backup_manifest_save($strBackupConfFile, \%oBackupManifest);
config_save($strBackupConfFile, \%oBackupManifest);
# After the backup has been stopped, need to make a copy of the archive logs need to make the db consistent
&log(DEBUG, "retrieving archive logs ${strArchiveStart}:${strArchiveStop}");
@ -1479,11 +1430,11 @@ sub backup
}
}
# Need some sort of backup validate - create a manifest and compare the backup manifest
# Need some sort of backup validate - create a manifest and compare the backup to manifest
# !!! DO IT
# Save the backup conf file final time
backup_manifest_save($strBackupConfFile, \%oBackupManifest);
config_save($strBackupConfFile, \%oBackupManifest);
# Rename the backup tmp path to complete the backup
&log(DEBUG, "moving ${strBackupTmpPath} to " . $oFile->path_get(PATH_BACKUP_CLUSTER, $strBackupPath));
@ -1514,15 +1465,17 @@ sub archive_list_get
&log(TRACE, "archive_list_get: post-9.3 database, including log FF");
}
# Get the timelines and make sure they match
my $strTimeline = substr($strArchiveStart, 0, 8);
my @stryArchive;
my $iArchiveIdx = 0;
if ($strTimeline ne substr($strArchiveStop, 0, 8))
{
confess "Timelines between ${strArchiveStart} and ${strArchiveStop} differ";
confess &log(ERROR, "Timelines between ${strArchiveStart} and ${strArchiveStop} differ");
}
# Iterate through all archive logs between start and stop
my $iStartMajor = hex substr($strArchiveStart, 8, 8);
my $iStartMinor = hex substr($strArchiveStart, 16, 8);
@ -1692,7 +1645,8 @@ sub backup_expire
# even though they are also in the pg_xlog directory (since they have been copied more than once).
&log(INFO, "archive retention based on backup " . $strArchiveRetentionBackup);
my %oManifest = backup_manifest_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/$strArchiveRetentionBackup/backup.manifest");
my %oManifest;
config_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/$strArchiveRetentionBackup/backup.manifest", \%oManifest);
my $strArchiveLast = ${oManifest}{backup}{"archive-start"};
if (!defined($strArchiveLast))

113
lib/BackRest/Utility.pm
View File

@ -11,6 +11,7 @@ use Carp;
use Fcntl qw(:DEFAULT :flock);
use File::Path qw(remove_tree);
use File::Basename;
use JSON;
use lib dirname($0) . "/../lib";
use BackRest::Exception;
@ -21,6 +22,7 @@ our @EXPORT = qw(version_get
data_hash_build trim common_prefix wait_for_file date_string_get file_size_format execute
log log_file_set log_level_set
lock_file_create lock_file_remove
config_save config_load
TRACE DEBUG ERROR ASSERT WARN INFO OFF true false);
# Global constants
@ -412,4 +414,115 @@ sub log
return $strMessage;
}
####################################################################################################################################
# CONFIG_LOAD - Load config file
####################################################################################################################################
sub config_load
{
my $strFile = shift;
my $oConfig = shift;
# Open the config file
my $hFile;
my $strSection;
open($hFile, '<', $strFile)
or confess &log(ERROR, "unable to open ${strFile}");
while (my $strLine = readline($hFile))
{
$strLine = trim($strLine);
# print "line:${strLine}\n";
if ($strLine ne '')
{
# Get the section
if (index($strLine, '[') == 0)
{
$strSection = substr($strLine, 1, length($strLine) - 2);
# print "found section ${strSection}\n";
}
else
{
# Get key and value
my $iIndex = index($strLine, '=');
if ($iIndex == -1)
{
confess &log(ERROR, "unable to read from ${strFile}: ${strLine}");
}
my $strKey = substr($strLine, 0, $iIndex);
my $strValue = substr($strLine, $iIndex + 1);
# print "found key ${strKey}:${strValue}\n";
# Try to store value as JSON
eval
{
${$oConfig}{"${strSection}"}{"${strKey}"} = decode_json($strValue);
};
# On error store value as a scalar
if ($@)
{
${$oConfig}{"${strSection}"}{"${strKey}"} = $strValue;
}
}
}
}
close($hFile);
}
####################################################################################################################################
# CONFIG_SAVE - Save config file
####################################################################################################################################
sub config_save
{
my $strBackupManifestFile = shift;
my $oConfig = shift;
my $hFile;
my $bFirst = true;
open($hFile, '>', $strBackupManifestFile)
or confess &log(ERROR, "unable to open ${strBackupManifestFile}");
foreach my $strSection (sort(keys $oConfig))
{
if (!$bFirst)
{
syswrite($hFile, "\n")
or confess "unable to write lf: $!";
}
syswrite($hFile, "[${strSection}]\n")
or confess "unable to write section ${strSection}: $!";
foreach my $strKey (sort(keys ${$oConfig}{"${strSection}"}))
{
my $strValue = ${$oConfig}{"${strSection}"}{"${strKey}"};
if (defined($strValue))
{
if (ref($strValue) eq "HASH")
{
syswrite($hFile, "${strKey}=" . encode_json($strValue) . "\n")
or confess "unable to write key ${strKey}: $!";
}
else
{
syswrite($hFile, "${strKey}=${strValue}\n")
or confess "unable to write key ${strKey}: $!";
}
}
}
$bFirst = false;
}
close($hFile);
}
1;

79
test/lib/BackRestTest/CommonTest.pm
View File

@ -16,6 +16,8 @@ use File::Basename;
use Cwd 'abs_path';
use Config::IniFiles;
use IPC::Open3;
use POSIX ":sys_wait_h";
use IO::Select;
use lib dirname($0) . "/../lib";
use BackRest::Utility;
@ -99,32 +101,74 @@ sub BackRestTestCommon_Execute
}
# system($strCommand);
my $strError;
# my $strError = '';
my $strErrorLog = '';
my $hError;
open($hError, '>', \$strError) or confess "unable to open handle to stderr string: $!\n";
# open($hError, '>', BackRestTestCommon_TestPathGet() . '/stderr.log');# or confess "unable to open handle to stderr string: $!\n";
# open($hError, '>', \$strError) or confess "unable to open handle to stderr string: $!\n";
my $strOut;
# my $strOut = '';
my $strOutLog = '';
my $hOut;
open($hOut, '>', \$strOut) or confess "unable to open handle to stdout string: $!\n";
# open($hOut, '>', BackRestTestCommon_TestPathGet() . '/stdout.log');# or confess "unable to open handle to stderr string: $!\n";
# open($hOut, '>', \$strOut) or confess "unable to open handle to stdout string: $!\n";
my $pId = open3(undef, $hOut, $hError, $strCommand);
my $oErrorSelect = IO::Select->new();
$oErrorSelect->add($hError);
my $oOutSelect = IO::Select->new();
$oOutSelect->add($hOut);
# Wait for the process to finish and report any errors
waitpid($pId, 0);
# my $iExitStatus;
# waitpid($pId, 0);
while(waitpid($pId, WNOHANG) == 0)
{
# print "stuck here\n";
if ($oErrorSelect->can_read(.1))
{
# print "read err\n";
while (my $strLine = readline($hError))
{
# print "out: ${strLine}";
$strErrorLog .= $strLine;
}
}
if ($oOutSelect->can_read(.1))
{
# print "read out begin\n";
while (my $strLine = readline($hOut))
{
# print "out: ${strLine}";
$strOutLog .= $strLine;
}
# print "read out end\n";
}
}
# print "got out\n";
my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
# $iExitStatus = $iExitStatus >> 8;
if ($iExitStatus != 0 && !$bSuppressError)
{
while (my $strLine = readline($hError))
{
print $strLine;
}
print "${strErrorLog}";
confess &log(ERROR, "command '${strCommand}' returned " . $iExitStatus);
}
close($hError);
close($hOut);
# print "${strOutLog}\n";
# close($hError);
# close($hOut);
# while (my $strLine = readline($hOut))
# {
@ -141,16 +185,19 @@ sub BackRestTestCommon_Setup
my $bDryRunParam = shift;
my $bNoCleanupParam = shift;
my $strBasePath = dirname(dirname(abs_path($0)));
$strCommonStanza = "db";
$strCommonCommandMain = '/Users/dsteele/pg_backrest/bin/pg_backrest.pl';
$strCommonCommandRemote = '/Users/dsteele/pg_backrest/bin/pg_backrest_remote.pl';
$strCommonCommandMain = "${strBasePath}/bin/pg_backrest.pl";
$strCommonCommandRemote = "${strBasePath}/bin/pg_backrest_remote.pl";
$strCommonCommandPsql = '/Library/PostgreSQL/9.3/bin/psql -X %option%';
# $strCommonCommandPsql = 'psql -X %option%';
$strCommonHost = '127.0.0.1';
$strCommonUser = getpwuid($<);
$strCommonGroup = getgrgid($();
$strCommonUserBackRest = 'backrest';
$strCommonTestPath = dirname(abs_path($0)) . '/test';
$strCommonDataPath = dirname(abs_path($0)) . '/data';
$strCommonTestPath = "${strBasePath}/test/test";
$strCommonDataPath = "${strBasePath}/test/data";
$strCommonBackupPath = "${strCommonTestPath}/backrest";
$strCommonArchivePath = "${strCommonTestPath}/archive";
$strCommonDbPath = "${strCommonTestPath}/db";
@ -197,7 +244,7 @@ sub BackRestTestCommon_ConfigCreate
$oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
}
$oParamHash{'global:log'}{'level-console'} = 'error';
$oParamHash{'global:log'}{'level-console'} = 'trace';
$oParamHash{'global:log'}{'level-file'} = 'trace';
if ($strLocal eq REMOTE_BACKUP)

61
test/lib/BackRestTest/UtilityTest.pm
View File

@ -13,10 +13,10 @@ use english;
use Carp;
use File::Basename;
use JSON; # remove when config functions go to utility
use lib dirname($0) . "/../lib";
use BackRest::Utility;
use BackRest::File;
use BackRestTest::CommonTest;
@ -68,7 +68,7 @@ sub BackRestTestUtility_Test
{
$iRun = 0;
$bCreate = true;
my $oFile;
my $oFile = BackRest::File->new();
&log(INFO, "Test config\n");
@ -78,6 +78,8 @@ sub BackRestTestUtility_Test
# Create the test directory
if ($bCreate)
{
# $oFile = BackRest::File->new();
# # Create the file object
# $oFile = (BackRest::File->new
# (
@ -92,6 +94,7 @@ sub BackRestTestUtility_Test
$bCreate = false;
}
# Generate a test config
my %oConfig;
$oConfig{test1}{key1} = 'value';
@ -103,48 +106,28 @@ sub BackRestTestUtility_Test
$oConfig{test3}{key2}{sub1} = 'value';
$oConfig{test3}{key2}{sub2} = 'value';
my $strFile = "$strTestPath/test.cfg";
my $hFile;
my $bFirst = true;
# Save the test config
my $strFile = "${strTestPath}/config.cfg";
config_save($strFile, \%oConfig);
open($hFile, '>', $strFile)
or confess "unable to open ${strFile}";
my $strConfigHash = $oFile->hash(PATH_ABSOLUTE, $strFile);
foreach my $strSection (sort(keys %oConfig))
# Reload the test config
my %oConfigTest;
config_load($strFile, \%oConfigTest);
# Resave the test config and compare hashes
my $strFileTest = "${strTestPath}/config-test.cfg";
config_save($strFileTest, \%oConfigTest);
my $strConfigTestHash = $oFile->hash(PATH_ABSOLUTE, $strFileTest);
if ($strConfigHash ne $strConfigTestHash)
{
if (!$bFirst)
{
syswrite($hFile, "\n")
or confess "unable to write lf: $!";
}
syswrite($hFile, "[${strSection}]\n")
or confess "unable to write section ${strSection}: $!";
foreach my $strKey (sort(keys $oConfig{$strSection}))
{
my $strValue = $oConfig{"${strSection}"}{"${strKey}"};
if (defined($strValue))
{
if (ref($strValue) eq "HASH")
{
syswrite($hFile, "${strKey}=" . encode_json($strValue) . "\n")
or confess "unable to write key ${strKey}: $!";
}
else
{
syswrite($hFile, "${strKey}=${strValue}\n")
or confess "unable to write key ${strKey}: $!";
}
}
}
$bFirst = false;
confess "config hash ${strConfigHash} != ${strConfigTestHash}";
}
close($hFile);
if (BackRestTestCommon_Cleanup())
{
&log(INFO, 'cleanup');