db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-06-20 01:17:49 +02:00
Code Issues Releases Activity

Comments and formatting.

Browse Source
This commit is contained in:
David Steele
2014-06-03 22:02:56 -04:00
parent 1b9782cb9e
commit 1793b7a5c4
9 changed files with 74 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -89,4 +89,4 @@ This version has been put into production at Resonate, so it does work, but ther
* Absolutely no documentation (outside the code). Well, excepting these release notes.
* Lots of other little things and not so little things. Much refactoring to follow.
* Lots of other little things and not so little things. Much refactoring to follow.
pg_backrest.conf
-2
View File
@@ -4,8 +4,6 @@ backrest_command=
compress=/usr/bin/gzip --stdout %file%
decompress=/usr/bin/gzip -dc %file%
#checksum=sha1sum %file% | awk '{print $1}' # Ubuntu Linux
checksum=/usr/bin/shasum %file% | awk '{print $1}'
manifest=/opt/local/bin/gfind %path% -ignore_readdir_race -printf '%P\t%y\t%u\t%g\t%m\t%T@\t%i\t%s\t%l\n'
psql=/Library/PostgreSQL/9.3/bin/psql -X %option%
[global:log]
pg_backrest.pl
+1 -9
View File
@@ -63,8 +63,6 @@ use constant
CONFIG_KEY_COMPRESS => "compress",
CONFIG_KEY_COMPRESS_ASYNC => "compress-async",
CONFIG_KEY_DECOMPRESS => "decompress",
CONFIG_KEY_CHECKSUM => "checksum",
CONFIG_KEY_MANIFEST => "manifest",
CONFIG_KEY_PSQL => "psql"
};
@@ -381,10 +379,8 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommand => $0,
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST),
strLockPath => $strLockPath
);
@@ -422,10 +418,8 @@ if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
bNoCompression => false,
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommand => $0,
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST)
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress)
);
backup_init
@@ -509,10 +503,8 @@ my $oFile = pg_backrest_file->new
strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER),
strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST),
strCommand => $0,
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST),
strCommandPsql => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL),
strLockPath => $strLockPath
);
pg_backrest_backup.pm
+21 -21
View File
@@ -65,7 +65,7 @@ sub backup_init
$iThreadMax = $iThreadMaxParam;
$bArchiveRequired = $bArchiveRequiredParam;
$iThreadTimeout = $iThreadTimeoutParam;
if (!defined($iThreadMax))
{
$iThreadMax = 1;
@@ -105,7 +105,7 @@ sub thread_init
$oThreadQueue[$iThreadIdx] = Thread::Queue->new();
$oMasterQueue[$iThreadIdx] = Thread::Queue->new();
}
return $iThreadActualTotal;
}
@@ -163,7 +163,7 @@ sub backup_thread_complete
while ($iThreadComplete < $iThreadLocalMax)
{
sleep(1);
# If a timeout has been defined, make sure we have not been running longer than that
if (defined($iTimeout))
{
@@ -176,7 +176,7 @@ sub backup_thread_complete
#confess &log(WARN, "all threads have exited, aborting...");
}
}
for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++)
{
if (defined($oThread[$iThreadIdx]))
@@ -232,7 +232,7 @@ sub archive_get
{
confess &log(ASSERT, (scalar @stryArchiveFile) . " archive files found for ${strSourceArchive}.");
}
# If there are no matching archive files then there are two possibilities:
# 1) The end of the archive stream has been reached, this is normal and a 1 will be returned
# 2) There is a hole in the archive stream so return a hard error (!!! However if turns out that due to race conditions this
@@ -248,7 +248,7 @@ sub archive_get
# Copy the archive file to the requested location
$oFile->file_copy(PATH_BACKUP_ARCHIVE, $stryArchiveFile[0], PATH_DB_ABSOLUTE, $strDestinationFile);
return 0;
}
@@ -261,7 +261,7 @@ sub archive_push
# Get the destination file
my $strDestinationFile = basename($strSourceFile);
# Determine if this is an archive file (don't want to do compression or checksum on .backup files)
my $bArchiveFile = basename($strSourceFile) =~ /^[0-F]{24}$/ ? true : false;
@@ -356,13 +356,13 @@ sub archive_pull
foreach my $strFile (sort @stryFile)
{
&log(INFO, "backing up archive file ${strFile}");
my $strArchiveFile = "${strArchivePath}/${strFile}";
# Copy the file
$oFile->file_copy(PATH_DB_ABSOLUTE, $strArchiveFile,
PATH_BACKUP_ARCHIVE, basename($strFile));
# Remove the source archive file
unlink($strArchiveFile) or confess &log(ERROR, "unable to remove ${strArchiveFile}");
}
@@ -379,7 +379,7 @@ sub archive_pull
&log(DEBUG, "removing local archive path ${strPath}");
rmdir($strArchivePath . "/" . $strPath) or &log(WARN, "unable to remove archive path ${strPath}, is it empty?");
}
# If the dir is not empty check if the files are in the manifest
# If they are error - there has been some issue
# If not, they are new - continue processing without error - they'll be picked up on the next run
@@ -498,7 +498,7 @@ sub archive_pull_compress_thread
while (my $strFile = $oThreadQueue[$iThreadIdx]->dequeue())
{
&log(INFO, "thread ${iThreadIdx} compressing archive file ${strFile}");
# Compress the file
$oFileThread->file_compress(PATH_DB_ABSOLUTE, "${strArchivePath}/${strFile}");
}
@@ -512,29 +512,29 @@ sub backup_regexp_get
my $bFull = shift;
my $bDifferential = shift;
my $bIncremental = shift;
if (!$bFull && !$bDifferential && !$bIncremental)
{
confess &log(ERROR, 'one parameter must be true');
}
my $strDateTimeRegExp = "[0-9]{8}\\-[0-9]{6}";
my $strRegExp = "^";
if ($bFull || $bDifferential || $bIncremental)
{
$strRegExp .= $strDateTimeRegExp . "F";
}
if ($bDifferential || $bIncremental)
{
if ($bFull)
{
$strRegExp .= "(\\_";
}
$strRegExp .= $strDateTimeRegExp;
if ($bDifferential && $bIncremental)
{
$strRegExp .= "(D|I)";
@@ -553,11 +553,11 @@ sub backup_regexp_get
$strRegExp .= "){0,1}";
}
}
$strRegExp .= "\$";
# &log(DEBUG, "backup_regexp_get($bFull, $bDifferential, $bIncremental): $strRegExp");
return $strRegExp;
}
@@ -579,7 +579,7 @@ sub backup_type_find
{
$strDirectory = ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"))[0];
}
return $strDirectory;
}
pg_backrest_db.pm
+5 -5
View File
@@ -36,7 +36,7 @@ sub BUILD
if (defined($self->{strDbHost}) && !defined($self->{oDbSSH}))
{
my $strOptionSSHRequestTTY = "RequestTTY=yes";
&log(TRACE, "connecting to database ssh host $self->{strDbHost}");
# !!! This could be improved by redirecting stderr to a file to get a better error message
@@ -108,13 +108,13 @@ sub tablespace_map_get
sub version_get
{
my $self = shift;
if (defined($self->{fVersion}))
{
return $self->{fVersion};
}
$self->{fVersion} =
$self->{fVersion} =
trim($self->psql_execute("copy (select (regexp_matches(split_part(version(), ' ', 2), '^[0-9]+\.[0-9]+'))[1]) to stdout"));
&log(DEBUG, "database version is $self->{fVersion}");
@@ -131,7 +131,7 @@ sub backup_start
my $strLabel = shift;
my $bStartFast = shift;
return trim($self->psql_execute("set client_min_messages = 'warning';" .
return trim($self->psql_execute("set client_min_messages = 'warning';" .
"copy (select pg_xlogfile_name(xlog) from pg_start_backup('${strLabel}'" .
($bStartFast ? ", true" : "") . ") as xlog) to stdout"));
}
@@ -148,4 +148,4 @@ sub backup_stop
}
no Moose;
__PACKAGE__->meta->make_immutable;
__PACKAGE__->meta->make_immutable;
pg_backrest_file.pm
+10 -16
View File
@@ -22,7 +22,8 @@ use lib dirname($0);
use pg_backrest_utility;
use Exporter qw(import);
our @EXPORT = qw(PATH_ABSOLUTE PATH_DB PATH_DB_ABSOLUTE PATH_BACKUP PATH_BACKUP_ABSOLUTE PATH_BACKUP_CLUSTER PATH_BACKUP_TMP PATH_BACKUP_ARCHIVE);
our @EXPORT = qw(PATH_ABSOLUTE PATH_DB PATH_DB_ABSOLUTE PATH_BACKUP PATH_BACKUP_ABSOLUTE PATH_BACKUP_CLUSTERPATH_BACKUP_TMP
PATH_BACKUP_ARCHIVE);
# Extension and permissions
has strCompressExtension => (is => 'ro', default => 'gz');
@@ -31,19 +32,13 @@ has strDefaultFilePermission => (is => 'ro', default => '0640');
# Command strings
has strCommand => (is => 'bare');
has strCommandChecksum => (is => 'bare');
has strCommandCompress => (is => 'bare');
has strCommandDecompress => (is => 'bare');
has strCommandCat => (is => 'bare', default => 'cat %file%');
has strCommandManifest => (is => 'bare');
# Lock path
has strLockPath => (is => 'bare');
# Files to hold stderr
#has strBackupStdErrFile => (is => 'bare');
#has strDbStdErrFile => (is => 'bare');
# Module variables
has strDbUser => (is => 'bare'); # Database user
has strDbHost => (is => 'bare'); # Database host
@@ -158,11 +153,9 @@ sub clone
strDefaultPathPermission => $self->{strDefaultPathPermission},
strDefaultFilePermission => $self->{strDefaultFilePermission},
strCommand => $self->{strCommand},
strCommandChecksum => $self->{strCommandChecksum},
strCommandCompress => $self->{strCommandCompress},
strCommandDecompress => $self->{strCommandDecompress},
strCommandCat => $self->{strCommandCat},
strCommandManifest => $self->{strCommandManifest},
strDbUser => $self->{strDbUser},
strDbHost => $self->{strDbHost},
strBackupUser => $self->{strBackupUser},
@@ -211,11 +204,12 @@ sub path_type_get
my $self = shift;
my $strType = shift;
# If db type
# If absolute type
if ($strType eq PATH_ABSOLUTE)
{
return PATH_ABSOLUTE;
}
# If db type
elsif ($strType =~ /^db(\:.*){0,1}/)
{
return PATH_DB;
@@ -405,7 +399,7 @@ sub remote_get
####################################################################################################################################
# LINK_CREATE
# LINK_CREATE !!! NEEDS TO BE CONVERTED
####################################################################################################################################
sub link_create
{
@@ -643,7 +637,7 @@ sub move
}
####################################################################################################################################
# FILE_COPY
# COPY !!! NEEDS TO BE CONVERTED
####################################################################################################################################
sub file_copy
{
@@ -830,7 +824,7 @@ sub file_copy
# Move the file from tmp to final destination
$self->file_move($self->path_type_get($strSourcePathType) . ":absolute", $strDestinationTmp,
$self->path_type_get($strDestinationPathType) . ":absolute", $strDestination, $bPathCreate);
return true;
}
@@ -1168,7 +1162,7 @@ sub remove
}
}
}
return $bRemoved;
}
@@ -1206,7 +1200,7 @@ sub manifest
{
confess &log(ERROR, "${strErrorPrefix} remote (${strCommand}): " . (defined($strOutput) ? $strOutput : $oSSH->error));
}
return data_hash_build($oManifestHashRef, $strOutput, "\t", ".");
}
# Run locally
@@ -1286,7 +1280,7 @@ sub manifest_recurse
confess &log(ERROR, "${strErrorPrefix}: " . $strError);
}
next;
}
pg_backrest_utility.pm
+23 -23
View File
@@ -66,9 +66,9 @@ sub lock_file_create
{
confess &lock(ASSERT, "${strLockFile} lock is already held");
}
$strLockPath = $strLockPathParam;
unless (-e $strLockPath)
{
if (system("mkdir -p ${strLockPath}") != 0)
@@ -85,7 +85,7 @@ sub lock_file_create
close($hLockFile);
return 0;
}
return $hLockFile;
}
@@ -97,9 +97,9 @@ sub lock_file_remove
if (defined($hLockFile))
{
close($hLockFile);
remove_tree($strLockPath) or confess &log(ERROR, "unable to delete lock path ${strLockPath}");
$hLockFile = undef;
$strLockPath = undef;
}
@@ -121,7 +121,7 @@ sub data_hash_build
my @stryFile = split("\n", $strData);
my @stryHeader = split($strDelimiter, $stryFile[0]);
for (my $iLineIdx = 1; $iLineIdx < scalar @stryFile; $iLineIdx++)
{
my @stryLine = split($strDelimiter, $stryFile[$iLineIdx]);
@@ -137,7 +137,7 @@ sub data_hash_build
{
confess "the first column must be unique to build the hash";
}
if (defined($stryLine[$iColumnIdx]) && $stryLine[$iColumnIdx] ne "")
{
${$oHashRef}{"$stryHeader[0]"}{"$stryLine[0]"}{"$stryHeader[$iColumnIdx]"} = $stryLine[$iColumnIdx];
@@ -171,7 +171,7 @@ sub wait_for_file
my $strDir = shift;
my $strRegEx = shift;
my $iSeconds = shift;
my $lTime = time();
my $hDir;
@@ -199,17 +199,17 @@ sub common_prefix
{
my $strString1 = shift;
my $strString2 = shift;
my $iCommonLen = 0;
my $iCompareLen = length($strString1) < length($strString2) ? length($strString1) : length($strString2);
for (my $iIndex = 0; $iIndex < $iCompareLen; $iIndex++)
{
if (substr($strString1, $iIndex, 1) ne substr($strString2, $iIndex, 1))
{
last;
}
$iCommonLen ++;
}
@@ -247,12 +247,12 @@ sub file_size_format
sub date_string_get
{
my $strFormat = shift;
if (!defined($strFormat))
{
$strFormat = "%4d%02d%02d-%02d%02d%02d";
}
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
return(sprintf($strFormat, $year+1900, $mon+1, $mday, $hour, $min, $sec));
@@ -324,7 +324,7 @@ sub log
{
confess &log(ASSERT, "log level ${strLevel} does not exist");
}
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
if (!defined($strMessage))
@@ -364,15 +364,15 @@ sub log
####################################################################################################################################
# EXECUTE - execute a command
####################################################################################################################################
sub execute
{
my $strCommand = shift;
my $strOutput;
#sub execute
#{
# my $strCommand = shift;
# my $strOutput;
#
# print("$strCommand");
$strOutput = capture($strCommand) or confess &log(ERROR, "unable to execute command ${strCommand}: " . $_);
return $strOutput;
}
# $strOutput = capture($strCommand) or confess &log(ERROR, "unable to execute command ${strCommand}: " . $_);
#
# return $strOutput;
#}
1;
test/test_old.pl
+12 -12
View File
@@ -26,21 +26,21 @@ sub execute
$strOutput = trim(capture($strCommand));
if ($strOutput eq "")
{
{
print(" ... complete\n\n");
}
else
{
print(" ... complete\n$strOutput\n\n");
}
return $strOutput;
}
sub pg_create
{
local($strPgBinPath, $strTestPath, $strTestDir, $strArchiveDir, $strBackupDir) = @_;
execute("mkdir $strTestPath");
execute("mkdir $strTestPath/$strTestDir");
execute("mkdir $strTestPath/$strTestDir/ts1");
@@ -54,7 +54,7 @@ sub pg_start
{
local($strPgBinPath, $strDbPath, $strPort, $strAchiveCommand) = @_;
my $strCommand = "$strPgBinPath/pg_ctl start -o \"-c port=$strPort -c checkpoint_segments=1 -c wal_level=archive -c archive_mode=on -c archive_command=\'$strAchiveCommand\'\" -D $strDbPath -l $strDbPath/postgresql.log -w -s";
execute($strCommand);
}
@@ -62,7 +62,7 @@ sub pg_password_set
{
local($strPgBinPath, $strPath, $strUser, $strPort) = @_;
my $strCommand = "$strPgBinPath/psql --port=$strPort -c \"alter user $strUser with password 'password'\" postgres";
execute($strCommand);
}
@@ -70,7 +70,7 @@ sub pg_stop
{
local($strPgBinPath, $strPath) = @_;
my $strCommand = "$strPgBinPath/pg_ctl stop -D $strPath -w -s -m fast";
execute($strCommand);
}
@@ -78,7 +78,7 @@ sub pg_drop
{
local($strTestPath) = @_;
my $strCommand = "rm -rf $strTestPath";
execute($strCommand);
}
@@ -100,9 +100,9 @@ sub archive_command_build
my $strDestinationPath = shift;
my $bCompression = shift;
my $bChecksum = shift;
my $strCommand = "$strBackRestBinPath/pg_backrest.pl --stanza=db --config=$strBackRestBinPath/pg_backrest.conf";
# if (!$bCompression)
# {
# $strCommand .= " --no-compression"
@@ -112,7 +112,7 @@ sub archive_command_build
# {
# $strCommand .= " --no-checksum"
# }
return $strCommand . " archive-push %p";
}
@@ -121,7 +121,7 @@ sub wait_for_file
my $strDir = shift;
my $strRegEx = shift;
my $iSeconds = shift;
my $lTime = time();
my $hDir;
@@ -146,7 +146,7 @@ sub pgbr_backup
{
my $strBackRestBinPath = shift;
my $strCluster = shift;
my $strCommand = "$strBackRestBinPath/pg_backrest.pl --config=$strBackRestBinPath/pg_backrest.conf backup $strCluster";
execute($strCommand);
util/clean_trailing_whitespace.sh
Executable
+1
View File
@@ -0,0 +1 @@
find . -type f -name "$1" -exec sh -c 'for i;do sed 's/[[:space:]]*$//' "$i">/tmp/.$$ && cat /tmp/.$$ > "$i";done' arg0 {} +