db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

Working on backup - mostly working except for the last archive step.

Browse Source
This commit is contained in:
David Steele 2014-06-22 16:05:00 -04:00
parent f91dce259f
commit 5d01aac92e
8 changed files with 98 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bin/pg_backrest_remote.pl
View File

@ -153,6 +153,31 @@ while ($strCommand ne OP_EXIT)
param_get(\%oParamHash, 'destination_compress'),
param_get(\%oParamHash, 'ignore_missing_source', false)) ? 'Y' : 'N');
}
elsif ($strCommand eq OP_FILE_MANIFEST)
{
my %oManifestHash;
$oFile->manifest(PATH_ABSOLUTE, param_get(\%oParamHash, 'path'), \%oManifestHash);
my $strOutput = "name\ttype\tuser\tgroup\tpermission\tmodification_time\tinode\tsize\tlink_destination";
foreach my $strName (sort(keys $oManifestHash{name}))
{
$strOutput .= "\n${strName}\t" .
$oManifestHash{name}{"${strName}"}{type} . "\t" .
(defined($oManifestHash{name}{"${strName}"}{user}) ? $oManifestHash{name}{"${strName}"}{user} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{group}) ? $oManifestHash{name}{"${strName}"}{group} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{permission}) ? $oManifestHash{name}{"${strName}"}{permission} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{modification_time}) ?
$oManifestHash{name}{"${strName}"}{modification_time} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{inode}) ? $oManifestHash{name}{"${strName}"}{inode} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{size}) ? $oManifestHash{name}{"${strName}"}{size} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{link_destination}) ?
$oManifestHash{name}{"${strName}"}{link_destination} : "");
}
$oRemote->output_write($strOutput);
}
else
{
if ($strCommand ne OP_NOOP)

53
lib/BackRest/Backup.pm
View File

@ -29,6 +29,7 @@ my $oDb;
my $oFile;
my $strType = "incremental"; # Type of backup: full, differential (diff), incremental (incr)
my $bHardLink;
my $bCompress = true;
my $bNoChecksum;
my $iThreadMax;
my $iThreadLocalMax;
@ -788,8 +789,7 @@ sub backup_manifest_build
${$oBackupManifestRef}{"${strSection}"}{"$strName"}{user} = $oManifestHash{name}{"${strName}"}{user};
${$oBackupManifestRef}{"${strSection}"}{"$strName"}{group} = $oManifestHash{name}{"${strName}"}{group};
${$oBackupManifestRef}{"${strSection}"}{"$strName"}{permission} = $oManifestHash{name}{"${strName}"}{permission};
${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} =
(split("\\.", $oManifestHash{name}{"${strName}"}{modification_time}))[0];
${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} = $oManifestHash{name}{"${strName}"}{modification_time};
if ($cType eq "f")
{
@ -1184,23 +1184,13 @@ sub backup_file_thread
file_size_format($oFileCopyMap{$strFile}{size}) .
($lSizeTotal > 0 ? ", " . int($lSize * 100 / $lSizeTotal) . "%" : "") . ")";
# Copy the file from the database to the backup
unless($oFileThread->file_copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
# Copy the file from the database to the backup (will return false if the source file is missing)
unless($oFileThread->copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file},
undef, $oFileCopyMap{$strFile}{modification_time},
undef, $bPathCreate, false))
false, # Source is not compressed since it is the db directory
$bCompress, # Destination should be compressed based on backup settings
true)) # Ignore missing files
{
&log(DEBUG, "thread ${iThreadIdx} unable to copy file: " . $oFileCopyMap{$strFile}{db_file});
# If the copy fails then then check if the file exists. The database frequently removes files so it is normal for
# files to be missing after the manifest is built. However, if the file exists then it means there was some other
# sort of fatal copy error and an abort is required to prevent a corrupted backup
if ($oFileThread->file_exists(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}))
{
# !!! Improve this error when able to retrieve error text from the File object
confess &log(ERROR, "unable to copy file $oFileCopyMap{$strFile}{db_file}");
}
# If file is missing assume the database removed it (else corruption and nothing we can do!)
&log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file});
@ -1216,20 +1206,20 @@ sub backup_file_thread
}
# Generate checksum for file if configured
if ($bChecksum && $lSize != 0)
{
# Generate the checksum
my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file});
# Write the checksum message into the master queue
$oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}");
&log(INFO, $strLog . " checksum ${strChecksum}");
}
else
{
# if ($bChecksum && $lSize != 0)
# {
# # Generate the checksum
# my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file});
#
# # Write the checksum message into the master queue
# $oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}");
#
# &log(INFO, $strLog . " checksum ${strChecksum}");
# }
# else
# {
&log(INFO, $strLog);
}
# }
}
&log(DEBUG, "thread ${iThreadIdx} exiting");
@ -1259,7 +1249,8 @@ sub backup
&log(DEBUG, "cluster path is $strDbClusterPath");
# Create the cluster backup path
$oFile->path_create(PATH_BACKUP, "backup", undef, true);
# $oFile->path_create(PATH_BACKUP, "backup", undef, true);
# $oFile->path_create(PATH_BACKUP, "temp", undef, true);
$oFile->path_create(PATH_BACKUP_CLUSTER, undef, undef, true);
# Find the previous backup based on the type

2
lib/BackRest/Db.pm
View File

@ -99,7 +99,7 @@ sub tablespace_map_get
my $self = shift;
my $oHashRef = shift;
return data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute(
data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute(
"copy (select oid, spcname from pg_tablespace) to stdout"), "\t");
}

43
lib/BackRest/File.pm
View File

@ -13,6 +13,7 @@ use Net::OpenSSH;
use IPC::Open3;
use File::Basename;
use File::Copy qw(cp);
use File::Path qw(make_path remove_tree);
use Digest::SHA;
use File::stat;
use Fcntl ':mode';
@ -579,18 +580,18 @@ sub path_create
if (!($bIgnoreExists && $self->exists($strPathType, $strPath)))
{
# Attempt the create the directory
my $bResult;
my $stryError;
if (defined($strPermission))
{
$bResult = mkdir($strPathOp, oct($strPermission));
make_path($strPathOp, {mode => oct($strPermission), error => \$stryError});
}
else
{
$bResult = mkdir($strPathOp);
make_path($strPathOp, {error => \$stryError});
}
if (!$bResult)
if (@$stryError)
{
# Capture the error
my $strError = "${strPathOp} could not be created: " . $!;
@ -602,7 +603,7 @@ sub path_create
}
# Error the normal way
confess &log(ERROR, "${strDebug}: " . $strError, COMMAND_ERR_PATH_CREATE);
confess &log(ERROR, "${strDebug}: " . $strError); #, COMMAND_ERR_PATH_CREATE);
}
}
}
@ -661,7 +662,7 @@ sub exists
}
else
{
confess &log(ERROR, "${strDebug}: " . $!, COMMAND_ERR_FILE_READ);
confess &log(ERROR, "${strDebug}: " . $!); #, COMMAND_ERR_FILE_READ);
}
}
@ -907,19 +908,30 @@ sub manifest
my $strPathOp = $self->path_get($strPathType, $strPath);
# Set operation and debug strings
my $strOperation = OP_FILE_EXISTS;
my $strOperation = OP_FILE_MANIFEST;
my $strDebug = "${strPathType}:${strPathOp}";
&log(DEBUG, "${strOperation}: ${strDebug}");
# Run remotely
if ($self->is_remote($strPathType))
{
confess &log(ASSERT, "${strDebug}: remote operation not supported");
# Build param hash
my %oParamHash;
$oParamHash{path} = $strPathOp;
# Add remote info to debug string
my $strRemote = "remote (" . $self->{oRemote}->command_param_string(\%oParamHash) . ")";
$strDebug = "${strOperation}: ${strRemote}: ${strDebug}";
&log(TRACE, "${strOperation}: ${strRemote}");
# Execute the command
data_hash_build($oManifestHashRef, $self->{oRemote}->command_execute($strOperation, \%oParamHash, true, $strDebug), "\t");
}
# Run locally
else
{
$self->manifest_recurse($strPathType, $strPathOp, undef, 0, $oManifestHashRef);
$self->manifest_recurse($strPathType, $strPathOp, undef, 0, $oManifestHashRef, $strDebug);
}
}
@ -931,8 +943,9 @@ sub manifest_recurse
my $strPathFileOp = shift;
my $iDepth = shift;
my $oManifestHashRef = shift;
my $strDebug = shift;
my $strErrorPrefix = "File->manifest";
$strDebug = $strDebug . (defined($strPathFileOp) ? " => ${strPathFileOp}" : "");
my $strPathRead = $strPathOp . (defined($strPathFileOp) ? "/${strPathFileOp}" : "");
my $hPath;
@ -952,7 +965,7 @@ sub manifest_recurse
confess &log(ERROR, $strError, $iErrorCode);
}
confess &log(ERROR, "${strErrorPrefix}: " . $strError);
confess &log(ERROR, "${strDebug}: " . $strError);
}
my @stryFileList = grep(!/^\..$/i, readdir($hPath));
@ -995,7 +1008,7 @@ sub manifest_recurse
confess &log(ERROR, $strError, COMMAND_ERR_FILE_READ);
}
confess &log(ERROR, "${strErrorPrefix}: " . $strError, COMMAND_ERR_FILE_READ);
confess &log(ERROR, "${strDebug}: " . $strError); #, COMMAND_ERR_FILE_READ);
}
# Check for regular file
@ -1037,7 +1050,7 @@ sub manifest_recurse
exit COMMAND_ERR_LINK_READ;
}
confess &log(ERROR, "${strErrorPrefix}: " . $strError);
confess &log(ERROR, "${strDebug}: " . $strError);
}
}
}
@ -1051,7 +1064,7 @@ sub manifest_recurse
exit COMMAND_ERR_FILE_TYPE;
}
confess &log(ERROR, "${strErrorPrefix}: " . $strError);
confess &log(ERROR, "${strDebug}: " . $strError);
}
# Get user name
@ -1069,7 +1082,7 @@ sub manifest_recurse
# Recurse into directories
if (${$oManifestHashRef}{name}{"${strFile}"}{type} eq "d" && !$bCurrentDir)
{
$self->manifest_recurse($strPathType, $strPathOp, $strFile, $iDepth + 1, $oManifestHashRef);
$self->manifest_recurse($strPathType, $strPathOp, $strFile, $iDepth + 1, $oManifestHashRef, $strDebug);
}
}
}

2
lib/BackRest/Remote.pm
View File

@ -24,7 +24,7 @@ use BackRest::Utility;
####################################################################################################################################
use constant
{
DEFAULT_BLOCK_SIZE => 8192
DEFAULT_BLOCK_SIZE => 1048576
};
####################################################################################################################################

6
test/lib/BackRestTest/BackupTest.pm
View File

@ -124,10 +124,10 @@ sub BackRestTestBackup_Test
BackRestTestBackup_Setup();
BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);
BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_DB);
BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_DB);#, REMOTE_BACKUP);
BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);#, REMOTE_DB);
BackRestTestCommon_ExecuteBackRest(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
"/pg_backrest.conf --type=incr --stanza=${strStanza} backup");
#-------------------------------------------------------------------------------------------------------------------------------

16
test/lib/BackRestTest/CommonTest.pm
View File

@ -98,6 +98,7 @@ sub BackRestTestCommon_Setup
sub BackRestTestCommon_ConfigCreate
{
my $strFile = shift;
my $strLocal = shift;
my $strRemote = shift;
my $oParamHashRef = shift;
@ -116,13 +117,26 @@ sub BackRestTestCommon_ConfigCreate
{
$oParamHash{$strCommonStanza}{'host'} = $strCommonHost;
$oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
$oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath;
}
if ($strLocal eq REMOTE_BACKUP)
{
$oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath;
$oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}";
}
elsif ($strLocal eq REMOTE_DB)
{
}
else
{
confess "invalid local type ${strLocal}";
}
$oParamHash{'global:backup'}{'path'} = $strCommonBackupPath;
$oParamHash{'global:log'}{'level-console'} = 'debug';
$oParamHash{'global:log'}{'level-file'} = 'trace';
tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}";
chmod(0770, $strFile) or die "unable to set permissions for ${strFile}";

2
test/lib/BackRestTest/FileTest.pm
View File

@ -498,7 +498,7 @@ sub BackRestTestFile_Test
# Execute in eval in case of error
my %oManifestHash;
my $bErrorExpected = !$bExists || $bError || $bRemote;
my $bErrorExpected = !$bExists || $bError;
eval
{