db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

Working on backup - mostly working except for the last archive step.

Browse Source
This commit is contained in:
David Steele 2014-06-22 16:05:00 -04:00
parent f91dce259f
commit 5d01aac92e
8 changed files with 98 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
bin/pg_backrest_remote.pl
View File

@ -153,6 +153,31 @@ while ($strCommand ne OP_EXIT)
param_get(\%oParamHash, 'destination_compress'), param_get(\%oParamHash, 'destination_compress'),
param_get(\%oParamHash, 'ignore_missing_source', false)) ? 'Y' : 'N'); param_get(\%oParamHash, 'ignore_missing_source', false)) ? 'Y' : 'N');
} }
elsif ($strCommand eq OP_FILE_MANIFEST)
{
my %oManifestHash;
$oFile->manifest(PATH_ABSOLUTE, param_get(\%oParamHash, 'path'), \%oManifestHash);
my $strOutput = "name\ttype\tuser\tgroup\tpermission\tmodification_time\tinode\tsize\tlink_destination";
foreach my $strName (sort(keys $oManifestHash{name}))
{
$strOutput .= "\n${strName}\t" .
$oManifestHash{name}{"${strName}"}{type} . "\t" .
(defined($oManifestHash{name}{"${strName}"}{user}) ? $oManifestHash{name}{"${strName}"}{user} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{group}) ? $oManifestHash{name}{"${strName}"}{group} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{permission}) ? $oManifestHash{name}{"${strName}"}{permission} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{modification_time}) ?
$oManifestHash{name}{"${strName}"}{modification_time} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{inode}) ? $oManifestHash{name}{"${strName}"}{inode} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{size}) ? $oManifestHash{name}{"${strName}"}{size} : "") . "\t" .
(defined($oManifestHash{name}{"${strName}"}{link_destination}) ?
$oManifestHash{name}{"${strName}"}{link_destination} : "");
}
$oRemote->output_write($strOutput);
}
else else
{ {
if ($strCommand ne OP_NOOP) if ($strCommand ne OP_NOOP)

55
lib/BackRest/Backup.pm
View File

@ -29,6 +29,7 @@ my $oDb;
my $oFile; my $oFile;
my $strType = "incremental"; # Type of backup: full, differential (diff), incremental (incr) my $strType = "incremental"; # Type of backup: full, differential (diff), incremental (incr)
my $bHardLink; my $bHardLink;
my $bCompress = true;
my $bNoChecksum; my $bNoChecksum;
my $iThreadMax; my $iThreadMax;
my $iThreadLocalMax; my $iThreadLocalMax;
@ -788,8 +789,7 @@ sub backup_manifest_build
${$oBackupManifestRef}{"${strSection}"}{"$strName"}{user} = $oManifestHash{name}{"${strName}"}{user}; ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{user} = $oManifestHash{name}{"${strName}"}{user};
${$oBackupManifestRef}{"${strSection}"}{"$strName"}{group} = $oManifestHash{name}{"${strName}"}{group}; ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{group} = $oManifestHash{name}{"${strName}"}{group};
${$oBackupManifestRef}{"${strSection}"}{"$strName"}{permission} = $oManifestHash{name}{"${strName}"}{permission}; ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{permission} = $oManifestHash{name}{"${strName}"}{permission};
${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} = ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} = $oManifestHash{name}{"${strName}"}{modification_time};
(split("\\.", $oManifestHash{name}{"${strName}"}{modification_time}))[0];
if ($cType eq "f") if ($cType eq "f")
{ {
@ -1184,23 +1184,13 @@ sub backup_file_thread
file_size_format($oFileCopyMap{$strFile}{size}) . file_size_format($oFileCopyMap{$strFile}{size}) .
($lSizeTotal > 0 ? ", " . int($lSize * 100 / $lSizeTotal) . "%" : "") . ")"; ($lSizeTotal > 0 ? ", " . int($lSize * 100 / $lSizeTotal) . "%" : "") . ")";
# Copy the file from the database to the backup # Copy the file from the database to the backup (will return false if the source file is missing)
unless($oFileThread->file_copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}, unless($oFileThread->copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file},
PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file},
undef, $oFileCopyMap{$strFile}{modification_time}, false, # Source is not compressed since it is the db directory
undef, $bPathCreate, false)) $bCompress, # Destination should be compressed based on backup settings
true)) # Ignore missing files
{ {
&log(DEBUG, "thread ${iThreadIdx} unable to copy file: " . $oFileCopyMap{$strFile}{db_file});
# If the copy fails then then check if the file exists. The database frequently removes files so it is normal for
# files to be missing after the manifest is built. However, if the file exists then it means there was some other
# sort of fatal copy error and an abort is required to prevent a corrupted backup
if ($oFileThread->file_exists(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}))
{
# !!! Improve this error when able to retrieve error text from the File object
confess &log(ERROR, "unable to copy file $oFileCopyMap{$strFile}{db_file}");
}
# If file is missing assume the database removed it (else corruption and nothing we can do!) # If file is missing assume the database removed it (else corruption and nothing we can do!)
&log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file}); &log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file});
@ -1216,20 +1206,20 @@ sub backup_file_thread
} }
# Generate checksum for file if configured # Generate checksum for file if configured
if ($bChecksum && $lSize != 0) # if ($bChecksum && $lSize != 0)
{ # {
# Generate the checksum # # Generate the checksum
my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}); # my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file});
#
# Write the checksum message into the master queue # # Write the checksum message into the master queue
$oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}"); # $oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}");
#
&log(INFO, $strLog . " checksum ${strChecksum}"); # &log(INFO, $strLog . " checksum ${strChecksum}");
} # }
else # else
{ # {
&log(INFO, $strLog); &log(INFO, $strLog);
} # }
} }
&log(DEBUG, "thread ${iThreadIdx} exiting"); &log(DEBUG, "thread ${iThreadIdx} exiting");
@ -1259,7 +1249,8 @@ sub backup
&log(DEBUG, "cluster path is $strDbClusterPath"); &log(DEBUG, "cluster path is $strDbClusterPath");
# Create the cluster backup path # Create the cluster backup path
$oFile->path_create(PATH_BACKUP, "backup", undef, true); # $oFile->path_create(PATH_BACKUP, "backup", undef, true);
# $oFile->path_create(PATH_BACKUP, "temp", undef, true);
$oFile->path_create(PATH_BACKUP_CLUSTER, undef, undef, true); $oFile->path_create(PATH_BACKUP_CLUSTER, undef, undef, true);
# Find the previous backup based on the type # Find the previous backup based on the type

4
lib/BackRest/Db.pm
View File

@ -99,8 +99,8 @@ sub tablespace_map_get
my $self = shift; my $self = shift;
my $oHashRef = shift; my $oHashRef = shift;
return data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute( data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute(
"copy (select oid, spcname from pg_tablespace) to stdout"), "\t"); "copy (select oid, spcname from pg_tablespace) to stdout"), "\t");
} }
#################################################################################################################################### ####################################################################################################################################

43
lib/BackRest/File.pm
View File

@ -13,6 +13,7 @@ use Net::OpenSSH;
use IPC::Open3; use IPC::Open3;
use File::Basename; use File::Basename;
use File::Copy qw(cp); use File::Copy qw(cp);
use File::Path qw(make_path remove_tree);
use Digest::SHA; use Digest::SHA;
use File::stat; use File::stat;
use Fcntl ':mode'; use Fcntl ':mode';
@ -579,18 +580,18 @@ sub path_create
if (!($bIgnoreExists && $self->exists($strPathType, $strPath))) if (!($bIgnoreExists && $self->exists($strPathType, $strPath)))
{ {
# Attempt the create the directory # Attempt the create the directory
my $bResult; my $stryError;
if (defined($strPermission)) if (defined($strPermission))
{ {
$bResult = mkdir($strPathOp, oct($strPermission)); make_path($strPathOp, {mode => oct($strPermission), error => \$stryError});
} }
else else
{ {
$bResult = mkdir($strPathOp); make_path($strPathOp, {error => \$stryError});
} }
if (!$bResult) if (@$stryError)
{ {
# Capture the error # Capture the error
my $strError = "${strPathOp} could not be created: " . $!; my $strError = "${strPathOp} could not be created: " . $!;
@ -602,7 +603,7 @@ sub path_create
} }
# Error the normal way # Error the normal way
confess &log(ERROR, "${strDebug}: " . $strError, COMMAND_ERR_PATH_CREATE); confess &log(ERROR, "${strDebug}: " . $strError); #, COMMAND_ERR_PATH_CREATE);
} }
} }
} }
@ -661,7 +662,7 @@ sub exists
} }
else else
{ {
confess &log(ERROR, "${strDebug}: " . $!, COMMAND_ERR_FILE_READ); confess &log(ERROR, "${strDebug}: " . $!); #, COMMAND_ERR_FILE_READ);
} }
} }
@ -907,19 +908,30 @@ sub manifest
my $strPathOp = $self->path_get($strPathType, $strPath); my $strPathOp = $self->path_get($strPathType, $strPath);
# Set operation and debug strings # Set operation and debug strings
my $strOperation = OP_FILE_EXISTS; my $strOperation = OP_FILE_MANIFEST;
my $strDebug = "${strPathType}:${strPathOp}"; my $strDebug = "${strPathType}:${strPathOp}";
&log(DEBUG, "${strOperation}: ${strDebug}"); &log(DEBUG, "${strOperation}: ${strDebug}");
# Run remotely # Run remotely
if ($self->is_remote($strPathType)) if ($self->is_remote($strPathType))
{ {
confess &log(ASSERT, "${strDebug}: remote operation not supported"); # Build param hash
my %oParamHash;
$oParamHash{path} = $strPathOp;
# Add remote info to debug string
my $strRemote = "remote (" . $self->{oRemote}->command_param_string(\%oParamHash) . ")";
$strDebug = "${strOperation}: ${strRemote}: ${strDebug}";
&log(TRACE, "${strOperation}: ${strRemote}");
# Execute the command
data_hash_build($oManifestHashRef, $self->{oRemote}->command_execute($strOperation, \%oParamHash, true, $strDebug), "\t");
} }
# Run locally # Run locally
else else
{ {
$self->manifest_recurse($strPathType, $strPathOp, undef, 0, $oManifestHashRef); $self->manifest_recurse($strPathType, $strPathOp, undef, 0, $oManifestHashRef, $strDebug);
} }
} }
@ -931,8 +943,9 @@ sub manifest_recurse
my $strPathFileOp = shift; my $strPathFileOp = shift;
my $iDepth = shift; my $iDepth = shift;
my $oManifestHashRef = shift; my $oManifestHashRef = shift;
my $strDebug = shift;
my $strErrorPrefix = "File->manifest"; $strDebug = $strDebug . (defined($strPathFileOp) ? " => ${strPathFileOp}" : "");
my $strPathRead = $strPathOp . (defined($strPathFileOp) ? "/${strPathFileOp}" : ""); my $strPathRead = $strPathOp . (defined($strPathFileOp) ? "/${strPathFileOp}" : "");
my $hPath; my $hPath;
@ -952,7 +965,7 @@ sub manifest_recurse
confess &log(ERROR, $strError, $iErrorCode); confess &log(ERROR, $strError, $iErrorCode);
} }
confess &log(ERROR, "${strErrorPrefix}: " . $strError); confess &log(ERROR, "${strDebug}: " . $strError);
} }
my @stryFileList = grep(!/^\..$/i, readdir($hPath)); my @stryFileList = grep(!/^\..$/i, readdir($hPath));
@ -995,7 +1008,7 @@ sub manifest_recurse
confess &log(ERROR, $strError, COMMAND_ERR_FILE_READ); confess &log(ERROR, $strError, COMMAND_ERR_FILE_READ);
} }
confess &log(ERROR, "${strErrorPrefix}: " . $strError, COMMAND_ERR_FILE_READ); confess &log(ERROR, "${strDebug}: " . $strError); #, COMMAND_ERR_FILE_READ);
} }
# Check for regular file # Check for regular file
@ -1037,7 +1050,7 @@ sub manifest_recurse
exit COMMAND_ERR_LINK_READ; exit COMMAND_ERR_LINK_READ;
} }
confess &log(ERROR, "${strErrorPrefix}: " . $strError); confess &log(ERROR, "${strDebug}: " . $strError);
} }
} }
} }
@ -1051,7 +1064,7 @@ sub manifest_recurse
exit COMMAND_ERR_FILE_TYPE; exit COMMAND_ERR_FILE_TYPE;
} }
confess &log(ERROR, "${strErrorPrefix}: " . $strError); confess &log(ERROR, "${strDebug}: " . $strError);
} }
# Get user name # Get user name
@ -1069,7 +1082,7 @@ sub manifest_recurse
# Recurse into directories # Recurse into directories
if (${$oManifestHashRef}{name}{"${strFile}"}{type} eq "d" && !$bCurrentDir) if (${$oManifestHashRef}{name}{"${strFile}"}{type} eq "d" && !$bCurrentDir)
{ {
$self->manifest_recurse($strPathType, $strPathOp, $strFile, $iDepth + 1, $oManifestHashRef); $self->manifest_recurse($strPathType, $strPathOp, $strFile, $iDepth + 1, $oManifestHashRef, $strDebug);
} }
} }
} }

2
lib/BackRest/Remote.pm
View File

@ -24,7 +24,7 @@ use BackRest::Utility;
#################################################################################################################################### ####################################################################################################################################
use constant use constant
{ {
DEFAULT_BLOCK_SIZE => 8192 DEFAULT_BLOCK_SIZE => 1048576
}; };
#################################################################################################################################### ####################################################################################################################################

6
test/lib/BackRestTest/BackupTest.pm
View File

@ -124,10 +124,10 @@ sub BackRestTestBackup_Test
BackRestTestBackup_Setup(); BackRestTestBackup_Setup();
BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_BACKUP); BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_DB);#, REMOTE_BACKUP);
BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_DB); BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);#, REMOTE_DB);
BackRestTestCommon_ExecuteBackRest(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() . BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() .
"/pg_backrest.conf --type=incr --stanza=${strStanza} backup"); "/pg_backrest.conf --type=incr --stanza=${strStanza} backup");
#------------------------------------------------------------------------------------------------------------------------------- #-------------------------------------------------------------------------------------------------------------------------------

16
test/lib/BackRestTest/CommonTest.pm
View File

@ -98,6 +98,7 @@ sub BackRestTestCommon_Setup
sub BackRestTestCommon_ConfigCreate sub BackRestTestCommon_ConfigCreate
{ {
my $strFile = shift; my $strFile = shift;
my $strLocal = shift;
my $strRemote = shift; my $strRemote = shift;
my $oParamHashRef = shift; my $oParamHashRef = shift;
@ -116,12 +117,25 @@ sub BackRestTestCommon_ConfigCreate
{ {
$oParamHash{$strCommonStanza}{'host'} = $strCommonHost; $oParamHash{$strCommonStanza}{'host'} = $strCommonHost;
$oParamHash{$strCommonStanza}{'user'} = $strCommonUser; $oParamHash{$strCommonStanza}{'user'} = $strCommonUser;
}
if ($strLocal eq REMOTE_BACKUP)
{
$oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath; $oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath;
$oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}"; $oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}";
} }
elsif ($strLocal eq REMOTE_DB)
{
}
else
{
confess "invalid local type ${strLocal}";
}
$oParamHash{'global:backup'}{'path'} = $strCommonBackupPath; $oParamHash{'global:backup'}{'path'} = $strCommonBackupPath;
$oParamHash{'global:log'}{'level-console'} = 'debug';
$oParamHash{'global:log'}{'level-file'} = 'trace';
tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}"; tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}";

2
test/lib/BackRestTest/FileTest.pm
View File

@ -498,7 +498,7 @@ sub BackRestTestFile_Test
# Execute in eval in case of error # Execute in eval in case of error
my %oManifestHash; my %oManifestHash;
my $bErrorExpected = !$bExists || $bError || $bRemote; my $bErrorExpected = !$bExists || $bError;
eval eval
{ {