diff --git a/bin/pg_backrest_remote.pl b/bin/pg_backrest_remote.pl index f299749fd..853695bdd 100755 --- a/bin/pg_backrest_remote.pl +++ b/bin/pg_backrest_remote.pl @@ -153,6 +153,31 @@ while ($strCommand ne OP_EXIT) param_get(\%oParamHash, 'destination_compress'), param_get(\%oParamHash, 'ignore_missing_source', false)) ? 'Y' : 'N'); } + elsif ($strCommand eq OP_FILE_MANIFEST) + { + my %oManifestHash; + + $oFile->manifest(PATH_ABSOLUTE, param_get(\%oParamHash, 'path'), \%oManifestHash); + + my $strOutput = "name\ttype\tuser\tgroup\tpermission\tmodification_time\tinode\tsize\tlink_destination"; + + foreach my $strName (sort(keys $oManifestHash{name})) + { + $strOutput .= "\n${strName}\t" . + $oManifestHash{name}{"${strName}"}{type} . "\t" . + (defined($oManifestHash{name}{"${strName}"}{user}) ? $oManifestHash{name}{"${strName}"}{user} : "") . "\t" . + (defined($oManifestHash{name}{"${strName}"}{group}) ? $oManifestHash{name}{"${strName}"}{group} : "") . "\t" . + (defined($oManifestHash{name}{"${strName}"}{permission}) ? $oManifestHash{name}{"${strName}"}{permission} : "") . "\t" . + (defined($oManifestHash{name}{"${strName}"}{modification_time}) ? + $oManifestHash{name}{"${strName}"}{modification_time} : "") . "\t" . + (defined($oManifestHash{name}{"${strName}"}{inode}) ? $oManifestHash{name}{"${strName}"}{inode} : "") . "\t" . + (defined($oManifestHash{name}{"${strName}"}{size}) ? $oManifestHash{name}{"${strName}"}{size} : "") . "\t" . + (defined($oManifestHash{name}{"${strName}"}{link_destination}) ? + $oManifestHash{name}{"${strName}"}{link_destination} : ""); + } + + $oRemote->output_write($strOutput); + } else { if ($strCommand ne OP_NOOP) diff --git a/lib/BackRest/Backup.pm b/lib/BackRest/Backup.pm index 3364595f4..8367ea6e8 100644 --- a/lib/BackRest/Backup.pm +++ b/lib/BackRest/Backup.pm @@ -29,6 +29,7 @@ my $oDb; my $oFile; my $strType = "incremental"; # Type of backup: full, differential (diff), incremental (incr) my $bHardLink; +my $bCompress = true; my $bNoChecksum; my $iThreadMax; my $iThreadLocalMax; @@ -788,8 +789,7 @@ sub backup_manifest_build ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{user} = $oManifestHash{name}{"${strName}"}{user}; ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{group} = $oManifestHash{name}{"${strName}"}{group}; ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{permission} = $oManifestHash{name}{"${strName}"}{permission}; - ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} = - (split("\\.", $oManifestHash{name}{"${strName}"}{modification_time}))[0]; + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} = $oManifestHash{name}{"${strName}"}{modification_time}; if ($cType eq "f") { @@ -1184,23 +1184,13 @@ sub backup_file_thread file_size_format($oFileCopyMap{$strFile}{size}) . ($lSizeTotal > 0 ? ", " . int($lSize * 100 / $lSizeTotal) . "%" : "") . ")"; - # Copy the file from the database to the backup - unless($oFileThread->file_copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}, - PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, - undef, $oFileCopyMap{$strFile}{modification_time}, - undef, $bPathCreate, false)) + # Copy the file from the database to the backup (will return false if the source file is missing) + unless($oFileThread->copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}, + PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, + false, # Source is not compressed since it is the db directory + $bCompress, # Destination should be compressed based on backup settings + true)) # Ignore missing files { - &log(DEBUG, "thread ${iThreadIdx} unable to copy file: " . $oFileCopyMap{$strFile}{db_file}); - - # If the copy fails then then check if the file exists. The database frequently removes files so it is normal for - # files to be missing after the manifest is built. However, if the file exists then it means there was some other - # sort of fatal copy error and an abort is required to prevent a corrupted backup - if ($oFileThread->file_exists(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file})) - { - # !!! Improve this error when able to retrieve error text from the File object - confess &log(ERROR, "unable to copy file $oFileCopyMap{$strFile}{db_file}"); - } - # If file is missing assume the database removed it (else corruption and nothing we can do!) &log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file}); @@ -1216,20 +1206,20 @@ sub backup_file_thread } # Generate checksum for file if configured - if ($bChecksum && $lSize != 0) - { - # Generate the checksum - my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}); - - # Write the checksum message into the master queue - $oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}"); - - &log(INFO, $strLog . " checksum ${strChecksum}"); - } - else - { + # if ($bChecksum && $lSize != 0) + # { + # # Generate the checksum + # my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}); + # + # # Write the checksum message into the master queue + # $oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}"); + # + # &log(INFO, $strLog . " checksum ${strChecksum}"); + # } + # else + # { &log(INFO, $strLog); - } + # } } &log(DEBUG, "thread ${iThreadIdx} exiting"); @@ -1259,7 +1249,8 @@ sub backup &log(DEBUG, "cluster path is $strDbClusterPath"); # Create the cluster backup path - $oFile->path_create(PATH_BACKUP, "backup", undef, true); + # $oFile->path_create(PATH_BACKUP, "backup", undef, true); + # $oFile->path_create(PATH_BACKUP, "temp", undef, true); $oFile->path_create(PATH_BACKUP_CLUSTER, undef, undef, true); # Find the previous backup based on the type diff --git a/lib/BackRest/Db.pm b/lib/BackRest/Db.pm index db009e123..30642fd3c 100644 --- a/lib/BackRest/Db.pm +++ b/lib/BackRest/Db.pm @@ -99,8 +99,8 @@ sub tablespace_map_get my $self = shift; my $oHashRef = shift; - return data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute( - "copy (select oid, spcname from pg_tablespace) to stdout"), "\t"); + data_hash_build($oHashRef, "oid\tname\n" . $self->psql_execute( + "copy (select oid, spcname from pg_tablespace) to stdout"), "\t"); } #################################################################################################################################### diff --git a/lib/BackRest/File.pm b/lib/BackRest/File.pm index 833372867..bf89ace1b 100644 --- a/lib/BackRest/File.pm +++ b/lib/BackRest/File.pm @@ -13,6 +13,7 @@ use Net::OpenSSH; use IPC::Open3; use File::Basename; use File::Copy qw(cp); +use File::Path qw(make_path remove_tree); use Digest::SHA; use File::stat; use Fcntl ':mode'; @@ -579,18 +580,18 @@ sub path_create if (!($bIgnoreExists && $self->exists($strPathType, $strPath))) { # Attempt the create the directory - my $bResult; + my $stryError; if (defined($strPermission)) { - $bResult = mkdir($strPathOp, oct($strPermission)); + make_path($strPathOp, {mode => oct($strPermission), error => \$stryError}); } else { - $bResult = mkdir($strPathOp); + make_path($strPathOp, {error => \$stryError}); } - if (!$bResult) + if (@$stryError) { # Capture the error my $strError = "${strPathOp} could not be created: " . $!; @@ -602,7 +603,7 @@ sub path_create } # Error the normal way - confess &log(ERROR, "${strDebug}: " . $strError, COMMAND_ERR_PATH_CREATE); + confess &log(ERROR, "${strDebug}: " . $strError); #, COMMAND_ERR_PATH_CREATE); } } } @@ -661,7 +662,7 @@ sub exists } else { - confess &log(ERROR, "${strDebug}: " . $!, COMMAND_ERR_FILE_READ); + confess &log(ERROR, "${strDebug}: " . $!); #, COMMAND_ERR_FILE_READ); } } @@ -907,19 +908,30 @@ sub manifest my $strPathOp = $self->path_get($strPathType, $strPath); # Set operation and debug strings - my $strOperation = OP_FILE_EXISTS; + my $strOperation = OP_FILE_MANIFEST; my $strDebug = "${strPathType}:${strPathOp}"; &log(DEBUG, "${strOperation}: ${strDebug}"); # Run remotely if ($self->is_remote($strPathType)) { - confess &log(ASSERT, "${strDebug}: remote operation not supported"); + # Build param hash + my %oParamHash; + + $oParamHash{path} = $strPathOp; + + # Add remote info to debug string + my $strRemote = "remote (" . $self->{oRemote}->command_param_string(\%oParamHash) . ")"; + $strDebug = "${strOperation}: ${strRemote}: ${strDebug}"; + &log(TRACE, "${strOperation}: ${strRemote}"); + + # Execute the command + data_hash_build($oManifestHashRef, $self->{oRemote}->command_execute($strOperation, \%oParamHash, true, $strDebug), "\t"); } # Run locally else { - $self->manifest_recurse($strPathType, $strPathOp, undef, 0, $oManifestHashRef); + $self->manifest_recurse($strPathType, $strPathOp, undef, 0, $oManifestHashRef, $strDebug); } } @@ -931,8 +943,9 @@ sub manifest_recurse my $strPathFileOp = shift; my $iDepth = shift; my $oManifestHashRef = shift; + my $strDebug = shift; - my $strErrorPrefix = "File->manifest"; + $strDebug = $strDebug . (defined($strPathFileOp) ? " => ${strPathFileOp}" : ""); my $strPathRead = $strPathOp . (defined($strPathFileOp) ? "/${strPathFileOp}" : ""); my $hPath; @@ -952,7 +965,7 @@ sub manifest_recurse confess &log(ERROR, $strError, $iErrorCode); } - confess &log(ERROR, "${strErrorPrefix}: " . $strError); + confess &log(ERROR, "${strDebug}: " . $strError); } my @stryFileList = grep(!/^\..$/i, readdir($hPath)); @@ -995,7 +1008,7 @@ sub manifest_recurse confess &log(ERROR, $strError, COMMAND_ERR_FILE_READ); } - confess &log(ERROR, "${strErrorPrefix}: " . $strError, COMMAND_ERR_FILE_READ); + confess &log(ERROR, "${strDebug}: " . $strError); #, COMMAND_ERR_FILE_READ); } # Check for regular file @@ -1037,7 +1050,7 @@ sub manifest_recurse exit COMMAND_ERR_LINK_READ; } - confess &log(ERROR, "${strErrorPrefix}: " . $strError); + confess &log(ERROR, "${strDebug}: " . $strError); } } } @@ -1051,7 +1064,7 @@ sub manifest_recurse exit COMMAND_ERR_FILE_TYPE; } - confess &log(ERROR, "${strErrorPrefix}: " . $strError); + confess &log(ERROR, "${strDebug}: " . $strError); } # Get user name @@ -1069,7 +1082,7 @@ sub manifest_recurse # Recurse into directories if (${$oManifestHashRef}{name}{"${strFile}"}{type} eq "d" && !$bCurrentDir) { - $self->manifest_recurse($strPathType, $strPathOp, $strFile, $iDepth + 1, $oManifestHashRef); + $self->manifest_recurse($strPathType, $strPathOp, $strFile, $iDepth + 1, $oManifestHashRef, $strDebug); } } } diff --git a/lib/BackRest/Remote.pm b/lib/BackRest/Remote.pm index 3d16b8532..2b4d52fee 100644 --- a/lib/BackRest/Remote.pm +++ b/lib/BackRest/Remote.pm @@ -24,7 +24,7 @@ use BackRest::Utility; #################################################################################################################################### use constant { - DEFAULT_BLOCK_SIZE => 8192 + DEFAULT_BLOCK_SIZE => 1048576 }; #################################################################################################################################### diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm index 21e0d96d9..0f8492242 100755 --- a/test/lib/BackRestTest/BackupTest.pm +++ b/test/lib/BackRestTest/BackupTest.pm @@ -124,10 +124,10 @@ sub BackRestTestBackup_Test BackRestTestBackup_Setup(); - BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_BACKUP); - BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_DB); + BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_DB);#, REMOTE_BACKUP); + BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);#, REMOTE_DB); - BackRestTestCommon_ExecuteBackRest(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() . + BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() . "/pg_backrest.conf --type=incr --stanza=${strStanza} backup"); #------------------------------------------------------------------------------------------------------------------------------- diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm index 43ebd03c2..21f1e2934 100755 --- a/test/lib/BackRestTest/CommonTest.pm +++ b/test/lib/BackRestTest/CommonTest.pm @@ -98,6 +98,7 @@ sub BackRestTestCommon_Setup sub BackRestTestCommon_ConfigCreate { my $strFile = shift; + my $strLocal = shift; my $strRemote = shift; my $oParamHashRef = shift; @@ -116,12 +117,25 @@ sub BackRestTestCommon_ConfigCreate { $oParamHash{$strCommonStanza}{'host'} = $strCommonHost; $oParamHash{$strCommonStanza}{'user'} = $strCommonUser; + } + + if ($strLocal eq REMOTE_BACKUP) + { $oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath; - $oParamHash{'db:command:option'}{'psql'} = "--port=${iCommonDbPort}"; } + elsif ($strLocal eq REMOTE_DB) + { + } + else + { + confess "invalid local type ${strLocal}"; + } $oParamHash{'global:backup'}{'path'} = $strCommonBackupPath; + + $oParamHash{'global:log'}{'level-console'} = 'debug'; + $oParamHash{'global:log'}{'level-file'} = 'trace'; tied(%oParamHash)->WriteConfig($strFile) or die "could not write config file ${strFile}"; diff --git a/test/lib/BackRestTest/FileTest.pm b/test/lib/BackRestTest/FileTest.pm index fc4b86c9d..fd37b7d96 100755 --- a/test/lib/BackRestTest/FileTest.pm +++ b/test/lib/BackRestTest/FileTest.pm @@ -498,7 +498,7 @@ sub BackRestTestFile_Test # Execute in eval in case of error my %oManifestHash; - my $bErrorExpected = !$bExists || $bError || $bRemote; + my $bErrorExpected = !$bExists || $bError; eval {