diff --git a/.gitignore b/.gitignore index ebec8ba8c..2a13959aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ **/*~ *~ + +*.swp diff --git a/README.md b/README.md index 41a55076e..f4a547a00 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,16 @@ v0.15: Added archive-get [PLANNED FUNCTIONALITY FOR THIS RELEASE] ------------- +v0.11: Minor fixes + +Tweaking a few settings after running backups for about a month. + +* Removed master_stderr_discard option on database SSH connections. There have been occasional lockups and they could be related issues originally seen in the file code. + +* Changed lock file conflicts on backup and expire commands to ERROR. They were set to DEBUG due to a copy-and-paste from the archive locks. + +------------- + v0.10: Backup and archiving are functional This version has been put into production at Resonate, so it does work, but there are a number of major caveats. diff --git a/pg_backrest.pl b/pg_backrest.pl index 8e65429c3..8f1036925 100755 --- a/pg_backrest.pl +++ b/pg_backrest.pl @@ -83,7 +83,7 @@ GetOptions ("config=s" => \$strConfigFile, # Global variables #################################################################################################################################### my %oConfig; # Configuration hash - + #################################################################################################################################### # CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false) #################################################################################################################################### @@ -522,6 +522,12 @@ backup_init config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT) ); +#################################################################################################################################### +# DEBUG +#################################################################################################################################### +#print "out: " . ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, "0000000100000000", ".*", 'asc'))[0]; +#exit 0; + #################################################################################################################################### # BACKUP #################################################################################################################################### @@ -531,15 +537,13 @@ if ($strOperation eq OP_BACKUP) if (!lock_file_create($strLockFile)) { - &log(DEBUG, "backup process is already running for stanza ${strStanza} - exiting"); + &log(ERROR, "backup process is already running for stanza ${strStanza} - exiting"); exit 0 } backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH)); $strOperation = OP_EXPIRE; - - sleep(30); lock_file_remove(); } @@ -553,7 +557,7 @@ if ($strOperation eq OP_EXPIRE) if (!lock_file_create($strLockFile)) { - &log(DEBUG, "expire process is already running for stanza ${strStanza} - exiting"); + &log(ERROR, "expire process is already running for stanza ${strStanza} - exiting"); exit 0 } diff --git a/pg_backrest_backup.pm b/pg_backrest_backup.pm index 12dc92b30..ce4afb44e 100644 --- a/pg_backrest_backup.pm +++ b/pg_backrest_backup.pm @@ -1440,8 +1440,6 @@ sub backup_expire while (defined($stryPath[$iIndex])) { - &log(INFO, "removed expired full backup: " . $stryPath[$iIndex]); - # Delete all backups that depend on the full backup. Done in reverse order so that remaining backups will still # be consistent if the process dies foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse")) @@ -1449,6 +1447,8 @@ sub backup_expire system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}"); } + &log(INFO, "removed expired full backup: " . $stryPath[$iIndex]); + $iIndex++; } } diff --git a/pg_backrest_db.pm b/pg_backrest_db.pm index 7341b4f61..69f6f6448 100644 --- a/pg_backrest_db.pm +++ b/pg_backrest_db.pm @@ -38,7 +38,7 @@ sub BUILD &log(TRACE, "connecting to database ssh host $self->{strDbHost}"); # !!! This could be improved by redirecting stderr to a file to get a better error message - $self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, master_stderr_discard => true, user => $self->{strDbUser}); + $self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, user => $self->{strDbUser}); $self->{oDbSSH}->error and confess &log(ERROR, "unable to connect to $self->{strDbHost}: " . $self->{oDbSSH}->error); } } diff --git a/pg_backrest_file.pm b/pg_backrest_file.pm index 8ff2ac972..bc9b31df3 100644 --- a/pg_backrest_file.pm +++ b/pg_backrest_file.pm @@ -762,66 +762,48 @@ sub file_list_get my $strExpression = shift; my $strSortOrder = shift; - # Get the root path for the manifest + # Get the root path for the file list my $strPathList = $self->path_get($strPathType, $strPath); - # Builds the exists command - my $strCommand = "ls ${strPathList} | egrep \"$strExpression\" 2> /dev/null"; + # Builds the file list command +# my $strCommand = "ls ${strPathList} | egrep \"$strExpression\" 2> /dev/null"; + my $strCommand = "ls ${strPathList} | cat 2> /dev/null"; - # Run the file exists command - my $strExists = ""; + # Run the file list command + my $strFileList = ""; # Run remotely if ($self->is_remote($strPathType)) { - &log(TRACE, "file_exists: remote ${strPathType}:${strPathExists}"); + &log(TRACE, "file_list_get: remote ${strPathType}:${strPathList} ${strCommand}"); my $oSSH = $self->remote_get($strPathType); - $strExists = $oSSH->capture($strCommand); + $strFileList = $oSSH->capture($strCommand); } # Run locally else { - &log(TRACE, "file_exists: local ${strPathType}:${strPathExists}"); - $strExists = capture($strCommand); + &log(TRACE, "file_list_get: local ${strPathType}:${strPathList} ${strCommand}"); + $strFileList = capture($strCommand) or confess("error in ${strCommand}"); } - # If the return from ls eq strPathExists then true - return ($strExists eq $strPathExists); + # If nothing was found return undef + if ($strFileList eq "") + { + return undef; + } - # # For now this operation is not supported remotely. Not currently needed. - # if ($self->is_remote($strPathType)) - # { - # confess &log(ASSERT, "remote operation not supported"); - # } - # - # my $strPathList = $self->path_get($strPathType, $strPath); - # my $hDir; - # - # opendir $hDir, $strPathList or confess &log(ERROR, "unable to open path ${strPathList}"); - # my @stryFileAll = readdir $hDir or confess &log(ERROR, "unable to get files for path ${strPathList}, expression ${strExpression}"); - # close $hDir; - # - # my @stryFile; - # - # if (@stryFileAll) - # { - # @stryFile = grep(/$strExpression/i, @stryFileAll) - # } - # - # if (@stryFile) - # { - # if (defined($strSortOrder) && $strSortOrder eq "reverse") - # { - # return sort {$b cmp $a} @stryFile; - # } - # else - # { - # return sort @stryFile; - # } - # } - # - # return @stryFile; + # Split the files into an array + my @stryFileList = grep(/$strExpression/i, split(/\n/, $strFileList)); + + # Return the array in reverse order if specified + if (defined($strSortOrder) && $strSortOrder eq "reverse") + { + return sort {$b cmp $a} @stryFileList; + } + + # Return in normal sorted order + return sort @stryFileList; } #################################################################################################################################### @@ -919,7 +901,7 @@ sub manifest_get # Builds the manifest command my $strCommand = $self->{strCommandManifest}; $strCommand =~ s/\%path\%/${strPathManifest}/g; - $strCommand .= " 2> /dev/null"; + $strCommand .= " 2> /dev/null"; # Run the manifest command my $strManifest;