file_list_get now works locally and remotely

Conflicts:
	README.md
	pg_backrest.pl
	pg_backrest_db.pm
	pg_backrest_file.pm

.gitignore

@@ -1,2 +1,4 @@
 **/*~
 *~
+
+*.swp

README.md

@@ -20,6 +20,16 @@ v0.15: Added archive-get [PLANNED FUNCTIONALITY FOR THIS RELEASE]
 
 -------------
 
+v0.11: Minor fixes
+
+Tweaking a few settings after running backups for about a month.
+
+* Removed master_stderr_discard option on database SSH connections.  There have been occasional lockups and they could be related issues originally seen in the file code.
+
+* Changed lock file conflicts on backup and expire commands to ERROR.  They were set to DEBUG due to a copy-and-paste from the archive locks.
+
+-------------
+
 v0.10: Backup and archiving are functional
 
 This version has been put into production at Resonate, so it does work, but there are a number of major caveats.

pg_backrest.pl

@@ -83,7 +83,7 @@ GetOptions ("config=s" => \$strConfigFile,
 # Global variables
 ####################################################################################################################################
 my %oConfig;            # Configuration hash
-
+    
 ####################################################################################################################################
 # CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false)
 ####################################################################################################################################
@@ -522,6 +522,12 @@ backup_init
     config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT)
 );
 
+####################################################################################################################################
+# DEBUG
+####################################################################################################################################
+#print "out: " . ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, "0000000100000000", ".*", 'asc'))[0];
+#exit 0;
+
 ####################################################################################################################################
 # BACKUP
 ####################################################################################################################################
@@ -531,15 +537,13 @@ if ($strOperation eq OP_BACKUP)
 
     if (!lock_file_create($strLockFile))
     {
-        &log(DEBUG, "backup process is already running for stanza ${strStanza} - exiting");
+        &log(ERROR, "backup process is already running for stanza ${strStanza} - exiting");
         exit 0
     }
 
     backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH));
 
     $strOperation = OP_EXPIRE;
-    
-    sleep(30);
 
     lock_file_remove();
 }
@@ -553,7 +557,7 @@ if ($strOperation eq OP_EXPIRE)
 
     if (!lock_file_create($strLockFile))
     {
-        &log(DEBUG, "expire process is already running for stanza ${strStanza} - exiting");
+        &log(ERROR, "expire process is already running for stanza ${strStanza} - exiting");
         exit 0
     }
 

pg_backrest_backup.pm

@@ -1440,8 +1440,6 @@ sub backup_expire
 
         while (defined($stryPath[$iIndex]))
         {
-            &log(INFO, "removed expired full backup: " . $stryPath[$iIndex]);
-
             # Delete all backups that depend on the full backup.  Done in reverse order so that remaining backups will still
             # be consistent if the process dies
             foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse"))
@@ -1449,6 +1447,8 @@ sub backup_expire
                 system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}");
             }
         
+            &log(INFO, "removed expired full backup: " . $stryPath[$iIndex]);
+
             $iIndex++;
         }
     }

pg_backrest_db.pm

@@ -38,7 +38,7 @@ sub BUILD
         &log(TRACE, "connecting to database ssh host $self->{strDbHost}");
 
         # !!! This could be improved by redirecting stderr to a file to get a better error message
-        $self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, master_stderr_discard => true, user => $self->{strDbUser});
+        $self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, user => $self->{strDbUser});
         $self->{oDbSSH}->error and confess &log(ERROR, "unable to connect to $self->{strDbHost}: " . $self->{oDbSSH}->error);
     }
 }

pg_backrest_file.pm

@@ -762,66 +762,48 @@ sub file_list_get
     my $strExpression = shift;
     my $strSortOrder = shift;
 
-    # Get the root path for the manifest
+    # Get the root path for the file list
     my $strPathList = $self->path_get($strPathType, $strPath);
 
-    # Builds the exists command
+    # Builds the file list command
-    my $strCommand = "ls ${strPathList} | egrep \"$strExpression\" 2> /dev/null";
+#    my $strCommand = "ls ${strPathList} | egrep \"$strExpression\" 2> /dev/null";
+    my $strCommand = "ls ${strPathList} | cat 2> /dev/null";
     
-    # Run the file exists command
+    # Run the file list command
-    my $strExists = "";
+    my $strFileList = "";
 
     # Run remotely
     if ($self->is_remote($strPathType))
     {
-        &log(TRACE, "file_exists: remote ${strPathType}:${strPathExists}");
+        &log(TRACE, "file_list_get: remote ${strPathType}:${strPathList} ${strCommand}");
 
         my $oSSH = $self->remote_get($strPathType);
-        $strExists = $oSSH->capture($strCommand);
+        $strFileList = $oSSH->capture($strCommand);
     }
     # Run locally
     else
     {
-        &log(TRACE, "file_exists: local ${strPathType}:${strPathExists}");
+        &log(TRACE, "file_list_get: local ${strPathType}:${strPathList} ${strCommand}");
-        $strExists = capture($strCommand);
+        $strFileList = capture($strCommand) or confess("error in ${strCommand}");
     }
 
-    # If the return from ls eq strPathExists then true
+    # If nothing was found return undef
-    return ($strExists eq $strPathExists);
+    if ($strFileList eq "")
+    {
+        return undef;
+    }
 
-    # # For now this operation is not supported remotely.  Not currently needed.
+    # Split the files into an array
-    # if ($self->is_remote($strPathType))
+    my @stryFileList = grep(/$strExpression/i, split(/\n/, $strFileList));
-    # {
+
-    #     confess &log(ASSERT, "remote operation not supported");
+    # Return the array in reverse order if specified
-    # }
+    if (defined($strSortOrder) && $strSortOrder eq "reverse")
-    # 
+    {
-    # my $strPathList = $self->path_get($strPathType, $strPath);
+        return sort {$b cmp $a} @stryFileList;
-    # my $hDir;
+    }
-    # 
+    
-    # opendir $hDir, $strPathList or confess &log(ERROR, "unable to open path ${strPathList}");
+    # Return in normal sorted order
-    # my @stryFileAll = readdir $hDir or confess &log(ERROR, "unable to get files for path ${strPathList}, expression ${strExpression}");
+    return sort @stryFileList;
-    # close $hDir;
-    # 
-    # my @stryFile;
-    # 
-    # if (@stryFileAll)
-    # {
-    #     @stryFile = grep(/$strExpression/i, @stryFileAll)
-    # }
-    # 
-    # if (@stryFile)
-    # {
-    #     if (defined($strSortOrder) && $strSortOrder eq "reverse")
-    #     {
-    #         return sort {$b cmp $a} @stryFile;
-    #     }
-    #     else
-    #     {
-    #         return sort @stryFile;
-    #     }
-    # }
-    # 
-    # return @stryFile;
 }
 
 ####################################################################################################################################
@@ -919,7 +901,7 @@ sub manifest_get
     # Builds the manifest command
     my $strCommand = $self->{strCommandManifest};
     $strCommand =~ s/\%path\%/${strPathManifest}/g;
-        $strCommand .= " 2> /dev/null";
+    $strCommand .= " 2> /dev/null";
     
     # Run the manifest command
     my $strManifest;