New implementation for archive logging

pg_backrest.pl

@@ -105,7 +105,6 @@ sub backup_type_find
         my @stryFile = sort {$b cmp $a} grep(/^[0-F]{8}\-[0-F]{6}F\_[0-F]{8}\-[0-F]{6}(I|D)$/i, readdir $hDir); 
         close $hDir;
         
-        #print "incremental: @stryFile\n";
         $strDirectory = $stryFile[0];
     }
 
@@ -115,7 +114,6 @@ sub backup_type_find
         my @stryFile = sort {$b cmp $a} grep(/^[0-F]{8}\-[0-F]{6}F$/i, readdir $hDir); 
         close $hDir;
 
-        #print "next: @stryFile\n";
         $strDirectory = $stryFile[0];
     }
     
@@ -576,97 +574,6 @@ my $strCommandCompress = config_load(\%oConfig, "command", "compress", !$bNoComp
 my $strCommandDecompress = config_load(\%oConfig, "command", "decompress", !$bNoCompression);
 my $strCommandCopy = config_load(\%oConfig, "command", "copy", $bNoCompression);
 
-####################################################################################################################################
-# ARCHIVE-PUSH Command
-####################################################################################################################################
-if ($strOperation eq "archive-push")
-{
-    # archive-push command must have three arguments
-    if (@ARGV != 3)
-    {
-        die "not enough arguments - show usage";
-    }
-
-    # Get the source dir/file
-    my $strSourceFile = $ARGV[1];
-    
-    unless (-e $strSourceFile)
-    {
-        die "source file does not exist - show usage";
-    }
-
-    # Get the destination dir/file
-    my $strDestinationFile = $ARGV[2];
-
-    # Make sure the destination directory exists
-    unless (-e dirname($strDestinationFile))
-    {
-        die "destination dir does not exist - show usage";
-    }
-
-    # Make sure the destination file does NOT exist - ignore checksum and extension in case they (or options) have changed
-    if (glob("$strDestinationFile*"))
-    {
-        die "destination file already exists";
-    }
-
-    # Setup the copy command
-    my $strCommand = "";
-
-    # !!! Modify this to skip compression and checksum for any file that is not a log file
-    if ($strSourceFile =~ /\.backup$/)
-    {
-        $strCommand = $strCommandCopy;
-        $strCommand =~ s/\%source\%/$strSourceFile/g;
-        $strCommand =~ s/\%destination\%/$strDestinationFile/g;
-    }
-    else
-    {
-        # Calculate sha1 hash for the file (unless disabled)
-        if (!$bNoChecksum)
-        {
-            $strDestinationFile .= "-" . file_hash_get($strCommandChecksum, $strSourceFile);
-        }
-    
-        if ($bNoCompression)
-        {
-            $strCommand = $strCommandCopy;
-            $strCommand =~ s/\%source\%/$strSourceFile/g;
-            $strCommand =~ s/\%destination\%/$strDestinationFile/g;
-        }
-        else
-        {
-            $strCommand = $strCommandCompress;
-            $strCommand =~ s/\%file\%/$strSourceFile/g;
-            $strCommand .= " > $strDestinationFile.gz";
-        }
-    }
-    
-    # Execute the copy
-    execute($strCommand);
-
-    exit 0;
-}
-
-####################################################################################################################################
-# GET MORE CONFIG INFO
-####################################################################################################################################
-# Check the backup type
-if ($strType eq "diff")
-{
-    $strType = "differential";
-}
-
-if ($strType eq "incr")
-{
-    $strType = "incremental";
-}
-
-if ($strType ne "full" && $strType ne "differential" && $strType ne "incremental")
-{
-    die &log(ERROR, "backup type must be full, differential (diff), incremental (incr)");
-}
-
 # Load and check the base backup path
 my $strBasePath = $oConfig{common}{backup_path};
 
@@ -694,6 +601,117 @@ unless (-e $strBackupClusterPath)
     mkdir $strBackupClusterPath or die &log(ERROR, "cluster backup path '${strBackupClusterPath}' create failed");
 }
 
+####################################################################################################################################
+# ARCHIVE-PUSH Command
+####################################################################################################################################
+if ($strOperation eq "archive-push")
+{
+    # archive-push command must have three arguments
+    if (@ARGV != 2)
+    {
+        die "not enough arguments - show usage";
+    }
+
+    my $strBackupClusterArchivePath = "${strBackupClusterPath}/archive";
+
+    unless (-e $strBackupClusterArchivePath)
+    {
+        &log (INFO, "creating cluster archive path ${strBackupClusterArchivePath}");
+        mkdir $strBackupClusterArchivePath or die &log(ERROR, "cluster backup archive path '${strBackupClusterArchivePath}' create failed");
+    }
+
+    # Get the source dir/file
+    my $strSourceFile = $ARGV[1];
+    
+    unless (-e $strSourceFile)
+    {
+        die "source file does not exist - show usage";
+    }
+
+    # Get the destination dir/file
+    my $strDestinationFile = basename($strSourceFile);
+    my $strDestinationTmpFile = "${strBackupClusterArchivePath}/archive.tmp";
+    my $strBackupClusterArchiveSubPath = "${strBackupClusterArchivePath}";
+
+    if (-e $strDestinationTmpFile)
+    {
+        unlink($strDestinationTmpFile);
+    }
+
+    # Setup the copy command
+    my $strCommand;
+
+    # !!! Modify this to skip compression and checksum for any file that is not a log file
+    if ($strDestinationFile =~ /^([0-9]|[A-F]){24}$/)
+    {
+        $strBackupClusterArchiveSubPath = "${strBackupClusterArchivePath}/" . substr($strDestinationFile, 0, 16);
+
+        unless (-e $strBackupClusterArchiveSubPath)
+        {
+            &log (INFO, "creating cluster archive sub path ${strBackupClusterArchiveSubPath}");
+            mkdir $strBackupClusterArchiveSubPath or die &log(ERROR, "cluster backup archive sub path '${strBackupClusterArchiveSubPath}' create failed");
+        }
+
+        # Make sure the destination file does NOT exist - ignore checksum and extension in case they (or options) have changed
+        if (glob("${strBackupClusterArchiveSubPath}/${strDestinationFile}*"))
+        {
+            die "destination file already exists";
+        }
+
+        # Calculate sha1 hash for the file (unless disabled)
+        if (!$bNoChecksum)
+        {
+            $strDestinationFile .= "-" . file_hash_get($strCommandChecksum, $strSourceFile);
+        }
+
+        if ($bNoCompression)
+        {
+            $strCommand = $strCommandCopy;
+            $strCommand =~ s/\%source\%/${strSourceFile}/g;
+            $strCommand =~ s/\%destination\%/${strDestinationTmpFile}/g;
+        }
+        else
+        {
+            $strCommand = $strCommandCompress;
+            $strCommand =~ s/\%file\%/${strSourceFile}/g;
+            $strCommand .= " > ${strDestinationTmpFile}";
+            $strDestinationFile .= ".gz";
+        }
+    }
+    else
+    {
+        $strCommand = $strCommandCopy;
+        $strCommand =~ s/\%source\%/$strSourceFile/g;
+        $strCommand =~ s/\%destination\%/${strDestinationTmpFile}/g;
+    }
+    
+    # Execute the copy
+    execute($strCommand);
+    rename($strDestinationTmpFile, "${strBackupClusterArchiveSubPath}/${strDestinationFile}")
+        or die &log(ERROR, "unable to rename '${strBackupClusterArchiveSubPath}/${strDestinationFile}'");
+
+    exit 0;
+}
+
+####################################################################################################################################
+# GET MORE CONFIG INFO
+####################################################################################################################################
+# Check the backup type
+if ($strType eq "diff")
+{
+    $strType = "differential";
+}
+
+if ($strType eq "incr")
+{
+    $strType = "incremental";
+}
+
+if ($strType ne "full" && $strType ne "differential" && $strType ne "incremental")
+{
+    die &log(ERROR, "backup type must be full, differential (diff), incremental (incr)");
+}
+
 # Load commands required for backup
 my $strCommandManifest = config_load(\%oConfig, "command", "manifest");
 my $strCommandPsql = config_load(\%oConfig, "command", "psql");

test/test.pl

@@ -44,8 +44,8 @@ sub pg_create
     execute("mkdir $strTestPath/$strTestDir/ts1");
     execute("mkdir $strTestPath/$strTestDir/ts2");
     execute($strPgBinPath . "/initdb -D $strTestPath/$strTestDir/common -A trust -k");
-    execute("mkdir $strTestPath/$strArchiveDir");
     execute("mkdir $strTestPath/$strBackupDir");
+#    execute("mkdir -p $strTestPath/$strArchiveDir");
 }
 
 sub pg_start
@@ -99,7 +99,7 @@ sub archive_command_build
     my $bCompression = shift;
     my $bChecksum = shift;
     
-    my $strCommand = "$strBackRestBinPath/pg_backrest.pl archive-push --config=$strBackRestBinPath/pg_backrest.conf";
+    my $strCommand = "$strBackRestBinPath/pg_backrest.pl --cluster=db --config=$strBackRestBinPath/pg_backrest.conf";
     
     if (!$bCompression)
     {
@@ -111,7 +111,7 @@ sub archive_command_build
         $strCommand .= " --no-checksum"
     }
     
-    return $strCommand . " %p $strDestinationPath/%f";
+    return $strCommand . " archive-push %p";
 }
 
 sub wait_for_file
@@ -158,7 +158,7 @@ my $strUser = execute('whoami');
 
 my $strTestPath = "/Users/dsteele/test";
 my $strDbDir = "db";
-my $strArchiveDir = "archive";
+my $strArchiveDir = "backup/db/archive";
 my $strBackupDir = "backup";
 
 my $strPgBinPath = "/Library/PostgreSQL/9.3/bin";
@@ -198,8 +198,10 @@ pg_execute($dbh, "create table test_ts2 (id int) tablespace ts1");
 pg_execute($dbh, "insert into test values (1)");
 pg_execute($dbh, "select pg_switch_xlog()");
 
+execute("mkdir -p $strTestPath/$strArchiveDir/0000000100000000");
+
 # Test for archive log file 000000010000000000000001
-wait_for_file("$strTestPath/$strArchiveDir", "^000000010000000000000001\$", 5);
+wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000001\$", 5);
 
 # Turn on log checksum for the next test
 $dbh->disconnect();
@@ -214,7 +216,7 @@ pg_execute($dbh, "insert into test values (2)");
 pg_execute($dbh, "select pg_switch_xlog()");
 
 # Test for archive log file 000000010000000000000002
-wait_for_file("$strTestPath/$strArchiveDir", "^000000010000000000000002-([a-f]|[0-9]){40}\$", 5);
+wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000002-([a-f]|[0-9]){40}\$", 5);
 
 # Turn on log compression and checksum for the next test
 $dbh->disconnect();
@@ -229,7 +231,7 @@ pg_execute($dbh, "insert into test values (3)");
 pg_execute($dbh, "select pg_switch_xlog()");
 
 # Test for archive log file 000000010000000000000003
-wait_for_file("$strTestPath/$strArchiveDir", "^000000010000000000000003-([a-f]|[0-9]){40}\\.gz\$", 5);
+wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000003-([a-f]|[0-9]){40}\\.gz\$", 5);
 
 $dbh->disconnect();