Include archive_status directory in online backups.

The archive_status directory is now recreated on restore to support PostgreSQL 8.3 which does not recreate it automatically like more recent versions do.

Also fixed log checking after PostgreSQL shuts down to include FATAL messages and disallow immediate shutdowns which can throw FATAL errors in the log.

Reported by Stephen Frost.

doc/xml/release.xml

@@ -165,6 +165,16 @@
     <release-list>
         <release date="XXXX-XX-XX" version="1.21dev" title="UNDER DEVELOPMENT">
             <release-core-list>
+                <release-bug-list>
+                    <release-item>
+                        <release-item-contributor-list>
+                            <release-item-ideator id="frost.stephen"/>
+                        </release-item-contributor-list>
+
+                        <p>The <path>archive_status</path> directory is now recreated on restore to support <postgres/> 8.3 which does not recreate it automatically like more recent versions do.</p>
+                    </release-item>
+                </release-bug-list>
+
                 <release-feature-list>
                     <release-item>
                         <release-item-contributor-list>
@@ -175,6 +185,14 @@
                     </release-item>
                 </release-feature-list>
             </release-core-list>
+
+            <release-test-list>
+                <release-bug-list>
+                    <release-item>
+                        <p>Fix log checking after <postgres/> shuts down to include <id>FATAL</id> messages and disallow immediate shutdowns which can throw <id>FATAL</id> errors in the log.</p>
+                    </release-item>
+                </release-bug-list>
+            </release-test-list>
         </release>
 
         <release date="2017-06-27" version="1.20" title="Critical 8.3/8.4 Bug Fix">

lib/pgBackRest/Manifest.pm

@@ -166,6 +166,8 @@ use constant MANIFEST_SUBKEY_USER                                   => 'user';
 ####################################################################################################################################
 # Database locations for important files/paths
 ####################################################################################################################################
+use constant DB_PATH_PGXLOG_ARCHIVESTATUS                           => 'pg_xlog/archive_status';
+    push @EXPORT, qw(DB_PATH_PGXLOG_ARCHIVESTATUS);
 use constant DB_PATH_BASE                                           => 'base';
     push @EXPORT, qw(DB_PATH_BASE);
 use constant DB_PATH_PGCLOG                                         => 'pg_clog';
@@ -224,6 +226,8 @@ use constant DB_FILE_PREFIX_TMP                                     => 'pgsql_tm
 ####################################################################################################################################
 # Manifest locations for important files/paths
 ####################################################################################################################################
+use constant MANIFEST_PATH_PGXLOG_ARCHIVESTATUS                     => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGXLOG_ARCHIVESTATUS;
+    push @EXPORT, qw(MANIFEST_PATH_PGXLOG_ARCHIVESTATUS);
 use constant MANIFEST_PATH_BASE                                     => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_BASE;
     push @EXPORT, qw(MANIFEST_PATH_BASE);
 use constant MANIFEST_PATH_PGCLOG                                   => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGCLOG;
@@ -684,7 +688,8 @@ sub build
 
         # Skip pg_xlog/* when doing an online backup.  WAL will be restored from the archive or stored in pg_xlog at the end of the
         # backup if the archive-copy option is set.
-        next if ($strFile =~ ('^' . MANIFEST_PATH_PGXLOG . '\/') && $bOnline);
+        next if ($bOnline && $strFile =~ ('^' . MANIFEST_PATH_PGXLOG . '\/') &&
+                 $strFile !~ ('^' . MANIFEST_PATH_PGXLOG_ARCHIVESTATUS . '$'));
 
         # Skip all directories and files that start with pgsql_tmp.  The files are removed when the server is restarted and the
         # directories are recreated.  Since temp files cannnot be created on the replica it makes sense to delete the directories

test/lib/pgBackRestTest/Env/Host/HostDbCommonTest.pm

@@ -698,11 +698,11 @@ sub restoreCompare
         }
     }
 
-    # Error when archive status exists in the manifest for an online backup
+    # Check that archive status exists in the manifest for an online backup
-    if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE} &&
+    if ($oExpectedManifestRef->{&MANIFEST_SECTION_BACKUP_OPTION}{&MANIFEST_KEY_ONLINE} &&
-        defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_TARGET_PATH}{MANIFEST_PATH_PGXLOG . '/archive_status'}))
+        !defined($oExpectedManifestRef->{&MANIFEST_SECTION_TARGET_PATH}{&MANIFEST_PATH_PGXLOG_ARCHIVESTATUS}))
     {
-        confess 'archive_status was backed up in pg_xlog - the filter did not work';
+        confess &log(ERROR, DB_PATH_PGXLOG_ARCHIVESTATUS . ' expected for online backup', ERROR_ASSERT);
     }
 
     # Delete the list of DBs

test/lib/pgBackRestTest/Env/Host/HostDbTest.pm

@@ -462,7 +462,6 @@ sub clusterStop
     my $hParam = shift;
 
     # Set defaults
-    my $bImmediate = defined($$hParam{bImmediate}) ? $$hParam{bImmediate} : false;
     my $bIgnoreLogError = defined($$hParam{bIgnoreLogError}) ? $$hParam{bIgnoreLogError} : false;
 
     # Disconnect user session
@@ -471,15 +470,15 @@ sub clusterStop
     # If postmaster process is running then stop the cluster
     if (-e $self->dbBasePath() . '/' . DB_FILE_POSTMASTERPID)
     {
-        $self->executeSimple(
+        $self->executeSimple($self->pgBinPath() . '/pg_ctl stop -D ' . $self->dbBasePath() . ' -w -s -m fast');
-            $self->pgBinPath() . '/pg_ctl stop -D ' . $self->dbBasePath() . ' -w -s -m ' .
-            ($bImmediate ? 'immediate' : 'fast'));
     }
 
     # Grep for errors in postgresql.log
     if (!$bIgnoreLogError && storageTest()->exists($self->pgLogFile()))
     {
-        $self->executeSimple('grep ERROR ' . $self->pgLogFile(), {iExpectedExitStatus => 1});
+        $self->executeSimple(
+            'grep -v "FATAL\:  57P03\: the database system is starting up" ' . $self->pgLogFile() . ' | grep "ERROR\|FATAL"',
+            {iExpectedExitStatus => 1});
     }
 
     # Remove the log file

test/lib/pgBackRestTest/Module/Real/RealAllTest.pm

@@ -473,6 +473,9 @@ sub run
 
             # Confirm the check command runs without error on a standby
             $oHostDbStandby->check('verify check command on standby');
+
+            # Shutdown the stanby before creating tablespaces (this will error since paths are different)
+            $oHostDbStandby->clusterStop({bIgnoreLogError => true});
         }
 
         # Execute stop and make sure the backup fails
@@ -954,12 +957,7 @@ sub run
 
         # Stop clusters to catch any errors in the postgres log
         #---------------------------------------------------------------------------------------------------------------------------
-        $oHostDbMaster->clusterStop({bImmediate => true});
+        $oHostDbMaster->clusterStop();
-
-        if (defined($oHostDbStandby))
-        {
-            $oHostDbStandby->clusterStop({bImmediate => true});
-        }
 
         # Test no-online backups
         #---------------------------------------------------------------------------------------------------------------------------