Closed #163: Document Generation Issue

Fixed an issue where document generation failed because some OSs are not tolerant of having multiple installed versions of PostgreSQL. A separate VM is now created for each version. Also added a sleep after database starts during document generation to ensure the database is running before the next command runs.

Reported by John Harvey.

CHANGELOG.md

@@ -7,6 +7,8 @@ __No Release Date Set__
 
 * Fixed an issue where a temp WAL file left over after a well-timed system crash could cause the next `archive-push` to fail.
 
+* Fixed an issue where document generation failed because some OSs are not tolerant of having multiple installed versions of PostgreSQL. A separate VM is now created for each version. Also added a sleep after database starts during document generation to ensure the database is running before the next command runs. _Reported by John Harvey_.
+
 * The `retention-archive` option can now be be safely set to less than backup retention (`retention-full` or `retention-diff`) without also specifying `archive-copy=n`. The WAL required to make the backups that fall outside of archive retention consistent will be preserved in the archive. However, in this case PITR will still not be possible for the backups that fall outside of archive retention.
 
 * Generate an error when `archive-check=y` but `archive_command` does not execute `pg_backrest`. _Contributed by Jason O'Donnell_.

doc/xml/change-log.xml

@@ -14,6 +14,9 @@
                 <release-feature>
                     <text>Fixed an issue where a temp WAL file left over after a well-timed system crash could cause the next <cmd>archive-push</cmd> to fail.</text>
                 </release-feature>
+                <release-feature>
+                    <text>Fixed an issue where document generation failed because some OSs are not tolerant of having multiple installed versions of <postgres/>.  A separate VM is now created for each version.  Also added a sleep after database starts during document generation to ensure the database is running before the next command runs. <i>Reported by John Harvey</i>.</text>
+                </release-feature>
                 <release-feature>
                     <text>The <setting>retention-archive</setting> option can now be be safely set to less than backup retention (<setting>retention-full</setting> or <setting>retention-diff</setting>) without also specifying <setting>archive-copy=n</setting>.  The WAL required to make the backups that fall outside of archive retention consistent will be preserved in the archive.  However, in this case PITR will still not be possible for the backups that fall outside of archive retention.</text>
                 </release-feature>

doc/xml/user-guide.xml

@@ -60,6 +60,8 @@
         <variable key="test-table-data">Important Data</variable>
 
         <!-- Database cluster commmands -->
+        <variable key="db-cluster-wait">sleep 1</variable>
+
         <variable key="db-cluster-create" keyword="default">pg_createcluster 9.4 {[postgres-cluster-demo]}</variable>
         <variable key="db-cluster-create" keyword="co6">service postgresql-9.4 initdb</variable>
 
@@ -341,6 +343,10 @@
                 <execute user="root">
                     <exe-cmd>{[db-cluster-restart]}</exe-cmd>
                 </execute>
+
+                <execute show="n">
+                    <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+                </execute>
             </execute-list>
 
 <!--        It would be good if the info command showed xlogs so we don't have to examine directories
@@ -438,6 +444,10 @@
                     <exe-highlight>could not find the database system</exe-highlight>
                 </execute>
 
+                <execute show="n">
+                    <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+                </execute>
+
                 <execute keyword="co6" user="root" show="n">
                     <exe-cmd>rm -f {[postgres-log-pgstartup-demo]}</exe-cmd>
                 </execute>
@@ -473,6 +483,10 @@
                 <execute user="root">
                     <exe-cmd>{[db-cluster-start]}</exe-cmd>
                 </execute>
+
+                <execute show="n">
+                    <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+                </execute>
             </execute-list>
 
             <p>This time the cluster started successfully since the restore replaced the missing <file>pg_control</file> file.</p>
@@ -711,6 +725,10 @@
                 <execute user="root">
                     <exe-cmd>{[db-cluster-start]}</exe-cmd>
                 </execute>
+
+                <execute show="n">
+                    <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+                </execute>
             </execute-list>
         </section>
     </section>
@@ -801,6 +819,10 @@
                 <exe-cmd>{[db-cluster-start]}</exe-cmd>
             </execute>
 
+            <execute show="n">
+                <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+            </execute>
+
             <execute output="y">
                 <exe-cmd>psql -c "select * from important_table"</exe-cmd>
                 <exe-highlight>{[test-table-data]}</exe-highlight>
@@ -862,6 +884,10 @@
                 <exe-cmd>{[db-cluster-start]}</exe-cmd>
             </execute>
 
+            <execute show="n">
+                <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+            </execute>
+
             <execute output="y" err-expect="1">
                 <exe-cmd>psql -c "select * from important_table"</exe-cmd>
                 <exe-highlight>does not exist</exe-highlight>
@@ -904,6 +930,10 @@
                 <exe-cmd>{[db-cluster-start]}</exe-cmd>
             </execute>
 
+            <execute show="n">
+                <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+            </execute>
+
             <execute output="y">
                 <exe-cmd>psql -c "select * from important_table"</exe-cmd>
                 <exe-highlight>{[test-table-data]}</exe-highlight>
@@ -1024,6 +1054,10 @@
                 <execute user="root">
                     <exe-cmd>{[db-cluster-start]}</exe-cmd>
                 </execute>
+
+                <execute show="n">
+                    <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+                </execute>
             </execute-list>
 
             <p>A new backup must be performed to due to the timeline switch.</p>
@@ -1234,6 +1268,10 @@
                 <execute user="root">
                     <exe-cmd>{[db-cluster-start]}</exe-cmd>
                 </execute>
+
+                <execute show="n">
+                    <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+                </execute>
             </execute-list>
 
             <p>The <postgres/> log gives valuable information about the recovery.  Note especially that the cluster has entered standby mode and is ready to accept read-only connections.</p>
@@ -1417,6 +1455,10 @@
                 <execute user="root">
                     <exe-cmd>{[db-cluster-start]}</exe-cmd>
                 </execute>
+
+                <execute show="n">
+                    <exe-cmd>{[db-cluster-wait]}</exe-cmd>
+                </execute>
             </execute-list>
 
             <p>The <postgres/> log will confirm that streaming replication has started.</p>

test/lib/BackRestTest/Common/VmTest.pm

@@ -42,7 +42,7 @@ my $oyVm =
     # Ubuntu 12.04
     &OS_U12 =>
     {
-        db => ['8.4', '9.0', '9.1', '9.2', '9.3', '9.4', '9.5']
+        db => ['8.3', '8.4', '9.0', '9.1', '9.2', '9.3', '9.4', '9.5']
     },
 
     # Ubuntu 14.04

test/lib/BackRestTest/Docker/ContainerTest.pm

@@ -352,7 +352,12 @@ sub containerBuild
 
         # Db image
         ###########################################################################################################################
-        $strImageName = "${strOS}-db";
+        foreach my $strDbVersion (@{$$oOS{db}})
+        {
+            my $strDbVersionNoDot = $strDbVersion;
+            $strDbVersionNoDot =~ s/\.//;
+
+            $strImageName = "${strOS}-db-${strDbVersionNoDot}";
             &log(INFO, "Building ${strImageName} image...");
 
             $strImage = "# Database Container\nFROM backrest/${strOS}-base";
@@ -368,42 +373,35 @@ sub containerBuild
             $strImage .=
                 "\n\n# Install PostgreSQL";
 
-        foreach my $strDbVersion (@{$$oOS{db}})
+            if ($strOS eq OS_U12 || $strOS eq OS_U14)
-        {
-            if ($strOS eq OS_CO6 || $strOS eq OS_CO7)
-            {
-                $strDbVersion =~ s/\.//;
-            }
-
-            if ($strOS eq OS_CO6)
-            {
-                $strImage .=
-                    "\nRUN yum -y install postgresql${strDbVersion}-server";
-            }
-            elsif ($strOS eq OS_CO7)
-            {
-                $strImage .=
-                    "\nRUN yum -y install postgresql${strDbVersion}-server";
-            }
-            elsif ($strOS eq OS_U12 || $strOS eq OS_U14)
             {
                 $strImage .=
                     "\nRUN apt-get install -y postgresql-${strDbVersion}" .
                     "\nRUN pg_dropcluster --stop ${strDbVersion} main";
             }
+            elsif ($strOS eq OS_CO6)
+            {
+                $strImage .=
+                    "\nRUN yum -y install postgresql${strDbVersionNoDot}-server";
+            }
+            elsif ($strOS eq OS_CO7)
+            {
+                $strImage .=
+                    "\nRUN yum -y install postgresql${strDbVersionNoDot}-server";
             }
 
             # Write the image
             fileStringWrite("${strTempPath}/${strImageName}", "${strImage}\n", false);
             executeTest("docker build -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}",
                         {bSuppressStdErr => true});
+        }
 
         # Db Doc image
         ###########################################################################################################################
         $strImageName = "${strOS}-db-doc";
         &log(INFO, "Building ${strImageName} image...");
 
-        $strImage = "# Database Doc Container\nFROM backrest/${strOS}-db";
+        $strImage = "# Database Doc Container\nFROM backrest/${strOS}-db-94";
 
         # Create pg_backrest.conf
         $strImage .=
@@ -462,10 +460,15 @@ sub containerBuild
 
         # Test image
         ###########################################################################################################################
-        $strImageName = "${strOS}-test";
+        foreach my $strDbVersion (@{$$oOS{db}})
+        {
+            my $strDbVersionNoDot = $strDbVersion;
+            $strDbVersionNoDot =~ s/\.//;
+
+            $strImageName = "${strOS}-test-${strDbVersionNoDot}";
             &log(INFO, "Building ${strImageName} image...");
 
-        $strImage = "# Test Container\nFROM backrest/${strOS}-db";
+            $strImage = "# Test Container\nFROM backrest/${strOS}-db-${strDbVersionNoDot}";
 
             # Create BackRest User
             $strImage .= "\n\n" . backrestUserCreate($strOS);
@@ -497,7 +500,7 @@ sub containerBuild
             fileStringWrite("${strTempPath}/${strImageName}", "${strImage}\n", false);
             executeTest("docker build -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}",
                         {bSuppressStdErr => true});
-
+        }
     }
 }
 

test/test.pl

@@ -541,6 +541,8 @@ eval
                                           (defined($$oTest{db}) ? ", db=$$oTest{db}" : '');
 
                     my $strImage = 'test-' . $iProcessIdx;
+                    my $strDbVersion = (defined($$oTest{db}) ? $$oTest{db} : '9.4');
+                    $strDbVersion =~ s/\.//;
 
                     &log($bDryRun && !$bVmOut || $bShowOutputAsync ? INFO : DEBUG, "${strTest}" .
                          ($bVmOut || $bShowOutputAsync ? "\n" : ''));
@@ -548,7 +550,7 @@ eval
                     if (!$bDryRun || $bVmOut)
                     {
                         executeTest("docker run -itd -h $$oTest{os}-test --name=${strImage}" .
-                                    " -v /backrest:/backrest backrest/$$oTest{os}-test");
+                                    " -v /backrest:/backrest backrest/$$oTest{os}-test-${strDbVersion}");
                     }
 
                     $strCommandLine =~ s/\-\-os\=\S*//g;