From 5d3c8e47f14c9f497866a899bed7d6e25f0fb67f Mon Sep 17 00:00:00 2001
From: David Steele <david@pgmasters.net>
Date: Thu, 29 Nov 2018 14:45:15 -0500
Subject: [PATCH] Pre-build containers for any execute elements marked pre.

This allows the documentation to be built more quickly and offline during development when --pre is specified on the command line.

Each host gets a pre-built container with all the execute elements marked pre. As long as the pre elements do not change the container will not need to be rebuilt.

The feature should not be used for CI builds as it may hide errors in the documentation.
---
 doc/doc.pl                                | 12 ++----
 doc/lib/BackRestDoc/Common/DocExecute.pm  | 48 ++++++++++++++++++++++-
 doc/lib/BackRestDoc/Common/DocManifest.pm |  2 +
 doc/lib/BackRestDoc/Common/DocRender.pm   | 29 +++++++++++++-
 doc/xml/dtd/doc.dtd                       |  1 +
 doc/xml/release.xml                       |  8 ++++
 doc/xml/user-guide.xml                    | 14 +++----
 7 files changed, 95 insertions(+), 19 deletions(-)

diff --git a/doc/doc.pl b/doc/doc.pl
index ae2a07c2a..01a5a1111 100755
--- a/doc/doc.pl
+++ b/doc/doc.pl
@@ -62,6 +62,7 @@ doc.pl [options]
    --no-exe         Should commands be executed when building help? (for testing only)
    --no-cache       Don't use execution cache
    --cache-only     Only use the execution cache - don't attempt to generate it
+   --pre            Pre-build containers for execute elements marked pre
    --var            Override variables defined in the XML
    --doc-path       Document path to render (manifest.xml should be located here)
    --out            Output types (html, pdf, markdown)
@@ -69,12 +70,11 @@ doc.pl [options]
    --include        Include source in generation (links will reference website)
    --exclude        Exclude source from generation (links will reference website)
 
-Keyword Options:
+ Keyword Options:
    --keyword        Keyword used to filter output
    --keyword-add    Add keyword without overriding 'default' keyword
    --dev            Add 'dev' keyword
    --debug          Add 'debug' keyword
-   --pre            Add 'pre' keyword
 =cut
 
 ####################################################################################################################################
@@ -191,12 +191,6 @@ eval
         push(@stryKeyword, 'debug');
     }
 
-    # If --pre passed then add the pre keyword
-    if ($bPre)
-    {
-        push(@stryKeyword, 'pre');
-    }
-
     # Doesn't make sense to pass include and exclude
     if (@stryInclude > 0 && @stryExclude > 0)
     {
@@ -228,7 +222,7 @@ eval
     # Load the manifest
     my $oManifest = new BackRestDoc::Common::DocManifest(
         $oStorageDoc, \@stryKeyword, \@stryRequire, \@stryInclude, \@stryExclude, $oVariableOverride, $strDocPath, $bDeploy,
-        $bCacheOnly);
+        $bCacheOnly, $bPre);
 
     if (!$bNoCache)
     {
diff --git a/doc/lib/BackRestDoc/Common/DocExecute.pm b/doc/lib/BackRestDoc/Common/DocExecute.pm
index d2e906cb9..db5ce9317 100644
--- a/doc/lib/BackRestDoc/Common/DocExecute.pm
+++ b/doc/lib/BackRestDoc/Common/DocExecute.pm
@@ -259,7 +259,8 @@ sub execute
 
     &log(DEBUG, ('    ' x $iIndent) . "execute: $strCommand");
 
-    if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y')
+    if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y' ||
+        $oCommand->paramGet('pre', false, 'n') eq 'y' && $self->{oManifest}->{bPre})
     {
         if ($self->{bExe} && $self->isRequired($oSection))
         {
@@ -1037,8 +1038,51 @@ sub sectionChildProcess
                 executeTest("rm -rf ~/data/$$hCacheKey{name}");
                 executeTest("mkdir -p ~/data/$$hCacheKey{name}/etc");
 
+                my $strHost = $hCacheKey->{name};
+                my $strImage = $hCacheKey->{image};
+
+                # Determine if a pre-built image should be created
+                if (defined($self->preExecute($strHost)))
+                {
+                    my $strPreImage = "${strImage}-${strHost}";
+                    my $strFrom = $strImage;
+
+                    &log(INFO, "Build vm '${strPreImage}' from '${strFrom}'");
+
+                    my $strCommandList;
+
+                    # Add all pre commands
+                    foreach my $oExecute ($self->preExecute($strHost))
+                    {
+                        my $hExecuteKey = $self->executeKey($strHost, $oExecute);
+                        my $strCommand =
+                            join("\n", @{$hExecuteKey->{cmd}}) .
+                            (defined($hExecuteKey->{'cmd-extra'}) ? ' ' . $hExecuteKey->{'cmd-extra'} : '');
+
+                        if (defined($strCommandList))
+                        {
+                            $strCommandList .= "\n";
+                        }
+
+                        $strCommandList .= "RUN ${strCommand}";
+
+                        &log(DETAIL, "    Pre command $strCommand");
+                    }
+
+                    # Build container
+                    my $strDockerfile = $self->{oManifest}{strDocPath} . "/output/doc-host.dockerfile";
+
+                    $self->{oManifest}{oStorage}->put(
+                        $strDockerfile,
+                        "FROM ${strFrom}\n\n" . trim($self->{oManifest}->variableReplace($strCommandList)) . "\n");
+                    executeTest("docker build -f ${strDockerfile} -t ${strPreImage} " . $self->{oManifest}{oStorage}->pathGet());
+
+                    # Use the pre-built image
+                    $strImage = $strPreImage;
+                }
+
                 my $oHost = new pgBackRestTest::Common::HostTest(
-                    $$hCacheKey{name}, "doc-$$hCacheKey{name}", $$hCacheKey{image},
+                    $$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage,
                     $self->{oManifest}->variableReplace($oChild->paramGet('user')), $$hCacheKey{os},
                     defined($oChild->paramGet('mount', false)) ?
                         [$self->{oManifest}->variableReplace($oChild->paramGet('mount'))] : undef,
diff --git a/doc/lib/BackRestDoc/Common/DocManifest.pm b/doc/lib/BackRestDoc/Common/DocManifest.pm
index a742425cf..566690f71 100644
--- a/doc/lib/BackRestDoc/Common/DocManifest.pm
+++ b/doc/lib/BackRestDoc/Common/DocManifest.pm
@@ -64,6 +64,7 @@ sub new
         $self->{strDocPath},
         $self->{bDeploy},
         $self->{bCacheOnly},
+        $self->{bPre},
     ) =
         logDebugParam
         (
@@ -77,6 +78,7 @@ sub new
             {name => 'strDocPath', required => false},
             {name => 'bDeploy', required => false},
             {name => 'bCacheOnly', required => false},
+            {name => 'bPre', required => false, default => false},
         );
 
     # Set the bin path
diff --git a/doc/lib/BackRestDoc/Common/DocRender.pm b/doc/lib/BackRestDoc/Common/DocRender.pm
index 0d940b7d6..a0ff7409e 100644
--- a/doc/lib/BackRestDoc/Common/DocRender.pm
+++ b/doc/lib/BackRestDoc/Common/DocRender.pm
@@ -306,6 +306,22 @@ sub variableGet
     return $self->{oManifest}->variableGet(shift);
 }
 
+####################################################################################################################################
+# Get pre-execute list for a host
+####################################################################################################################################
+sub preExecute
+{
+    my $self = shift;
+    my $strHost = shift;
+
+    if (defined($self->{preExecute}{$strHost}))
+    {
+        return @{$self->{preExecute}{$strHost}};
+    }
+
+    return;
+}
+
 ####################################################################################################################################
 # build
 #
@@ -337,6 +353,7 @@ sub build
             &log(DEBUG, "            filtered ${strName}" . (defined($strDescription) ? ": ${strDescription}" : ''));
 
             $oParent->nodeRemove($oNode);
+            return;
         }
     }
     else
@@ -527,6 +544,16 @@ sub build
             $iChildIdx++;
         }
     }
+    # Check for pre-execute statements
+    elsif ($strName eq 'execute')
+    {
+        if ($self->{oManifest}->{bPre} && $oNode->paramGet('pre', false, 'n') eq 'y')
+        {
+            # Add to pre-execute list
+            my $strHost = $self->variableReplace($oParent->paramGet('host'));
+            push(@{$self->{preExecute}{$strHost}}, $oNode);
+        }
+    }
 
     # Iterate all text nodes
     if (defined($oNode->textGet(false)))
@@ -548,7 +575,7 @@ sub build
             $self->build($oChild, $oNode, $strPath, $strPathPrefix);
 
             # If the child should be logged then log the parent as well so the hierarchy is complete
-            if ($oChild->nameGet() eq 'section' && $oChild->paramGet('log'))
+            if ($oChild->nameGet() eq 'section' && $oChild->paramGet('log', false, false))
             {
                 $oNode->paramSet('log', true);
             }
diff --git a/doc/xml/dtd/doc.dtd b/doc/xml/dtd/doc.dtd
index 34afd5a0d..17e87807a 100644
--- a/doc/xml/dtd/doc.dtd
+++ b/doc/xml/dtd/doc.dtd
@@ -61,6 +61,7 @@
 <!ELEMENT execute (exe-cmd, exe-cmd-extra?, exe-user?,  exe-var?, exe-retry?, exe-output?, (exe-highlight-type?, exe-highlight)?,
                    exe-err-expect?, exe-err-suppress?, exe-err-suppress-stderr?)>
     <!ATTLIST execute keyword CDATA "">
+    <!ATTLIST execute pre CDATA "">
     <!ATTLIST execute user CDATA "">
     <!ATTLIST execute filter CDATA "">
     <!ATTLIST execute filter-context CDATA "">
diff --git a/doc/xml/release.xml b/doc/xml/release.xml
index 31f0dfa83..0006e81a2 100644
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -109,6 +109,14 @@
                 </release-development-list>
             </release-core-list>
 
+            <release-doc-list>
+                <release-development-list>
+                    <release-item>
+                        <p>Pre-build containers for any <id>execute</id> elements marked <id>pre</id>.</p>
+                    </release-item>
+                </release-development-list>
+            </release-doc-list>
+
             <release-test-list>
                 <release-development-list>
                     <release-item>
diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml
index d82f0a570..684bc9161 100644
--- a/doc/xml/user-guide.xml
+++ b/doc/xml/user-guide.xml
@@ -91,7 +91,7 @@
         <variable key="host-os" keyword="default">u16</variable>
         <variable key="host-os" keyword="co6">co6</variable>
 
-        <variable key="host-image">pgbackrest/doc:{[host-os]}-base</variable>
+        <variable key="host-image">pgbackrest/doc:{[host-os]}</variable>
 
         <variable key="host-user" eval="y">use English; getpwuid($UID) eq 'root' ? 'vagrant' : getpwuid($UID) . ''</variable>
         <variable key="host-mount">{[pgbackrest-base-dir]}:/backrest</variable>
@@ -329,12 +329,12 @@
         <execute-list host="{[br-install-host]}">
             <title>Install required Perl packages</title>
 
-            <execute keyword="default" user="root">
+            <execute keyword="default" user="root" pre="y">
                 <exe-cmd>apt-get install libdbd-pg-perl libio-socket-ssl-perl libxml-libxml-perl</exe-cmd>
                 <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
             </execute>
 
-            <execute keyword="co6" user="root">
+            <execute keyword="co6" user="root" pre="y">
                 <exe-cmd>yum install perl perl-Time-HiRes perl-parent perl-JSON
                     perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL</exe-cmd>
                 <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
@@ -519,16 +519,16 @@
         <execute-list host="{[host-build]}">
             <title>Install build dependencies</title>
 
-            <execute keyword="default" user="root" show="n">
+            <execute keyword="default" user="root" show="n" pre="y">
                 <exe-cmd>apt-get update</exe-cmd>
             </execute>
 
-            <execute keyword="default" user="root">
+            <execute keyword="default" user="root" pre="y">
                 <exe-cmd>apt-get install build-essential libssl-dev libxml2-dev libperl-dev</exe-cmd>
                 <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
             </execute>
 
-            <execute keyword="co6" user="root">
+            <execute keyword="co6" user="root" pre="y">
                 <exe-cmd>
                     yum install build-essential gcc openssl-devel libxml2-devel
                         perl-ExtUtils-Embed
@@ -1190,7 +1190,7 @@
             <execute-list host="{[host-pg1]}">
                 <title>Install <proper>jq</proper> utility</title>
 
-                <execute user="root">
+                <execute user="root" pre="y">
                     <exe-cmd>apt-get install jq</exe-cmd>
                     <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
                 </execute>