Pre-build containers for any execute elements marked pre.

This allows the documentation to be built more quickly and offline during development when --pre is specified on the command line.

Each host gets a pre-built container with all the execute elements marked pre. As long as the pre elements do not change the container will not need to be rebuilt.

The feature should not be used for CI builds as it may hide errors in the documentation.

doc/doc.pl

@@ -62,6 +62,7 @@ doc.pl [options]
    --no-exe         Should commands be executed when building help? (for testing only)
    --no-cache       Don't use execution cache
    --cache-only     Only use the execution cache - don't attempt to generate it
+   --pre            Pre-build containers for execute elements marked pre
    --var            Override variables defined in the XML
    --doc-path       Document path to render (manifest.xml should be located here)
    --out            Output types (html, pdf, markdown)
@@ -69,12 +70,11 @@ doc.pl [options]
    --include        Include source in generation (links will reference website)
    --exclude        Exclude source from generation (links will reference website)
 
-Keyword Options:
+ Keyword Options:
    --keyword        Keyword used to filter output
    --keyword-add    Add keyword without overriding 'default' keyword
    --dev            Add 'dev' keyword
    --debug          Add 'debug' keyword
-   --pre            Add 'pre' keyword
 =cut
 
 ####################################################################################################################################
@@ -191,12 +191,6 @@ eval
         push(@stryKeyword, 'debug');
     }
 
-    # If --pre passed then add the pre keyword
-    if ($bPre)
-    {
-        push(@stryKeyword, 'pre');
-    }
-
     # Doesn't make sense to pass include and exclude
     if (@stryInclude > 0 && @stryExclude > 0)
     {
@@ -228,7 +222,7 @@ eval
     # Load the manifest
     my $oManifest = new BackRestDoc::Common::DocManifest(
         $oStorageDoc, \@stryKeyword, \@stryRequire, \@stryInclude, \@stryExclude, $oVariableOverride, $strDocPath, $bDeploy,
-        $bCacheOnly);
+        $bCacheOnly, $bPre);
 
     if (!$bNoCache)
     {

doc/lib/BackRestDoc/Common/DocExecute.pm

@@ -259,7 +259,8 @@ sub execute
 
     &log(DEBUG, ('    ' x $iIndent) . "execute: $strCommand");
 
-    if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y')
+    if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y' ||
+        $oCommand->paramGet('pre', false, 'n') eq 'y' && $self->{oManifest}->{bPre})
     {
         if ($self->{bExe} && $self->isRequired($oSection))
         {
@@ -1037,8 +1038,51 @@ sub sectionChildProcess
                 executeTest("rm -rf ~/data/$$hCacheKey{name}");
                 executeTest("mkdir -p ~/data/$$hCacheKey{name}/etc");
 
+                my $strHost = $hCacheKey->{name};
+                my $strImage = $hCacheKey->{image};
+
+                # Determine if a pre-built image should be created
+                if (defined($self->preExecute($strHost)))
+                {
+                    my $strPreImage = "${strImage}-${strHost}";
+                    my $strFrom = $strImage;
+
+                    &log(INFO, "Build vm '${strPreImage}' from '${strFrom}'");
+
+                    my $strCommandList;
+
+                    # Add all pre commands
+                    foreach my $oExecute ($self->preExecute($strHost))
+                    {
+                        my $hExecuteKey = $self->executeKey($strHost, $oExecute);
+                        my $strCommand =
+                            join("\n", @{$hExecuteKey->{cmd}}) .
+                            (defined($hExecuteKey->{'cmd-extra'}) ? ' ' . $hExecuteKey->{'cmd-extra'} : '');
+
+                        if (defined($strCommandList))
+                        {
+                            $strCommandList .= "\n";
+                        }
+
+                        $strCommandList .= "RUN ${strCommand}";
+
+                        &log(DETAIL, "    Pre command $strCommand");
+                    }
+
+                    # Build container
+                    my $strDockerfile = $self->{oManifest}{strDocPath} . "/output/doc-host.dockerfile";
+
+                    $self->{oManifest}{oStorage}->put(
+                        $strDockerfile,
+                        "FROM ${strFrom}\n\n" . trim($self->{oManifest}->variableReplace($strCommandList)) . "\n");
+                    executeTest("docker build -f ${strDockerfile} -t ${strPreImage} " . $self->{oManifest}{oStorage}->pathGet());
+
+                    # Use the pre-built image
+                    $strImage = $strPreImage;
+                }
+
                 my $oHost = new pgBackRestTest::Common::HostTest(
-                    $$hCacheKey{name}, "doc-$$hCacheKey{name}", $$hCacheKey{image},
+                    $$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage,
                     $self->{oManifest}->variableReplace($oChild->paramGet('user')), $$hCacheKey{os},
                     defined($oChild->paramGet('mount', false)) ?
                         [$self->{oManifest}->variableReplace($oChild->paramGet('mount'))] : undef,

doc/lib/BackRestDoc/Common/DocManifest.pm

@@ -64,6 +64,7 @@ sub new
         $self->{strDocPath},
         $self->{bDeploy},
         $self->{bCacheOnly},
+        $self->{bPre},
     ) =
         logDebugParam
         (
@@ -77,6 +78,7 @@ sub new
             {name => 'strDocPath', required => false},
             {name => 'bDeploy', required => false},
             {name => 'bCacheOnly', required => false},
+            {name => 'bPre', required => false, default => false},
         );
 
     # Set the bin path

doc/lib/BackRestDoc/Common/DocRender.pm

@@ -306,6 +306,22 @@ sub variableGet
     return $self->{oManifest}->variableGet(shift);
 }
 
+####################################################################################################################################
+# Get pre-execute list for a host
+####################################################################################################################################
+sub preExecute
+{
+    my $self = shift;
+    my $strHost = shift;
+
+    if (defined($self->{preExecute}{$strHost}))
+    {
+        return @{$self->{preExecute}{$strHost}};
+    }
+
+    return;
+}
+
 ####################################################################################################################################
 # build
 #
@@ -337,6 +353,7 @@ sub build
             &log(DEBUG, "            filtered ${strName}" . (defined($strDescription) ? ": ${strDescription}" : ''));
 
             $oParent->nodeRemove($oNode);
+            return;
         }
     }
     else
@@ -527,6 +544,16 @@ sub build
             $iChildIdx++;
         }
     }
+    # Check for pre-execute statements
+    elsif ($strName eq 'execute')
+    {
+        if ($self->{oManifest}->{bPre} && $oNode->paramGet('pre', false, 'n') eq 'y')
+        {
+            # Add to pre-execute list
+            my $strHost = $self->variableReplace($oParent->paramGet('host'));
+            push(@{$self->{preExecute}{$strHost}}, $oNode);
+        }
+    }
 
     # Iterate all text nodes
     if (defined($oNode->textGet(false)))
@@ -548,7 +575,7 @@ sub build
             $self->build($oChild, $oNode, $strPath, $strPathPrefix);
 
             # If the child should be logged then log the parent as well so the hierarchy is complete
-            if ($oChild->nameGet() eq 'section' && $oChild->paramGet('log'))
+            if ($oChild->nameGet() eq 'section' && $oChild->paramGet('log', false, false))
             {
                 $oNode->paramSet('log', true);
             }

doc/xml/dtd/doc.dtd

@@ -61,6 +61,7 @@
 <!ELEMENT execute (exe-cmd, exe-cmd-extra?, exe-user?,  exe-var?, exe-retry?, exe-output?, (exe-highlight-type?, exe-highlight)?,
                    exe-err-expect?, exe-err-suppress?, exe-err-suppress-stderr?)>
     <!ATTLIST execute keyword CDATA "">
+    <!ATTLIST execute pre CDATA "">
     <!ATTLIST execute user CDATA "">
     <!ATTLIST execute filter CDATA "">
     <!ATTLIST execute filter-context CDATA "">

doc/xml/release.xml

@@ -109,6 +109,14 @@
                 </release-development-list>
             </release-core-list>
 
+            <release-doc-list>
+                <release-development-list>
+                    <release-item>
+                        <p>Pre-build containers for any <id>execute</id> elements marked <id>pre</id>.</p>
+                    </release-item>
+                </release-development-list>
+            </release-doc-list>
+
             <release-test-list>
                 <release-development-list>
                     <release-item>

doc/xml/user-guide.xml

@@ -91,7 +91,7 @@
         <variable key="host-os" keyword="default">u16</variable>
         <variable key="host-os" keyword="co6">co6</variable>
 
-        <variable key="host-image">pgbackrest/doc:{[host-os]}-base</variable>
+        <variable key="host-image">pgbackrest/doc:{[host-os]}</variable>
 
         <variable key="host-user" eval="y">use English; getpwuid($UID) eq 'root' ? 'vagrant' : getpwuid($UID) . ''</variable>
         <variable key="host-mount">{[pgbackrest-base-dir]}:/backrest</variable>
@@ -329,12 +329,12 @@
         <execute-list host="{[br-install-host]}">
             <title>Install required Perl packages</title>
 
-            <execute keyword="default" user="root">
+            <execute keyword="default" user="root" pre="y">
                 <exe-cmd>apt-get install libdbd-pg-perl libio-socket-ssl-perl libxml-libxml-perl</exe-cmd>
                 <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
             </execute>
 
-            <execute keyword="co6" user="root">
+            <execute keyword="co6" user="root" pre="y">
                 <exe-cmd>yum install perl perl-Time-HiRes perl-parent perl-JSON
                     perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL</exe-cmd>
                 <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
@@ -519,16 +519,16 @@
         <execute-list host="{[host-build]}">
             <title>Install build dependencies</title>
 
-            <execute keyword="default" user="root" show="n">
+            <execute keyword="default" user="root" show="n" pre="y">
                 <exe-cmd>apt-get update</exe-cmd>
             </execute>
 
-            <execute keyword="default" user="root">
+            <execute keyword="default" user="root" pre="y">
                 <exe-cmd>apt-get install build-essential libssl-dev libxml2-dev libperl-dev</exe-cmd>
                 <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
             </execute>
 
-            <execute keyword="co6" user="root">
+            <execute keyword="co6" user="root" pre="y">
                 <exe-cmd>
                     yum install build-essential gcc openssl-devel libxml2-devel
                         perl-ExtUtils-Embed
@@ -1190,7 +1190,7 @@
             <execute-list host="{[host-pg1]}">
                 <title>Install <proper>jq</proper> utility</title>
 
-                <execute user="root">
+                <execute user="root" pre="y">
                     <exe-cmd>apt-get install jq</exe-cmd>
                     <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
                 </execute>