Pre-build containers for any execute elements marked pre.

This allows the documentation to be built more quickly and offline during development when --pre is specified on the command line. Each host gets a pre-built container with all the execute elements marked pre. As long as the pre elements do not change the container will not need to be rebuilt. The feature should not be used for CI builds as it may hide errors in the documentation.
2024-12-14 10:13:05 +02:00 · 2018-11-29 14:45:15 -05:00 · 2018-11-29 14:45:15 -05:00 · 5d3c8e47f1
commit 5d3c8e47f1
parent 74b72df9db
7 changed files with 95 additions and 19 deletions
--- a/doc/doc.pl
+++ b/doc/doc.pl
@ -62,6 +62,7 @@ doc.pl [options]
   --no-exe         Should commands be executed when building help? (for testing only)
   --no-cache       Don't use execution cache
   --cache-only     Only use the execution cache - don't attempt to generate it
+   --pre            Pre-build containers for execute elements marked pre
   --var            Override variables defined in the XML
   --doc-path       Document path to render (manifest.xml should be located here)
   --out            Output types (html, pdf, markdown)
@ -69,12 +70,11 @@ doc.pl [options]
   --include        Include source in generation (links will reference website)
   --exclude        Exclude source from generation (links will reference website)

-Keyword Options:
+ Keyword Options:
   --keyword        Keyword used to filter output
   --keyword-add    Add keyword without overriding 'default' keyword
   --dev            Add 'dev' keyword
   --debug          Add 'debug' keyword
-   --pre            Add 'pre' keyword
 =cut

 ####################################################################################################################################
@ -191,12 +191,6 @@ eval
        push(@stryKeyword, 'debug');
    }

-    # If --pre passed then add the pre keyword
-    if ($bPre)
-    {
-        push(@stryKeyword, 'pre');
-    }
-
    # Doesn't make sense to pass include and exclude
    if (@stryInclude > 0 && @stryExclude > 0)
    {
@ -228,7 +222,7 @@ eval
    # Load the manifest
    my $oManifest = new BackRestDoc::Common::DocManifest(
        $oStorageDoc, \@stryKeyword, \@stryRequire, \@stryInclude, \@stryExclude, $oVariableOverride, $strDocPath, $bDeploy,
-        $bCacheOnly);
+        $bCacheOnly, $bPre);

    if (!$bNoCache)
    {
--- a/doc/lib/BackRestDoc/Common/DocExecute.pm
+++ b/doc/lib/BackRestDoc/Common/DocExecute.pm
@ -259,7 +259,8 @@ sub execute

    &log(DEBUG, ('    ' x $iIndent) . "execute: $strCommand");

-    if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y')
+    if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y' ||
+        $oCommand->paramGet('pre', false, 'n') eq 'y' && $self->{oManifest}->{bPre})
    {
        if ($self->{bExe} && $self->isRequired($oSection))
        {
@ -1037,8 +1038,51 @@ sub sectionChildProcess
                executeTest("rm -rf ~/data/$$hCacheKey{name}");
                executeTest("mkdir -p ~/data/$$hCacheKey{name}/etc");

+                my $strHost = $hCacheKey->{name};
+                my $strImage = $hCacheKey->{image};
+
+                # Determine if a pre-built image should be created
+                if (defined($self->preExecute($strHost)))
+                {
+                    my $strPreImage = "${strImage}-${strHost}";
+                    my $strFrom = $strImage;
+
+                    &log(INFO, "Build vm '${strPreImage}' from '${strFrom}'");
+
+                    my $strCommandList;
+
+                    # Add all pre commands
+                    foreach my $oExecute ($self->preExecute($strHost))
+                    {
+                        my $hExecuteKey = $self->executeKey($strHost, $oExecute);
+                        my $strCommand =
+                            join("\n", @{$hExecuteKey->{cmd}}) .
+                            (defined($hExecuteKey->{'cmd-extra'}) ? ' ' . $hExecuteKey->{'cmd-extra'} : '');
+
+                        if (defined($strCommandList))
+                        {
+                            $strCommandList .= "\n";
+                        }
+
+                        $strCommandList .= "RUN ${strCommand}";
+
+                        &log(DETAIL, "    Pre command $strCommand");
+                    }
+
+                    # Build container
+                    my $strDockerfile = $self->{oManifest}{strDocPath} . "/output/doc-host.dockerfile";
+
+                    $self->{oManifest}{oStorage}->put(
+                        $strDockerfile,
+                        "FROM ${strFrom}\n\n" . trim($self->{oManifest}->variableReplace($strCommandList)) . "\n");
+                    executeTest("docker build -f ${strDockerfile} -t ${strPreImage} " . $self->{oManifest}{oStorage}->pathGet());
+
+                    # Use the pre-built image
+                    $strImage = $strPreImage;
+                }
+
                my $oHost = new pgBackRestTest::Common::HostTest(
-                    $$hCacheKey{name}, "doc-$$hCacheKey{name}", $$hCacheKey{image},
+                    $$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage,
                    $self->{oManifest}->variableReplace($oChild->paramGet('user')), $$hCacheKey{os},
                    defined($oChild->paramGet('mount', false)) ?
                        [$self->{oManifest}->variableReplace($oChild->paramGet('mount'))] : undef,
--- a/doc/lib/BackRestDoc/Common/DocManifest.pm
+++ b/doc/lib/BackRestDoc/Common/DocManifest.pm
@ -64,6 +64,7 @@ sub new
        $self->{strDocPath},
        $self->{bDeploy},
        $self->{bCacheOnly},
+        $self->{bPre},
    ) =
        logDebugParam
        (
@ -77,6 +78,7 @@ sub new
            {name => 'strDocPath', required => false},
            {name => 'bDeploy', required => false},
            {name => 'bCacheOnly', required => false},
+            {name => 'bPre', required => false, default => false},
        );

    # Set the bin path
--- a/doc/lib/BackRestDoc/Common/DocRender.pm
+++ b/doc/lib/BackRestDoc/Common/DocRender.pm
@ -306,6 +306,22 @@ sub variableGet
    return $self->{oManifest}->variableGet(shift);
 }

+####################################################################################################################################
+# Get pre-execute list for a host
+####################################################################################################################################
+sub preExecute
+{
+    my $self = shift;
+    my $strHost = shift;
+
+    if (defined($self->{preExecute}{$strHost}))
+    {
+        return @{$self->{preExecute}{$strHost}};
+    }
+
+    return;
+}
+
 ####################################################################################################################################
 # build
 #
@ -337,6 +353,7 @@ sub build
            &log(DEBUG, "            filtered ${strName}" . (defined($strDescription) ? ": ${strDescription}" : ''));

            $oParent->nodeRemove($oNode);
+            return;
        }
    }
    else
@ -527,6 +544,16 @@ sub build
            $iChildIdx++;
        }
    }
+    # Check for pre-execute statements
+    elsif ($strName eq 'execute')
+    {
+        if ($self->{oManifest}->{bPre} && $oNode->paramGet('pre', false, 'n') eq 'y')
+        {
+            # Add to pre-execute list
+            my $strHost = $self->variableReplace($oParent->paramGet('host'));
+            push(@{$self->{preExecute}{$strHost}}, $oNode);
+        }
+    }

    # Iterate all text nodes
    if (defined($oNode->textGet(false)))
@ -548,7 +575,7 @@ sub build
            $self->build($oChild, $oNode, $strPath, $strPathPrefix);

            # If the child should be logged then log the parent as well so the hierarchy is complete
-            if ($oChild->nameGet() eq 'section' && $oChild->paramGet('log'))
+            if ($oChild->nameGet() eq 'section' && $oChild->paramGet('log', false, false))
            {
                $oNode->paramSet('log', true);
            }
--- a/doc/xml/dtd/doc.dtd
+++ b/doc/xml/dtd/doc.dtd
@ -61,6 +61,7 @@
 <!ELEMENT execute (exe-cmd, exe-cmd-extra?, exe-user?,  exe-var?, exe-retry?, exe-output?, (exe-highlight-type?, exe-highlight)?,
                   exe-err-expect?, exe-err-suppress?, exe-err-suppress-stderr?)>
    <!ATTLIST execute keyword CDATA "">
+    <!ATTLIST execute pre CDATA "">
    <!ATTLIST execute user CDATA "">
    <!ATTLIST execute filter CDATA "">
    <!ATTLIST execute filter-context CDATA "">
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@ -109,6 +109,14 @@
                </release-development-list>
            </release-core-list>

+            <release-doc-list>
+                <release-development-list>
+                    <release-item>
+                        <p>Pre-build containers for any <id>execute</id> elements marked <id>pre</id>.</p>
+                    </release-item>
+                </release-development-list>
+            </release-doc-list>
+
            <release-test-list>
                <release-development-list>
                    <release-item>
--- a/doc/xml/user-guide.xml
+++ b/doc/xml/user-guide.xml
@ -91,7 +91,7 @@
        <variable key="host-os" keyword="default">u16</variable>
        <variable key="host-os" keyword="co6">co6</variable>

-        <variable key="host-image">pgbackrest/doc:{[host-os]}-base</variable>
+        <variable key="host-image">pgbackrest/doc:{[host-os]}</variable>

        <variable key="host-user" eval="y">use English; getpwuid($UID) eq 'root' ? 'vagrant' : getpwuid($UID) . ''</variable>
        <variable key="host-mount">{[pgbackrest-base-dir]}:/backrest</variable>
@ -329,12 +329,12 @@
        <execute-list host="{[br-install-host]}">
            <title>Install required Perl packages</title>

-            <execute keyword="default" user="root">
+            <execute keyword="default" user="root" pre="y">
                <exe-cmd>apt-get install libdbd-pg-perl libio-socket-ssl-perl libxml-libxml-perl</exe-cmd>
                <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
            </execute>

-            <execute keyword="co6" user="root">
+            <execute keyword="co6" user="root" pre="y">
                <exe-cmd>yum install perl perl-Time-HiRes perl-parent perl-JSON
                    perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL</exe-cmd>
                <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
@ -519,16 +519,16 @@
        <execute-list host="{[host-build]}">
            <title>Install build dependencies</title>

-            <execute keyword="default" user="root" show="n">
+            <execute keyword="default" user="root" show="n" pre="y">
                <exe-cmd>apt-get update</exe-cmd>
            </execute>

-            <execute keyword="default" user="root">
+            <execute keyword="default" user="root" pre="y">
                <exe-cmd>apt-get install build-essential libssl-dev libxml2-dev libperl-dev</exe-cmd>
                <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
            </execute>

-            <execute keyword="co6" user="root">
+            <execute keyword="co6" user="root" pre="y">
                <exe-cmd>
                    yum install build-essential gcc openssl-devel libxml2-devel
                        perl-ExtUtils-Embed
@ -1190,7 +1190,7 @@
            <execute-list host="{[host-pg1]}">
                <title>Install <proper>jq</proper> utility</title>

-                <execute user="root">
+                <execute user="root" pre="y">
                    <exe-cmd>apt-get install jq</exe-cmd>
                    <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
                </execute>