diff --git a/doc/doc.pl b/doc/doc.pl index ae2a07c2a..01a5a1111 100755 --- a/doc/doc.pl +++ b/doc/doc.pl @@ -62,6 +62,7 @@ doc.pl [options] --no-exe Should commands be executed when building help? (for testing only) --no-cache Don't use execution cache --cache-only Only use the execution cache - don't attempt to generate it + --pre Pre-build containers for execute elements marked pre --var Override variables defined in the XML --doc-path Document path to render (manifest.xml should be located here) --out Output types (html, pdf, markdown) @@ -69,12 +70,11 @@ doc.pl [options] --include Include source in generation (links will reference website) --exclude Exclude source from generation (links will reference website) -Keyword Options: + Keyword Options: --keyword Keyword used to filter output --keyword-add Add keyword without overriding 'default' keyword --dev Add 'dev' keyword --debug Add 'debug' keyword - --pre Add 'pre' keyword =cut #################################################################################################################################### @@ -191,12 +191,6 @@ eval push(@stryKeyword, 'debug'); } - # If --pre passed then add the pre keyword - if ($bPre) - { - push(@stryKeyword, 'pre'); - } - # Doesn't make sense to pass include and exclude if (@stryInclude > 0 && @stryExclude > 0) { @@ -228,7 +222,7 @@ eval # Load the manifest my $oManifest = new BackRestDoc::Common::DocManifest( $oStorageDoc, \@stryKeyword, \@stryRequire, \@stryInclude, \@stryExclude, $oVariableOverride, $strDocPath, $bDeploy, - $bCacheOnly); + $bCacheOnly, $bPre); if (!$bNoCache) { diff --git a/doc/lib/BackRestDoc/Common/DocExecute.pm b/doc/lib/BackRestDoc/Common/DocExecute.pm index d2e906cb9..db5ce9317 100644 --- a/doc/lib/BackRestDoc/Common/DocExecute.pm +++ b/doc/lib/BackRestDoc/Common/DocExecute.pm @@ -259,7 +259,8 @@ sub execute &log(DEBUG, (' ' x $iIndent) . "execute: $strCommand"); - if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y') + if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y' || + $oCommand->paramGet('pre', false, 'n') eq 'y' && $self->{oManifest}->{bPre}) { if ($self->{bExe} && $self->isRequired($oSection)) { @@ -1037,8 +1038,51 @@ sub sectionChildProcess executeTest("rm -rf ~/data/$$hCacheKey{name}"); executeTest("mkdir -p ~/data/$$hCacheKey{name}/etc"); + my $strHost = $hCacheKey->{name}; + my $strImage = $hCacheKey->{image}; + + # Determine if a pre-built image should be created + if (defined($self->preExecute($strHost))) + { + my $strPreImage = "${strImage}-${strHost}"; + my $strFrom = $strImage; + + &log(INFO, "Build vm '${strPreImage}' from '${strFrom}'"); + + my $strCommandList; + + # Add all pre commands + foreach my $oExecute ($self->preExecute($strHost)) + { + my $hExecuteKey = $self->executeKey($strHost, $oExecute); + my $strCommand = + join("\n", @{$hExecuteKey->{cmd}}) . + (defined($hExecuteKey->{'cmd-extra'}) ? ' ' . $hExecuteKey->{'cmd-extra'} : ''); + + if (defined($strCommandList)) + { + $strCommandList .= "\n"; + } + + $strCommandList .= "RUN ${strCommand}"; + + &log(DETAIL, " Pre command $strCommand"); + } + + # Build container + my $strDockerfile = $self->{oManifest}{strDocPath} . "/output/doc-host.dockerfile"; + + $self->{oManifest}{oStorage}->put( + $strDockerfile, + "FROM ${strFrom}\n\n" . trim($self->{oManifest}->variableReplace($strCommandList)) . "\n"); + executeTest("docker build -f ${strDockerfile} -t ${strPreImage} " . $self->{oManifest}{oStorage}->pathGet()); + + # Use the pre-built image + $strImage = $strPreImage; + } + my $oHost = new pgBackRestTest::Common::HostTest( - $$hCacheKey{name}, "doc-$$hCacheKey{name}", $$hCacheKey{image}, + $$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage, $self->{oManifest}->variableReplace($oChild->paramGet('user')), $$hCacheKey{os}, defined($oChild->paramGet('mount', false)) ? [$self->{oManifest}->variableReplace($oChild->paramGet('mount'))] : undef, diff --git a/doc/lib/BackRestDoc/Common/DocManifest.pm b/doc/lib/BackRestDoc/Common/DocManifest.pm index a742425cf..566690f71 100644 --- a/doc/lib/BackRestDoc/Common/DocManifest.pm +++ b/doc/lib/BackRestDoc/Common/DocManifest.pm @@ -64,6 +64,7 @@ sub new $self->{strDocPath}, $self->{bDeploy}, $self->{bCacheOnly}, + $self->{bPre}, ) = logDebugParam ( @@ -77,6 +78,7 @@ sub new {name => 'strDocPath', required => false}, {name => 'bDeploy', required => false}, {name => 'bCacheOnly', required => false}, + {name => 'bPre', required => false, default => false}, ); # Set the bin path diff --git a/doc/lib/BackRestDoc/Common/DocRender.pm b/doc/lib/BackRestDoc/Common/DocRender.pm index 0d940b7d6..a0ff7409e 100644 --- a/doc/lib/BackRestDoc/Common/DocRender.pm +++ b/doc/lib/BackRestDoc/Common/DocRender.pm @@ -306,6 +306,22 @@ sub variableGet return $self->{oManifest}->variableGet(shift); } +#################################################################################################################################### +# Get pre-execute list for a host +#################################################################################################################################### +sub preExecute +{ + my $self = shift; + my $strHost = shift; + + if (defined($self->{preExecute}{$strHost})) + { + return @{$self->{preExecute}{$strHost}}; + } + + return; +} + #################################################################################################################################### # build # @@ -337,6 +353,7 @@ sub build &log(DEBUG, " filtered ${strName}" . (defined($strDescription) ? ": ${strDescription}" : '')); $oParent->nodeRemove($oNode); + return; } } else @@ -527,6 +544,16 @@ sub build $iChildIdx++; } } + # Check for pre-execute statements + elsif ($strName eq 'execute') + { + if ($self->{oManifest}->{bPre} && $oNode->paramGet('pre', false, 'n') eq 'y') + { + # Add to pre-execute list + my $strHost = $self->variableReplace($oParent->paramGet('host')); + push(@{$self->{preExecute}{$strHost}}, $oNode); + } + } # Iterate all text nodes if (defined($oNode->textGet(false))) @@ -548,7 +575,7 @@ sub build $self->build($oChild, $oNode, $strPath, $strPathPrefix); # If the child should be logged then log the parent as well so the hierarchy is complete - if ($oChild->nameGet() eq 'section' && $oChild->paramGet('log')) + if ($oChild->nameGet() eq 'section' && $oChild->paramGet('log', false, false)) { $oNode->paramSet('log', true); } diff --git a/doc/xml/dtd/doc.dtd b/doc/xml/dtd/doc.dtd index 34afd5a0d..17e87807a 100644 --- a/doc/xml/dtd/doc.dtd +++ b/doc/xml/dtd/doc.dtd @@ -61,6 +61,7 @@ + diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 31f0dfa83..0006e81a2 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -109,6 +109,14 @@ + + + +

Pre-build containers for any execute elements marked pre.

+
+
+
+ diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index d82f0a570..684bc9161 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -91,7 +91,7 @@ u16 co6 - pgbackrest/doc:{[host-os]}-base + pgbackrest/doc:{[host-os]} use English; getpwuid($UID) eq 'root' ? 'vagrant' : getpwuid($UID) . '' {[pgbackrest-base-dir]}:/backrest @@ -329,12 +329,12 @@ Install required Perl packages - + apt-get install libdbd-pg-perl libio-socket-ssl-perl libxml-libxml-perl -y 2>&1 - + yum install perl perl-Time-HiRes perl-parent perl-JSON perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL -y 2>&1 @@ -519,16 +519,16 @@ Install build dependencies - + apt-get update - + apt-get install build-essential libssl-dev libxml2-dev libperl-dev -y 2>&1 - + yum install build-essential gcc openssl-devel libxml2-devel perl-ExtUtils-Embed @@ -1190,7 +1190,7 @@ Install <proper>jq</proper> utility - + apt-get install jq -y 2>&1