Added execution cache for document generation.

Added an execution cache so that documentation can be generated without setting up the full container environment. This is useful for packaging, keeps the documentation consistent for a release, and speeds up generation when no changes are made in the execution list.

README.md

@@ -72,7 +72,7 @@ pgBackRest includes support for versions down to 8.3, since older versions of Po
 
 pgBackRest strives to be easy to configure and operate:
 
-- [User guide](http://www.pgbackrest.org/user-guide.html) for Ubuntu 12.04 & 14.04 / PostgreSQL 9.4.
+- [User guide](http://www.pgbackrest.org/user-guide.html) for Debian & Ubuntu / PostgreSQL 9.4.
 - [Command reference](http://www.pgbackrest.org/command.html) for command-line operations.
 - [Configuration reference](http://www.pgbackrest.org/configuration.html) for creating pgBackRest configurations.
 

doc/doc.pl

@@ -16,6 +16,7 @@ use Cwd qw(abs_path);
 use File::Basename qw(dirname);
 use Getopt::Long qw(GetOptions);
 use Pod::Usage qw(pod2usage);
+use Scalar::Util qw(blessed);
 use Storable;
 
 use lib dirname($0) . '/lib';
@@ -51,13 +52,16 @@ doc.pl [options]
    --version        Display pgBackRest version
    --quiet          Sets log level to ERROR
    --log-level      Log level for execution (e.g. ERROR, WARN, INFO, DEBUG)
+   --deploy         Write exe.cache into resource for persistence
    --no-exe         Should commands be executed when building help? (for testing only)
-   --use-cache      Use cached data to generate the docs (for testing textual changes only)
+   --no-cache       Don't use execution cache
+   --cache-only     Only use the execution cache - don't attempt to generate it
    --var            Override variables defined in the XML
    --doc-path       Document path to render (manifest.xml should be located here)
    --out            Output types (html, pdf, markdown)
    --keyword        Keyword used to filter output
    --require        Require only certain sections of the document (to speed testing)
+   --exclude        Exclude source from generation (links will reference website)
 =cut
 
 ####################################################################################################################################
@@ -68,12 +72,15 @@ my $bVersion = false;
 my $bQuiet = false;
 my $strLogLevel = 'info';
 my $bNoExe = false;
-my $bUseCache = false;
+my $bNoCache = false;
+my $bCacheOnly = false;
 my $oVariableOverride = {};
 my $strDocPath;
 my @stryOutput;
 my @stryKeyword;
 my @stryRequire;
+my @stryExclude;
+my $bDeploy = false;
 
 GetOptions ('help' => \$bHelp,
             'version' => \$bVersion,
@@ -82,12 +89,21 @@ GetOptions ('help' => \$bHelp,
             'out=s@' => \@stryOutput,
             'keyword=s@' => \@stryKeyword,
             'require=s@' => \@stryRequire,
+            'exclude=s@' => \@stryExclude,
             'no-exe', \$bNoExe,
-            'use-cache', \$bUseCache,
+            'deploy', \$bDeploy,
+            'no-cache', \$bNoCache,
+            'cache-only', \$bCacheOnly,
             'var=s%', $oVariableOverride,
             'doc-path=s', \$strDocPath)
     or pod2usage(2);
 
+####################################################################################################################################
+# Run in eval block to catch errors
+####################################################################################################################################
+eval
+{
+
     # Display version and exit if requested
     if ($bHelp || $bVersion)
     {
@@ -102,10 +118,31 @@ if ($bHelp || $bVersion)
         exit 0;
     }
 
-# Set no-exe if use-cache is set
+    # Disable cache when no exe
-if ($bUseCache)
+    if ($bNoExe)
     {
-    $bNoExe = true;
+        $bNoCache = true;
+    }
+
+    # Make sure options are set correctly for deploy
+    if ($bDeploy)
+    {
+        my $strError = 'cannot be specified for deploy';
+
+        !$bNoExe
+            or confess "--no-exe ${strError}";
+
+        !$bNoCache
+            or confess "--no-cache ${strError}";
+
+        !@stryRequire
+            or confess "--require ${strError}";
+
+        !@stryOutput
+            or confess "--out ${strError}";
+
+        !@stryExclude
+            or confess "--exclude ${strError}";
     }
 
     # Set console log level
@@ -140,16 +177,12 @@ if (!-e $strOutputPath)
     }
 
     # Load the manifest
-my $oManifest;
+    my $oManifest =
-my $strManifestCache = "${strOutputPath}/manifest.cache";
+        new BackRestDoc::Common::DocManifest(\@stryKeyword, \@stryRequire, \@stryExclude, $oVariableOverride, $strDocPath, $bDeploy, $bCacheOnly);
 
-if ($bUseCache && -e $strManifestCache)
+    if (!$bNoCache)
     {
-    $oManifest = retrieve($strManifestCache);
+        $oManifest->cacheRead();
-}
-else
-{
-    $oManifest = new BackRestDoc::Common::DocManifest(\@stryKeyword, \@stryRequire, $oVariableOverride, $strDocPath);
     }
 
     # If no outputs were given
@@ -239,7 +272,24 @@ for my $strOutput (@stryOutput)
     }
 
     # Cache the manifest (mostly useful for testing rendering changes in the code)
-if (!$bUseCache)
+    if (!$bNoCache && !$bCacheOnly)
     {
-    store($oManifest, $strManifestCache);
+        $oManifest->cacheWrite();
+    }
+};
+
+####################################################################################################################################
+# Check for errors
+####################################################################################################################################
+if ($@)
+{
+    my $oMessage = $@;
+
+    # If a backrest exception then return the code - don't confess
+    if (blessed($oMessage) && $oMessage->isa('pgBackRest::Common::Exception'))
+    {
+        exit $oMessage->code();
+    }
+
+    confess $oMessage;
 }

doc/lib/BackRestDoc/Common/DocExecute.pm

@@ -19,6 +19,7 @@ use pgBackRest::Common::Log;
 use pgBackRest::Common::String;
 use pgBackRest::Config::Config;
 use pgBackRest::FileCommon;
+use pgBackRest::Version;
 
 use lib dirname($0) . '/../test/lib';
 use pgBackRestTest::Common::ExecuteTest;
@@ -55,6 +56,16 @@ sub new
     my $self = $class->SUPER::new($strType, $oManifest, $strRenderOutKey);
     bless $self, $class;
 
+    if (defined($self->{oSource}{hyCache}))
+    {
+        $self->{bCache} = true;
+        $self->{iCacheIdx} = 0;
+    }
+    else
+    {
+        $self->{bCache} = false;
+    }
+
     $self->{bExe} = $bExe;
 
     # Return from function and log return values if any
@@ -65,6 +76,75 @@ sub new
     );
 }
 
+####################################################################################################################################
+# executeKey
+#
+# Get a unique key for the execution step to determine if the cache is valid.
+####################################################################################################################################
+sub executeKey
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strHostName,
+        $oCommand,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->executeKey', \@_,
+            {name => 'strHostName', trace => true},
+            {name => 'oCommand', trace => true},
+        );
+
+    # Add user to command
+    my $strCommand = $self->{oManifest}->variableReplace(trim($oCommand->fieldGet('exe-cmd')));
+    my $strUser = $self->{oManifest}->variableReplace($oCommand->paramGet('user', false, 'postgres'));
+    $strCommand = ($strUser eq 'vagrant' ? '' : ('sudo ' . ($strUser eq 'root' ? '' : "-u $strUser "))) . $strCommand;
+
+    # Format and split command
+    $strCommand =~ s/[ ]*\n[ ]*/ \\\n    /smg;
+    my @stryCommand = split("\n", $strCommand);
+
+    my $hCacheKey =
+    {
+        host => $strHostName,
+        cmd => \@stryCommand,
+        output => JSON::PP::false,
+    };
+
+    if (defined($oCommand->paramGet('err-expect', false)))
+    {
+        $$hCacheKey{'err-expect'} = $oCommand->paramGet('err-expect');
+    }
+
+    if ($oCommand->paramTest('output', 'y') || $oCommand->paramTest('show', 'y') || $oCommand->paramTest('variable-key'))
+    {
+        $$hCacheKey{'output'} = JSON::PP::true;
+    }
+
+    if (defined($oCommand->fieldGet('exe-highlight', false)))
+    {
+        $$hCacheKey{'output'} = JSON::PP::true;
+        $$hCacheKey{highlight}{'filter'} = $oCommand->paramTest('filter', 'n') ? JSON::PP::false : JSON::PP::true;
+        $$hCacheKey{highlight}{'filter-context'} = $oCommand->paramGet('filter-context', false, 2);
+
+        my @stryHighlight;
+        $stryHighlight[0] = $self->{oManifest}->variableReplace($oCommand->fieldGet('exe-highlight'));
+
+        $$hCacheKey{highlight}{list} = \@stryHighlight;
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'hExecuteKey', value => $hCacheKey, trace => true}
+    );
+}
+
 ####################################################################################################################################
 # execute
 ####################################################################################################################################
@@ -79,7 +159,8 @@ sub execute
         $oSection,
         $strHostName,
         $oCommand,
-        $iIndent
+        $iIndent,
+        $bCache,
     ) =
         logDebugParam
         (
@@ -87,34 +168,15 @@ sub execute
             {name => 'oSection'},
             {name => 'strHostName'},
             {name => 'oCommand'},
-            {name => 'iIndent', default => 1}
+            {name => 'iIndent', default => 1},
+            {name => 'bCache', default => true},
         );
 
     # Working variables
-    my $strCommand;
+    my $hCacheKey = $self->executeKey($strHostName, $oCommand);
+    my $strCommand = join("\n", @{$$hCacheKey{cmd}});
     my $strOutput;
 
-    if ($oCommand->fieldTest('actual-command'))
-    {
-        $strCommand = $oCommand->fieldGet('actual-command');
-        $strOutput = $oCommand->fieldGet('actual-output', false);
-    }
-    else
-    {
-        # Command variables
-        $strCommand = trim($oCommand->fieldGet('exe-cmd'));
-        my $strUser = $self->{oManifest}->variableReplace($oCommand->paramGet('user', false, 'postgres'));
-        my $bExeOutput = $oCommand->paramTest('output', 'y');
-        my $strVariableKey = $oCommand->paramGet('variable-key', false);
-        my $iExeExpectedError = $oCommand->paramGet('err-expect', false);
-
-        $strCommand = $self->{oManifest}->variableReplace(
-            ($strUser eq 'vagrant' ? '' :
-                ('sudo ' . ($strUser eq 'root' ? '' : "-u ${strUser} "))) . $strCommand);
-
-        # Add continuation chars and proper spacing
-        $strCommand =~ s/[ ]*\n[ ]*/ \\\n    /smg;
-
     if (!$oCommand->paramTest('show', 'n') && $self->{bExe} && $self->isRequired($oSection))
     {
         # Make sure that no lines are greater than 80 chars
@@ -132,6 +194,14 @@ sub execute
     if (!$oCommand->paramTest('skip', 'y'))
     {
         if ($self->{bExe} && $self->isRequired($oSection))
+        {
+            my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('exe', $hCacheKey);
+
+            if ($bCacheHit)
+            {
+                $strOutput = defined($$hCacheValue{output}) ? join("\n", @{$$hCacheValue{output}}) : undef;
+            }
+            else
             {
                 # Check that the host is valid
                 my $oHost = $self->{host}{$strHostName};
@@ -142,13 +212,13 @@ sub execute
                 }
 
                 my $oExec = $oHost->execute($strCommand,
-                                            {iExpectedExitStatus => $iExeExpectedError,
+                                            {iExpectedExitStatus => $$hCacheKey{'err-expect'},
                                              bSuppressError => $oCommand->paramTest('err-suppress', 'y'),
                                              iRetrySeconds => $oCommand->paramGet('retry', false)});
                 $oExec->begin();
                 $oExec->end();
 
-                if ($bExeOutput && defined($oExec->{strOutLog}) && $oExec->{strOutLog} ne '')
+                if (defined($oExec->{strOutLog}) && $oExec->{strOutLog} ne '')
                 {
                     $strOutput = $oExec->{strOutLog};
 
@@ -160,28 +230,36 @@ sub execute
                         $strOutput =~ s/^                             //smg;
                         $strOutput =~ s/^[0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-6][0-9]:[0-6][0-9]\.[0-9]{3} T[0-9]{2} //smg;
                     }
+
+                    # my @stryOutput = split("\n", $strOutput);
+                    # $$hCacheValue{stdout} = \@stryOutput;
                 }
 
-                if (defined($iExeExpectedError))
+                if (defined($$hCacheKey{'err-expect'}) && defined($oExec->{strErrorLog}) && $oExec->{strErrorLog} ne '')
                 {
-                    $strOutput .= trim($oExec->{strErrorLog});
+                    # my $strError = $oExec->{strErrorLog};
+                    # $strError =~ s/^\n+|\n$//g;
+                    # my @stryError = split("\n", $strError);
+                    # $$hCacheValue{stderr} = \@stryError;
+
+                    $strOutput .= $oExec->{strErrorLog};
                 }
 
-                # Output is assigned to a var
+                # if (defined($$hCacheValue{stderr}))
-                if (defined($strVariableKey))
+                # {
+                #     $strOutput .= join("\n", @{$$hCacheValue{stderr}});
+                # }
+
+                if ($$hCacheKey{output} && defined($$hCacheKey{highlight}) && $$hCacheKey{highlight}{filter} && defined($strOutput))
                 {
-                    $self->{oManifest}->variableSet($strVariableKey, trim($oExec->{strOutLog}));
+                    my $strHighLight = @{$$hCacheKey{highlight}{list}}[0];
-                }
-                elsif (!$oCommand->paramTest('filter', 'n') && $bExeOutput && defined($strOutput))
-                {
-                    my $strHighLight = $self->{oManifest}->variableReplace($oCommand->fieldGet('exe-highlight', false));
 
                     if (!defined($strHighLight))
                     {
                         confess &log(ERROR, 'filter requires highlight definition: ' . $strCommand);
                     }
 
-                    my $iFilterContext = $oCommand->paramGet('filter-context', false, 2);
+                    my $iFilterContext = $$hCacheKey{highlight}{'filter-context'};
 
                     my @stryOutput = split("\n", $strOutput);
                     undef($strOutput);
@@ -248,20 +326,40 @@ sub execute
                         }
                     }
                 }
+
+                if (!$$hCacheKey{output})
+                {
+                    $strOutput = undef;
                 }
-            elsif ($bExeOutput)
+
+                if (defined($strOutput))
+                {
+                    my @stryOutput = split("\n", $strOutput);
+                    $$hCacheValue{output} = \@stryOutput;
+                }
+
+                if ($bCache)
+                {
+                    $self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
+                }
+            }
+
+            # Output is assigned to a var
+            if ($oCommand->paramTest('variable-key'))
+            {
+                $self->{oManifest}->variableSet($oCommand->paramGet('variable-key'), trim($strOutput), true);
+            }
+        }
+        elsif ($$hCacheKey{output})
         {
             $strOutput = 'Output suppressed for testing';
         }
     }
 
-        if (defined($strVariableKey) && !defined($self->{oManifest}->variableGet($strVariableKey)))
+    # Default variable output when it was not set by execution
+    if ($oCommand->paramTest('variable-key') && !defined($self->{oManifest}->variableGet($oCommand->paramGet('variable-key'))))
     {
-            $self->{oManifest}->variableSet($strVariableKey, '[Test Variable]');
+        $self->{oManifest}->variableSet($oCommand->paramGet('variable-key'), '[Test Variable]', true);
-        }
-
-        $oCommand->fieldSet('actual-command', $strCommand);
-        $oCommand->fieldSet('actual-output', $strOutput);
     }
 
     # Return from function and log return values if any
@@ -273,6 +371,76 @@ sub execute
     );
 }
 
+
+####################################################################################################################################
+# configKey
+####################################################################################################################################
+sub configKey
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $oConfig,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->hostKey', \@_,
+            {name => 'oConfig', trace => true},
+        );
+
+    my $hCacheKey =
+    {
+        host => $self->{oManifest}->variableReplace($oConfig->paramGet('host')),
+        file => $self->{oManifest}->variableReplace($oConfig->paramGet('file')),
+    };
+
+    if ($oConfig->paramTest('reset', 'y'))
+    {
+        $$hCacheKey{reset} = JSON::PP::true;
+    }
+
+    # Add all options to the key
+    my $strOptionTag = $oConfig->nameGet() eq 'backrest-config' ? 'backrest-config-option' : 'postgres-config-option';
+
+    foreach my $oOption ($oConfig->nodeList($strOptionTag))
+    {
+        my $hOption = {};
+
+        if ($oOption->paramTest('remove', 'y'))
+        {
+            $$hOption{remove} = JSON::PP::true;
+        }
+
+        if (defined($oOption->valueGet(false)))
+        {
+            $$hOption{value} = $self->{oManifest}->variableReplace($oOption->valueGet());
+        }
+
+        my $strKey = $self->{oManifest}->variableReplace($oOption->paramGet('key'));
+
+        if ($oConfig->nameGet() eq 'backrest-config')
+        {
+            my $strSection = $self->{oManifest}->variableReplace($oOption->paramGet('section'));
+
+            $$hCacheKey{option}{$strSection}{$strKey} = $hOption;
+        }
+        else
+        {
+            $$hCacheKey{option}{$strKey} = $hOption;
+        }
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'hCacheKey', value => $hCacheKey, trace => true}
+    );
+}
+
 ####################################################################################################################################
 # backrestConfig
 ####################################################################################################################################
@@ -297,22 +465,21 @@ sub backrestConfig
         );
 
     # Working variables
-    my $strFile;
+    my $hCacheKey = $self->configKey($oConfig);
-    my $strConfig;
+    my $strFile = $$hCacheKey{file};
+    my $strConfig = undef;
 
-    if ($oConfig->fieldTest('actual-file'))
+    &log(DEBUG, ('    ' x $iDepth) . 'process backrest config: ' . $$hCacheKey{file});
-    {
-        $strFile = $oConfig->fieldGet('actual-file');
-        $strConfig = $oConfig->fieldGet('actual-config');
-    }
-    else
-    {
-        # Get filename
-        $strFile = $self->{oManifest}->variableReplace($oConfig->paramGet('file'));
-
-        &log(DEBUG, ('    ' x $iDepth) . 'process backrest config: ' . $strFile);
 
     if ($self->{bExe} && $self->isRequired($oSection))
+    {
+        my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('cfg-' . BACKREST_EXE, $hCacheKey);
+
+        if ($bCacheHit)
+        {
+            $strConfig = defined($$hCacheValue{config}) ? join("\n", @{$$hCacheValue{config}}) : undef;
+        }
+        else
         {
             # Check that the host is valid
             my $strHostName = $self->{oManifest}->variableReplace($oConfig->paramGet('host'));
@@ -326,7 +493,7 @@ sub backrestConfig
             # Reset all options
             if ($oConfig->paramTest('reset', 'y'))
             {
-                delete(${$self->{config}}{$strHostName}{$strFile})
+                delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}})
             }
 
             foreach my $oOption ($oConfig->nodeList('backrest-config-option'))
@@ -342,11 +509,11 @@ sub backrestConfig
 
                 if (!defined($strValue))
                 {
-                    delete(${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey});
+                    delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey});
 
-                    if (keys(%{${$self->{config}}{$strHostName}{$strFile}{$strSection}}) == 0)
+                    if (keys(%{${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}}) == 0)
                     {
-                        delete(${$self->{config}}{$strHostName}{$strFile}{$strSection});
+                        delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection});
                     }
 
                     &log(DEBUG, ('    ' x ($iDepth + 1)) . "reset ${strSection}->${strKey}");
@@ -364,10 +531,10 @@ sub backrestConfig
 
                     # If this option is a hash and the value is already set then append to the array
                     if ($$oOption{$strKey}{&OPTION_RULE_TYPE} eq OPTION_TYPE_HASH &&
-                        defined(${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey}))
+                        defined(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey}))
                     {
                         my @oValue = ();
-                        my $strHashValue = ${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey};
+                        my $strHashValue = ${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey};
 
                         # If there is only one key/value
                         if (ref(\$strHashValue) eq 'SCALAR')
@@ -381,12 +548,12 @@ sub backrestConfig
                         }
 
                         push(@oValue, $strValue);
-                        ${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey} = \@oValue;
+                        ${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey} = \@oValue;
                     }
                     # else just set the value
                     else
                     {
-                        ${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey} = $strValue;
+                        ${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey} = $strValue;
                     }
 
                     &log(DEBUG, ('    ' x ($iDepth + 1)) . "set ${strSection}->${strKey} = ${strValue}");
@@ -396,21 +563,27 @@ sub backrestConfig
             my $strLocalFile = "/home/vagrant/data/db-master/etc/pgbackrest.conf";
 
             # Save the ini file
-            iniSave($strLocalFile, $self->{config}{$strHostName}{$strFile}, true);
+            iniSave($strLocalFile, $self->{config}{$strHostName}{$$hCacheKey{file}}, true);
 
-            $strConfig = fileStringRead($strLocalFile);
+            $oHost->copyTo($strLocalFile, $$hCacheKey{file}, $oConfig->paramGet('owner', false, 'postgres:postgres'), '640');
 
-            $oHost->copyTo($strLocalFile, $strFile, $oConfig->paramGet('owner', false, 'postgres:postgres'), '640');
+            my $strConfig = fileStringRead($strLocalFile);
+            my @stryConfig = undef;
+
+            if (trim($strConfig) ne '')
+            {
+                @stryConfig = split("\n", $strConfig);
+            }
+
+            $$hCacheValue{config} = \@stryConfig;
+            $self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
+        }
     }
     else
     {
         $strConfig = 'Config suppressed for testing';
     }
 
-        $oConfig->fieldSet('actual-file', $strFile);
-        $oConfig->fieldSet('actual-config', $strConfig);
-    }
-
     # Return from function and log return values if any
     return logDebugReturn
     (
@@ -445,20 +618,19 @@ sub postgresConfig
         );
 
     # Working variables
-    my $strFile;
+    my $hCacheKey = $self->configKey($oConfig);
+    my $strFile = $$hCacheKey{file};
     my $strConfig;
 
-    if ($oConfig->fieldTest('actual-file'))
+    if ($self->{bExe} && $self->isRequired($oSection))
     {
-        $strFile = $oConfig->fieldGet('actual-file');
+        my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('cfg-postgresql', $hCacheKey);
-        $strConfig = $oConfig->fieldGet('actual-config');
+
+        if ($bCacheHit)
+        {
+            $strConfig = defined($$hCacheValue{config}) ? join("\n", @{$$hCacheValue{config}}) : undef;
         }
         else
-    {
-        # Get filename
-        $strFile = $self->{oManifest}->variableReplace($oConfig->paramGet('file'));
-
-        if ($self->{bExe} && $self->isRequired($oSection))
         {
             # Check that the host is valid
             my $strHostName = $self->{oManifest}->variableReplace($oConfig->paramGet('host'));
@@ -470,14 +642,14 @@ sub postgresConfig
             }
 
             my $strLocalFile = '/home/vagrant/data/db-master/etc/postgresql.conf';
-            $oHost->copyFrom($strFile, $strLocalFile);
+            $oHost->copyFrom($$hCacheKey{file}, $strLocalFile);
 
-            if (!defined(${$self->{'pg-config'}}{$strHostName}{$strFile}{base}) && $self->{bExe})
+            if (!defined(${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base}) && $self->{bExe})
             {
-                ${$self->{'pg-config'}}{$strHostName}{$strFile}{base} = fileStringRead($strLocalFile);
+                ${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base} = fileStringRead($strLocalFile);
             }
 
-            my $oConfigHash = $self->{'pg-config'}{$strHostName}{$strFile};
+            my $oConfigHash = $self->{'pg-config'}{$strHostName}{$$hCacheKey{file}};
             my $oConfigHashNew;
 
             if (!defined($$oConfigHash{old}))
@@ -490,7 +662,7 @@ sub postgresConfig
                 $oConfigHashNew = dclone($$oConfigHash{old});
             }
 
-            &log(DEBUG, ('    ' x $iDepth) . 'process postgres config: ' . $strFile);
+            &log(DEBUG, ('    ' x $iDepth) . 'process postgres config: ' . $$hCacheKey{file});
 
             foreach my $oOption ($oConfig->nodeList('postgres-config-option'))
             {
@@ -527,20 +699,27 @@ sub postgresConfig
                 fileStringWrite($strLocalFile, $$oConfigHash{base} .
                                 (defined($strConfig) ? "\n# pgBackRest Configuration\n${strConfig}\n" : ''));
 
-                $oHost->copyTo($strLocalFile, $strFile, 'postgres:postgres', '640');
+                $oHost->copyTo($strLocalFile, $$hCacheKey{file}, 'postgres:postgres', '640');
             }
 
             $$oConfigHash{old} = $oConfigHashNew;
+
+            my @stryConfig = undef;
+
+            if (trim($strConfig) ne '')
+            {
+                @stryConfig = split("\n", $strConfig);
+            }
+
+            $$hCacheValue{config} = \@stryConfig;
+            $self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
+        }
     }
     else
     {
         $strConfig = 'Config suppressed for testing';
     }
 
-        $oConfig->fieldSet('actual-file', $strFile);
-        $oConfig->fieldSet('actual-config', $strConfig);
-    }
-
     # Return from function and log return values if any
     return logDebugReturn
     (
@@ -551,6 +730,178 @@ sub postgresConfig
     );
 }
 
+####################################################################################################################################
+# hostKey
+####################################################################################################################################
+sub hostKey
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $oHost,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->hostKey', \@_,
+            {name => 'oHost', trace => true},
+        );
+
+    my $hCacheKey =
+    {
+        name => $self->{oManifest}->variableReplace($oHost->paramGet('name')),
+        user => $self->{oManifest}->variableReplace($oHost->paramGet('user')),
+        image => $self->{oManifest}->variableReplace($oHost->paramGet('image')),
+    };
+
+    if (defined($oHost->paramGet('os', false)))
+    {
+        $$hCacheKey{os} = $self->{oManifest}->variableReplace($oHost->paramGet('os'));
+    }
+
+    if (defined($oHost->paramGet('mount', false)))
+    {
+        $$hCacheKey{mount} = $self->{oManifest}->variableReplace($oHost->paramGet('mount'));
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'hCacheKey', value => $hCacheKey, trace => true}
+    );
+}
+
+####################################################################################################################################
+# cachePop
+####################################################################################################################################
+sub cachePop
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strCacheType,
+        $hCacheKey,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->hostKey', \@_,
+            {name => 'strCacheType', trace => true},
+            {name => 'hCacheKey', trace => true},
+        );
+
+    my $bCacheHit = false;
+    my $oCacheValue = undef;
+
+    if ($self->{bCache})
+    {
+        my $oJSON = JSON::PP->new()->canonical()->allow_nonref();
+        # &log(WARN, "checking cache for\ncurrent key: " . $oJSON->encode($hCacheKey));
+
+        my $hCache = ${$self->{oSource}{hyCache}}[$self->{iCacheIdx}];
+
+        if (!defined($hCache))
+        {
+            confess &log(ERROR, 'unable to get index from cache', -1);
+        }
+
+        if (!defined($$hCache{key}))
+        {
+            confess &log(ERROR, 'unable to get key from cache', -1);
+        }
+
+        if (!defined($$hCache{type}))
+        {
+            confess &log(ERROR, 'unable to get type from cache', -1);
+        }
+
+        if ($$hCache{type} ne $strCacheType)
+        {
+            confess &log(ERROR, 'types do not match, cache is invalid', -1);
+        }
+
+        if ($oJSON->encode($$hCache{key}) ne $oJSON->encode($hCacheKey))
+        {
+            confess &log(ERROR,
+                "keys at index $self->{iCacheIdx} do not match, cache is invalid." .
+                "\ncache key: " . $oJSON->encode($$hCache{key}) .
+                "\ncurrent key: " . $oJSON->encode($hCacheKey), -1);
+        }
+
+        $bCacheHit = true;
+        $oCacheValue = $$hCache{value};
+        $self->{iCacheIdx}++;
+    }
+    else
+    {
+        if ($self->{oManifest}{bCacheOnly})
+        {
+            confess &log(ERROR, 'Cache only operation forced by --cache-only option');
+        }
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'bCacheHit', value => $bCacheHit, trace => true},
+        {name => 'strCacheType', value => $strCacheType, trace => true},
+        {name => 'hCacheKey', value => $hCacheKey, trace => true},
+        {name => 'oCacheValue', value => $oCacheValue, trace => true},
+    );
+}
+
+####################################################################################################################################
+# cachePush
+####################################################################################################################################
+sub cachePush
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strType,
+        $hCacheKey,
+        $oCacheValue,
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->hostKey', \@_,
+            {name => 'strType', trace => true},
+            {name => 'hCacheKey', trace => true},
+            {name => 'oCacheValue', required => false, trace => true},
+        );
+
+    if ($self->{bCache})
+    {
+        confess &log(ASSERT, "cachePush should not be called when cache is already present");
+    }
+
+    # Create the cache entry
+    my $hCache =
+    {
+        key => $hCacheKey,
+        type => $strType,
+    };
+
+    if (defined($oCacheValue))
+    {
+        $$hCache{value} = $oCacheValue;
+    }
+
+    push @{$self->{oSource}{hyCache}}, $hCache;
+
+    # Return from function and log return values if any
+    return logDebugReturn($strOperation);
+}
+
 ####################################################################################################################################
 # sectionChildProcesss
 ####################################################################################################################################
@@ -579,27 +930,33 @@ sub sectionChildProcess
     # Execute a command
     if ($oChild->nameGet() eq 'host-add')
     {
-        if ($self->{bExe} && $self->isRequired($oSection) && !$oChild->paramTest('created', true))
+        if ($self->{bExe} && $self->isRequired($oSection))
         {
-            my $strName = $self->{oManifest}->variableReplace($oChild->paramGet('name'));
+            my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('host', $self->hostKey($oChild));
-            my $strUser = $self->{oManifest}->variableReplace($oChild->paramGet('user'));
-            my $strImage = $self->{oManifest}->variableReplace($oChild->paramGet('image'));
-            my $strOS = $self->{oManifest}->variableReplace($oChild->paramGet('os', false));
-            my $strMount = $self->{oManifest}->variableReplace($oChild->paramGet('mount', false));
 
-            if (defined($self->{host}{$strName}))
+            if ($bCacheHit)
+            {
+                $self->{oManifest}->variableSet("host-$$hCacheKey{name}-ip", $$hCacheValue{ip}, true);
+            }
+            else
+            {
+                if (defined($self->{host}{$$hCacheKey{name}}))
                 {
                     confess &log(ERROR, 'cannot add host ${strName} because the host already exists');
                 }
 
-            my $oHost = new pgBackRestTest::Common::HostTest($strName, $strImage, $strUser, $strOS, $strMount);
+                my $oHost =
-            $self->{host}{$strName} = $oHost;
+                    new pgBackRestTest::Common::HostTest(
-            $self->{oManifest}->variableSet("host-${strName}-ip", $oHost->{strIP});
+                        $$hCacheKey{name}, $$hCacheKey{image}, $$hCacheKey{user}, $$hCacheKey{os}, $$hCacheKey{mount});
+
+                $self->{host}{$$hCacheKey{name}} = $oHost;
+                $self->{oManifest}->variableSet("host-$$hCacheKey{name}-ip", $oHost->{strIP}, true);
+                $$hCacheValue{ip} = $oHost->{strIP};
 
                 # Execute cleanup commands
                 foreach my $oExecute ($oChild->nodeList('execute'))
                 {
-                $self->execute($oSection, $strName, $oExecute, $iDepth + 1);
+                    $self->execute($oSection, $$hCacheKey{name}, $oExecute, $iDepth + 1, false);
                 }
 
                 $oHost->executeSimple("sh -c 'echo \"\" >> /etc/hosts\'", undef, 'root');
@@ -608,7 +965,7 @@ sub sectionChildProcess
                 # Add all other host IPs to this host
                 foreach my $strOtherHostName (sort(keys(%{$self->{host}})))
                 {
-                if ($strOtherHostName ne $strName)
+                    if ($strOtherHostName ne $$hCacheKey{name})
                     {
                         my $oOtherHost = $self->{host}{$strOtherHostName};
 
@@ -619,15 +976,16 @@ sub sectionChildProcess
                 # Add this host IP to all other hosts
                 foreach my $strOtherHostName (sort(keys(%{$self->{host}})))
                 {
-                if ($strOtherHostName ne $strName)
+                    if ($strOtherHostName ne $$hCacheKey{name})
                     {
                         my $oOtherHost = $self->{host}{$strOtherHostName};
 
-                    $oOtherHost->executeSimple("sh -c 'echo \"$oHost->{strIP} ${strName}\" >> /etc/hosts'", undef, 'root');
+                        $oOtherHost->executeSimple("sh -c 'echo \"$oHost->{strIP} $$hCacheKey{name}\" >> /etc/hosts'", undef, 'root');
                     }
                 }
 
-            $oChild->paramSet('created', true);
+                $self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
+            }
         }
     }
     # Skip children that have already been processed and error on others

doc/lib/BackRestDoc/Common/DocManifest.pm

@@ -11,6 +11,7 @@ use Cwd qw(abs_path);
 use Exporter qw(import);
     our @EXPORT = qw();
 use File::Basename qw(dirname);
+use JSON::PP;
 use Scalar::Util qw(blessed);
 
 use lib dirname($0) . '/../lib';
@@ -53,16 +54,22 @@ sub new
         my $strOperation,
         $self->{stryKeyword},
         $self->{stryRequire},
+        $self->{stryExclude},
         my $oVariableOverride,
-        $self->{strDocPath}
+        $self->{strDocPath},
+        $self->{bDeploy},
+        $self->{bCacheOnly},
     ) =
         logDebugParam
         (
             __PACKAGE__ . '->new', \@_,
             {name => 'stryKeyword'},
             {name => 'stryRequire'},
+            {name => 'stryExclude'},
             {name => 'oVariableOverride', required => false},
             {name => 'strDocPath', required => false},
+            {name => 'bDeploy', required => false},
+            {name => 'bCacheOnly', required => false},
         );
 
     # Set the base path if it was not passed in
@@ -71,6 +78,10 @@ sub new
         $self->{strDocPath} = abs_path(dirname($0));
     }
 
+    # Set cache file names
+    $self->{strExeCacheLocal} = $self->{strDocPath} . "/output/exe.cache";
+    $self->{strExeCacheDeploy} = $self->{strDocPath} . "/resource/exe.cache";
+
     # Load the manifest
     $self->{oManifestXml} = new BackRestDoc::Common::Doc("$self->{strDocPath}/manifest.xml");
 
@@ -90,6 +101,12 @@ sub new
             {name => 'strSourceType', value => $strSourceType}
         );
 
+        # Skip sources in exclude list
+        if (grep(/^$strKey$/, @{$self->{stryExclude}}))
+        {
+            next;
+        }
+
         $$oSourceHash{doc} = new BackRestDoc::Common::Doc("$self->{strDocPath}/xml/${strKey}.xml");
 
         # Read variables from source
@@ -134,6 +151,12 @@ sub new
             my $strKey = $oRenderOut->paramGet('key');
             my $strSource = $oRenderOut->paramGet('source', false, $strKey);
 
+            # Skip sources in exclude list
+            if (grep(/^$strSource$/, @{$self->{stryExclude}}))
+            {
+                next;
+            }
+
             $$oRenderOutHash{source} = $strSource;
 
             # Get the menu caption
@@ -263,7 +286,7 @@ sub variableListParse
             if ($self->keywordMatch($oVariable->paramGet('keyword', false)))
             {
                 my $strKey = $oVariable->paramGet('key');
-                my $strValue = $oVariable->valueGet();
+                my $strValue = $self->variableReplace($oVariable->valueGet());
 
                 if ($oVariable->paramTest('eval', 'y'))
                 {
@@ -526,18 +549,20 @@ sub renderOutGet
     (
         $strOperation,
         $strType,
-        $strKey
+        $strKey,
+        $bIgnoreMissing,
     ) =
         logDebugParam
         (
             __PACKAGE__ . '->renderOutGet', \@_,
             {name => 'strType', trace => true},
-            {name => 'strKey', trace => true}
+            {name => 'strKey', trace => true},
+            {name => 'bIgnoreMissing', default => false, trace => true},
         );
 
     # use Data::Dumper; print Dumper(${$self->{oManifest}}{render});
 
-    if (!defined(${$self->{oManifest}}{render}{$strType}{out}{$strKey}))
+    if (!defined(${$self->{oManifest}}{render}{$strType}{out}{$strKey}) && !$bIgnoreMissing)
     {
         confess &log(ERROR, "render out ${strKey} does not exist");
     }
@@ -550,4 +575,133 @@ sub renderOutGet
     );
 }
 
+####################################################################################################################################
+# cacheKey
+####################################################################################################################################
+sub cacheKey
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my ($strOperation) = logDebugParam(__PACKAGE__ . '->cacheKey');
+
+    my $strKeyword = join("\n", @{$self->{stryKeyword}});
+    my $strRequire = defined($self->{stryRequire}) && @{$self->{stryRequire}} > 0 ?
+        join("\n", @{$self->{stryRequire}}) : 'all';
+
+    # Return from function and log return values if any
+    return logDebugReturn
+    (
+        $strOperation,
+        {name => 'strKeyword', value => $strKeyword},
+        {name => 'strRequire', value => $strRequire},
+    );
+}
+
+####################################################################################################################################
+# cacheRead
+####################################################################################################################################
+sub cacheRead
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my ($strOperation) = logDebugParam(__PACKAGE__ . '->cacheRead');
+
+    $self->{hCache} = undef;
+
+    my $strCacheFile = $self->{bDeploy} ? $self->{strExeCacheDeploy} : $self->{strExeCacheLocal};
+
+    if (!fileExists($strCacheFile) && !$self->{bDeploy})
+    {
+        $strCacheFile = $self->{strExeCacheDeploy};
+    }
+
+    if (fileExists($strCacheFile))
+    {
+        my ($strKeyword, $strRequire) = $self->cacheKey();
+        my $oJSON = JSON::PP->new()->allow_nonref();
+        $self->{hCache} = $oJSON->decode(fileStringRead($strCacheFile));
+
+        foreach my $strSource (sort(keys(%{${$self->{oManifest}}{source}})))
+        {
+            my $hSource = ${$self->{oManifest}}{source}{$strSource};
+
+            if (defined(${$self->{hCache}}{$strKeyword}{$strRequire}{$strSource}))
+            {
+                $$hSource{hyCache} = ${$self->{hCache}}{$strKeyword}{$strRequire}{$strSource};
+                &log(DETAIL, "cache load $strSource (keyword = ${strKeyword}, require = ${strRequire})");
+            }
+        }
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# cacheWrite
+####################################################################################################################################
+sub cacheWrite
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my ($strOperation) = logDebugParam(__PACKAGE__ . '->cacheWrite');
+
+    my $strCacheFile = $self->{bDeploy} ? $self->{strExeCacheDeploy} : $self->{strExeCacheLocal};
+    my ($strKeyword, $strRequire) = $self->cacheKey();
+
+    foreach my $strSource (sort(keys(%{${$self->{oManifest}}{source}})))
+    {
+        my $hSource = ${$self->{oManifest}}{source}{$strSource};
+
+        if (defined($$hSource{hyCache}))
+        {
+            ${$self->{hCache}}{$strKeyword}{$strRequire}{$strSource} = $$hSource{hyCache};
+            &log(DETAIL, "cache load $strSource (keyword = ${strKeyword}, require = ${strRequire})");
+        }
+    }
+
+    if (defined($self->{hCache}))
+    {
+        my $oJSON = JSON::PP->new()->canonical()->allow_nonref()->pretty();
+        fileStringWrite($strCacheFile, $oJSON->encode($self->{hCache}), false);
+    }
+
+    # Return from function and log return values if any
+    return logDebugReturn($strOperation);
+}
+
+####################################################################################################################################
+# cacheReset
+####################################################################################################################################
+sub cacheReset
+{
+    my $self = shift;
+
+    # Assign function parameters, defaults, and log debug info
+    my
+    (
+        $strOperation,
+        $strSource
+    ) =
+        logDebugParam
+        (
+            __PACKAGE__ . '->cacheReset', \@_,
+            {name => 'strSource', trace => true}
+        );
+
+    if ($self->{bCacheOnly})
+    {
+        confess &log(ERROR, 'Cache reset disabled by --cache-only option');
+    }
+
+    &log(WARN, "Cache will be reset for source ${strSource} and rendering retried automatically");
+    delete(${$self->{oManifest}}{source}{$strSource}{hyCache});
+
+    # Return from function and log return values if any
+    return logDebugReturn($strOperation);
+}
+
 1;

doc/lib/BackRestDoc/Common/DocRender.pm

@@ -187,16 +187,10 @@ sub new
             {
                 $self->{oReference} =
                     new BackRestDoc::Common::DocConfig(${$self->{oManifest}->sourceGet('reference')}{doc}, $self);
-
-                require BackRestDoc::Custom::DocCustomRelease;
-                BackRestDoc::Custom::DocCustomRelease->import();
-
-                $self->{oRelease} =
-                    new BackRestDoc::Custom::DocCustomRelease(${$self->{oManifest}->sourceGet('release')}{doc}, $self);
             }
         }
 
-        if (defined($$oRenderOut{source}) && $$oRenderOut{source} eq 'reference')
+        if (defined($$oRenderOut{source}) && $$oRenderOut{source} eq 'reference' && $self->{oManifest}->isBackRest())
         {
             if ($self->{strRenderOutKey} eq 'configuration')
             {
@@ -211,14 +205,20 @@ sub new
                 confess &log(ERROR, "cannot render $self->{strRenderOutKey} from source $$oRenderOut{source}");
             }
         }
-        elsif (defined($$oRenderOut{source}) && $$oRenderOut{source} eq 'release')
+        elsif (defined($$oRenderOut{source}) && $$oRenderOut{source} eq 'release' && $self->{oManifest}->isBackRest())
         {
-            $self->{oDoc} = $self->{oRelease}->docGet();
+            require BackRestDoc::Custom::DocCustomRelease;
+            BackRestDoc::Custom::DocCustomRelease->import();
+
+            $self->{oDoc} =
+                (new BackRestDoc::Custom::DocCustomRelease(${$self->{oManifest}->sourceGet('release')}{doc}, $self))->docGet();
         }
         else
         {
             $self->{oDoc} = ${$self->{oManifest}->sourceGet($self->{strRenderOutKey})}{doc};
         }
+
+        $self->{oSource} = $self->{oManifest}->sourceGet($$oRenderOut{source});
     }
 
     if (defined($self->{strRenderOutKey}))
@@ -459,7 +459,7 @@ sub processTag
         {
             if (!defined($strUrl))
             {
-                $strUrl = '{[backrest-url-base]}/' . $oTag->paramGet('page');
+                $strUrl = '{[backrest-url-base]}/' . $oTag->paramGet('page') . '.html';
             }
 
             $strBuffer = '[' . $oTag->valueGet() . '](' . $strUrl . ')';
@@ -468,7 +468,22 @@ sub processTag
         {
             if (!defined($strUrl))
             {
-                $strUrl = $oTag->paramGet('page', false);
+                my $strPage = $self->variableReplace($oTag->paramGet('page', false));
+
+                # If this is a page URL
+                if (defined($strPage))
+                {
+                    # If the page wasn't rendered then point at the website
+                    if (!defined($self->{oManifest}->renderOutGet('html', $strPage, true)))
+                    {
+                        $strUrl = '{[backrest-url-base]}/' . $oTag->paramGet('page') . '.html';
+                    }
+                    # Else point locally
+                    else
+                    {
+                        $strUrl = $oTag->paramGet('page', false) . '.html';
+                    }
+                }
 
                 if (!defined($strUrl))
                 {

doc/lib/BackRestDoc/Custom/DocCustomRelease.pm

@@ -96,6 +96,30 @@ sub new
     );
 }
 
+####################################################################################################################################
+# currentStableVersion
+#
+# Return the current stable version.
+####################################################################################################################################
+sub currentStableVersion
+{
+    my $self = shift;
+
+    my $oDoc = $self->{oDoc};
+
+    foreach my $oRelease ($oDoc->nodeGet('release-list')->nodeList('release'))
+    {
+        my $strVersion = $oRelease->paramGet('version');
+
+        if ($strVersion !~ /dev$/)
+        {
+            return $strVersion;
+        }
+    }
+
+    confess &log(ERROR, "unable to find non-development version");
+}
+
 ####################################################################################################################################
 # contributorTextGet
 #

doc/lib/BackRestDoc/Html/DocHtmlSite.pm

@@ -13,6 +13,7 @@ use Exporter qw(import);
 use File::Basename qw(dirname);
 use File::Copy;
 use POSIX qw(strftime);
+use Scalar::Util qw(blessed);
 use Storable qw(dclone);
 
 use lib dirname($0) . '/../lib';
@@ -121,11 +122,37 @@ sub process
     {
         &log(INFO, "    render out: ${strPageId}");
 
+        my $strHtml;
+
+        eval
+        {
+            $strHtml =
+                $self->{oManifest}->variableReplace(
+                    (new BackRestDoc::Html::DocHtmlPage($self->{oManifest}, $strPageId, $self->{bExe}))->process());
+        };
+
+        if ($@)
+        {
+            my $oMessage = $@;
+
+            # If a backrest exception then return the code - don't confess
+            if (blessed($oMessage) && $oMessage->isa('pgBackRest::Common::Exception') && $oMessage->code() == -1)
+            {
+                my $oRenderOut = $self->{oManifest}->renderOutGet(RENDER_TYPE_HTML, $strPageId);
+                $self->{oManifest}->cacheReset($$oRenderOut{source});
+
+                $strHtml =
+                    $self->{oManifest}->variableReplace(
+                        (new BackRestDoc::Html::DocHtmlPage($self->{oManifest}, $strPageId, $self->{bExe}))->process());
+            }
+            else
+            {
+                confess $@;
+            }
+        }
+
         # Save the html page
-        fileStringWrite("$self->{strHtmlPath}/${strPageId}.html",
+        fileStringWrite("$self->{strHtmlPath}/${strPageId}.html", $strHtml, false);
-                        $self->{oManifest}->variableReplace((new BackRestDoc::Html::DocHtmlPage($self->{oManifest},
-                            $strPageId, $self->{bExe}))->process()),
-                        false);
     }
 
     # Return from function and log return values if any

doc/lib/BackRestDoc/Latex/DocLatex.pm

@@ -13,6 +13,7 @@ use Exporter qw(import);
 use File::Basename qw(dirname);
 use File::Copy;
 use POSIX qw(strftime);
+use Scalar::Util qw(blessed);
 use Storable qw(dclone);
 
 use lib dirname($0) . '/../lib';
@@ -104,11 +105,32 @@ sub process
     {
         &log(INFO, "    render out: ${strPageId}");
 
+        eval
+        {
             my $oDocLatexSection =
                 new BackRestDoc::Latex::DocLatexSection($self->{oManifest}, $strPageId, $self->{bExe});
 
             # Save the html page
             $strLatex .= $oDocLatexSection->process();
+        };
+
+        if ($@)
+        {
+            my $oMessage = $@;
+
+            # If a backrest exception then return the code - don't confess
+            if (blessed($oMessage) && $oMessage->isa('pgBackRest::Common::Exception') && $oMessage->code() == -1)
+            {
+                my $oRenderOut = $self->{oManifest}->renderOutGet(RENDER_TYPE_HTML, $strPageId);
+                $self->{oManifest}->cacheReset($$oRenderOut{source});
+
+                my $oDocLatexSection =
+                    new BackRestDoc::Latex::DocLatexSection($self->{oManifest}, $strPageId, $self->{bExe});
+
+                # Save the html page
+                $strLatex .= $oDocLatexSection->process();
+            }
+        }
     }
 
     $strLatex .= "\n% " . ('-' x 130) . "\n% End document\n% " . ('-' x 130) . "\n\\end{document}\n";

doc/lib/BackRestDoc/Latex/DocLatexSection.pm

@@ -392,7 +392,7 @@ sub configProcess
         $strLatex =
             "\n\\begin\{lstlisting\}[title=\{\\textnormal{\\textbf\{${strHostName}}}:\\textnormal{\\texttt\{${strFile}}} --- " .
             $self->processText($oConfig->nodeGet('title')->textGet()) . "}]\n" .
-            ${strConfig} .
+            (defined($strConfig) ? $strConfig : '') .
             "\\end{lstlisting}\n";
     }
 

doc/manifest.xml

@@ -5,6 +5,12 @@
     <variable-list>
         <variable key="project">pgBackRest</variable>
         <variable key="version" eval="y">use pgBackRest::Version; $VERSION</variable>
+        <variable key="version-stable" eval="y">
+            use BackRestDoc::Custom::DocCustomRelease;
+
+            (new BackRestDoc::Custom::DocCustomRelease(
+                new BackRestDoc::Common::Doc("{[doc-path]}/xml/release.xml")))->currentStableVersion();
+        </variable>
         <variable key="project-exe">pgbackrest</variable>
         <variable key="project-url-root">/</variable>
         <variable key="postgres">PostgreSQL</variable>

doc/xml/index.xml

@@ -13,11 +13,11 @@
         <variable key="github-url-license">{[github-url-master]}/LICENSE</variable>
 
         <variable key="backrest-url-base">http://www.pgbackrest.org</variable>
-        <variable key="backrest-page-user-guide">user-guide.html</variable>
+        <variable key="backrest-page-user-guide">user-guide</variable>
-        <variable key="backrest-page-configuration">configuration.html</variable>
+        <variable key="backrest-page-configuration">configuration</variable>
-        <variable key="backrest-page-command">command.html</variable>
+        <variable key="backrest-page-command">command</variable>
-        <variable key="backrest-page-feature-backlog">backlog.html</variable>
+        <variable key="backrest-page-feature-backlog">backlog</variable>
-        <variable key="backrest-page-release">release.html</variable>
+        <variable key="backrest-page-release">release</variable>
 
         <variable key="crunchy-url-base">http://www.crunchydata.com</variable>
         <variable key="crunchy-url-cbm">{[crunchy-url-base]}/crunchy-backup-manager</variable>
@@ -33,7 +33,7 @@
 
         <p>Instead of relying on traditional backup tools like tar and rsync, <backrest/> implements all backup features internally and uses a custom protocol for communicating with remote systems.  Removing reliance on tar and rsync allows for better solutions to database-specific backup challenges.  The custom remote protocol allows for more flexibility and limits the types of connections that are required to perform a backup which increases security.</p>
 
-        <p><backrest/> <link url="{[github-url-base]}/releases/tag/release/{[version]}">v{[version]}</link> is the current stable release.  Release notes are on the <link page="{[backrest-page-release]}">Releases</link> page.</p>
+        <p><backrest/> <link url="{[github-url-base]}/releases/tag/release/{[version-stable]}">v{[version-stable]}</link> is the current stable release.  Release notes are on the <link page="{[backrest-page-release]}">Releases</link> page.</p>
     </section>
 
     <section id="features">
@@ -126,7 +126,7 @@
         <p><backrest/> strives to be easy to configure and operate:</p>
 
         <list>
-          <list-item><link page="{[backrest-page-user-guide]}">User guide</link> for Ubuntu 12.04 &amp; 14.04 / <postgres/> 9.4.</list-item>
+          <list-item><link page="{[backrest-page-user-guide]}">User guide</link> for {[user-guide-subtitle]} / <postgres/> 9.4.</list-item>
           <list-item><link page="{[backrest-page-command]}">Command reference</link> for command-line operations.</list-item>
           <list-item><link page="{[backrest-page-configuration]}">Configuration reference</link> for creating <backrest/> configurations.</list-item>
       </list>

doc/xml/release.xml

@@ -138,6 +138,10 @@
 
                         <p>Added man page generation.</p>
                     </release-item>
+
+                    <release-item>
+                        <p>Added an execution cache so that documentation can be generated without setting up the full container environment.  This is useful for packaging, keeps the documentation consistent for a release, and speeds up generation when no changes are made in the execution list.</p>
+                    </release-item>
                 </release-feature-list>
 
                  <release-refactor-list>

doc/xml/user-guide.xml

@@ -1,10 +1,16 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE doc SYSTEM "doc.dtd">
-<doc title="User Guide" subtitle="Ubuntu 12.04 &amp; 14.04 / {[postgres]} 9.4">
+<doc title="User Guide" subtitle="{[user-guide-subtitle]} / {[postgres]} 9.4">
     <description>The {[project]} User Guide demonstrates how to quickly and easily setup {[project]} for your {[postgres]} database. Step-by-step instructions lead the user through all the important features of the fastest, most reliable {[postgres]} backup and restore solution.</description>
 
     <variable-list>
         <!-- Variables used by the rest of the script -->
+        <variable key="user-guide-subtitle" keyword="default">Debian &amp; Ubuntu</variable>
+        <variable key="user-guide-subtitle" keyword="co6">RHEL &amp; CentOS 6</variable>
+
+        <variable key="user-guide-os" keyword="default">Debian/Ubuntu</variable>
+        <variable key="user-guide-os" keyword="co6">RHEL/CentOS 6</variable>
+
         <variable key="perl-lib-path" keyword="default">/usr/lib/perl5</variable>
         <variable key="perl-lib-path" keyword="co6">/usr/share/perl5</variable>
         <variable key="perl-bin-path">/usr/bin</variable>
@@ -13,6 +19,9 @@
         <variable key="postgres-cluster-demo">demo</variable>
         <variable key="backrest-config-demo">/etc/{[project-exe]}.conf</variable>
 
+        <variable key="db-path-default" keyword="default">/var/lib/postgresql/[version]/[cluster]</variable>
+        <variable key="db-path-default" keyword="co6">/var/lib/pgsql/[version]/data</variable>
+
         <variable key="db-path" keyword="default">/var/lib/postgresql/9.4/{[postgres-cluster-demo]}</variable>
         <variable key="db-path" keyword="co6">/var/lib/pgsql/9.4/data</variable>
 
@@ -86,7 +95,7 @@
 
         <p>This user guide is intended to be followed sequentially from beginning to end &amp;mdash; each section depends on the last.  For example, the <link section="/backup">Backup</link> section relies on setup that is performed in the <link section="/quickstart">Quick Start</link> section.  Once <backrest/> is up and running then skipping around is possible but following the user guide in order is recommended the first time through.</p>
 
-        <p>Although the examples are targeted at Ubuntu and <postgres/> 9.4 they will also work fine on Debian and it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version, but note that only 64-bit distributions are currently supported due to 64-bit operations in the Perl code.  The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters.  The <backrest/> commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary.
+        <p>Although the examples are targeted at {[user-guide-os]} and <postgres/> 9.4, it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version. Note that only 64-bit distributions are currently supported due to 64-bit operations in the Perl code.  The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters.  The <backrest/> commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary.
 
         Configuration information and documentation for PostgreSQL can be found in the <postgres/> <link url='http://www.postgresql.org/docs/9.4/static/index.html'>Manual</link>.</p>
 
@@ -148,7 +157,7 @@
             </execute>
         </host-add>
 
-        <p keyword="default"><backrest/> is written in Perl which is included with Ubuntu by default.  A few additional modules are required which are all available as packages.</p>
+        <p keyword="default"><backrest/> is written in Perl which is included with {[user-guide-os]} by default.  A few additional modules are required which are all available as packages.</p>
 
         <execute-list host="{[host-db-master]}" keyword="default">
             <title>Install required Perl packages</title>
@@ -158,7 +167,7 @@
             </execute>
         </execute-list>
 
-        <p keyword="co6"><backrest/> is written in Perl, which is not included with RHEL/CentOS by default, however all required modules are all available as standard packages.</p>
+        <p keyword="co6"><backrest/> is written in Perl which is not included with {[user-guide-os]} by default, however all required modules are all available as standard packages.</p>
 
         <execute-list host="{[host-db-master]}" keyword="co6">
             <title>Install required Perl packages</title>
@@ -169,9 +178,9 @@
             </execute>
         </execute-list>
 
-        <p keyword="default">No Debian/Ubuntu packages are currently available for <backrest/> but it is easy to download the source and install manually.</p>
+        <p keyword="default">No {[user-guide-os]} packages are currently available for <backrest/> but it is easy to download the source and install manually.</p>
 
-        <p keyword="co6">No RHEL/CentOS packages are currently available for <backrest/> but it is easy to download the source and install manually.</p>
+        <p keyword="co6">No {[user-guide-os]} packages are currently available for <backrest/> but it is easy to download the source and install manually.</p>
 
         <execute-list host="{[host-db-master]}">
             <title>Download version <id>{[version]}</id> of <backrest/></title>
@@ -257,7 +266,7 @@
                 <postgres-config-option key="log_line_prefix">''</postgres-config-option>
             </postgres-config>
 
-            <p keyword="co6">By default CentOS/RHEL includes the day of the week in the log filename.  This makes automating the user guide a bit more complicated so the <pg-option>log_filename</pg-option> is set to a constant.</p>
+            <p keyword="co6">By default {[user-guide-os]} includes the day of the week in the log filename.  This makes automating the user guide a bit more complicated so the <pg-option>log_filename</pg-option> is set to a constant.</p>
 
             <postgres-config host="{[host-db-master]}" keyword="co6" file="{[postgres-config-demo]}">
                 <title>Set <pg-option>log_filename</pg-option></title>
@@ -276,7 +285,7 @@
 
             <p><backrest/> needs to know where the base data directory for the <postgres/> cluster is located.  The path can be requested from <postgres/> directly but in a recovery scenario the <postgres/> process will not be available.  During backups the value supplied to <backrest/> will be compared against the path that <postgres/> is running on and they must be equal or the backup will return an error.  Make sure that <br-option>db-path</br-option> is exactly equal to <pg-option>data_directory</pg-option> in <file>postgresql.conf</file>.</p>
 
-            <p>By default Ubuntu stores clusters in <path>/var/lib/postgresql/[version]/[cluster]</path> so it is easy to determine the correct path for the data directory.</p>
+            <p>By default {[user-guide-os]} stores clusters in <path>{[db-path-default]}</path> so it is easy to determine the correct path for the data directory.</p>
 
             <p>When creating the <file>{[backrest-config-demo]}</file> file, the database owner (usually <id>postgres</id>) must be granted read privileges.</p>
 
@@ -386,7 +395,7 @@
 
                 <execute output="y">
                     <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]}
-                        --log-level-console=detail backup</exe-cmd>
+                        --log-level-console=info backup</exe-cmd>
                     <exe-highlight>no prior backup exists|full backup size</exe-highlight>
                 </execute>
 
@@ -404,7 +413,7 @@
 
                 <execute output="y">
                     <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=diff
-                        --log-level-console=detail backup</exe-cmd>
+                        --log-level-console=info backup</exe-cmd>
                     <exe-highlight>diff backup size</exe-highlight>
                 </execute>
             </execute-list>
@@ -534,7 +543,7 @@
 
                 <execute output="y">
                     <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
-                        --log-level-console=detail backup</exe-cmd>
+                        --log-level-console=info backup</exe-cmd>
                     <exe-highlight>backup begins after the next regular checkpoint completes</exe-highlight>
                 </execute>
             </execute-list>
@@ -552,7 +561,7 @@
 
                 <execute output="y">
                     <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
-                        --log-level-console=detail backup</exe-cmd>
+                        --log-level-console=info backup</exe-cmd>
                     <exe-highlight>backup begins after the requested immediate checkpoint completes</exe-highlight>
                 </execute>
             </execute-list>
@@ -575,7 +584,7 @@
 
                 <execute output="y" err-expect="122">
                     <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
-                        --log-level-console=detail backup</exe-cmd>
+                        --log-level-console=info backup</exe-cmd>
                     <exe-highlight>ERROR:</exe-highlight>
                 </execute>
             </execute-list>
@@ -591,7 +600,7 @@
 
                 <execute output="y" err-expect="132">
                     <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
-                        --log-level-console=detail backup</exe-cmd>
+                        --log-level-console=info backup</exe-cmd>
                     <exe-highlight>ERROR:</exe-highlight>
                 </execute>
             </execute-list>
@@ -611,7 +620,7 @@
 
                 <execute output="y">
                     <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
-                        --log-level-console=detail backup</exe-cmd>
+                        --log-level-console=info backup</exe-cmd>
                     <exe-highlight>cluster is already in backup mode|backup begins after the requested immediate checkpoint completes</exe-highlight>
                 </execute>
             </execute-list>
@@ -663,7 +672,7 @@
 
                 <execute output="y">
                     <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=full
-                        --log-level-console=detail backup</exe-cmd>
+                        --log-level-console=info backup</exe-cmd>
                     <exe-highlight>expire full backup set\: {[backup-full-first]}|archive retention on backup {[backup-full-second]}|remove archive</exe-highlight>
                 </execute>
             </execute-list>
@@ -708,7 +717,7 @@
 
                 <execute output="y">
                     <exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=diff
-                        --log-level-console=detail backup</exe-cmd>
+                        --log-level-console=info backup</exe-cmd>
                     <exe-highlight>expire diff backup set: {[backup-diff-second]}</exe-highlight>
                 </execute>
             </execute-list>
@@ -1640,7 +1649,7 @@
                 </execute>
             </execute-list>
 
-            <p keyword="co6">By default CentOS/RHEL stores the <file>postgresql.conf</file> file in the <postgres/> data directory.  That means the change made to <file>postgresql.conf</file> was overwritten by the last restore and the <pg-option>hot_standby</pg-option> setting must be enabled again.  Other solutions to this problem are to store the <file>postgresql.conf</file> file elsewhere or to enable the <pg-option>hot_standby</pg-option> setting on the <host>db-master</host> host where it will be ignored.</p>
+            <p keyword="co6">By default {[user-guide-os]} stores the <file>postgresql.conf</file> file in the <postgres/> data directory.  That means the change made to <file>postgresql.conf</file> was overwritten by the last restore and the <pg-option>hot_standby</pg-option> setting must be enabled again.  Other solutions to this problem are to store the <file>postgresql.conf</file> file elsewhere or to enable the <pg-option>hot_standby</pg-option> setting on the <host>db-master</host> host where it will be ignored.</p>
 
             <postgres-config host="{[host-db-standby]}" keyword="co6" file="{[postgres-config-demo]}">
                 <title>Enable <pg-option>hot_standby</pg-option></title>

test/lib/pgBackRestTest/Docker/ContainerTest.pm

@@ -212,12 +212,7 @@ sub containerBuild
 
     # Create temp path
     my $strTempPath = dirname(abs_path($0)) . '/.vagrant/docker';
-
+    filePathCreate($strTempPath, '0770', true, true);
-    if (!-e $strTempPath)
-    {
-        mkdir $strTempPath
-            or confess &log(ERROR, "unable to create ${strTempPath}");
-    }
 
     # Create SSH key (if it does not already exist)
     if (-e "${strTempPath}/id_rsa")

test/test.pl

@@ -197,6 +197,9 @@ if (!defined($strTestPath))
     $strTestPath = cwd() . '/test';
 }
 
+# Get the base backrest path
+my $strBackRestBase = dirname(dirname(abs_path($0)));
+
 ####################################################################################################################################
 # Build Docker containers
 ####################################################################################################################################
@@ -320,12 +323,12 @@ eval
     if ($strVm ne 'none')
     {
         # Load the doc module dynamically since it is not supported on all systems
-        use lib dirname($0) . '/../doc/lib';
+        use lib dirname(abs_path($0)) . '/../doc/lib';
         require BackRestDoc::Common::Doc;
         BackRestDoc::Common::Doc->import();
 
         # Make sure version number matches the latest release
-        my $strReleaseFile = dirname(dirname($0)) . '/doc/xml/release.xml';
+        my $strReleaseFile = dirname(dirname(abs_path($0))) . '/doc/xml/release.xml';
         my $oReleaseDoc = new BackRestDoc::Common::Doc($strReleaseFile);
 
         foreach my $oRelease ($oReleaseDoc->nodeGet('release-list')->nodeList('release'))
@@ -619,24 +622,27 @@ eval
 
                         executeTest("docker run -itd -h $$oTest{os}-test --name=${strImage}" .
                                     " -v ${strHostTestPath}:${strVmTestPath}" .
-                                    " -v /backrest:/backrest backrest/$$oTest{os}-test-${strDbVersion}");
+                                    " -v ${strBackRestBase}:${strBackRestBase} backrest/$$oTest{os}-test-${strDbVersion}");
                     }
 
                     # Build up command line for the individual test
                     $strCommandLine =~ s/\-\-os\=\S*//g;
-                    $strCommandLine =~ s/\-\-test-path\=\S*//g;
+                    $strCommandLine =~ s/\-\-test\-path\=\S*//g;
                     $strCommandLine =~ s/\-\-module\=\S*//g;
                     $strCommandLine =~ s/\-\-test\=\S*//g;
                     $strCommandLine =~ s/\-\-run\=\S*//g;
                     $strCommandLine =~ s/\-\-db\-version\=\S*//g;
+                    $strCommandLine =~ s/\-\-no\-lint\S*//g;
+                    $strCommandLine =~ s/\-\-process\-max\=\S*//g;
 
                     my $strCommand =
-                        "docker exec -i -u vagrant ${strImage} $0 ${strCommandLine} --test-path=${strVmTestPath}" .
+                        "docker exec -i -u vagrant ${strImage} ${strBackRestBase}/test/test.pl ${strCommandLine}" .
                         " --vm=none --module=$$oTest{module} --test=$$oTest{test}" .
                         (defined($$oTest{run}) ? " --run=$$oTest{run}" : '') .
                         (defined($$oTest{thread}) ? " --thread-max=$$oTest{thread}" : '') .
                         (defined($$oTest{db}) ? " --db-version=$$oTest{db}" : '') .
                         ($bDryRun ? " --dry-run" : '') .
+                        " --test-path=${strVmTestPath}" .
                         " --no-cleanup --vm-out";
 
                     &log(DEBUG, $strCommand);