db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-06-14 23:44:58 +02:00
Code Issues Releases Activity

Added execution cache for document generation.

Browse Source 
Added an execution cache so that documentation can be generated without setting up the full container environment. This is useful for packaging, keeps the documentation consistent for a release, and speeds up generation when no changes are made in the execution list.
This commit is contained in:
David Steele
2016-06-02 09:32:56 -04:00
parent df6086bd24
commit eaa981c5aa
15 changed files with 1017 additions and 347 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -72,7 +72,7 @@ pgBackRest includes support for versions down to 8.3, since older versions of Po
pgBackRest strives to be easy to configure and operate:
- [User guide](http://www.pgbackrest.org/user-guide.html) for Ubuntu 12.04 & 14.04 / PostgreSQL 9.4.
- [User guide](http://www.pgbackrest.org/user-guide.html) for Debian & Ubuntu / PostgreSQL 9.4.
- [Command reference](http://www.pgbackrest.org/command.html) for command-line operations.
- [Configuration reference](http://www.pgbackrest.org/configuration.html) for creating pgBackRest configurations.

152
doc/doc.pl
View File

@ -16,6 +16,7 @@ use Cwd qw(abs_path);
use File::Basename qw(dirname);
use Getopt::Long qw(GetOptions);
use Pod::Usage qw(pod2usage);
use Scalar::Util qw(blessed);
use Storable;
use lib dirname($0) . '/lib';
@ -51,13 +52,16 @@ doc.pl [options]
--version Display pgBackRest version
--quiet Sets log level to ERROR
--log-level Log level for execution (e.g. ERROR, WARN, INFO, DEBUG)
--deploy Write exe.cache into resource for persistence
--no-exe Should commands be executed when building help? (for testing only)
--use-cache Use cached data to generate the docs (for testing textual changes only)
--no-cache Don't use execution cache
--cache-only Only use the execution cache - don't attempt to generate it
--var Override variables defined in the XML
--doc-path Document path to render (manifest.xml should be located here)
--out Output types (html, pdf, markdown)
--keyword Keyword used to filter output
--require Require only certain sections of the document (to speed testing)
--exclude Exclude source from generation (links will reference website)
=cut
####################################################################################################################################
@ -68,12 +72,15 @@ my $bVersion = false;
my $bQuiet = false;
my $strLogLevel = 'info';
my $bNoExe = false;
my $bUseCache = false;
my $bNoCache = false;
my $bCacheOnly = false;
my $oVariableOverride = {};
my $strDocPath;
my @stryOutput;
my @stryKeyword;
my @stryRequire;
my @stryExclude;
my $bDeploy = false;
GetOptions ('help' => \$bHelp,
'version' => \$bVersion,
@ -82,15 +89,24 @@ GetOptions ('help' => \$bHelp,
'out=s@' => \@stryOutput,
'keyword=s@' => \@stryKeyword,
'require=s@' => \@stryRequire,
'exclude=s@' => \@stryExclude,
'no-exe', \$bNoExe,
'use-cache', \$bUseCache,
'deploy', \$bDeploy,
'no-cache', \$bNoCache,
'cache-only', \$bCacheOnly,
'var=s%', $oVariableOverride,
'doc-path=s', \$strDocPath)
or pod2usage(2);
# Display version and exit if requested
if ($bHelp || $bVersion)
####################################################################################################################################
# Run in eval block to catch errors
####################################################################################################################################
eval
{
# Display version and exit if requested
if ($bHelp || $bVersion)
{
print BACKREST_NAME . ' ' . $VERSION . " Documentation Builder\n";
if ($bHelp)
@ -100,61 +116,78 @@ if ($bHelp || $bVersion)
}
exit 0;
}
}
# Set no-exe if use-cache is set
if ($bUseCache)
{
$bNoExe = true;
}
# Disable cache when no exe
if ($bNoExe)
{
$bNoCache = true;
}
# Set console log level
if ($bQuiet)
{
# Make sure options are set correctly for deploy
if ($bDeploy)
{
my $strError = 'cannot be specified for deploy';
!$bNoExe
or confess "--no-exe ${strError}";
!$bNoCache
or confess "--no-cache ${strError}";
!@stryRequire
or confess "--require ${strError}";
!@stryOutput
or confess "--out ${strError}";
!@stryExclude
or confess "--exclude ${strError}";
}
# Set console log level
if ($bQuiet)
{
$strLogLevel = 'error';
}
}
# If no keyword was passed then use default
if (@stryKeyword == 0)
{
# If no keyword was passed then use default
if (@stryKeyword == 0)
{
@stryKeyword = ('default');
}
}
logLevelSet(undef, uc($strLogLevel));
logLevelSet(undef, uc($strLogLevel));
# Get the base path
my $strBasePath = abs_path(dirname($0));
# Get the base path
my $strBasePath = abs_path(dirname($0));
if (!defined($strDocPath))
{
if (!defined($strDocPath))
{
$strDocPath = $strBasePath;
}
}
my $strOutputPath = "${strDocPath}/output";
my $strOutputPath = "${strDocPath}/output";
# Create the out path if it does not exist
if (!-e $strOutputPath)
{
# Create the out path if it does not exist
if (!-e $strOutputPath)
{
mkdir($strOutputPath)
or confess &log(ERROR, "unable to create path ${strOutputPath}");
}
}
# Load the manifest
my $oManifest;
my $strManifestCache = "${strOutputPath}/manifest.cache";
# Load the manifest
my $oManifest =
new BackRestDoc::Common::DocManifest(\@stryKeyword, \@stryRequire, \@stryExclude, $oVariableOverride, $strDocPath, $bDeploy, $bCacheOnly);
if ($bUseCache && -e $strManifestCache)
{
$oManifest = retrieve($strManifestCache);
}
else
{
$oManifest = new BackRestDoc::Common::DocManifest(\@stryKeyword, \@stryRequire, $oVariableOverride, $strDocPath);
}
if (!$bNoCache)
{
$oManifest->cacheRead();
}
# If no outputs were given
if (@stryOutput == 0)
{
# If no outputs were given
if (@stryOutput == 0)
{
@stryOutput = $oManifest->renderList();
if ($oManifest->isBackRest())
@ -162,10 +195,10 @@ if (@stryOutput == 0)
push(@stryOutput, 'help');
push(@stryOutput, 'man');
}
}
}
for my $strOutput (@stryOutput)
{
for my $strOutput (@stryOutput)
{
if (!(($strOutput eq 'help' || $strOutput eq 'man') && $oManifest->isBackRest()))
{
$oManifest->renderGet($strOutput);
@ -236,10 +269,27 @@ for my $strOutput (@stryOutput)
$oLatex->process();
}
}
}
# Cache the manifest (mostly useful for testing rendering changes in the code)
if (!$bUseCache)
# Cache the manifest (mostly useful for testing rendering changes in the code)
if (!$bNoCache && !$bCacheOnly)
{
$oManifest->cacheWrite();
}
};
####################################################################################################################################
# Check for errors
####################################################################################################################################
if ($@)
{
store($oManifest, $strManifestCache);
my $oMessage = $@;
# If a backrest exception then return the code - don't confess
if (blessed($oMessage) && $oMessage->isa('pgBackRest::Common::Exception'))
{
exit $oMessage->code();
}
confess $oMessage;
}

568
doc/lib/BackRestDoc/Common/DocExecute.pm
View File

@ -19,6 +19,7 @@ use pgBackRest::Common::Log;
use pgBackRest::Common::String;
use pgBackRest::Config::Config;
use pgBackRest::FileCommon;
use pgBackRest::Version;
use lib dirname($0) . '/../test/lib';
use pgBackRestTest::Common::ExecuteTest;
@ -55,6 +56,16 @@ sub new
my $self = $class->SUPER::new($strType, $oManifest, $strRenderOutKey);
bless $self, $class;
if (defined($self->{oSource}{hyCache}))
{
$self->{bCache} = true;
$self->{iCacheIdx} = 0;
}
else
{
$self->{bCache} = false;
}
$self->{bExe} = $bExe;
# Return from function and log return values if any
@ -65,6 +76,75 @@ sub new
);
}
####################################################################################################################################
# executeKey
#
# Get a unique key for the execution step to determine if the cache is valid.
####################################################################################################################################
sub executeKey
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strHostName,
$oCommand,
) =
logDebugParam
(
__PACKAGE__ . '->executeKey', \@_,
{name => 'strHostName', trace => true},
{name => 'oCommand', trace => true},
);
# Add user to command
my $strCommand = $self->{oManifest}->variableReplace(trim($oCommand->fieldGet('exe-cmd')));
my $strUser = $self->{oManifest}->variableReplace($oCommand->paramGet('user', false, 'postgres'));
$strCommand = ($strUser eq 'vagrant' ? '' : ('sudo ' . ($strUser eq 'root' ? '' : "-u $strUser "))) . $strCommand;
# Format and split command
$strCommand =~ s/[ ]*\n[ ]*/ \\\n /smg;
my @stryCommand = split("\n", $strCommand);
my $hCacheKey =
{
host => $strHostName,
cmd => \@stryCommand,
output => JSON::PP::false,
};
if (defined($oCommand->paramGet('err-expect', false)))
{
$$hCacheKey{'err-expect'} = $oCommand->paramGet('err-expect');
}
if ($oCommand->paramTest('output', 'y') || $oCommand->paramTest('show', 'y') || $oCommand->paramTest('variable-key'))
{
$$hCacheKey{'output'} = JSON::PP::true;
}
if (defined($oCommand->fieldGet('exe-highlight', false)))
{
$$hCacheKey{'output'} = JSON::PP::true;
$$hCacheKey{highlight}{'filter'} = $oCommand->paramTest('filter', 'n') ? JSON::PP::false : JSON::PP::true;
$$hCacheKey{highlight}{'filter-context'} = $oCommand->paramGet('filter-context', false, 2);
my @stryHighlight;
$stryHighlight[0] = $self->{oManifest}->variableReplace($oCommand->fieldGet('exe-highlight'));
$$hCacheKey{highlight}{list} = \@stryHighlight;
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hExecuteKey', value => $hCacheKey, trace => true}
);
}
####################################################################################################################################
# execute
####################################################################################################################################
@ -79,7 +159,8 @@ sub execute
$oSection,
$strHostName,
$oCommand,
$iIndent
$iIndent,
$bCache,
) =
logDebugParam
(
@ -87,34 +168,15 @@ sub execute
{name => 'oSection'},
{name => 'strHostName'},
{name => 'oCommand'},
{name => 'iIndent', default => 1}
{name => 'iIndent', default => 1},
{name => 'bCache', default => true},
);
# Working variables
my $strCommand;
my $hCacheKey = $self->executeKey($strHostName, $oCommand);
my $strCommand = join("\n", @{$$hCacheKey{cmd}});
my $strOutput;
if ($oCommand->fieldTest('actual-command'))
{
$strCommand = $oCommand->fieldGet('actual-command');
$strOutput = $oCommand->fieldGet('actual-output', false);
}
else
{
# Command variables
$strCommand = trim($oCommand->fieldGet('exe-cmd'));
my $strUser = $self->{oManifest}->variableReplace($oCommand->paramGet('user', false, 'postgres'));
my $bExeOutput = $oCommand->paramTest('output', 'y');
my $strVariableKey = $oCommand->paramGet('variable-key', false);
my $iExeExpectedError = $oCommand->paramGet('err-expect', false);
$strCommand = $self->{oManifest}->variableReplace(
($strUser eq 'vagrant' ? '' :
('sudo ' . ($strUser eq 'root' ? '' : "-u ${strUser} "))) . $strCommand);
# Add continuation chars and proper spacing
$strCommand =~ s/[ ]*\n[ ]*/ \\\n /smg;
if (!$oCommand->paramTest('show', 'n') && $self->{bExe} && $self->isRequired($oSection))
{
# Make sure that no lines are greater than 80 chars
@ -132,6 +194,14 @@ sub execute
if (!$oCommand->paramTest('skip', 'y'))
{
if ($self->{bExe} && $self->isRequired($oSection))
{
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('exe', $hCacheKey);
if ($bCacheHit)
{
$strOutput = defined($$hCacheValue{output}) ? join("\n", @{$$hCacheValue{output}}) : undef;
}
else
{
# Check that the host is valid
my $oHost = $self->{host}{$strHostName};
@ -142,13 +212,13 @@ sub execute
}
my $oExec = $oHost->execute($strCommand,
{iExpectedExitStatus => $iExeExpectedError,
{iExpectedExitStatus => $$hCacheKey{'err-expect'},
bSuppressError => $oCommand->paramTest('err-suppress', 'y'),
iRetrySeconds => $oCommand->paramGet('retry', false)});
$oExec->begin();
$oExec->end();
if ($bExeOutput && defined($oExec->{strOutLog}) && $oExec->{strOutLog} ne '')
if (defined($oExec->{strOutLog}) && $oExec->{strOutLog} ne '')
{
$strOutput = $oExec->{strOutLog};
@ -160,28 +230,36 @@ sub execute
$strOutput =~ s/^ //smg;
$strOutput =~ s/^[0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-6][0-9]:[0-6][0-9]\.[0-9]{3} T[0-9]{2} //smg;
}
# my @stryOutput = split("\n", $strOutput);
# $$hCacheValue{stdout} = \@stryOutput;
}
if (defined($iExeExpectedError))
if (defined($$hCacheKey{'err-expect'}) && defined($oExec->{strErrorLog}) && $oExec->{strErrorLog} ne '')
{
$strOutput .= trim($oExec->{strErrorLog});
# my $strError = $oExec->{strErrorLog};
# $strError =~ s/^\n+|\n$//g;
# my @stryError = split("\n", $strError);
# $$hCacheValue{stderr} = \@stryError;
$strOutput .= $oExec->{strErrorLog};
}
# Output is assigned to a var
if (defined($strVariableKey))
# if (defined($$hCacheValue{stderr}))
# {
# $strOutput .= join("\n", @{$$hCacheValue{stderr}});
# }
if ($$hCacheKey{output} && defined($$hCacheKey{highlight}) && $$hCacheKey{highlight}{filter} && defined($strOutput))
{
$self->{oManifest}->variableSet($strVariableKey, trim($oExec->{strOutLog}));
}
elsif (!$oCommand->paramTest('filter', 'n') && $bExeOutput && defined($strOutput))
{
my $strHighLight = $self->{oManifest}->variableReplace($oCommand->fieldGet('exe-highlight', false));
my $strHighLight = @{$$hCacheKey{highlight}{list}}[0];
if (!defined($strHighLight))
{
confess &log(ERROR, 'filter requires highlight definition: ' . $strCommand);
}
my $iFilterContext = $oCommand->paramGet('filter-context', false, 2);
my $iFilterContext = $$hCacheKey{highlight}{'filter-context'};
my @stryOutput = split("\n", $strOutput);
undef($strOutput);
@ -248,20 +326,40 @@ sub execute
}
}
}
if (!$$hCacheKey{output})
{
$strOutput = undef;
}
elsif ($bExeOutput)
if (defined($strOutput))
{
my @stryOutput = split("\n", $strOutput);
$$hCacheValue{output} = \@stryOutput;
}
if ($bCache)
{
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
}
}
# Output is assigned to a var
if ($oCommand->paramTest('variable-key'))
{
$self->{oManifest}->variableSet($oCommand->paramGet('variable-key'), trim($strOutput), true);
}
}
elsif ($$hCacheKey{output})
{
$strOutput = 'Output suppressed for testing';
}
}
if (defined($strVariableKey) && !defined($self->{oManifest}->variableGet($strVariableKey)))
# Default variable output when it was not set by execution
if ($oCommand->paramTest('variable-key') && !defined($self->{oManifest}->variableGet($oCommand->paramGet('variable-key'))))
{
$self->{oManifest}->variableSet($strVariableKey, '[Test Variable]');
}
$oCommand->fieldSet('actual-command', $strCommand);
$oCommand->fieldSet('actual-output', $strOutput);
$self->{oManifest}->variableSet($oCommand->paramGet('variable-key'), '[Test Variable]', true);
}
# Return from function and log return values if any
@ -273,6 +371,76 @@ sub execute
);
}
####################################################################################################################################
# configKey
####################################################################################################################################
sub configKey
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oConfig,
) =
logDebugParam
(
__PACKAGE__ . '->hostKey', \@_,
{name => 'oConfig', trace => true},
);
my $hCacheKey =
{
host => $self->{oManifest}->variableReplace($oConfig->paramGet('host')),
file => $self->{oManifest}->variableReplace($oConfig->paramGet('file')),
};
if ($oConfig->paramTest('reset', 'y'))
{
$$hCacheKey{reset} = JSON::PP::true;
}
# Add all options to the key
my $strOptionTag = $oConfig->nameGet() eq 'backrest-config' ? 'backrest-config-option' : 'postgres-config-option';
foreach my $oOption ($oConfig->nodeList($strOptionTag))
{
my $hOption = {};
if ($oOption->paramTest('remove', 'y'))
{
$$hOption{remove} = JSON::PP::true;
}
if (defined($oOption->valueGet(false)))
{
$$hOption{value} = $self->{oManifest}->variableReplace($oOption->valueGet());
}
my $strKey = $self->{oManifest}->variableReplace($oOption->paramGet('key'));
if ($oConfig->nameGet() eq 'backrest-config')
{
my $strSection = $self->{oManifest}->variableReplace($oOption->paramGet('section'));
$$hCacheKey{option}{$strSection}{$strKey} = $hOption;
}
else
{
$$hCacheKey{option}{$strKey} = $hOption;
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hCacheKey', value => $hCacheKey, trace => true}
);
}
####################################################################################################################################
# backrestConfig
####################################################################################################################################
@ -297,22 +465,21 @@ sub backrestConfig
);
# Working variables
my $strFile;
my $strConfig;
my $hCacheKey = $self->configKey($oConfig);
my $strFile = $$hCacheKey{file};
my $strConfig = undef;
if ($oConfig->fieldTest('actual-file'))
{
$strFile = $oConfig->fieldGet('actual-file');
$strConfig = $oConfig->fieldGet('actual-config');
}
else
{
# Get filename
$strFile = $self->{oManifest}->variableReplace($oConfig->paramGet('file'));
&log(DEBUG, (' ' x $iDepth) . 'process backrest config: ' . $strFile);
&log(DEBUG, (' ' x $iDepth) . 'process backrest config: ' . $$hCacheKey{file});
if ($self->{bExe} && $self->isRequired($oSection))
{
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('cfg-' . BACKREST_EXE, $hCacheKey);
if ($bCacheHit)
{
$strConfig = defined($$hCacheValue{config}) ? join("\n", @{$$hCacheValue{config}}) : undef;
}
else
{
# Check that the host is valid
my $strHostName = $self->{oManifest}->variableReplace($oConfig->paramGet('host'));
@ -326,7 +493,7 @@ sub backrestConfig
# Reset all options
if ($oConfig->paramTest('reset', 'y'))
{
delete(${$self->{config}}{$strHostName}{$strFile})
delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}})
}
foreach my $oOption ($oConfig->nodeList('backrest-config-option'))
@ -342,11 +509,11 @@ sub backrestConfig
if (!defined($strValue))
{
delete(${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey});
delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey});
if (keys(%{${$self->{config}}{$strHostName}{$strFile}{$strSection}}) == 0)
if (keys(%{${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}}) == 0)
{
delete(${$self->{config}}{$strHostName}{$strFile}{$strSection});
delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection});
}
&log(DEBUG, (' ' x ($iDepth + 1)) . "reset ${strSection}->${strKey}");
@ -364,10 +531,10 @@ sub backrestConfig
# If this option is a hash and the value is already set then append to the array
if ($$oOption{$strKey}{&OPTION_RULE_TYPE} eq OPTION_TYPE_HASH &&
defined(${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey}))
defined(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey}))
{
my @oValue = ();
my $strHashValue = ${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey};
my $strHashValue = ${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey};
# If there is only one key/value
if (ref(\$strHashValue) eq 'SCALAR')
@ -381,12 +548,12 @@ sub backrestConfig
}
push(@oValue, $strValue);
${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey} = \@oValue;
${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey} = \@oValue;
}
# else just set the value
else
{
${$self->{config}}{$strHostName}{$strFile}{$strSection}{$strKey} = $strValue;
${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey} = $strValue;
}
&log(DEBUG, (' ' x ($iDepth + 1)) . "set ${strSection}->${strKey} = ${strValue}");
@ -396,21 +563,27 @@ sub backrestConfig
my $strLocalFile = "/home/vagrant/data/db-master/etc/pgbackrest.conf";
# Save the ini file
iniSave($strLocalFile, $self->{config}{$strHostName}{$strFile}, true);
iniSave($strLocalFile, $self->{config}{$strHostName}{$$hCacheKey{file}}, true);
$strConfig = fileStringRead($strLocalFile);
$oHost->copyTo($strLocalFile, $$hCacheKey{file}, $oConfig->paramGet('owner', false, 'postgres:postgres'), '640');
$oHost->copyTo($strLocalFile, $strFile, $oConfig->paramGet('owner', false, 'postgres:postgres'), '640');
my $strConfig = fileStringRead($strLocalFile);
my @stryConfig = undef;
if (trim($strConfig) ne '')
{
@stryConfig = split("\n", $strConfig);
}
$$hCacheValue{config} = \@stryConfig;
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
}
}
else
{
$strConfig = 'Config suppressed for testing';
}
$oConfig->fieldSet('actual-file', $strFile);
$oConfig->fieldSet('actual-config', $strConfig);
}
# Return from function and log return values if any
return logDebugReturn
(
@ -445,20 +618,19 @@ sub postgresConfig
);
# Working variables
my $strFile;
my $hCacheKey = $self->configKey($oConfig);
my $strFile = $$hCacheKey{file};
my $strConfig;
if ($oConfig->fieldTest('actual-file'))
if ($self->{bExe} && $self->isRequired($oSection))
{
$strFile = $oConfig->fieldGet('actual-file');
$strConfig = $oConfig->fieldGet('actual-config');
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('cfg-postgresql', $hCacheKey);
if ($bCacheHit)
{
$strConfig = defined($$hCacheValue{config}) ? join("\n", @{$$hCacheValue{config}}) : undef;
}
else
{
# Get filename
$strFile = $self->{oManifest}->variableReplace($oConfig->paramGet('file'));
if ($self->{bExe} && $self->isRequired($oSection))
{
# Check that the host is valid
my $strHostName = $self->{oManifest}->variableReplace($oConfig->paramGet('host'));
@ -470,14 +642,14 @@ sub postgresConfig
}
my $strLocalFile = '/home/vagrant/data/db-master/etc/postgresql.conf';
$oHost->copyFrom($strFile, $strLocalFile);
$oHost->copyFrom($$hCacheKey{file}, $strLocalFile);
if (!defined(${$self->{'pg-config'}}{$strHostName}{$strFile}{base}) && $self->{bExe})
if (!defined(${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base}) && $self->{bExe})
{
${$self->{'pg-config'}}{$strHostName}{$strFile}{base} = fileStringRead($strLocalFile);
${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base} = fileStringRead($strLocalFile);
}
my $oConfigHash = $self->{'pg-config'}{$strHostName}{$strFile};
my $oConfigHash = $self->{'pg-config'}{$strHostName}{$$hCacheKey{file}};
my $oConfigHashNew;
if (!defined($$oConfigHash{old}))
@ -490,7 +662,7 @@ sub postgresConfig
$oConfigHashNew = dclone($$oConfigHash{old});
}
&log(DEBUG, (' ' x $iDepth) . 'process postgres config: ' . $strFile);
&log(DEBUG, (' ' x $iDepth) . 'process postgres config: ' . $$hCacheKey{file});
foreach my $oOption ($oConfig->nodeList('postgres-config-option'))
{
@ -527,20 +699,27 @@ sub postgresConfig
fileStringWrite($strLocalFile, $$oConfigHash{base} .
(defined($strConfig) ? "\n# pgBackRest Configuration\n${strConfig}\n" : ''));
$oHost->copyTo($strLocalFile, $strFile, 'postgres:postgres', '640');
$oHost->copyTo($strLocalFile, $$hCacheKey{file}, 'postgres:postgres', '640');
}
$$oConfigHash{old} = $oConfigHashNew;
my @stryConfig = undef;
if (trim($strConfig) ne '')
{
@stryConfig = split("\n", $strConfig);
}
$$hCacheValue{config} = \@stryConfig;
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
}
}
else
{
$strConfig = 'Config suppressed for testing';
}
$oConfig->fieldSet('actual-file', $strFile);
$oConfig->fieldSet('actual-config', $strConfig);
}
# Return from function and log return values if any
return logDebugReturn
(
@ -551,6 +730,178 @@ sub postgresConfig
);
}
####################################################################################################################################
# hostKey
####################################################################################################################################
sub hostKey
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$oHost,
) =
logDebugParam
(
__PACKAGE__ . '->hostKey', \@_,
{name => 'oHost', trace => true},
);
my $hCacheKey =
{
name => $self->{oManifest}->variableReplace($oHost->paramGet('name')),
user => $self->{oManifest}->variableReplace($oHost->paramGet('user')),
image => $self->{oManifest}->variableReplace($oHost->paramGet('image')),
};
if (defined($oHost->paramGet('os', false)))
{
$$hCacheKey{os} = $self->{oManifest}->variableReplace($oHost->paramGet('os'));
}
if (defined($oHost->paramGet('mount', false)))
{
$$hCacheKey{mount} = $self->{oManifest}->variableReplace($oHost->paramGet('mount'));
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'hCacheKey', value => $hCacheKey, trace => true}
);
}
####################################################################################################################################
# cachePop
####################################################################################################################################
sub cachePop
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strCacheType,
$hCacheKey,
) =
logDebugParam
(
__PACKAGE__ . '->hostKey', \@_,
{name => 'strCacheType', trace => true},
{name => 'hCacheKey', trace => true},
);
my $bCacheHit = false;
my $oCacheValue = undef;
if ($self->{bCache})
{
my $oJSON = JSON::PP->new()->canonical()->allow_nonref();
# &log(WARN, "checking cache for\ncurrent key: " . $oJSON->encode($hCacheKey));
my $hCache = ${$self->{oSource}{hyCache}}[$self->{iCacheIdx}];
if (!defined($hCache))
{
confess &log(ERROR, 'unable to get index from cache', -1);
}
if (!defined($$hCache{key}))
{
confess &log(ERROR, 'unable to get key from cache', -1);
}
if (!defined($$hCache{type}))
{
confess &log(ERROR, 'unable to get type from cache', -1);
}
if ($$hCache{type} ne $strCacheType)
{
confess &log(ERROR, 'types do not match, cache is invalid', -1);
}
if ($oJSON->encode($$hCache{key}) ne $oJSON->encode($hCacheKey))
{
confess &log(ERROR,
"keys at index $self->{iCacheIdx} do not match, cache is invalid." .
"\ncache key: " . $oJSON->encode($$hCache{key}) .
"\ncurrent key: " . $oJSON->encode($hCacheKey), -1);
}
$bCacheHit = true;
$oCacheValue = $$hCache{value};
$self->{iCacheIdx}++;
}
else
{
if ($self->{oManifest}{bCacheOnly})
{
confess &log(ERROR, 'Cache only operation forced by --cache-only option');
}
}
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'bCacheHit', value => $bCacheHit, trace => true},
{name => 'strCacheType', value => $strCacheType, trace => true},
{name => 'hCacheKey', value => $hCacheKey, trace => true},
{name => 'oCacheValue', value => $oCacheValue, trace => true},
);
}
####################################################################################################################################
# cachePush
####################################################################################################################################
sub cachePush
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strType,
$hCacheKey,
$oCacheValue,
) =
logDebugParam
(
__PACKAGE__ . '->hostKey', \@_,
{name => 'strType', trace => true},
{name => 'hCacheKey', trace => true},
{name => 'oCacheValue', required => false, trace => true},
);
if ($self->{bCache})
{
confess &log(ASSERT, "cachePush should not be called when cache is already present");
}
# Create the cache entry
my $hCache =
{
key => $hCacheKey,
type => $strType,
};
if (defined($oCacheValue))
{
$$hCache{value} = $oCacheValue;
}
push @{$self->{oSource}{hyCache}}, $hCache;
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# sectionChildProcesss
####################################################################################################################################
@ -579,27 +930,33 @@ sub sectionChildProcess
# Execute a command
if ($oChild->nameGet() eq 'host-add')
{
if ($self->{bExe} && $self->isRequired($oSection) && !$oChild->paramTest('created', true))
if ($self->{bExe} && $self->isRequired($oSection))
{
my $strName = $self->{oManifest}->variableReplace($oChild->paramGet('name'));
my $strUser = $self->{oManifest}->variableReplace($oChild->paramGet('user'));
my $strImage = $self->{oManifest}->variableReplace($oChild->paramGet('image'));
my $strOS = $self->{oManifest}->variableReplace($oChild->paramGet('os', false));
my $strMount = $self->{oManifest}->variableReplace($oChild->paramGet('mount', false));
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('host', $self->hostKey($oChild));
if (defined($self->{host}{$strName}))
if ($bCacheHit)
{
$self->{oManifest}->variableSet("host-$$hCacheKey{name}-ip", $$hCacheValue{ip}, true);
}
else
{
if (defined($self->{host}{$$hCacheKey{name}}))
{
confess &log(ERROR, 'cannot add host ${strName} because the host already exists');
}
my $oHost = new pgBackRestTest::Common::HostTest($strName, $strImage, $strUser, $strOS, $strMount);
$self->{host}{$strName} = $oHost;
$self->{oManifest}->variableSet("host-${strName}-ip", $oHost->{strIP});
my $oHost =
new pgBackRestTest::Common::HostTest(
$$hCacheKey{name}, $$hCacheKey{image}, $$hCacheKey{user}, $$hCacheKey{os}, $$hCacheKey{mount});
$self->{host}{$$hCacheKey{name}} = $oHost;
$self->{oManifest}->variableSet("host-$$hCacheKey{name}-ip", $oHost->{strIP}, true);
$$hCacheValue{ip} = $oHost->{strIP};
# Execute cleanup commands
foreach my $oExecute ($oChild->nodeList('execute'))
{
$self->execute($oSection, $strName, $oExecute, $iDepth + 1);
$self->execute($oSection, $$hCacheKey{name}, $oExecute, $iDepth + 1, false);
}
$oHost->executeSimple("sh -c 'echo \"\" >> /etc/hosts\'", undef, 'root');
@ -608,7 +965,7 @@ sub sectionChildProcess
# Add all other host IPs to this host
foreach my $strOtherHostName (sort(keys(%{$self->{host}})))
{
if ($strOtherHostName ne $strName)
if ($strOtherHostName ne $$hCacheKey{name})
{
my $oOtherHost = $self->{host}{$strOtherHostName};
@ -619,15 +976,16 @@ sub sectionChildProcess
# Add this host IP to all other hosts
foreach my $strOtherHostName (sort(keys(%{$self->{host}})))
{
if ($strOtherHostName ne $strName)
if ($strOtherHostName ne $$hCacheKey{name})
{
my $oOtherHost = $self->{host}{$strOtherHostName};
$oOtherHost->executeSimple("sh -c 'echo \"$oHost->{strIP} ${strName}\" >> /etc/hosts'", undef, 'root');
$oOtherHost->executeSimple("sh -c 'echo \"$oHost->{strIP} $$hCacheKey{name}\" >> /etc/hosts'", undef, 'root');
}
}
$oChild->paramSet('created', true);
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
}
}
}
# Skip children that have already been processed and error on others

164
doc/lib/BackRestDoc/Common/DocManifest.pm
View File

@ -11,6 +11,7 @@ use Cwd qw(abs_path);
use Exporter qw(import);
our @EXPORT = qw();
use File::Basename qw(dirname);
use JSON::PP;
use Scalar::Util qw(blessed);
use lib dirname($0) . '/../lib';
@ -53,16 +54,22 @@ sub new
my $strOperation,
$self->{stryKeyword},
$self->{stryRequire},
$self->{stryExclude},
my $oVariableOverride,
$self->{strDocPath}
$self->{strDocPath},
$self->{bDeploy},
$self->{bCacheOnly},
) =
logDebugParam
(
__PACKAGE__ . '->new', \@_,
{name => 'stryKeyword'},
{name => 'stryRequire'},
{name => 'stryExclude'},
{name => 'oVariableOverride', required => false},
{name => 'strDocPath', required => false},
{name => 'bDeploy', required => false},
{name => 'bCacheOnly', required => false},
);
# Set the base path if it was not passed in
@ -71,6 +78,10 @@ sub new
$self->{strDocPath} = abs_path(dirname($0));
}
# Set cache file names
$self->{strExeCacheLocal} = $self->{strDocPath} . "/output/exe.cache";
$self->{strExeCacheDeploy} = $self->{strDocPath} . "/resource/exe.cache";
# Load the manifest
$self->{oManifestXml} = new BackRestDoc::Common::Doc("$self->{strDocPath}/manifest.xml");
@ -90,6 +101,12 @@ sub new
{name => 'strSourceType', value => $strSourceType}
);
# Skip sources in exclude list
if (grep(/^$strKey$/, @{$self->{stryExclude}}))
{
next;
}
$$oSourceHash{doc} = new BackRestDoc::Common::Doc("$self->{strDocPath}/xml/${strKey}.xml");
# Read variables from source
@ -134,6 +151,12 @@ sub new
my $strKey = $oRenderOut->paramGet('key');
my $strSource = $oRenderOut->paramGet('source', false, $strKey);
# Skip sources in exclude list
if (grep(/^$strSource$/, @{$self->{stryExclude}}))
{
next;
}
$$oRenderOutHash{source} = $strSource;
# Get the menu caption
@ -263,7 +286,7 @@ sub variableListParse
if ($self->keywordMatch($oVariable->paramGet('keyword', false)))
{
my $strKey = $oVariable->paramGet('key');
my $strValue = $oVariable->valueGet();
my $strValue = $self->variableReplace($oVariable->valueGet());
if ($oVariable->paramTest('eval', 'y'))
{
@ -526,18 +549,20 @@ sub renderOutGet
(
$strOperation,
$strType,
$strKey
$strKey,
$bIgnoreMissing,
) =
logDebugParam
(
__PACKAGE__ . '->renderOutGet', \@_,
{name => 'strType', trace => true},
{name => 'strKey', trace => true}
{name => 'strKey', trace => true},
{name => 'bIgnoreMissing', default => false, trace => true},
);
# use Data::Dumper; print Dumper(${$self->{oManifest}}{render});
if (!defined(${$self->{oManifest}}{render}{$strType}{out}{$strKey}))
if (!defined(${$self->{oManifest}}{render}{$strType}{out}{$strKey}) && !$bIgnoreMissing)
{
confess &log(ERROR, "render out ${strKey} does not exist");
}
@ -550,4 +575,133 @@ sub renderOutGet
);
}
####################################################################################################################################
# cacheKey
####################################################################################################################################
sub cacheKey
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->cacheKey');
my $strKeyword = join("\n", @{$self->{stryKeyword}});
my $strRequire = defined($self->{stryRequire}) && @{$self->{stryRequire}} > 0 ?
join("\n", @{$self->{stryRequire}}) : 'all';
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strKeyword', value => $strKeyword},
{name => 'strRequire', value => $strRequire},
);
}
####################################################################################################################################
# cacheRead
####################################################################################################################################
sub cacheRead
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->cacheRead');
$self->{hCache} = undef;
my $strCacheFile = $self->{bDeploy} ? $self->{strExeCacheDeploy} : $self->{strExeCacheLocal};
if (!fileExists($strCacheFile) && !$self->{bDeploy})
{
$strCacheFile = $self->{strExeCacheDeploy};
}
if (fileExists($strCacheFile))
{
my ($strKeyword, $strRequire) = $self->cacheKey();
my $oJSON = JSON::PP->new()->allow_nonref();
$self->{hCache} = $oJSON->decode(fileStringRead($strCacheFile));
foreach my $strSource (sort(keys(%{${$self->{oManifest}}{source}})))
{
my $hSource = ${$self->{oManifest}}{source}{$strSource};
if (defined(${$self->{hCache}}{$strKeyword}{$strRequire}{$strSource}))
{
$$hSource{hyCache} = ${$self->{hCache}}{$strKeyword}{$strRequire}{$strSource};
&log(DETAIL, "cache load $strSource (keyword = ${strKeyword}, require = ${strRequire})");
}
}
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# cacheWrite
####################################################################################################################################
sub cacheWrite
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my ($strOperation) = logDebugParam(__PACKAGE__ . '->cacheWrite');
my $strCacheFile = $self->{bDeploy} ? $self->{strExeCacheDeploy} : $self->{strExeCacheLocal};
my ($strKeyword, $strRequire) = $self->cacheKey();
foreach my $strSource (sort(keys(%{${$self->{oManifest}}{source}})))
{
my $hSource = ${$self->{oManifest}}{source}{$strSource};
if (defined($$hSource{hyCache}))
{
${$self->{hCache}}{$strKeyword}{$strRequire}{$strSource} = $$hSource{hyCache};
&log(DETAIL, "cache load $strSource (keyword = ${strKeyword}, require = ${strRequire})");
}
}
if (defined($self->{hCache}))
{
my $oJSON = JSON::PP->new()->canonical()->allow_nonref()->pretty();
fileStringWrite($strCacheFile, $oJSON->encode($self->{hCache}), false);
}
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
####################################################################################################################################
# cacheReset
####################################################################################################################################
sub cacheReset
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strSource
) =
logDebugParam
(
__PACKAGE__ . '->cacheReset', \@_,
{name => 'strSource', trace => true}
);
if ($self->{bCacheOnly})
{
confess &log(ERROR, 'Cache reset disabled by --cache-only option');
}
&log(WARN, "Cache will be reset for source ${strSource} and rendering retried automatically");
delete(${$self->{oManifest}}{source}{$strSource}{hyCache});
# Return from function and log return values if any
return logDebugReturn($strOperation);
}
1;

37
doc/lib/BackRestDoc/Common/DocRender.pm
View File

@ -187,16 +187,10 @@ sub new
{
$self->{oReference} =
new BackRestDoc::Common::DocConfig(${$self->{oManifest}->sourceGet('reference')}{doc}, $self);
require BackRestDoc::Custom::DocCustomRelease;
BackRestDoc::Custom::DocCustomRelease->import();
$self->{oRelease} =
new BackRestDoc::Custom::DocCustomRelease(${$self->{oManifest}->sourceGet('release')}{doc}, $self);
}
}
if (defined($$oRenderOut{source}) && $$oRenderOut{source} eq 'reference')
if (defined($$oRenderOut{source}) && $$oRenderOut{source} eq 'reference' && $self->{oManifest}->isBackRest())
{
if ($self->{strRenderOutKey} eq 'configuration')
{
@ -211,14 +205,20 @@ sub new
confess &log(ERROR, "cannot render $self->{strRenderOutKey} from source $$oRenderOut{source}");
}
}
elsif (defined($$oRenderOut{source}) && $$oRenderOut{source} eq 'release')
elsif (defined($$oRenderOut{source}) && $$oRenderOut{source} eq 'release' && $self->{oManifest}->isBackRest())
{
$self->{oDoc} = $self->{oRelease}->docGet();
require BackRestDoc::Custom::DocCustomRelease;
BackRestDoc::Custom::DocCustomRelease->import();
$self->{oDoc} =
(new BackRestDoc::Custom::DocCustomRelease(${$self->{oManifest}->sourceGet('release')}{doc}, $self))->docGet();
}
else
{
$self->{oDoc} = ${$self->{oManifest}->sourceGet($self->{strRenderOutKey})}{doc};
}
$self->{oSource} = $self->{oManifest}->sourceGet($$oRenderOut{source});
}
if (defined($self->{strRenderOutKey}))
@ -459,7 +459,7 @@ sub processTag
{
if (!defined($strUrl))
{
$strUrl = '{[backrest-url-base]}/' . $oTag->paramGet('page');
$strUrl = '{[backrest-url-base]}/' . $oTag->paramGet('page') . '.html';
}
$strBuffer = '[' . $oTag->valueGet() . '](' . $strUrl . ')';
@ -468,7 +468,22 @@ sub processTag
{
if (!defined($strUrl))
{
$strUrl = $oTag->paramGet('page', false);
my $strPage = $self->variableReplace($oTag->paramGet('page', false));
# If this is a page URL
if (defined($strPage))
{
# If the page wasn't rendered then point at the website
if (!defined($self->{oManifest}->renderOutGet('html', $strPage, true)))
{
$strUrl = '{[backrest-url-base]}/' . $oTag->paramGet('page') . '.html';
}
# Else point locally
else
{
$strUrl = $oTag->paramGet('page', false) . '.html';
}
}
if (!defined($strUrl))
{

24
doc/lib/BackRestDoc/Custom/DocCustomRelease.pm
View File

@ -96,6 +96,30 @@ sub new
);
}
####################################################################################################################################
# currentStableVersion
#
# Return the current stable version.
####################################################################################################################################
sub currentStableVersion
{
my $self = shift;
my $oDoc = $self->{oDoc};
foreach my $oRelease ($oDoc->nodeGet('release-list')->nodeList('release'))
{
my $strVersion = $oRelease->paramGet('version');
if ($strVersion !~ /dev$/)
{
return $strVersion;
}
}
confess &log(ERROR, "unable to find non-development version");
}
####################################################################################################################################
# contributorTextGet
#

35
doc/lib/BackRestDoc/Html/DocHtmlSite.pm
View File

@ -13,6 +13,7 @@ use Exporter qw(import);
use File::Basename qw(dirname);
use File::Copy;
use POSIX qw(strftime);
use Scalar::Util qw(blessed);
use Storable qw(dclone);
use lib dirname($0) . '/../lib';
@ -121,11 +122,37 @@ sub process
{
&log(INFO, " render out: ${strPageId}");
my $strHtml;
eval
{
$strHtml =
$self->{oManifest}->variableReplace(
(new BackRestDoc::Html::DocHtmlPage($self->{oManifest}, $strPageId, $self->{bExe}))->process());
};
if ($@)
{
my $oMessage = $@;
# If a backrest exception then return the code - don't confess
if (blessed($oMessage) && $oMessage->isa('pgBackRest::Common::Exception') && $oMessage->code() == -1)
{
my $oRenderOut = $self->{oManifest}->renderOutGet(RENDER_TYPE_HTML, $strPageId);
$self->{oManifest}->cacheReset($$oRenderOut{source});
$strHtml =
$self->{oManifest}->variableReplace(
(new BackRestDoc::Html::DocHtmlPage($self->{oManifest}, $strPageId, $self->{bExe}))->process());
}
else
{
confess $@;
}
}
# Save the html page
fileStringWrite("$self->{strHtmlPath}/${strPageId}.html",
$self->{oManifest}->variableReplace((new BackRestDoc::Html::DocHtmlPage($self->{oManifest},
$strPageId, $self->{bExe}))->process()),
false);
fileStringWrite("$self->{strHtmlPath}/${strPageId}.html", $strHtml, false);
}
# Return from function and log return values if any

22
doc/lib/BackRestDoc/Latex/DocLatex.pm
View File

@ -13,6 +13,7 @@ use Exporter qw(import);
use File::Basename qw(dirname);
use File::Copy;
use POSIX qw(strftime);
use Scalar::Util qw(blessed);
use Storable qw(dclone);
use lib dirname($0) . '/../lib';
@ -104,11 +105,32 @@ sub process
{
&log(INFO, " render out: ${strPageId}");
eval
{
my $oDocLatexSection =
new BackRestDoc::Latex::DocLatexSection($self->{oManifest}, $strPageId, $self->{bExe});
# Save the html page
$strLatex .= $oDocLatexSection->process();
};
if ($@)
{
my $oMessage = $@;
# If a backrest exception then return the code - don't confess
if (blessed($oMessage) && $oMessage->isa('pgBackRest::Common::Exception') && $oMessage->code() == -1)
{
my $oRenderOut = $self->{oManifest}->renderOutGet(RENDER_TYPE_HTML, $strPageId);
$self->{oManifest}->cacheReset($$oRenderOut{source});
my $oDocLatexSection =
new BackRestDoc::Latex::DocLatexSection($self->{oManifest}, $strPageId, $self->{bExe});
# Save the html page
$strLatex .= $oDocLatexSection->process();
}
}
}
$strLatex .= "\n% " . ('-' x 130) . "\n% End document\n% " . ('-' x 130) . "\n\\end{document}\n";

2
doc/lib/BackRestDoc/Latex/DocLatexSection.pm
View File

@ -392,7 +392,7 @@ sub configProcess
$strLatex =
"\n\\begin\{lstlisting\}[title=\{\\textnormal{\\textbf\{${strHostName}}}:\\textnormal{\\texttt\{${strFile}}} --- " .
$self->processText($oConfig->nodeGet('title')->textGet()) . "}]\n" .
${strConfig} .
(defined($strConfig) ? $strConfig : '') .
"\\end{lstlisting}\n";
}

6
doc/manifest.xml
View File

@ -5,6 +5,12 @@
<variable-list>
<variable key="project">pgBackRest</variable>
<variable key="version" eval="y">use pgBackRest::Version; $VERSION</variable>
<variable key="version-stable" eval="y">
use BackRestDoc::Custom::DocCustomRelease;
(new BackRestDoc::Custom::DocCustomRelease(
new BackRestDoc::Common::Doc("{[doc-path]}/xml/release.xml")))->currentStableVersion();
</variable>
<variable key="project-exe">pgbackrest</variable>
<variable key="project-url-root">/</variable>
<variable key="postgres">PostgreSQL</variable>

14
doc/xml/index.xml
View File

@ -13,11 +13,11 @@
<variable key="github-url-license">{[github-url-master]}/LICENSE</variable>
<variable key="backrest-url-base">http://www.pgbackrest.org</variable>
<variable key="backrest-page-user-guide">user-guide.html</variable>
<variable key="backrest-page-configuration">configuration.html</variable>
<variable key="backrest-page-command">command.html</variable>
<variable key="backrest-page-feature-backlog">backlog.html</variable>
<variable key="backrest-page-release">release.html</variable>
<variable key="backrest-page-user-guide">user-guide</variable>
<variable key="backrest-page-configuration">configuration</variable>
<variable key="backrest-page-command">command</variable>
<variable key="backrest-page-feature-backlog">backlog</variable>
<variable key="backrest-page-release">release</variable>
<variable key="crunchy-url-base">http://www.crunchydata.com</variable>
<variable key="crunchy-url-cbm">{[crunchy-url-base]}/crunchy-backup-manager</variable>
@ -33,7 +33,7 @@
<p>Instead of relying on traditional backup tools like tar and rsync, <backrest/> implements all backup features internally and uses a custom protocol for communicating with remote systems. Removing reliance on tar and rsync allows for better solutions to database-specific backup challenges. The custom remote protocol allows for more flexibility and limits the types of connections that are required to perform a backup which increases security.</p>
<p><backrest/> <link url="{[github-url-base]}/releases/tag/release/{[version]}">v{[version]}</link> is the current stable release. Release notes are on the <link page="{[backrest-page-release]}">Releases</link> page.</p>
<p><backrest/> <link url="{[github-url-base]}/releases/tag/release/{[version-stable]}">v{[version-stable]}</link> is the current stable release. Release notes are on the <link page="{[backrest-page-release]}">Releases</link> page.</p>
</section>
<section id="features">
@ -126,7 +126,7 @@
<p><backrest/> strives to be easy to configure and operate:</p>
<list>
<list-item><link page="{[backrest-page-user-guide]}">User guide</link> for Ubuntu 12.04 &amp; 14.04 / <postgres/> 9.4.</list-item>
<list-item><link page="{[backrest-page-user-guide]}">User guide</link> for {[user-guide-subtitle]} / <postgres/> 9.4.</list-item>
<list-item><link page="{[backrest-page-command]}">Command reference</link> for command-line operations.</list-item>
<list-item><link page="{[backrest-page-configuration]}">Configuration reference</link> for creating <backrest/> configurations.</list-item>
</list>

4
doc/xml/release.xml
View File

@ -138,6 +138,10 @@
<p>Added man page generation.</p>
</release-item>
<release-item>
<p>Added an execution cache so that documentation can be generated without setting up the full container environment. This is useful for packaging, keeps the documentation consistent for a release, and speeds up generation when no changes are made in the execution list.</p>
</release-item>
</release-feature-list>
<release-refactor-list>

45
doc/xml/user-guide.xml
View File

@ -1,10 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE doc SYSTEM "doc.dtd">
<doc title="User Guide" subtitle="Ubuntu 12.04 &amp; 14.04 / {[postgres]} 9.4">
<doc title="User Guide" subtitle="{[user-guide-subtitle]} / {[postgres]} 9.4">
<description>The {[project]} User Guide demonstrates how to quickly and easily setup {[project]} for your {[postgres]} database. Step-by-step instructions lead the user through all the important features of the fastest, most reliable {[postgres]} backup and restore solution.</description>
<variable-list>
<!-- Variables used by the rest of the script -->
<variable key="user-guide-subtitle" keyword="default">Debian &amp; Ubuntu</variable>
<variable key="user-guide-subtitle" keyword="co6">RHEL &amp; CentOS 6</variable>
<variable key="user-guide-os" keyword="default">Debian/Ubuntu</variable>
<variable key="user-guide-os" keyword="co6">RHEL/CentOS 6</variable>
<variable key="perl-lib-path" keyword="default">/usr/lib/perl5</variable>
<variable key="perl-lib-path" keyword="co6">/usr/share/perl5</variable>
<variable key="perl-bin-path">/usr/bin</variable>
@ -13,6 +19,9 @@
<variable key="postgres-cluster-demo">demo</variable>
<variable key="backrest-config-demo">/etc/{[project-exe]}.conf</variable>
<variable key="db-path-default" keyword="default">/var/lib/postgresql/[version]/[cluster]</variable>
<variable key="db-path-default" keyword="co6">/var/lib/pgsql/[version]/data</variable>
<variable key="db-path" keyword="default">/var/lib/postgresql/9.4/{[postgres-cluster-demo]}</variable>
<variable key="db-path" keyword="co6">/var/lib/pgsql/9.4/data</variable>
@ -86,7 +95,7 @@
<p>This user guide is intended to be followed sequentially from beginning to end &amp;mdash; each section depends on the last. For example, the <link section="/backup">Backup</link> section relies on setup that is performed in the <link section="/quickstart">Quick Start</link> section. Once <backrest/> is up and running then skipping around is possible but following the user guide in order is recommended the first time through.</p>
<p>Although the examples are targeted at Ubuntu and <postgres/> 9.4 they will also work fine on Debian and it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version, but note that only 64-bit distributions are currently supported due to 64-bit operations in the Perl code. The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters. The <backrest/> commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary.
<p>Although the examples are targeted at {[user-guide-os]} and <postgres/> 9.4, it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version. Note that only 64-bit distributions are currently supported due to 64-bit operations in the Perl code. The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters. The <backrest/> commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary.
Configuration information and documentation for PostgreSQL can be found in the <postgres/> <link url='http://www.postgresql.org/docs/9.4/static/index.html'>Manual</link>.</p>
@ -148,7 +157,7 @@
</execute>
</host-add>
<p keyword="default"><backrest/> is written in Perl which is included with Ubuntu by default. A few additional modules are required which are all available as packages.</p>
<p keyword="default"><backrest/> is written in Perl which is included with {[user-guide-os]} by default. A few additional modules are required which are all available as packages.</p>
<execute-list host="{[host-db-master]}" keyword="default">
<title>Install required Perl packages</title>
@ -158,7 +167,7 @@
</execute>
</execute-list>
<p keyword="co6"><backrest/> is written in Perl, which is not included with RHEL/CentOS by default, however all required modules are all available as standard packages.</p>
<p keyword="co6"><backrest/> is written in Perl which is not included with {[user-guide-os]} by default, however all required modules are all available as standard packages.</p>
<execute-list host="{[host-db-master]}" keyword="co6">
<title>Install required Perl packages</title>
@ -169,9 +178,9 @@
</execute>
</execute-list>
<p keyword="default">No Debian/Ubuntu packages are currently available for <backrest/> but it is easy to download the source and install manually.</p>
<p keyword="default">No {[user-guide-os]} packages are currently available for <backrest/> but it is easy to download the source and install manually.</p>
<p keyword="co6">No RHEL/CentOS packages are currently available for <backrest/> but it is easy to download the source and install manually.</p>
<p keyword="co6">No {[user-guide-os]} packages are currently available for <backrest/> but it is easy to download the source and install manually.</p>
<execute-list host="{[host-db-master]}">
<title>Download version <id>{[version]}</id> of <backrest/></title>
@ -257,7 +266,7 @@
<postgres-config-option key="log_line_prefix">''</postgres-config-option>
</postgres-config>
<p keyword="co6">By default CentOS/RHEL includes the day of the week in the log filename. This makes automating the user guide a bit more complicated so the <pg-option>log_filename</pg-option> is set to a constant.</p>
<p keyword="co6">By default {[user-guide-os]} includes the day of the week in the log filename. This makes automating the user guide a bit more complicated so the <pg-option>log_filename</pg-option> is set to a constant.</p>
<postgres-config host="{[host-db-master]}" keyword="co6" file="{[postgres-config-demo]}">
<title>Set <pg-option>log_filename</pg-option></title>
@ -276,7 +285,7 @@
<p><backrest/> needs to know where the base data directory for the <postgres/> cluster is located. The path can be requested from <postgres/> directly but in a recovery scenario the <postgres/> process will not be available. During backups the value supplied to <backrest/> will be compared against the path that <postgres/> is running on and they must be equal or the backup will return an error. Make sure that <br-option>db-path</br-option> is exactly equal to <pg-option>data_directory</pg-option> in <file>postgresql.conf</file>.</p>
<p>By default Ubuntu stores clusters in <path>/var/lib/postgresql/[version]/[cluster]</path> so it is easy to determine the correct path for the data directory.</p>
<p>By default {[user-guide-os]} stores clusters in <path>{[db-path-default]}</path> so it is easy to determine the correct path for the data directory.</p>
<p>When creating the <file>{[backrest-config-demo]}</file> file, the database owner (usually <id>postgres</id>) must be granted read privileges.</p>
@ -386,7 +395,7 @@
<execute output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]}
--log-level-console=detail backup</exe-cmd>
--log-level-console=info backup</exe-cmd>
<exe-highlight>no prior backup exists|full backup size</exe-highlight>
</execute>
@ -404,7 +413,7 @@
<execute output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=diff
--log-level-console=detail backup</exe-cmd>
--log-level-console=info backup</exe-cmd>
<exe-highlight>diff backup size</exe-highlight>
</execute>
</execute-list>
@ -534,7 +543,7 @@
<execute output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
--log-level-console=detail backup</exe-cmd>
--log-level-console=info backup</exe-cmd>
<exe-highlight>backup begins after the next regular checkpoint completes</exe-highlight>
</execute>
</execute-list>
@ -552,7 +561,7 @@
<execute output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
--log-level-console=detail backup</exe-cmd>
--log-level-console=info backup</exe-cmd>
<exe-highlight>backup begins after the requested immediate checkpoint completes</exe-highlight>
</execute>
</execute-list>
@ -575,7 +584,7 @@
<execute output="y" err-expect="122">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
--log-level-console=detail backup</exe-cmd>
--log-level-console=info backup</exe-cmd>
<exe-highlight>ERROR:</exe-highlight>
</execute>
</execute-list>
@ -591,7 +600,7 @@
<execute output="y" err-expect="132">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
--log-level-console=detail backup</exe-cmd>
--log-level-console=info backup</exe-cmd>
<exe-highlight>ERROR:</exe-highlight>
</execute>
</execute-list>
@ -611,7 +620,7 @@
<execute output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr
--log-level-console=detail backup</exe-cmd>
--log-level-console=info backup</exe-cmd>
<exe-highlight>cluster is already in backup mode|backup begins after the requested immediate checkpoint completes</exe-highlight>
</execute>
</execute-list>
@ -663,7 +672,7 @@
<execute output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=full
--log-level-console=detail backup</exe-cmd>
--log-level-console=info backup</exe-cmd>
<exe-highlight>expire full backup set\: {[backup-full-first]}|archive retention on backup {[backup-full-second]}|remove archive</exe-highlight>
</execute>
</execute-list>
@ -708,7 +717,7 @@
<execute output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=diff
--log-level-console=detail backup</exe-cmd>
--log-level-console=info backup</exe-cmd>
<exe-highlight>expire diff backup set: {[backup-diff-second]}</exe-highlight>
</execute>
</execute-list>
@ -1640,7 +1649,7 @@
</execute>
</execute-list>
<p keyword="co6">By default CentOS/RHEL stores the <file>postgresql.conf</file> file in the <postgres/> data directory. That means the change made to <file>postgresql.conf</file> was overwritten by the last restore and the <pg-option>hot_standby</pg-option> setting must be enabled again. Other solutions to this problem are to store the <file>postgresql.conf</file> file elsewhere or to enable the <pg-option>hot_standby</pg-option> setting on the <host>db-master</host> host where it will be ignored.</p>
<p keyword="co6">By default {[user-guide-os]} stores the <file>postgresql.conf</file> file in the <postgres/> data directory. That means the change made to <file>postgresql.conf</file> was overwritten by the last restore and the <pg-option>hot_standby</pg-option> setting must be enabled again. Other solutions to this problem are to store the <file>postgresql.conf</file> file elsewhere or to enable the <pg-option>hot_standby</pg-option> setting on the <host>db-master</host> host where it will be ignored.</p>
<postgres-config host="{[host-db-standby]}" keyword="co6" file="{[postgres-config-demo]}">
<title>Enable <pg-option>hot_standby</pg-option></title>

7
test/lib/pgBackRestTest/Docker/ContainerTest.pm
View File

@ -212,12 +212,7 @@ sub containerBuild
# Create temp path
my $strTempPath = dirname(abs_path($0)) . '/.vagrant/docker';
if (!-e $strTempPath)
{
mkdir $strTempPath
or confess &log(ERROR, "unable to create ${strTempPath}");
}
filePathCreate($strTempPath, '0770', true, true);
# Create SSH key (if it does not already exist)
if (-e "${strTempPath}/id_rsa")

16
test/test.pl
View File

@ -197,6 +197,9 @@ if (!defined($strTestPath))
$strTestPath = cwd() . '/test';
}
# Get the base backrest path
my $strBackRestBase = dirname(dirname(abs_path($0)));
####################################################################################################################################
# Build Docker containers
####################################################################################################################################
@ -320,12 +323,12 @@ eval
if ($strVm ne 'none')
{
# Load the doc module dynamically since it is not supported on all systems
use lib dirname($0) . '/../doc/lib';
use lib dirname(abs_path($0)) . '/../doc/lib';
require BackRestDoc::Common::Doc;
BackRestDoc::Common::Doc->import();
# Make sure version number matches the latest release
my $strReleaseFile = dirname(dirname($0)) . '/doc/xml/release.xml';
my $strReleaseFile = dirname(dirname(abs_path($0))) . '/doc/xml/release.xml';
my $oReleaseDoc = new BackRestDoc::Common::Doc($strReleaseFile);
foreach my $oRelease ($oReleaseDoc->nodeGet('release-list')->nodeList('release'))
@ -619,24 +622,27 @@ eval
executeTest("docker run -itd -h $$oTest{os}-test --name=${strImage}" .
" -v ${strHostTestPath}:${strVmTestPath}" .
" -v /backrest:/backrest backrest/$$oTest{os}-test-${strDbVersion}");
" -v ${strBackRestBase}:${strBackRestBase} backrest/$$oTest{os}-test-${strDbVersion}");
}
# Build up command line for the individual test
$strCommandLine =~ s/\-\-os\=\S*//g;
$strCommandLine =~ s/\-\-test-path\=\S*//g;
$strCommandLine =~ s/\-\-test\-path\=\S*//g;
$strCommandLine =~ s/\-\-module\=\S*//g;
$strCommandLine =~ s/\-\-test\=\S*//g;
$strCommandLine =~ s/\-\-run\=\S*//g;
$strCommandLine =~ s/\-\-db\-version\=\S*//g;
$strCommandLine =~ s/\-\-no\-lint\S*//g;
$strCommandLine =~ s/\-\-process\-max\=\S*//g;
my $strCommand =
"docker exec -i -u vagrant ${strImage} $0 ${strCommandLine} --test-path=${strVmTestPath}" .
"docker exec -i -u vagrant ${strImage} ${strBackRestBase}/test/test.pl ${strCommandLine}" .
" --vm=none --module=$$oTest{module} --test=$$oTest{test}" .
(defined($$oTest{run}) ? " --run=$$oTest{run}" : '') .
(defined($$oTest{thread}) ? " --thread-max=$$oTest{thread}" : '') .
(defined($$oTest{db}) ? " --db-version=$$oTest{db}" : '') .
($bDryRun ? " --dry-run" : '') .
" --test-path=${strVmTestPath}" .
" --no-cleanup --vm-out";
&log(DEBUG, $strCommand);