mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-04-21 11:57:01 +02:00
02b06aa495
The prior limitations were based on using getopt_long() to parse command-line options, which required a static list of allowed options. Setting index max too high bloated the binary unacceptably. 45a4e80 replaced the functionality of getopt_long() but the static list remained. Improve cfgParseOption() to use available option data and remove the need for a static list. This also allows the option deprecations to be represented more compactly. Index max is still capped at 256 because a large enough index could cause parseOptionIdxValue() to run out of memory since it allocates a static list based on the highest index found. If that function were improved with a map of found index values then index max could be set to UINT64_MAX. Note that deprecations no longer set an index max or define whether reset is valid. These were space-saving measures which are no longer required. This means that indexed deprecated options will also be valid up to 256 and always allow reset, but it doesn't seem worth additional code to limit this behavior. cfgParseOptionId() is no longer needed because calling cfgParseOption() with .ignoreMissingIndex = true duplicates the functionality of cfgParseOptionId(). This leads to some simplification in the help code.
1093 lines
38 KiB
Perl
1093 lines
38 KiB
Perl
####################################################################################################################################
|
|
# DOC EXECUTE MODULE
|
|
####################################################################################################################################
|
|
package pgBackRestDoc::Common::DocExecute;
|
|
use parent 'pgBackRestDoc::Common::DocRender';
|
|
|
|
use strict;
|
|
use warnings FATAL => qw(all);
|
|
use Carp qw(confess);
|
|
use English '-no_match_vars';
|
|
|
|
use Cwd qw(abs_path);
|
|
use Exporter qw(import);
|
|
our @EXPORT = qw();
|
|
use File::Basename qw(dirname);
|
|
use Storable qw(dclone);
|
|
|
|
use pgBackRestBuild::Config::Data;
|
|
|
|
use pgBackRestTest::Common::ExecuteTest;
|
|
use pgBackRestTest::Common::HostTest;
|
|
use pgBackRestTest::Common::HostGroupTest;
|
|
|
|
use pgBackRestDoc::Common::DocManifest;
|
|
use pgBackRestDoc::Common::Exception;
|
|
use pgBackRestDoc::Common::Ini;
|
|
use pgBackRestDoc::Common::Log;
|
|
use pgBackRestDoc::Common::String;
|
|
use pgBackRestDoc::ProjectInfo;
|
|
|
|
####################################################################################################################################
|
|
# User that's building the docs
|
|
####################################################################################################################################
|
|
use constant DOC_USER => getpwuid($UID) eq 'root' ? 'ubuntu' : getpwuid($UID) . '';
|
|
|
|
####################################################################################################################################
|
|
# CONSTRUCTOR
|
|
####################################################################################################################################
|
|
sub new
|
|
{
|
|
my $class = shift; # Class name
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strType,
|
|
$oManifest,
|
|
$strRenderOutKey,
|
|
$bExe
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->new', \@_,
|
|
{name => 'strType'},
|
|
{name => 'oManifest'},
|
|
{name => 'strRenderOutKey'},
|
|
{name => 'bExe'}
|
|
);
|
|
|
|
# Create the class hash
|
|
my $self = $class->SUPER::new($strType, $oManifest, $bExe, $strRenderOutKey);
|
|
bless $self, $class;
|
|
|
|
if (defined($self->{oSource}{hyCache}))
|
|
{
|
|
$self->{bCache} = true;
|
|
$self->{iCacheIdx} = 0;
|
|
}
|
|
else
|
|
{
|
|
$self->{bCache} = false;
|
|
}
|
|
|
|
$self->{bExe} = $bExe;
|
|
|
|
$self->{iCmdLineLen} = $self->{oDoc}->paramGet('cmd-line-len', false, 80);
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'self', value => $self}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# executeKey
|
|
#
|
|
# Get a unique key for the execution step to determine if the cache is valid.
|
|
####################################################################################################################################
|
|
sub executeKey
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strHostName,
|
|
$oCommand,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->executeKey', \@_,
|
|
{name => 'strHostName', trace => true},
|
|
{name => 'oCommand', trace => true},
|
|
);
|
|
|
|
# Add user to command
|
|
my $bUserForce = $oCommand->paramTest('user-force', 'y') ? true : false;
|
|
my $strCommand = $self->{oManifest}->variableReplace(trim($oCommand->fieldGet('exe-cmd')));
|
|
my $strUser = $self->{oManifest}->variableReplace($oCommand->paramGet('user', false, DOC_USER));
|
|
$strCommand = ($strUser eq DOC_USER || $bUserForce ? '' : ('sudo ' . ($strUser eq 'root' ? '' : "-u $strUser "))) . $strCommand;
|
|
|
|
# Format and split command
|
|
$strCommand =~ s/[ ]*\n[ ]*/ \\\n /smg;
|
|
$strCommand =~ s/ \\\@ \\//smg;
|
|
my @stryCommand = split("\n", $strCommand);
|
|
|
|
my $hCacheKey =
|
|
{
|
|
host => $strHostName,
|
|
cmd => \@stryCommand,
|
|
output => JSON::PP::false,
|
|
};
|
|
|
|
$$hCacheKey{'run-as-user'} = $bUserForce ? $strUser : undef;
|
|
|
|
if (defined($oCommand->fieldGet('exe-cmd-extra', false)))
|
|
{
|
|
$$hCacheKey{'cmd-extra'} = $self->{oManifest}->variableReplace($oCommand->fieldGet('exe-cmd-extra'));
|
|
}
|
|
|
|
if (defined($oCommand->paramGet('err-expect', false)))
|
|
{
|
|
$$hCacheKey{'err-expect'} = $oCommand->paramGet('err-expect');
|
|
}
|
|
|
|
if ($oCommand->paramTest('output', 'y') || $oCommand->paramTest('show', 'y') || $oCommand->paramTest('variable-key'))
|
|
{
|
|
$$hCacheKey{'output'} = JSON::PP::true;
|
|
}
|
|
|
|
$$hCacheKey{'load-env'} = $oCommand->paramTest('load-env', 'n') ? JSON::PP::false : JSON::PP::true;
|
|
$$hCacheKey{'bash-wrap'} = $oCommand->paramTest('bash-wrap', 'n') ? JSON::PP::false : JSON::PP::true;
|
|
|
|
if (defined($oCommand->fieldGet('exe-highlight', false)))
|
|
{
|
|
$$hCacheKey{'output'} = JSON::PP::true;
|
|
$$hCacheKey{highlight}{'filter'} = $oCommand->paramTest('filter', 'n') ? JSON::PP::false : JSON::PP::true;
|
|
$$hCacheKey{highlight}{'filter-context'} = $oCommand->paramGet('filter-context', false, 2);
|
|
|
|
my @stryHighlight;
|
|
$stryHighlight[0] = $self->{oManifest}->variableReplace($oCommand->fieldGet('exe-highlight'));
|
|
|
|
$$hCacheKey{highlight}{list} = \@stryHighlight;
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'hExecuteKey', value => $hCacheKey, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# execute
|
|
####################################################################################################################################
|
|
sub execute
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oSection,
|
|
$strHostName,
|
|
$oCommand,
|
|
$iIndent,
|
|
$bCache,
|
|
$bShow,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->execute', \@_,
|
|
{name => 'oSection'},
|
|
{name => 'strHostName'},
|
|
{name => 'oCommand'},
|
|
{name => 'iIndent', optional => true, default => 1},
|
|
{name => 'bCache', optional => true, default => true},
|
|
{name => 'bShow', optional => true, default => true},
|
|
);
|
|
|
|
# Working variables
|
|
my $hCacheKey = $self->executeKey($strHostName, $oCommand);
|
|
my $strCommand = join("\n", @{$$hCacheKey{cmd}});
|
|
my $strOutput;
|
|
|
|
if ($bShow && $self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
# Make sure that no lines are greater than 80 chars
|
|
foreach my $strLine (split("\n", $strCommand))
|
|
{
|
|
if (length(trim($strLine)) > $self->{iCmdLineLen})
|
|
{
|
|
confess &log(ERROR,
|
|
"command has a line > $self->{iCmdLineLen} characters:\n${strCommand}\noffending line: ${strLine}");
|
|
}
|
|
}
|
|
}
|
|
|
|
&log(DEBUG, (' ' x $iIndent) . "execute: $strCommand");
|
|
|
|
if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y')
|
|
{
|
|
if ($self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('exe', $hCacheKey);
|
|
|
|
if ($bCacheHit)
|
|
{
|
|
$strOutput = defined($$hCacheValue{output}) ? join("\n", @{$$hCacheValue{output}}) : undef;
|
|
}
|
|
else
|
|
{
|
|
# Check that the host is valid
|
|
my $oHost = $self->{host}{$strHostName};
|
|
|
|
if (!defined($oHost))
|
|
{
|
|
confess &log(ERROR, "cannot execute on host ${strHostName} because the host does not exist");
|
|
}
|
|
|
|
my $oExec = $oHost->execute(
|
|
$strCommand . (defined($$hCacheKey{'cmd-extra'}) ? ' ' . $$hCacheKey{'cmd-extra'} : ''),
|
|
{iExpectedExitStatus => $$hCacheKey{'err-expect'},
|
|
bSuppressError => $oCommand->paramTest('err-suppress', 'y'),
|
|
iRetrySeconds => $oCommand->paramGet('retry', false)}, $hCacheKey->{'run-as-user'},
|
|
{bLoadEnv => $hCacheKey->{'load-env'}, bBashWrap => $hCacheKey->{'bash-wrap'}});
|
|
$oExec->begin();
|
|
$oExec->end();
|
|
|
|
if (defined($oExec->{strOutLog}) && $oExec->{strOutLog} ne '')
|
|
{
|
|
$strOutput = $oExec->{strOutLog};
|
|
|
|
# Trim off extra linefeeds before and after
|
|
$strOutput =~ s/^\n+|\n$//g;
|
|
}
|
|
|
|
if (defined($$hCacheKey{'err-expect'}) && defined($oExec->{strErrorLog}) && $oExec->{strErrorLog} ne '')
|
|
{
|
|
$strOutput .= $oExec->{strErrorLog};
|
|
}
|
|
|
|
if ($$hCacheKey{output} && defined($$hCacheKey{highlight}) && $$hCacheKey{highlight}{filter} && defined($strOutput))
|
|
{
|
|
my $strHighLight = @{$$hCacheKey{highlight}{list}}[0];
|
|
|
|
if (!defined($strHighLight))
|
|
{
|
|
confess &log(ERROR, 'filter requires highlight definition: ' . $strCommand);
|
|
}
|
|
|
|
my $iFilterContext = $$hCacheKey{highlight}{'filter-context'};
|
|
|
|
my @stryOutput = split("\n", $strOutput);
|
|
undef($strOutput);
|
|
# my $iFiltered = 0;
|
|
my $iLastOutput = -1;
|
|
|
|
for (my $iIndex = 0; $iIndex < @stryOutput; $iIndex++)
|
|
{
|
|
if ($stryOutput[$iIndex] =~ /$strHighLight/)
|
|
{
|
|
# Determine the first line to output
|
|
my $iFilterFirst = $iIndex - $iFilterContext;
|
|
|
|
# Don't go past the beginning
|
|
$iFilterFirst = $iFilterFirst < 0 ? 0 : $iFilterFirst;
|
|
|
|
# Don't repeat lines that have already been output
|
|
$iFilterFirst = $iFilterFirst <= $iLastOutput ? $iLastOutput + 1 : $iFilterFirst;
|
|
|
|
# Determine the last line to output
|
|
my $iFilterLast = $iIndex + $iFilterContext;
|
|
|
|
# Don't got past the end
|
|
$iFilterLast = $iFilterLast >= @stryOutput ? @stryOutput -1 : $iFilterLast;
|
|
|
|
# Mark filtered lines if any
|
|
if ($iFilterFirst > $iLastOutput + 1)
|
|
{
|
|
my $iFiltered = $iFilterFirst - ($iLastOutput + 1);
|
|
|
|
if ($iFiltered > 1)
|
|
{
|
|
$strOutput .= (defined($strOutput) ? "\n" : '') .
|
|
" [filtered ${iFiltered} lines of output]";
|
|
}
|
|
else
|
|
{
|
|
$iFilterFirst -= 1;
|
|
}
|
|
}
|
|
|
|
# Output the lines
|
|
for (my $iOutputIndex = $iFilterFirst; $iOutputIndex <= $iFilterLast; $iOutputIndex++)
|
|
{
|
|
$strOutput .= (defined($strOutput) ? "\n" : '') . $stryOutput[$iOutputIndex];
|
|
}
|
|
|
|
$iLastOutput = $iFilterLast;
|
|
}
|
|
}
|
|
|
|
if (@stryOutput - 1 > $iLastOutput + 1)
|
|
{
|
|
my $iFiltered = (@stryOutput - 1) - ($iLastOutput + 1);
|
|
|
|
if ($iFiltered > 1)
|
|
{
|
|
$strOutput .= (defined($strOutput) ? "\n" : '') .
|
|
" [filtered ${iFiltered} lines of output]";
|
|
}
|
|
else
|
|
{
|
|
$strOutput .= (defined($strOutput) ? "\n" : '') . $stryOutput[-1];
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!$$hCacheKey{output})
|
|
{
|
|
$strOutput = undef;
|
|
}
|
|
|
|
if (defined($strOutput))
|
|
{
|
|
my @stryOutput = split("\n", $strOutput);
|
|
$$hCacheValue{output} = \@stryOutput;
|
|
}
|
|
|
|
if ($bCache)
|
|
{
|
|
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
|
|
}
|
|
}
|
|
|
|
# Output is assigned to a var
|
|
if ($oCommand->paramTest('variable-key'))
|
|
{
|
|
$self->{oManifest}->variableSet($oCommand->paramGet('variable-key'), trim($strOutput), true);
|
|
}
|
|
}
|
|
elsif ($$hCacheKey{output})
|
|
{
|
|
$strOutput = 'Output suppressed for testing';
|
|
}
|
|
}
|
|
|
|
# Default variable output when it was not set by execution
|
|
if ($oCommand->paramTest('variable-key') && !defined($self->{oManifest}->variableGet($oCommand->paramGet('variable-key'))))
|
|
{
|
|
$self->{oManifest}->variableSet($oCommand->paramGet('variable-key'), '[Test Variable]', true);
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strCommand', value => $strCommand, trace => true},
|
|
{name => 'strOutput', value => $strOutput, trace => true}
|
|
);
|
|
}
|
|
|
|
|
|
####################################################################################################################################
|
|
# configKey
|
|
####################################################################################################################################
|
|
sub configKey
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oConfig,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->hostKey', \@_,
|
|
{name => 'oConfig', trace => true},
|
|
);
|
|
|
|
my $hCacheKey =
|
|
{
|
|
host => $self->{oManifest}->variableReplace($oConfig->paramGet('host')),
|
|
file => $self->{oManifest}->variableReplace($oConfig->paramGet('file')),
|
|
};
|
|
|
|
if ($oConfig->paramTest('reset', 'y'))
|
|
{
|
|
$$hCacheKey{reset} = JSON::PP::true;
|
|
}
|
|
|
|
# Add all options to the key
|
|
my $strOptionTag = $oConfig->nameGet() eq 'backrest-config' ? 'backrest-config-option' : 'postgres-config-option';
|
|
|
|
foreach my $oOption ($oConfig->nodeList($strOptionTag))
|
|
{
|
|
my $hOption = {};
|
|
|
|
if ($oOption->paramTest('remove', 'y'))
|
|
{
|
|
$$hOption{remove} = JSON::PP::true;
|
|
}
|
|
|
|
if (defined($oOption->valueGet(false)))
|
|
{
|
|
$$hOption{value} = $self->{oManifest}->variableReplace($oOption->valueGet());
|
|
}
|
|
|
|
my $strKey = $self->{oManifest}->variableReplace($oOption->paramGet('key'));
|
|
|
|
if ($oConfig->nameGet() eq 'backrest-config')
|
|
{
|
|
my $strSection = $self->{oManifest}->variableReplace($oOption->paramGet('section'));
|
|
|
|
$$hCacheKey{option}{$strSection}{$strKey} = $hOption;
|
|
}
|
|
else
|
|
{
|
|
$$hCacheKey{option}{$strKey} = $hOption;
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'hCacheKey', value => $hCacheKey, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# backrestConfig
|
|
####################################################################################################################################
|
|
sub backrestConfig
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oSection,
|
|
$oConfig,
|
|
$iDepth
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->backrestConfig', \@_,
|
|
{name => 'oSection'},
|
|
{name => 'oConfig'},
|
|
{name => 'iDepth'}
|
|
);
|
|
|
|
# Working variables
|
|
my $hCacheKey = $self->configKey($oConfig);
|
|
my $strFile = $$hCacheKey{file};
|
|
my $strConfig = undef;
|
|
|
|
&log(DEBUG, (' ' x $iDepth) . 'process backrest config: ' . $$hCacheKey{file});
|
|
|
|
if ($self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('cfg-' . PROJECT_EXE, $hCacheKey);
|
|
|
|
if ($bCacheHit)
|
|
{
|
|
$strConfig = defined($$hCacheValue{config}) ? join("\n", @{$$hCacheValue{config}}) : undef;
|
|
}
|
|
else
|
|
{
|
|
# Check that the host is valid
|
|
my $strHostName = $self->{oManifest}->variableReplace($oConfig->paramGet('host'));
|
|
my $oHost = $self->{host}{$strHostName};
|
|
|
|
if (!defined($oHost))
|
|
{
|
|
confess &log(ERROR, "cannot configure backrest on host ${strHostName} because the host does not exist");
|
|
}
|
|
|
|
# Reset all options
|
|
if ($oConfig->paramTest('reset', 'y'))
|
|
{
|
|
delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}})
|
|
}
|
|
|
|
foreach my $oOption ($oConfig->nodeList('backrest-config-option'))
|
|
{
|
|
my $strSection = $self->{oManifest}->variableReplace($oOption->paramGet('section'));
|
|
my $strKey = $self->{oManifest}->variableReplace($oOption->paramGet('key'));
|
|
my $strValue;
|
|
|
|
if (!$oOption->paramTest('remove', 'y'))
|
|
{
|
|
$strValue = $self->{oManifest}->variableReplace(trim($oOption->valueGet(false)));
|
|
}
|
|
|
|
if (!defined($strValue))
|
|
{
|
|
delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey});
|
|
|
|
if (keys(%{${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}}) == 0)
|
|
{
|
|
delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection});
|
|
}
|
|
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . "reset ${strSection}->${strKey}");
|
|
}
|
|
else
|
|
{
|
|
# If this option is a hash and the value is already set then append to the array
|
|
if (defined(cfgDefine()->{$strKey}) &&
|
|
cfgDefine()->{$strKey}{&CFGDEF_TYPE} eq CFGDEF_TYPE_HASH &&
|
|
defined(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey}))
|
|
{
|
|
my @oValue = ();
|
|
my $strHashValue = ${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey};
|
|
|
|
# If there is only one key/value
|
|
if (ref(\$strHashValue) eq 'SCALAR')
|
|
{
|
|
push(@oValue, $strHashValue);
|
|
}
|
|
# Else if there is an array of values
|
|
else
|
|
{
|
|
@oValue = @{$strHashValue};
|
|
}
|
|
|
|
push(@oValue, $strValue);
|
|
${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey} = \@oValue;
|
|
}
|
|
# else just set the value
|
|
else
|
|
{
|
|
${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey} = $strValue;
|
|
}
|
|
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . "set ${strSection}->${strKey} = ${strValue}");
|
|
}
|
|
}
|
|
|
|
my $strLocalFile = '/home/' . DOC_USER . '/data/pgbackrest.conf';
|
|
|
|
# Save the ini file
|
|
$self->{oManifest}->storage()->put($strLocalFile, iniRender($self->{config}{$strHostName}{$$hCacheKey{file}}, true));
|
|
|
|
$oHost->copyTo(
|
|
$strLocalFile, $$hCacheKey{file},
|
|
$self->{oManifest}->variableReplace($oConfig->paramGet('owner', false, 'postgres:postgres')), '640');
|
|
|
|
# Remove the log-console-stderr option before pushing into the cache
|
|
# ??? This is not very pretty and should be replaced with a general way to hide config options
|
|
my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}});
|
|
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_LEVEL_STDERR});
|
|
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_TIMESTAMP});
|
|
|
|
# Don't show repo1-storage-host option. Since Azure behaves differently with Azurite (which we use for local testing)
|
|
# and the actual service we can't just fake /etc/hosts like we do for S3.
|
|
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{'repo1-storage-host'});
|
|
|
|
if (keys(%{$$oConfigClean{&CFGDEF_SECTION_GLOBAL}}) == 0)
|
|
{
|
|
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL});
|
|
}
|
|
|
|
$self->{oManifest}->storage()->put("${strLocalFile}.clean", iniRender($oConfigClean, true));
|
|
|
|
# Push config file into the cache
|
|
$strConfig = ${$self->{oManifest}->storage()->get("${strLocalFile}.clean")};
|
|
|
|
my @stryConfig = undef;
|
|
|
|
if (trim($strConfig) ne '')
|
|
{
|
|
@stryConfig = split("\n", $strConfig);
|
|
}
|
|
|
|
$$hCacheValue{config} = \@stryConfig;
|
|
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
$strConfig = 'Config suppressed for testing';
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strFile', value => $strFile, trace => true},
|
|
{name => 'strConfig', value => $strConfig, trace => true},
|
|
{name => 'bShow', value => $oConfig->paramTest('show', 'n') ? false : true, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# postgresConfig
|
|
####################################################################################################################################
|
|
sub postgresConfig
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oSection,
|
|
$oConfig,
|
|
$iDepth
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->postgresConfig', \@_,
|
|
{name => 'oSection'},
|
|
{name => 'oConfig'},
|
|
{name => 'iDepth'}
|
|
);
|
|
|
|
# Working variables
|
|
my $hCacheKey = $self->configKey($oConfig);
|
|
my $strFile = $$hCacheKey{file};
|
|
my $strConfig;
|
|
|
|
if ($self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('cfg-postgresql', $hCacheKey);
|
|
|
|
if ($bCacheHit)
|
|
{
|
|
$strConfig = defined($$hCacheValue{config}) ? join("\n", @{$$hCacheValue{config}}) : undef;
|
|
}
|
|
else
|
|
{
|
|
# Check that the host is valid
|
|
my $strHostName = $self->{oManifest}->variableReplace($oConfig->paramGet('host'));
|
|
my $oHost = $self->{host}{$strHostName};
|
|
|
|
if (!defined($oHost))
|
|
{
|
|
confess &log(ERROR, "cannot configure postgres on host ${strHostName} because the host does not exist");
|
|
}
|
|
|
|
my $strLocalFile = '/home/' . DOC_USER . '/data/postgresql.conf';
|
|
$oHost->copyFrom($$hCacheKey{file}, $strLocalFile);
|
|
|
|
if (!defined(${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base}) && $self->{bExe})
|
|
{
|
|
${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base} =
|
|
${$self->{oManifest}->storage()->get($strLocalFile)};
|
|
}
|
|
|
|
my $oConfigHash = $self->{'pg-config'}{$strHostName}{$$hCacheKey{file}};
|
|
my $oConfigHashNew;
|
|
|
|
if (!defined($$oConfigHash{old}))
|
|
{
|
|
$oConfigHashNew = {};
|
|
$$oConfigHash{old} = {}
|
|
}
|
|
else
|
|
{
|
|
$oConfigHashNew = dclone($$oConfigHash{old});
|
|
}
|
|
|
|
&log(DEBUG, (' ' x $iDepth) . 'process postgres config: ' . $$hCacheKey{file});
|
|
|
|
foreach my $oOption ($oConfig->nodeList('postgres-config-option'))
|
|
{
|
|
my $strKey = $oOption->paramGet('key');
|
|
my $strValue = $self->{oManifest}->variableReplace(trim($oOption->valueGet()));
|
|
|
|
if ($strValue eq '')
|
|
{
|
|
delete($$oConfigHashNew{$strKey});
|
|
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . "reset ${strKey}");
|
|
}
|
|
else
|
|
{
|
|
$$oConfigHashNew{$strKey} = $strValue;
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . "set ${strKey} = ${strValue}");
|
|
}
|
|
}
|
|
|
|
# Generate config text
|
|
foreach my $strKey (sort(keys(%$oConfigHashNew)))
|
|
{
|
|
if (defined($strConfig))
|
|
{
|
|
$strConfig .= "\n";
|
|
}
|
|
|
|
$strConfig .= "${strKey} = $$oConfigHashNew{$strKey}";
|
|
}
|
|
|
|
# Save the conf file
|
|
if ($self->{bExe})
|
|
{
|
|
$self->{oManifest}->storage()->put($strLocalFile, $$oConfigHash{base} .
|
|
(defined($strConfig) ? "\n# pgBackRest Configuration\n${strConfig}\n" : ''));
|
|
|
|
$oHost->copyTo($strLocalFile, $$hCacheKey{file}, 'postgres:postgres', '640');
|
|
}
|
|
|
|
$$oConfigHash{old} = $oConfigHashNew;
|
|
|
|
my @stryConfig = undef;
|
|
|
|
if (trim($strConfig) ne '')
|
|
{
|
|
@stryConfig = split("\n", $strConfig);
|
|
}
|
|
|
|
$$hCacheValue{config} = \@stryConfig;
|
|
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
$strConfig = 'Config suppressed for testing';
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strFile', value => $strFile, trace => true},
|
|
{name => 'strConfig', value => $strConfig, trace => true},
|
|
{name => 'bShow', value => $oConfig->paramTest('show', 'n') ? false : true, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# hostKey
|
|
####################################################################################################################################
|
|
sub hostKey
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oHost,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->hostKey', \@_,
|
|
{name => 'oHost', trace => true},
|
|
);
|
|
|
|
my $hCacheKey =
|
|
{
|
|
name => $self->{oManifest}->variableReplace($oHost->paramGet('name')),
|
|
image => $self->{oManifest}->variableReplace($oHost->paramGet('image')),
|
|
};
|
|
|
|
if (defined($oHost->paramGet('id', false)))
|
|
{
|
|
$hCacheKey->{id} = $self->{oManifest}->variableReplace($oHost->paramGet('id'));
|
|
}
|
|
else
|
|
{
|
|
$hCacheKey->{id} = $hCacheKey->{name};
|
|
}
|
|
|
|
if (defined($oHost->paramGet('option', false)))
|
|
{
|
|
$$hCacheKey{option} = $self->{oManifest}->variableReplace($oHost->paramGet('option'));
|
|
}
|
|
|
|
if (defined($oHost->paramGet('param', false)))
|
|
{
|
|
$$hCacheKey{param} = $self->{oManifest}->variableReplace($oHost->paramGet('param'));
|
|
}
|
|
|
|
if (defined($oHost->paramGet('os', false)))
|
|
{
|
|
$$hCacheKey{os} = $self->{oManifest}->variableReplace($oHost->paramGet('os'));
|
|
}
|
|
|
|
$$hCacheKey{'update-hosts'} = $oHost->paramTest('update-hosts', 'n') ? JSON::PP::false : JSON::PP::true;
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'hCacheKey', value => $hCacheKey, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# cachePop
|
|
####################################################################################################################################
|
|
sub cachePop
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strCacheType,
|
|
$hCacheKey,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->hostKey', \@_,
|
|
{name => 'strCacheType', trace => true},
|
|
{name => 'hCacheKey', trace => true},
|
|
);
|
|
|
|
my $bCacheHit = false;
|
|
my $oCacheValue = undef;
|
|
|
|
if ($self->{bCache})
|
|
{
|
|
my $oJSON = JSON::PP->new()->canonical()->allow_nonref();
|
|
# &log(WARN, "checking cache for\ncurrent key: " . $oJSON->encode($hCacheKey));
|
|
|
|
my $hCache = ${$self->{oSource}{hyCache}}[$self->{iCacheIdx}];
|
|
|
|
if (!defined($hCache))
|
|
{
|
|
confess &log(ERROR, 'unable to get index from cache', ERROR_FILE_INVALID);
|
|
}
|
|
|
|
if (!defined($$hCache{key}))
|
|
{
|
|
confess &log(ERROR, 'unable to get key from cache', ERROR_FILE_INVALID);
|
|
}
|
|
|
|
if (!defined($$hCache{type}))
|
|
{
|
|
confess &log(ERROR, 'unable to get type from cache', ERROR_FILE_INVALID);
|
|
}
|
|
|
|
if ($$hCache{type} ne $strCacheType)
|
|
{
|
|
confess &log(ERROR, 'types do not match, cache is invalid', ERROR_FILE_INVALID);
|
|
}
|
|
|
|
if ($oJSON->encode($$hCache{key}) ne $oJSON->encode($hCacheKey))
|
|
{
|
|
confess &log(ERROR,
|
|
"keys at index $self->{iCacheIdx} do not match, cache is invalid." .
|
|
"\n cache key: " . $oJSON->encode($$hCache{key}) .
|
|
"\ncurrent key: " . $oJSON->encode($hCacheKey), ERROR_FILE_INVALID);
|
|
}
|
|
|
|
$bCacheHit = true;
|
|
$oCacheValue = $$hCache{value};
|
|
$self->{iCacheIdx}++;
|
|
}
|
|
else
|
|
{
|
|
if ($self->{oManifest}{bCacheOnly})
|
|
{
|
|
confess &log(ERROR, 'Cache only operation forced by --cache-only option');
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'bCacheHit', value => $bCacheHit, trace => true},
|
|
{name => 'strCacheType', value => $strCacheType, trace => true},
|
|
{name => 'hCacheKey', value => $hCacheKey, trace => true},
|
|
{name => 'oCacheValue', value => $oCacheValue, trace => true},
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# cachePush
|
|
####################################################################################################################################
|
|
sub cachePush
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strType,
|
|
$hCacheKey,
|
|
$oCacheValue,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->hostKey', \@_,
|
|
{name => 'strType', trace => true},
|
|
{name => 'hCacheKey', trace => true},
|
|
{name => 'oCacheValue', required => false, trace => true},
|
|
);
|
|
|
|
if ($self->{bCache})
|
|
{
|
|
confess &log(ASSERT, "cachePush should not be called when cache is already present");
|
|
}
|
|
|
|
# Create the cache entry
|
|
my $hCache =
|
|
{
|
|
key => $hCacheKey,
|
|
type => $strType,
|
|
};
|
|
|
|
if (defined($oCacheValue))
|
|
{
|
|
$$hCache{value} = $oCacheValue;
|
|
}
|
|
|
|
push @{$self->{oSource}{hyCache}}, $hCache;
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn($strOperation);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# sectionChildProcesss
|
|
####################################################################################################################################
|
|
sub sectionChildProcess
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oSection,
|
|
$oChild,
|
|
$iDepth
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->sectionChildProcess', \@_,
|
|
{name => 'oSection'},
|
|
{name => 'oChild'},
|
|
{name => 'iDepth'}
|
|
);
|
|
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . 'process child: ' . $oChild->nameGet());
|
|
|
|
# Execute a command
|
|
if ($oChild->nameGet() eq 'host-add')
|
|
{
|
|
if ($self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('host', $self->hostKey($oChild));
|
|
|
|
if ($bCacheHit)
|
|
{
|
|
$self->{oManifest}->variableSet('host-' . $hCacheKey->{id} . '-ip', $hCacheValue->{ip}, true);
|
|
}
|
|
else
|
|
{
|
|
if (defined($self->{host}{$$hCacheKey{name}}))
|
|
{
|
|
confess &log(ERROR, 'cannot add host ${strName} because the host already exists');
|
|
}
|
|
|
|
executeTest("rm -rf ~/data/$$hCacheKey{name}");
|
|
executeTest("mkdir -p ~/data/$$hCacheKey{name}/etc");
|
|
|
|
my $strHost = $hCacheKey->{name};
|
|
my $strImage = $hCacheKey->{image};
|
|
my $strHostUser = $self->{oManifest}->variableReplace($oChild->paramGet('user'));
|
|
|
|
# Determine if a pre-built image should be created
|
|
if (defined($self->preExecute($strHost)))
|
|
{
|
|
my $strPreImage = "${strImage}-${strHost}";
|
|
my $strFrom = $strImage;
|
|
|
|
&log(INFO, "Build vm '${strPreImage}' from '${strFrom}'");
|
|
|
|
my $strCommandList;
|
|
|
|
# Add all pre commands
|
|
foreach my $oExecute ($self->preExecute($strHost))
|
|
{
|
|
my $hExecuteKey = $self->executeKey($strHost, $oExecute);
|
|
|
|
my $strCommand =
|
|
join("\n", @{$hExecuteKey->{cmd}}) .
|
|
(defined($hExecuteKey->{'cmd-extra'}) ? ' ' . $hExecuteKey->{'cmd-extra'} : '');
|
|
$strCommand =~ s/'/'\\''/g;
|
|
|
|
$strCommand =
|
|
"sudo -u ${strHostUser}" .
|
|
($hCacheKey->{'bash-wrap'} ?
|
|
" bash" . ($hCacheKey->{'load-env'} ? ' -l' : '') . " -c '${strCommand}'" : " ${strCommand}");
|
|
|
|
if (defined($strCommandList))
|
|
{
|
|
$strCommandList .= "\n";
|
|
}
|
|
|
|
$strCommandList .= "RUN ${strCommand}";
|
|
|
|
&log(DETAIL, " Pre command $strCommand");
|
|
}
|
|
|
|
# Build container
|
|
my $strDockerfile = $self->{oManifest}{strDocPath} . "/output/doc-host.dockerfile";
|
|
|
|
$self->{oManifest}{oStorage}->put(
|
|
$strDockerfile,
|
|
"FROM ${strFrom}\n\n" . trim($self->{oManifest}->variableReplace($strCommandList)) . "\n");
|
|
executeTest("docker build -f ${strDockerfile} -t ${strPreImage} " . $self->{oManifest}{oStorage}->pathGet());
|
|
|
|
# Use the pre-built image
|
|
$strImage = $strPreImage;
|
|
}
|
|
|
|
my $strHostRepoPath = dirname(dirname(abs_path($0)));
|
|
|
|
# Replace host repo path in mounts with if present
|
|
my $strMount = undef;
|
|
|
|
if (defined($oChild->paramGet('mount', false)))
|
|
{
|
|
$strMount = $self->{oManifest}->variableReplace($oChild->paramGet('mount'));
|
|
$strMount =~ s/\{\[host\-repo\-path\]\}/${strHostRepoPath}/g;
|
|
}
|
|
|
|
# Replace host repo mount in params if present
|
|
my $strOption = $$hCacheKey{option};
|
|
|
|
if (defined($strOption))
|
|
{
|
|
$strOption =~ s/\{\[host\-repo\-path\]\}/${strHostRepoPath}/g;
|
|
}
|
|
|
|
my $oHost = new pgBackRestTest::Common::HostTest(
|
|
$$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage, $strHostUser, $$hCacheKey{os},
|
|
defined($strMount) ? [$strMount] : undef, $strOption, $$hCacheKey{param}, $$hCacheKey{'update-hosts'});
|
|
|
|
$self->{host}{$$hCacheKey{name}} = $oHost;
|
|
$self->{oManifest}->variableSet('host-' . $hCacheKey->{id} . '-ip', $oHost->{strIP}, true);
|
|
$$hCacheValue{ip} = $oHost->{strIP};
|
|
|
|
# Add to the host group
|
|
my $oHostGroup = hostGroupGet();
|
|
$oHostGroup->hostAdd($oHost);
|
|
|
|
# Execute initialize commands
|
|
foreach my $oExecute ($oChild->nodeList('execute', false))
|
|
{
|
|
$self->execute(
|
|
$oSection, $$hCacheKey{name}, $oExecute, {iIndent => $iDepth + 1, bCache => false, bShow => false});
|
|
}
|
|
|
|
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
|
|
}
|
|
}
|
|
}
|
|
# Skip children that have already been processed and error on others
|
|
elsif ($oChild->nameGet() ne 'title')
|
|
{
|
|
confess &log(ASSERT, 'unable to process child type ' . $oChild->nameGet());
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
1;
|