mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-04-15 11:36:40 +02:00
653ffcf8d9
Azurite released another breaking change (see fbd018cd, 096829b3, c38d6926, and Azurite issue 1039) so make adjustments as needed to documentation and tests. Also remove some dead code that hid the repo-storage-host option and was made obsolete by all these changes.
1088 lines
38 KiB
Perl
1088 lines
38 KiB
Perl
####################################################################################################################################
|
|
# DOC EXECUTE MODULE
|
|
####################################################################################################################################
|
|
package pgBackRestDoc::Common::DocExecute;
|
|
use parent 'pgBackRestDoc::Common::DocRender';
|
|
|
|
use strict;
|
|
use warnings FATAL => qw(all);
|
|
use Carp qw(confess);
|
|
use English '-no_match_vars';
|
|
|
|
use Cwd qw(abs_path);
|
|
use Exporter qw(import);
|
|
our @EXPORT = qw();
|
|
use File::Basename qw(dirname);
|
|
use Storable qw(dclone);
|
|
|
|
use pgBackRestTest::Common::ExecuteTest;
|
|
use pgBackRestTest::Common::HostTest;
|
|
use pgBackRestTest::Common::HostGroupTest;
|
|
|
|
use pgBackRestDoc::Common::DocManifest;
|
|
use pgBackRestDoc::Common::Exception;
|
|
use pgBackRestDoc::Common::Ini;
|
|
use pgBackRestDoc::Common::Log;
|
|
use pgBackRestDoc::Common::String;
|
|
use pgBackRestDoc::Custom::DocConfigData;
|
|
use pgBackRestDoc::ProjectInfo;
|
|
|
|
####################################################################################################################################
|
|
# User that's building the docs
|
|
####################################################################################################################################
|
|
use constant DOC_USER => getpwuid($UID) eq 'root' ? 'ubuntu' : getpwuid($UID) . '';
|
|
|
|
####################################################################################################################################
|
|
# CONSTRUCTOR
|
|
####################################################################################################################################
|
|
sub new
|
|
{
|
|
my $class = shift; # Class name
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strType,
|
|
$oManifest,
|
|
$strRenderOutKey,
|
|
$bExe
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->new', \@_,
|
|
{name => 'strType'},
|
|
{name => 'oManifest'},
|
|
{name => 'strRenderOutKey'},
|
|
{name => 'bExe'}
|
|
);
|
|
|
|
# Create the class hash
|
|
my $self = $class->SUPER::new($strType, $oManifest, $bExe, $strRenderOutKey);
|
|
bless $self, $class;
|
|
|
|
if (defined($self->{oSource}{hyCache}))
|
|
{
|
|
$self->{bCache} = true;
|
|
$self->{iCacheIdx} = 0;
|
|
}
|
|
else
|
|
{
|
|
$self->{bCache} = false;
|
|
}
|
|
|
|
$self->{bExe} = $bExe;
|
|
|
|
$self->{iCmdLineLen} = $self->{oDoc}->paramGet('cmd-line-len', false, 80);
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'self', value => $self}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# executeKey
|
|
#
|
|
# Get a unique key for the execution step to determine if the cache is valid.
|
|
####################################################################################################################################
|
|
sub executeKey
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strHostName,
|
|
$oCommand,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->executeKey', \@_,
|
|
{name => 'strHostName', trace => true},
|
|
{name => 'oCommand', trace => true},
|
|
);
|
|
|
|
# Add user to command
|
|
my $bUserForce = $oCommand->paramTest('user-force', 'y') ? true : false;
|
|
my $strCommand = $self->{oManifest}->variableReplace(trim($oCommand->fieldGet('exe-cmd')));
|
|
my $strUser = $self->{oManifest}->variableReplace($oCommand->paramGet('user', false, DOC_USER));
|
|
$strCommand = ($strUser eq DOC_USER || $bUserForce ? '' : ('sudo ' . ($strUser eq 'root' ? '' : "-u $strUser "))) . $strCommand;
|
|
|
|
# Format and split command
|
|
$strCommand =~ s/[ ]*\n[ ]*/ \\\n /smg;
|
|
$strCommand =~ s/ \\\@ \\//smg;
|
|
my @stryCommand = split("\n", $strCommand);
|
|
|
|
my $hCacheKey =
|
|
{
|
|
host => $strHostName,
|
|
cmd => \@stryCommand,
|
|
output => JSON::PP::false,
|
|
};
|
|
|
|
$$hCacheKey{'run-as-user'} = $bUserForce ? $strUser : undef;
|
|
|
|
if (defined($oCommand->fieldGet('exe-cmd-extra', false)))
|
|
{
|
|
$$hCacheKey{'cmd-extra'} = $self->{oManifest}->variableReplace($oCommand->fieldGet('exe-cmd-extra'));
|
|
}
|
|
|
|
if (defined($oCommand->paramGet('err-expect', false)))
|
|
{
|
|
$$hCacheKey{'err-expect'} = $oCommand->paramGet('err-expect');
|
|
}
|
|
|
|
if ($oCommand->paramTest('output', 'y') || $oCommand->paramTest('show', 'y') || $oCommand->paramTest('variable-key'))
|
|
{
|
|
$$hCacheKey{'output'} = JSON::PP::true;
|
|
}
|
|
|
|
$$hCacheKey{'load-env'} = $oCommand->paramTest('load-env', 'n') ? JSON::PP::false : JSON::PP::true;
|
|
$$hCacheKey{'bash-wrap'} = $oCommand->paramTest('bash-wrap', 'n') ? JSON::PP::false : JSON::PP::true;
|
|
|
|
if (defined($oCommand->fieldGet('exe-highlight', false)))
|
|
{
|
|
$$hCacheKey{'output'} = JSON::PP::true;
|
|
$$hCacheKey{highlight}{'filter'} = $oCommand->paramTest('filter', 'n') ? JSON::PP::false : JSON::PP::true;
|
|
$$hCacheKey{highlight}{'filter-context'} = $oCommand->paramGet('filter-context', false, 2);
|
|
|
|
my @stryHighlight;
|
|
$stryHighlight[0] = $self->{oManifest}->variableReplace($oCommand->fieldGet('exe-highlight'));
|
|
|
|
$$hCacheKey{highlight}{list} = \@stryHighlight;
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'hExecuteKey', value => $hCacheKey, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# execute
|
|
####################################################################################################################################
|
|
sub execute
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oSection,
|
|
$strHostName,
|
|
$oCommand,
|
|
$iIndent,
|
|
$bCache,
|
|
$bShow,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->execute', \@_,
|
|
{name => 'oSection'},
|
|
{name => 'strHostName'},
|
|
{name => 'oCommand'},
|
|
{name => 'iIndent', optional => true, default => 1},
|
|
{name => 'bCache', optional => true, default => true},
|
|
{name => 'bShow', optional => true, default => true},
|
|
);
|
|
|
|
# Working variables
|
|
my $hCacheKey = $self->executeKey($strHostName, $oCommand);
|
|
my $strCommand = join("\n", @{$$hCacheKey{cmd}});
|
|
my $strOutput;
|
|
|
|
if ($bShow && $self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
# Make sure that no lines are greater than 80 chars
|
|
foreach my $strLine (split("\n", $strCommand))
|
|
{
|
|
if (length(trim($strLine)) > $self->{iCmdLineLen})
|
|
{
|
|
confess &log(ERROR,
|
|
"command has a line > $self->{iCmdLineLen} characters:\n${strCommand}\noffending line: ${strLine}");
|
|
}
|
|
}
|
|
}
|
|
|
|
&log(DEBUG, (' ' x $iIndent) . "execute: $strCommand");
|
|
|
|
if ($self->{oManifest}->variableReplace($oCommand->paramGet('skip', false, 'n')) ne 'y')
|
|
{
|
|
if ($self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('exe', $hCacheKey);
|
|
|
|
if ($bCacheHit)
|
|
{
|
|
$strOutput = defined($$hCacheValue{output}) ? join("\n", @{$$hCacheValue{output}}) : undef;
|
|
}
|
|
else
|
|
{
|
|
# Check that the host is valid
|
|
my $oHost = $self->{host}{$strHostName};
|
|
|
|
if (!defined($oHost))
|
|
{
|
|
confess &log(ERROR, "cannot execute on host ${strHostName} because the host does not exist");
|
|
}
|
|
|
|
my $oExec = $oHost->execute(
|
|
$strCommand . (defined($$hCacheKey{'cmd-extra'}) ? ' ' . $$hCacheKey{'cmd-extra'} : ''),
|
|
{iExpectedExitStatus => $$hCacheKey{'err-expect'},
|
|
bSuppressError => $oCommand->paramTest('err-suppress', 'y'),
|
|
iRetrySeconds => $oCommand->paramGet('retry', false)}, $hCacheKey->{'run-as-user'},
|
|
{bLoadEnv => $hCacheKey->{'load-env'}, bBashWrap => $hCacheKey->{'bash-wrap'}});
|
|
$oExec->begin();
|
|
$oExec->end();
|
|
|
|
if (defined($oExec->{strOutLog}) && $oExec->{strOutLog} ne '')
|
|
{
|
|
$strOutput = $oExec->{strOutLog};
|
|
|
|
# Trim off extra linefeeds before and after
|
|
$strOutput =~ s/^\n+|\n$//g;
|
|
}
|
|
|
|
if (defined($$hCacheKey{'err-expect'}) && defined($oExec->{strErrorLog}) && $oExec->{strErrorLog} ne '')
|
|
{
|
|
$strOutput .= $oExec->{strErrorLog};
|
|
}
|
|
|
|
if ($$hCacheKey{output} && defined($$hCacheKey{highlight}) && $$hCacheKey{highlight}{filter} && defined($strOutput))
|
|
{
|
|
my $strHighLight = @{$$hCacheKey{highlight}{list}}[0];
|
|
|
|
if (!defined($strHighLight))
|
|
{
|
|
confess &log(ERROR, 'filter requires highlight definition: ' . $strCommand);
|
|
}
|
|
|
|
my $iFilterContext = $$hCacheKey{highlight}{'filter-context'};
|
|
|
|
my @stryOutput = split("\n", $strOutput);
|
|
undef($strOutput);
|
|
# my $iFiltered = 0;
|
|
my $iLastOutput = -1;
|
|
|
|
for (my $iIndex = 0; $iIndex < @stryOutput; $iIndex++)
|
|
{
|
|
if ($stryOutput[$iIndex] =~ /$strHighLight/)
|
|
{
|
|
# Determine the first line to output
|
|
my $iFilterFirst = $iIndex - $iFilterContext;
|
|
|
|
# Don't go past the beginning
|
|
$iFilterFirst = $iFilterFirst < 0 ? 0 : $iFilterFirst;
|
|
|
|
# Don't repeat lines that have already been output
|
|
$iFilterFirst = $iFilterFirst <= $iLastOutput ? $iLastOutput + 1 : $iFilterFirst;
|
|
|
|
# Determine the last line to output
|
|
my $iFilterLast = $iIndex + $iFilterContext;
|
|
|
|
# Don't got past the end
|
|
$iFilterLast = $iFilterLast >= @stryOutput ? @stryOutput -1 : $iFilterLast;
|
|
|
|
# Mark filtered lines if any
|
|
if ($iFilterFirst > $iLastOutput + 1)
|
|
{
|
|
my $iFiltered = $iFilterFirst - ($iLastOutput + 1);
|
|
|
|
if ($iFiltered > 1)
|
|
{
|
|
$strOutput .= (defined($strOutput) ? "\n" : '') .
|
|
" [filtered ${iFiltered} lines of output]";
|
|
}
|
|
else
|
|
{
|
|
$iFilterFirst -= 1;
|
|
}
|
|
}
|
|
|
|
# Output the lines
|
|
for (my $iOutputIndex = $iFilterFirst; $iOutputIndex <= $iFilterLast; $iOutputIndex++)
|
|
{
|
|
$strOutput .= (defined($strOutput) ? "\n" : '') . $stryOutput[$iOutputIndex];
|
|
}
|
|
|
|
$iLastOutput = $iFilterLast;
|
|
}
|
|
}
|
|
|
|
if (@stryOutput - 1 > $iLastOutput + 1)
|
|
{
|
|
my $iFiltered = (@stryOutput - 1) - ($iLastOutput + 1);
|
|
|
|
if ($iFiltered > 1)
|
|
{
|
|
$strOutput .= (defined($strOutput) ? "\n" : '') .
|
|
" [filtered ${iFiltered} lines of output]";
|
|
}
|
|
else
|
|
{
|
|
$strOutput .= (defined($strOutput) ? "\n" : '') . $stryOutput[-1];
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!$$hCacheKey{output})
|
|
{
|
|
$strOutput = undef;
|
|
}
|
|
|
|
if (defined($strOutput))
|
|
{
|
|
my @stryOutput = split("\n", $strOutput);
|
|
$$hCacheValue{output} = \@stryOutput;
|
|
}
|
|
|
|
if ($bCache)
|
|
{
|
|
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
|
|
}
|
|
}
|
|
|
|
# Output is assigned to a var
|
|
if ($oCommand->paramTest('variable-key'))
|
|
{
|
|
$self->{oManifest}->variableSet($oCommand->paramGet('variable-key'), trim($strOutput), true);
|
|
}
|
|
}
|
|
elsif ($$hCacheKey{output})
|
|
{
|
|
$strOutput = 'Output suppressed for testing';
|
|
}
|
|
}
|
|
|
|
# Default variable output when it was not set by execution
|
|
if ($oCommand->paramTest('variable-key') && !defined($self->{oManifest}->variableGet($oCommand->paramGet('variable-key'))))
|
|
{
|
|
$self->{oManifest}->variableSet($oCommand->paramGet('variable-key'), '[Test Variable]', true);
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strCommand', value => $strCommand, trace => true},
|
|
{name => 'strOutput', value => $strOutput, trace => true}
|
|
);
|
|
}
|
|
|
|
|
|
####################################################################################################################################
|
|
# configKey
|
|
####################################################################################################################################
|
|
sub configKey
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oConfig,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->hostKey', \@_,
|
|
{name => 'oConfig', trace => true},
|
|
);
|
|
|
|
my $hCacheKey =
|
|
{
|
|
host => $self->{oManifest}->variableReplace($oConfig->paramGet('host')),
|
|
file => $self->{oManifest}->variableReplace($oConfig->paramGet('file')),
|
|
};
|
|
|
|
if ($oConfig->paramTest('reset', 'y'))
|
|
{
|
|
$$hCacheKey{reset} = JSON::PP::true;
|
|
}
|
|
|
|
# Add all options to the key
|
|
my $strOptionTag = $oConfig->nameGet() eq 'backrest-config' ? 'backrest-config-option' : 'postgres-config-option';
|
|
|
|
foreach my $oOption ($oConfig->nodeList($strOptionTag))
|
|
{
|
|
my $hOption = {};
|
|
|
|
if ($oOption->paramTest('remove', 'y'))
|
|
{
|
|
$$hOption{remove} = JSON::PP::true;
|
|
}
|
|
|
|
if (defined($oOption->valueGet(false)))
|
|
{
|
|
$$hOption{value} = $self->{oManifest}->variableReplace($oOption->valueGet());
|
|
}
|
|
|
|
my $strKey = $self->{oManifest}->variableReplace($oOption->paramGet('key'));
|
|
|
|
if ($oConfig->nameGet() eq 'backrest-config')
|
|
{
|
|
my $strSection = $self->{oManifest}->variableReplace($oOption->paramGet('section'));
|
|
|
|
$$hCacheKey{option}{$strSection}{$strKey} = $hOption;
|
|
}
|
|
else
|
|
{
|
|
$$hCacheKey{option}{$strKey} = $hOption;
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'hCacheKey', value => $hCacheKey, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# backrestConfig
|
|
####################################################################################################################################
|
|
sub backrestConfig
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oSection,
|
|
$oConfig,
|
|
$iDepth
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->backrestConfig', \@_,
|
|
{name => 'oSection'},
|
|
{name => 'oConfig'},
|
|
{name => 'iDepth'}
|
|
);
|
|
|
|
# Working variables
|
|
my $hCacheKey = $self->configKey($oConfig);
|
|
my $strFile = $$hCacheKey{file};
|
|
my $strConfig = undef;
|
|
|
|
&log(DEBUG, (' ' x $iDepth) . 'process backrest config: ' . $$hCacheKey{file});
|
|
|
|
if ($self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('cfg-' . PROJECT_EXE, $hCacheKey);
|
|
|
|
if ($bCacheHit)
|
|
{
|
|
$strConfig = defined($$hCacheValue{config}) ? join("\n", @{$$hCacheValue{config}}) : undef;
|
|
}
|
|
else
|
|
{
|
|
# Check that the host is valid
|
|
my $strHostName = $self->{oManifest}->variableReplace($oConfig->paramGet('host'));
|
|
my $oHost = $self->{host}{$strHostName};
|
|
|
|
if (!defined($oHost))
|
|
{
|
|
confess &log(ERROR, "cannot configure backrest on host ${strHostName} because the host does not exist");
|
|
}
|
|
|
|
# Reset all options
|
|
if ($oConfig->paramTest('reset', 'y'))
|
|
{
|
|
delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}})
|
|
}
|
|
|
|
foreach my $oOption ($oConfig->nodeList('backrest-config-option'))
|
|
{
|
|
my $strSection = $self->{oManifest}->variableReplace($oOption->paramGet('section'));
|
|
my $strKey = $self->{oManifest}->variableReplace($oOption->paramGet('key'));
|
|
my $strValue;
|
|
|
|
if (!$oOption->paramTest('remove', 'y'))
|
|
{
|
|
$strValue = $self->{oManifest}->variableReplace(trim($oOption->valueGet(false)));
|
|
}
|
|
|
|
if (!defined($strValue))
|
|
{
|
|
delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey});
|
|
|
|
if (keys(%{${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}}) == 0)
|
|
{
|
|
delete(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection});
|
|
}
|
|
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . "reset ${strSection}->${strKey}");
|
|
}
|
|
else
|
|
{
|
|
# If this option is a hash and the value is already set then append to the array
|
|
if (defined(cfgDefine()->{$strKey}) &&
|
|
cfgDefine()->{$strKey}{&CFGDEF_TYPE} eq CFGDEF_TYPE_HASH &&
|
|
defined(${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey}))
|
|
{
|
|
my @oValue = ();
|
|
my $strHashValue = ${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey};
|
|
|
|
# If there is only one key/value
|
|
if (ref(\$strHashValue) eq 'SCALAR')
|
|
{
|
|
push(@oValue, $strHashValue);
|
|
}
|
|
# Else if there is an array of values
|
|
else
|
|
{
|
|
@oValue = @{$strHashValue};
|
|
}
|
|
|
|
push(@oValue, $strValue);
|
|
${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey} = \@oValue;
|
|
}
|
|
# else just set the value
|
|
else
|
|
{
|
|
${$self->{config}}{$strHostName}{$$hCacheKey{file}}{$strSection}{$strKey} = $strValue;
|
|
}
|
|
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . "set ${strSection}->${strKey} = ${strValue}");
|
|
}
|
|
}
|
|
|
|
my $strLocalFile = '/home/' . DOC_USER . '/data/pgbackrest.conf';
|
|
|
|
# Save the ini file
|
|
$self->{oManifest}->storage()->put($strLocalFile, iniRender($self->{config}{$strHostName}{$$hCacheKey{file}}, true));
|
|
|
|
$oHost->copyTo(
|
|
$strLocalFile, $$hCacheKey{file},
|
|
$self->{oManifest}->variableReplace($oConfig->paramGet('owner', false, 'postgres:postgres')), '640');
|
|
|
|
# Remove the log-console-stderr option before pushing into the cache
|
|
# ??? This is not very pretty and should be replaced with a general way to hide config options
|
|
my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}});
|
|
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_LEVEL_STDERR});
|
|
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_TIMESTAMP});
|
|
|
|
if (keys(%{$$oConfigClean{&CFGDEF_SECTION_GLOBAL}}) == 0)
|
|
{
|
|
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL});
|
|
}
|
|
|
|
$self->{oManifest}->storage()->put("${strLocalFile}.clean", iniRender($oConfigClean, true));
|
|
|
|
# Push config file into the cache
|
|
$strConfig = ${$self->{oManifest}->storage()->get("${strLocalFile}.clean")};
|
|
|
|
my @stryConfig = undef;
|
|
|
|
if (trim($strConfig) ne '')
|
|
{
|
|
@stryConfig = split("\n", $strConfig);
|
|
}
|
|
|
|
$$hCacheValue{config} = \@stryConfig;
|
|
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
$strConfig = 'Config suppressed for testing';
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strFile', value => $strFile, trace => true},
|
|
{name => 'strConfig', value => $strConfig, trace => true},
|
|
{name => 'bShow', value => $oConfig->paramTest('show', 'n') ? false : true, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# postgresConfig
|
|
####################################################################################################################################
|
|
sub postgresConfig
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oSection,
|
|
$oConfig,
|
|
$iDepth
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->postgresConfig', \@_,
|
|
{name => 'oSection'},
|
|
{name => 'oConfig'},
|
|
{name => 'iDepth'}
|
|
);
|
|
|
|
# Working variables
|
|
my $hCacheKey = $self->configKey($oConfig);
|
|
my $strFile = $$hCacheKey{file};
|
|
my $strConfig;
|
|
|
|
if ($self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('cfg-postgresql', $hCacheKey);
|
|
|
|
if ($bCacheHit)
|
|
{
|
|
$strConfig = defined($$hCacheValue{config}) ? join("\n", @{$$hCacheValue{config}}) : undef;
|
|
}
|
|
else
|
|
{
|
|
# Check that the host is valid
|
|
my $strHostName = $self->{oManifest}->variableReplace($oConfig->paramGet('host'));
|
|
my $oHost = $self->{host}{$strHostName};
|
|
|
|
if (!defined($oHost))
|
|
{
|
|
confess &log(ERROR, "cannot configure postgres on host ${strHostName} because the host does not exist");
|
|
}
|
|
|
|
my $strLocalFile = '/home/' . DOC_USER . '/data/postgresql.conf';
|
|
$oHost->copyFrom($$hCacheKey{file}, $strLocalFile);
|
|
|
|
if (!defined(${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base}) && $self->{bExe})
|
|
{
|
|
${$self->{'pg-config'}}{$strHostName}{$$hCacheKey{file}}{base} =
|
|
${$self->{oManifest}->storage()->get($strLocalFile)};
|
|
}
|
|
|
|
my $oConfigHash = $self->{'pg-config'}{$strHostName}{$$hCacheKey{file}};
|
|
my $oConfigHashNew;
|
|
|
|
if (!defined($$oConfigHash{old}))
|
|
{
|
|
$oConfigHashNew = {};
|
|
$$oConfigHash{old} = {}
|
|
}
|
|
else
|
|
{
|
|
$oConfigHashNew = dclone($$oConfigHash{old});
|
|
}
|
|
|
|
&log(DEBUG, (' ' x $iDepth) . 'process postgres config: ' . $$hCacheKey{file});
|
|
|
|
foreach my $oOption ($oConfig->nodeList('postgres-config-option'))
|
|
{
|
|
my $strKey = $oOption->paramGet('key');
|
|
my $strValue = $self->{oManifest}->variableReplace(trim($oOption->valueGet()));
|
|
|
|
if ($strValue eq '')
|
|
{
|
|
delete($$oConfigHashNew{$strKey});
|
|
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . "reset ${strKey}");
|
|
}
|
|
else
|
|
{
|
|
$$oConfigHashNew{$strKey} = $strValue;
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . "set ${strKey} = ${strValue}");
|
|
}
|
|
}
|
|
|
|
# Generate config text
|
|
foreach my $strKey (sort(keys(%$oConfigHashNew)))
|
|
{
|
|
if (defined($strConfig))
|
|
{
|
|
$strConfig .= "\n";
|
|
}
|
|
|
|
$strConfig .= "${strKey} = $$oConfigHashNew{$strKey}";
|
|
}
|
|
|
|
# Save the conf file
|
|
if ($self->{bExe})
|
|
{
|
|
$self->{oManifest}->storage()->put($strLocalFile, $$oConfigHash{base} .
|
|
(defined($strConfig) ? "\n# pgBackRest Configuration\n${strConfig}\n" : ''));
|
|
|
|
$oHost->copyTo($strLocalFile, $$hCacheKey{file}, 'postgres:postgres', '640');
|
|
}
|
|
|
|
$$oConfigHash{old} = $oConfigHashNew;
|
|
|
|
my @stryConfig = undef;
|
|
|
|
if (trim($strConfig) ne '')
|
|
{
|
|
@stryConfig = split("\n", $strConfig);
|
|
}
|
|
|
|
$$hCacheValue{config} = \@stryConfig;
|
|
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
$strConfig = 'Config suppressed for testing';
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'strFile', value => $strFile, trace => true},
|
|
{name => 'strConfig', value => $strConfig, trace => true},
|
|
{name => 'bShow', value => $oConfig->paramTest('show', 'n') ? false : true, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# hostKey
|
|
####################################################################################################################################
|
|
sub hostKey
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oHost,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->hostKey', \@_,
|
|
{name => 'oHost', trace => true},
|
|
);
|
|
|
|
my $hCacheKey =
|
|
{
|
|
name => $self->{oManifest}->variableReplace($oHost->paramGet('name')),
|
|
image => $self->{oManifest}->variableReplace($oHost->paramGet('image')),
|
|
};
|
|
|
|
if (defined($oHost->paramGet('id', false)))
|
|
{
|
|
$hCacheKey->{id} = $self->{oManifest}->variableReplace($oHost->paramGet('id'));
|
|
}
|
|
else
|
|
{
|
|
$hCacheKey->{id} = $hCacheKey->{name};
|
|
}
|
|
|
|
if (defined($oHost->paramGet('option', false)))
|
|
{
|
|
$$hCacheKey{option} = $self->{oManifest}->variableReplace($oHost->paramGet('option'));
|
|
}
|
|
|
|
if (defined($oHost->paramGet('param', false)))
|
|
{
|
|
$$hCacheKey{param} = $self->{oManifest}->variableReplace($oHost->paramGet('param'));
|
|
}
|
|
|
|
if (defined($oHost->paramGet('os', false)))
|
|
{
|
|
$$hCacheKey{os} = $self->{oManifest}->variableReplace($oHost->paramGet('os'));
|
|
}
|
|
|
|
$$hCacheKey{'update-hosts'} = $oHost->paramTest('update-hosts', 'n') ? JSON::PP::false : JSON::PP::true;
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'hCacheKey', value => $hCacheKey, trace => true}
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# cachePop
|
|
####################################################################################################################################
|
|
sub cachePop
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strCacheType,
|
|
$hCacheKey,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->hostKey', \@_,
|
|
{name => 'strCacheType', trace => true},
|
|
{name => 'hCacheKey', trace => true},
|
|
);
|
|
|
|
my $bCacheHit = false;
|
|
my $oCacheValue = undef;
|
|
|
|
if ($self->{bCache})
|
|
{
|
|
my $oJSON = JSON::PP->new()->canonical()->allow_nonref();
|
|
# &log(WARN, "checking cache for\ncurrent key: " . $oJSON->encode($hCacheKey));
|
|
|
|
my $hCache = ${$self->{oSource}{hyCache}}[$self->{iCacheIdx}];
|
|
|
|
if (!defined($hCache))
|
|
{
|
|
confess &log(ERROR, 'unable to get index from cache', ERROR_FILE_INVALID);
|
|
}
|
|
|
|
if (!defined($$hCache{key}))
|
|
{
|
|
confess &log(ERROR, 'unable to get key from cache', ERROR_FILE_INVALID);
|
|
}
|
|
|
|
if (!defined($$hCache{type}))
|
|
{
|
|
confess &log(ERROR, 'unable to get type from cache', ERROR_FILE_INVALID);
|
|
}
|
|
|
|
if ($$hCache{type} ne $strCacheType)
|
|
{
|
|
confess &log(ERROR, 'types do not match, cache is invalid', ERROR_FILE_INVALID);
|
|
}
|
|
|
|
if ($oJSON->encode($$hCache{key}) ne $oJSON->encode($hCacheKey))
|
|
{
|
|
confess &log(ERROR,
|
|
"keys at index $self->{iCacheIdx} do not match, cache is invalid." .
|
|
"\n cache key: " . $oJSON->encode($$hCache{key}) .
|
|
"\ncurrent key: " . $oJSON->encode($hCacheKey), ERROR_FILE_INVALID);
|
|
}
|
|
|
|
$bCacheHit = true;
|
|
$oCacheValue = $$hCache{value};
|
|
$self->{iCacheIdx}++;
|
|
}
|
|
else
|
|
{
|
|
if ($self->{oManifest}{bCacheOnly})
|
|
{
|
|
confess &log(ERROR, 'Cache only operation forced by --cache-only option');
|
|
}
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation,
|
|
{name => 'bCacheHit', value => $bCacheHit, trace => true},
|
|
{name => 'strCacheType', value => $strCacheType, trace => true},
|
|
{name => 'hCacheKey', value => $hCacheKey, trace => true},
|
|
{name => 'oCacheValue', value => $oCacheValue, trace => true},
|
|
);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# cachePush
|
|
####################################################################################################################################
|
|
sub cachePush
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$strType,
|
|
$hCacheKey,
|
|
$oCacheValue,
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->hostKey', \@_,
|
|
{name => 'strType', trace => true},
|
|
{name => 'hCacheKey', trace => true},
|
|
{name => 'oCacheValue', required => false, trace => true},
|
|
);
|
|
|
|
if ($self->{bCache})
|
|
{
|
|
confess &log(ASSERT, "cachePush should not be called when cache is already present");
|
|
}
|
|
|
|
# Create the cache entry
|
|
my $hCache =
|
|
{
|
|
key => $hCacheKey,
|
|
type => $strType,
|
|
};
|
|
|
|
if (defined($oCacheValue))
|
|
{
|
|
$$hCache{value} = $oCacheValue;
|
|
}
|
|
|
|
push @{$self->{oSource}{hyCache}}, $hCache;
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn($strOperation);
|
|
}
|
|
|
|
####################################################################################################################################
|
|
# sectionChildProcesss
|
|
####################################################################################################################################
|
|
sub sectionChildProcess
|
|
{
|
|
my $self = shift;
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
my
|
|
(
|
|
$strOperation,
|
|
$oSection,
|
|
$oChild,
|
|
$iDepth
|
|
) =
|
|
logDebugParam
|
|
(
|
|
__PACKAGE__ . '->sectionChildProcess', \@_,
|
|
{name => 'oSection'},
|
|
{name => 'oChild'},
|
|
{name => 'iDepth'}
|
|
);
|
|
|
|
&log(DEBUG, (' ' x ($iDepth + 1)) . 'process child: ' . $oChild->nameGet());
|
|
|
|
# Execute a command
|
|
if ($oChild->nameGet() eq 'host-add')
|
|
{
|
|
if ($self->{bExe} && $self->isRequired($oSection))
|
|
{
|
|
my ($bCacheHit, $strCacheType, $hCacheKey, $hCacheValue) = $self->cachePop('host', $self->hostKey($oChild));
|
|
|
|
if ($bCacheHit)
|
|
{
|
|
$self->{oManifest}->variableSet('host-' . $hCacheKey->{id} . '-ip', $hCacheValue->{ip}, true);
|
|
}
|
|
else
|
|
{
|
|
if (defined($self->{host}{$$hCacheKey{name}}))
|
|
{
|
|
confess &log(ERROR, 'cannot add host ${strName} because the host already exists');
|
|
}
|
|
|
|
executeTest("rm -rf ~/data/$$hCacheKey{name}");
|
|
executeTest("mkdir -p ~/data/$$hCacheKey{name}/etc");
|
|
|
|
my $strHost = $hCacheKey->{name};
|
|
my $strImage = $hCacheKey->{image};
|
|
my $strHostUser = $self->{oManifest}->variableReplace($oChild->paramGet('user'));
|
|
|
|
# Determine if a pre-built image should be created
|
|
if (defined($self->preExecute($strHost)))
|
|
{
|
|
my $strPreImage = "${strImage}-${strHost}";
|
|
my $strFrom = $strImage;
|
|
|
|
&log(INFO, "Build vm '${strPreImage}' from '${strFrom}'");
|
|
|
|
my $strCommandList;
|
|
|
|
# Add all pre commands
|
|
foreach my $oExecute ($self->preExecute($strHost))
|
|
{
|
|
my $hExecuteKey = $self->executeKey($strHost, $oExecute);
|
|
|
|
my $strCommand =
|
|
join("\n", @{$hExecuteKey->{cmd}}) .
|
|
(defined($hExecuteKey->{'cmd-extra'}) ? ' ' . $hExecuteKey->{'cmd-extra'} : '');
|
|
$strCommand =~ s/'/'\\''/g;
|
|
|
|
$strCommand =
|
|
"sudo -u ${strHostUser}" .
|
|
($hCacheKey->{'bash-wrap'} ?
|
|
" bash" . ($hCacheKey->{'load-env'} ? ' -l' : '') . " -c '${strCommand}'" : " ${strCommand}");
|
|
|
|
if (defined($strCommandList))
|
|
{
|
|
$strCommandList .= "\n";
|
|
}
|
|
|
|
$strCommandList .= "RUN ${strCommand}";
|
|
|
|
&log(DETAIL, " Pre command $strCommand");
|
|
}
|
|
|
|
# Build container
|
|
my $strDockerfile = $self->{oManifest}{strDocPath} . "/output/doc-host.dockerfile";
|
|
|
|
$self->{oManifest}{oStorage}->put(
|
|
$strDockerfile,
|
|
"FROM ${strFrom}\n\n" . trim($self->{oManifest}->variableReplace($strCommandList)) . "\n");
|
|
executeTest("docker build -f ${strDockerfile} -t ${strPreImage} " . $self->{oManifest}{oStorage}->pathGet());
|
|
|
|
# Use the pre-built image
|
|
$strImage = $strPreImage;
|
|
}
|
|
|
|
my $strHostRepoPath = dirname(dirname(abs_path($0)));
|
|
|
|
# Replace host repo path in mounts with if present
|
|
my $strMount = undef;
|
|
|
|
if (defined($oChild->paramGet('mount', false)))
|
|
{
|
|
$strMount = $self->{oManifest}->variableReplace($oChild->paramGet('mount'));
|
|
$strMount =~ s/\{\[host\-repo\-path\]\}/${strHostRepoPath}/g;
|
|
}
|
|
|
|
# Replace host repo mount in params if present
|
|
my $strOption = $$hCacheKey{option};
|
|
|
|
if (defined($strOption))
|
|
{
|
|
$strOption =~ s/\{\[host\-repo\-path\]\}/${strHostRepoPath}/g;
|
|
}
|
|
|
|
my $oHost = new pgBackRestTest::Common::HostTest(
|
|
$$hCacheKey{name}, "doc-$$hCacheKey{name}", $strImage, $strHostUser, $$hCacheKey{os},
|
|
defined($strMount) ? [$strMount] : undef, $strOption, $$hCacheKey{param}, $$hCacheKey{'update-hosts'});
|
|
|
|
$self->{host}{$$hCacheKey{name}} = $oHost;
|
|
$self->{oManifest}->variableSet('host-' . $hCacheKey->{id} . '-ip', $oHost->{strIP}, true);
|
|
$$hCacheValue{ip} = $oHost->{strIP};
|
|
|
|
# Add to the host group
|
|
my $oHostGroup = hostGroupGet();
|
|
$oHostGroup->hostAdd($oHost);
|
|
|
|
# Execute initialize commands
|
|
foreach my $oExecute ($oChild->nodeList('execute', false))
|
|
{
|
|
$self->execute(
|
|
$oSection, $$hCacheKey{name}, $oExecute, {iIndent => $iDepth + 1, bCache => false, bShow => false});
|
|
}
|
|
|
|
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
|
|
}
|
|
}
|
|
}
|
|
# Skip children that have already been processed and error on others
|
|
elsif ($oChild->nameGet() ne 'title')
|
|
{
|
|
confess &log(ASSERT, 'unable to process child type ' . $oChild->nameGet());
|
|
}
|
|
|
|
# Return from function and log return values if any
|
|
return logDebugReturn
|
|
(
|
|
$strOperation
|
|
);
|
|
}
|
|
|
|
1;
|