1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00

Use a prebuilt s3 server container for documentation and tests.

This commit is contained in:
David Steele 2018-06-12 13:43:15 -04:00
parent 350b30fa49
commit d55e609959
8 changed files with 122 additions and 149 deletions

View File

@ -227,6 +227,7 @@ sub execute
$oCommand,
$iIndent,
$bCache,
$bShow,
) =
logDebugParam
(
@ -234,8 +235,9 @@ sub execute
{name => 'oSection'},
{name => 'strHostName'},
{name => 'oCommand'},
{name => 'iIndent', default => 1},
{name => 'bCache', default => true},
{name => 'iIndent', optional => true, default => 1},
{name => 'bCache', optional => true, default => true},
{name => 'bShow', optional => true, default => true},
);
# Working variables
@ -243,7 +245,7 @@ sub execute
my $strCommand = join("\n", @{$$hCacheKey{cmd}});
my $strOutput;
if (!$oCommand->paramTest('show', 'n') && $self->{bExe} && $self->isRequired($oSection))
if ($bShow && $self->{bExe} && $self->isRequired($oSection))
{
# Make sure that no lines are greater than 80 chars
foreach my $strLine (split("\n", $strCommand))
@ -624,7 +626,7 @@ sub backrestConfig
my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}});
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_LEVEL_STDERR});
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_TIMESTAMP});
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_REPO_S3_VERIFY_SSL});
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'});
if (keys(%{$$oConfigClean{&CFGDEF_SECTION_GLOBAL}}) == 0)
{
@ -1050,7 +1052,8 @@ sub sectionChildProcess
# Execute initialize commands
foreach my $oExecute ($oChild->nodeList('execute', false))
{
$self->execute($oSection, $$hCacheKey{name}, $oExecute, $iDepth + 1, false);
$self->execute(
$oSection, $$hCacheKey{name}, $oExecute, {iIndent => $iDepth + 1, bCache => false, bShow => false});
}
$self->cachePush($strCacheType, $hCacheKey, $hCacheValue);

View File

@ -287,25 +287,30 @@ sub sectionProcess
# Execute a command
if ($oChild->nameGet() eq 'execute-list')
{
my $oSectionBodyExecute = $oSectionBodyElement->addNew(HTML_DIV, "execute");
my $bShow = $oChild->paramTest('show', 'n') ? false : true;
my $oExecuteBodyElement;
my $bFirst = true;
my $strHostName = $self->{oManifest}->variableReplace($oChild->paramGet('host'));
$oSectionBodyExecute->
addNew(HTML_DIV, "execute-title",
{strContent => "<span class=\"host\">${strHostName}</span> <b>&#x21d2;</b> " .
$self->processText($oChild->nodeGet('title')->textGet())});
my $oExecuteBodyElement = $oSectionBodyExecute->addNew(HTML_DIV, "execute-body");
if ($bShow)
{
my $oSectionBodyExecute = $oSectionBodyElement->addNew(HTML_DIV, "execute");
$oSectionBodyExecute->
addNew(HTML_DIV, "execute-title",
{strContent => "<span class=\"host\">${strHostName}</span> <b>&#x21d2;</b> " .
$self->processText($oChild->nodeGet('title')->textGet())});
$oExecuteBodyElement = $oSectionBodyExecute->addNew(HTML_DIV, "execute-body");
}
foreach my $oExecute ($oChild->nodeList('execute'))
{
my $bExeShow = !$oExecute->paramTest('show', 'n');
my $bExeExpectedError = defined($oExecute->paramGet('err-expect', false));
my ($strCommand, $strOutput) = $self->execute($oSection, $strHostName, $oExecute, $iDepth + 3);
my ($strCommand, $strOutput) = $self->execute(
$oSection, $strHostName, $oExecute, {iIndent => $iDepth + 3, bShow => $bShow && $bExeShow});
if ($bExeShow)
if ($bShow && $bExeShow)
{
# Add continuation chars and proper spacing
$strCommand =~ s/\n/\n /smg;

View File

@ -160,20 +160,24 @@ sub sectionProcess
# Execute a command
if ($oChild->nameGet() eq 'execute-list')
{
my $bShow = $oChild->paramTest('show', 'n') ? false : true;
my $strHostName = $self->{oManifest}->variableReplace($oChild->paramGet('host'));
$strLatex .=
"\n\\begin\{lstlisting\}[title=\{\\textnormal{\\textbf\{${strHostName}}} --- " .
$self->processText($oChild->nodeGet('title')->textGet()) . "}]\n";
if ($bShow)
{
$strLatex .=
"\n\\begin\{lstlisting\}[title=\{\\textnormal{\\textbf\{${strHostName}}} --- " .
$self->processText($oChild->nodeGet('title')->textGet()) . "}]\n";
}
foreach my $oExecute ($oChild->nodeList('execute'))
{
my $bExeShow = !$oExecute->paramTest('show', 'n');
my ($strCommand, $strOutput) = $self->execute($oSection,
$self->{oManifest}->variableReplace($oChild->paramGet('host')),
$oExecute, $iDepth + 3);
my ($strCommand, $strOutput) = $self->execute(
$oSection, $self->{oManifest}->variableReplace($oChild->paramGet('host')), $oExecute,
{iIndent => $iDepth + 3, bShow => $bShow && $bExeShow});
if ($bExeShow)
if ($bShow && $bExeShow)
{
$strLatex .= "${strCommand}\n";
@ -184,8 +188,11 @@ sub sectionProcess
}
}
$strLatex .=
"\\end{lstlisting}\n";
if ($bShow)
{
$strLatex .=
"\\end{lstlisting}\n";
}
}
# Add code block
elsif ($oChild->nameGet() eq 'code-block')

View File

@ -57,6 +57,7 @@
<!ELEMENT execute-list (title, execute+)>
<!ATTLIST execute-list host CDATA #REQUIRED>
<!ATTLIST execute-list keyword CDATA "">
<!ATTLIST execute-list show CDATA "">
<!ELEMENT execute (exe-cmd, exe-cmd-extra?, exe-user?, exe-var?, exe-retry?, exe-output?, (exe-highlight-type?, exe-highlight)?,
exe-err-expect?, exe-err-suppress?, exe-err-suppress-stderr?)>
<!ATTLIST execute keyword CDATA "">

View File

@ -46,11 +46,11 @@
<release-doc-list>
<release-development-list>
<release-item>
<p>Build containers from scratch for more accurate testing.</p>
<p>Build containers from scratch for more accurate testing. Use a prebuilt s3 server container.</p>
</release-item>
<release-item>
<p>Document generator improvements. Allow parameters to be passed when a container is created. Allow <file>/etc/hosts</file> update to be skipped (for containers without bash). Allow environment load to be skipped. Allow bash wrapping to be skipped. Allow forcing a command to run as a user without sudo.</p>
<p>Document generator improvements. Allow parameters to be passed when a container is created. Allow <file>/etc/hosts</file> update to be skipped (for containers without bash). Allow environment load to be skipped. Allow bash wrapping to be skipped. Allow forcing a command to run as a user without sudo. Allow an entire execute list to be hidden.</p>
</release-item>
</release-development-list>
</release-doc-list>

View File

@ -241,16 +241,16 @@
<!-- ======================================================================================================================= -->
<block-define id="host-install">
<execute-list keyword="default" host="{[host-install-host]}">
<execute-list keyword="default" host="{[host-install-host]}" show="n">
<title>Update apt repository and install packages</title>
<!-- Fix root tty -->
<execute user="root" user-force="y" load-env="n" show="n">
<execute user="root" user-force="y" load-env="n">
<exe-cmd>sed -i 's/^mesg n/tty -s \&amp;\&amp; mesg n/g' /root/.profile</exe-cmd>
</execute>
<!-- Suppress dpkg interactive output -->
<execute user="root" user-force="y" show="n">
<execute user="root" user-force="y">
<exe-cmd>rm /etc/apt/apt.conf.d/70debconf</exe-cmd>
</execute>
@ -260,7 +260,7 @@
</execute>
<!-- Create an ssh key for root so all hosts can ssh to each other as root -->
<execute user="root" user-force="y" show="n">
<execute user="root" user-force="y">
<exe-cmd>{[ssh-key-install]}</exe-cmd>
</execute>
@ -276,19 +276,19 @@
</execute>
<!-- add doc user -->
<execute user="root" user-force="y" show="n">
<execute user="root" user-force="y">
<exe-cmd>adduser --disabled-password --gecos "" {[host-user]}</exe-cmd>
</execute>
<execute user="root" user-force="y" show="n">
<execute user="root" user-force="y">
<exe-cmd>echo '%{[host-user]} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers</exe-cmd>
</execute>
</execute-list>
<execute-list keyword="co6" host="{[host-install-host]}">
<execute-list keyword="co6" host="{[host-install-host]}" show="n">
<title>Update yum repository and install packages</title>
<!-- Create an ssh key for root so all hosts can ssh to each other as root -->
<execute user="root" user-force="y" show="n">
<execute user="root" user-force="y">
<exe-cmd>{[ssh-key-install]}</exe-cmd>
</execute>
@ -299,10 +299,10 @@
</execute>
<!-- add doc user -->
<execute user="root" user-force="y" show="n">
<execute user="root" user-force="y">
<exe-cmd>adduser -n {[host-user]}</exe-cmd>
</execute>
<execute user="root" user-force="y" show="n">
<execute user="root" user-force="y">
<exe-cmd>echo '{[host-user]} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/{[host-user]}</exe-cmd>
</execute>
@ -591,65 +591,10 @@
<title>Installation</title>
<!-- Create S3 server first to allow it time to boot before being used -->
<host-add id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="ubuntu:16.04" os="{[host-os]}">
<!-- Suppress dpkg interactive output -->
<execute user="root" user-force="y" load-env="n" show="n">
<exe-cmd>rm /etc/apt/apt.conf.d/70debconf</exe-cmd>
</execute>
<!-- Install OpenSSL -->
<execute user="root" user-force="y" load-env="n" show="n">
<exe-cmd>
apt-get update &amp;&amp;
apt-get -y install build-essential openssl wget git 2>&amp;1
</exe-cmd>
</execute>
<!-- Create self-signed cert -->
<execute user="root" user-force="y" load-env="n" show="n">
<exe-cmd>
mkdir -p -m 755 /etc/fake-cert &amp;&amp;
cd /etc/fake-cert &amp;&amp;
openssl genrsa -out ca.key 2048 2>&amp;1 &amp;&amp;
openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999
-subj "/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org" 2>&amp;1 &amp;&amp;
openssl genrsa -out server.key 2048 2>&amp;1 &amp;&amp;
openssl req -new -key server.key -out server.csr
-subj "/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org" 2>&amp;1 &amp;&amp;
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999
-sha256 2>&amp;1 &amp;&amp;
chmod 644 /etc/fake-cert/*
</exe-cmd>
</execute>
<!-- Install node.js -->
<execute user="root" user-force="y" load-env="n" show="n">
<exe-cmd>
wget -q -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x &amp;&amp;
bash /root/nodejs.sh &amp;&amp;
apt-get install -y nodejs
</exe-cmd>
</execute>
<!-- Install ScalityS3 -->
<execute user="root" user-force="y" load-env="n" show="n">
<exe-cmd>
wget -q -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz &amp;&amp;
mkdir /root/scalitys3 &amp;&amp;
tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz &amp;&amp;
cd /root/scalitys3 &amp;&amp;
npm install 2>&amp;1 &amp;&amp;
sed -i "0,/,/s//,\n \"certFilePaths\":{\"key\":\"\/etc\/fake\-cert\/server.key\",\"cert\":\"\/etc\/fake\-cert\/server.crt\",\"ca\":\"\/etc\/fake\-cert\/ca.crt\"},/"
./config.json &amp;&amp;
sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json
</exe-cmd>
</execute>
<!-- Run ScalityS3 -->
<execute user="root" user-force="y" load-env="n" bash-wrap="n" show="n">
<exe-cmd>
nohup npm start --prefix /root/scalitys3 &amp;
</exe-cmd>
<host-add id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="pgbackrest/test:s3-server-20180612A" os="{[host-os]}">
<!-- Set host entries to redirect AWS to local s3 server -->
<execute user="root" user-force="y">
<exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
</execute>
</host-add>
@ -1897,6 +1842,14 @@
<p><backrest/> supports storing repositories in <proper>Amazon S3</proper>. The bucket used to store the repository must be created in advance &amp;mdash; <backrest/> will not do it automatically.</p>
<execute-list host="{[host-s3]}" show="n">
<title>Create the bucket</title>
<execute show='n'>
<exe-cmd>aws s3 --no-verify-ssl mb s3://demo-bucket 2>&amp;1</exe-cmd>
</execute>
</execute-list>
<backrest-config host="{[host-pg1]}" show="n" file="{[backrest-config-demo]}">
<title>Clear the cipher settings</title>
@ -1924,49 +1877,9 @@
<execute-list host="{[host-pg1]}">
<title>Create the stanza</title>
<!-- Install AWS CLI -->
<execute keyword="default" user="root" show='n'>
<exe-cmd>apt-get -y install python 2>&amp;1</exe-cmd>
</execute>
<execute keyword="co6" user="root" show='n'>
<exe-cmd>yum -y install epel-release 2>&amp;1</exe-cmd>
</execute>
<execute keyword="co6" user="root" show='n'>
<exe-cmd>yum -y install python-pip 2>&amp;1</exe-cmd>
</execute>
<execute keyword="default" user="root" show='n'>
<exe-cmd>
wget -q --no-check-certificate -O /root/get-pip.py
https://bootstrap.pypa.io/get-pip.py
</exe-cmd>
</execute>
<execute keyword="default" user="root" show='n'>
<exe-cmd>-H python /root/get-pip.py</exe-cmd>
</execute>
<execute keyword="co6" user="root" show='n'>
<exe-cmd>pip install {[dash]}-upgrade {[dash]}-no-cache-dir pip==9.0.3 2>&amp;1</exe-cmd>
</execute>
<execute user="root" show='n'>
<exe-cmd>-H pip install --upgrade awscli 2>&amp;1</exe-cmd>
</execute>
<execute show='n'>
<exe-cmd>aws configure set region us-east-1</exe-cmd>
</execute>
<execute show='n'>
<exe-cmd>aws configure set aws_access_key_id accessKey1</exe-cmd>
</execute>
<execute show='n'>
<exe-cmd>aws configure set aws_secret_access_key verySecretKey1</exe-cmd>
</execute>
<!-- Set host entries to redirect AWS to local s3 server -->
<execute user="root" show='n'>
<exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" |
sudo tee -a /etc/hosts</exe-cmd>
</execute>
<execute show='n'>
<exe-cmd>aws s3 --no-verify-ssl mb s3://demo-bucket 2>&amp;1</exe-cmd>
<execute user="root" user-force="y" show="n">
<exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
</execute>
<execute user="postgres" output="y">

View File

@ -0,0 +1,50 @@
# S3 Server used for testing and documentation
# docker build -f s3-server.docker -t pgbackrest/test:s3-server-20180612A .
FROM ubuntu:18.04
# Suppress dpkg interactive output
RUN rm /etc/apt/apt.conf.d/70debconf && \
# Install base packages
apt-get update && \
apt-get -y install build-essential openssl wget git python-pip && \
# Fix root tty
sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile && \
# Generate fake certs
mkdir -p -m 755 /etc/fake-cert && \
cd /etc/fake-cert && \
openssl genrsa -out ca.key 2048 && \
openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 \
-subj "/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org" && \
openssl genrsa -out server.key 2048 && \
openssl req -new -key server.key -out server.csr \
-subj "/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org" && \
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 \
-sha256 && \
chmod 644 /etc/fake-cert/* && \
# Install AWS CLI
pip install --upgrade awscli && \
aws configure set region us-east-1 && \
aws configure set aws_access_key_id accessKey1 && \
aws configure set aws_secret_access_key verySecretKey1 && \
# Install node.js
wget -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x && \
bash /root/nodejs.sh && \
apt-get install -y nodejs npm && \
# Install Scality S3
wget -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz && \
mkdir /root/scalitys3 && \
tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz && \
cd /root/scalitys3 && \
npm install && \
sed -i "0,/,/s//,\n \"certFilePaths\":{\"key\":\"\/etc\/fake\-cert\/server.key\",\"cert\":\"\/etc\/fake\-cert\/server.crt\",\"ca\":\"\/etc\/fake\-cert\/ca.crt\"},/" \
./config.json && \
sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json
# Start SSH when container starts
ENTRYPOINT npm start --prefix /root/scalitys3

View File

@ -69,6 +69,11 @@ use constant CERT_FAKE_SERVER_KEY => CERT_FAKE
####################################################################################################################################
use constant CONTAINER_DEBUG => false;
####################################################################################################################################
# Container Debug - speeds container debugging by splitting each section into a separate intermediate container
####################################################################################################################################
use constant CONTAINER_S3_SERVER_TAG => 's3-server-20180612A';
####################################################################################################################################
# Generate Devel::Cover package name
####################################################################################################################################
@ -631,19 +636,8 @@ sub containerBuild
}
else
{
$strImageParent = $oVm->{&VM_U18}{&VM_IMAGE};
$strScript = sectionHeader() .
"# Install required packages\n" .
" apt-get update && \\\n" .
" apt-get install -y wget git gnupg";
$strScript .= certSetup();
$strScript .= s3ServerSetup(VM_U18);
$strScript .= sectionHeader() .
"# Fix root tty\n" .
" echo 'tty -s && mesg n || true' > /root/.profile";
$strImageParent = containerRepo() . ':' . CONTAINER_S3_SERVER_TAG;
$strScript = '';
}
$strScript .= "\n\nENTRYPOINT npm start --prefix /root/scalitys3";