Use a prebuilt s3 server container for documentation and tests.

2025-10-30 23:37:45 +02:00 · 2018-06-12 13:43:15 -04:00
parent 350b30fa49
commit d55e609959
8 changed files with 122 additions and 149 deletions
--- a/doc/lib/BackRestDoc/Common/DocExecute.pm
+++ b/doc/lib/BackRestDoc/Common/DocExecute.pm
@@ -227,6 +227,7 @@ sub execute
        $oCommand,
        $iIndent,
        $bCache,
+        $bShow,
    ) =
        logDebugParam
        (
@@ -234,8 +235,9 @@ sub execute
            {name => 'oSection'},
            {name => 'strHostName'},
            {name => 'oCommand'},
-            {name => 'iIndent', default => 1},
-            {name => 'bCache', default => true},
+            {name => 'iIndent', optional => true, default => 1},
+            {name => 'bCache', optional => true, default => true},
+            {name => 'bShow', optional => true, default => true},
        );

    # Working variables
@@ -243,7 +245,7 @@ sub execute
    my $strCommand = join("\n", @{$$hCacheKey{cmd}});
    my $strOutput;

-    if (!$oCommand->paramTest('show', 'n') && $self->{bExe} && $self->isRequired($oSection))
+    if ($bShow && $self->{bExe} && $self->isRequired($oSection))
    {
        # Make sure that no lines are greater than 80 chars
        foreach my $strLine (split("\n", $strCommand))
@@ -624,7 +626,7 @@ sub backrestConfig
            my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}});
            delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_LEVEL_STDERR});
            delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_TIMESTAMP});
-            delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_REPO_S3_VERIFY_SSL});
+            delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'});

            if (keys(%{$$oConfigClean{&CFGDEF_SECTION_GLOBAL}}) == 0)
            {
@@ -1050,7 +1052,8 @@ sub sectionChildProcess
                # Execute initialize commands
                foreach my $oExecute ($oChild->nodeList('execute', false))
                {
-                    $self->execute($oSection, $$hCacheKey{name}, $oExecute, $iDepth + 1, false);
+                    $self->execute(
+                        $oSection, $$hCacheKey{name}, $oExecute, {iIndent => $iDepth + 1, bCache => false, bShow => false});
                }

                $self->cachePush($strCacheType, $hCacheKey, $hCacheValue);
--- a/doc/lib/BackRestDoc/Html/DocHtmlPage.pm
+++ b/doc/lib/BackRestDoc/Html/DocHtmlPage.pm
@@ -287,25 +287,30 @@ sub sectionProcess
        # Execute a command
        if ($oChild->nameGet() eq 'execute-list')
        {
-            my $oSectionBodyExecute = $oSectionBodyElement->addNew(HTML_DIV, "execute");
+            my $bShow = $oChild->paramTest('show', 'n') ? false : true;
+            my $oExecuteBodyElement;
            my $bFirst = true;
            my $strHostName = $self->{oManifest}->variableReplace($oChild->paramGet('host'));

-            $oSectionBodyExecute->
-                addNew(HTML_DIV, "execute-title",
-                       {strContent => "<span class=\"host\">${strHostName}</span> <b>&#x21d2;</b> " .
-                                      $self->processText($oChild->nodeGet('title')->textGet())});
-
-            my $oExecuteBodyElement = $oSectionBodyExecute->addNew(HTML_DIV, "execute-body");
+            if ($bShow)
+            {
+                my $oSectionBodyExecute = $oSectionBodyElement->addNew(HTML_DIV, "execute");
+                $oSectionBodyExecute->
+                    addNew(HTML_DIV, "execute-title",
+                           {strContent => "<span class=\"host\">${strHostName}</span> <b>&#x21d2;</b> " .
+                                          $self->processText($oChild->nodeGet('title')->textGet())});
+                $oExecuteBodyElement = $oSectionBodyExecute->addNew(HTML_DIV, "execute-body");
+            }

            foreach my $oExecute ($oChild->nodeList('execute'))
            {
                my $bExeShow = !$oExecute->paramTest('show', 'n');
                my $bExeExpectedError = defined($oExecute->paramGet('err-expect', false));

-                my ($strCommand, $strOutput) = $self->execute($oSection, $strHostName, $oExecute, $iDepth + 3);
+                my ($strCommand, $strOutput) = $self->execute(
+                    $oSection, $strHostName, $oExecute, {iIndent => $iDepth + 3, bShow => $bShow && $bExeShow});

-                if ($bExeShow)
+                if ($bShow && $bExeShow)
                {
                    # Add continuation chars and proper spacing
                    $strCommand =~ s/\n/\n   /smg;
--- a/doc/lib/BackRestDoc/Latex/DocLatexSection.pm
+++ b/doc/lib/BackRestDoc/Latex/DocLatexSection.pm
@@ -160,20 +160,24 @@ sub sectionProcess
        # Execute a command
        if ($oChild->nameGet() eq 'execute-list')
        {
+            my $bShow = $oChild->paramTest('show', 'n') ? false : true;
            my $strHostName = $self->{oManifest}->variableReplace($oChild->paramGet('host'));

-            $strLatex .=
-                "\n\\begin\{lstlisting\}[title=\{\\textnormal{\\textbf\{${strHostName}}} --- " .
-                $self->processText($oChild->nodeGet('title')->textGet()) . "}]\n";
+            if ($bShow)
+            {
+                $strLatex .=
+                    "\n\\begin\{lstlisting\}[title=\{\\textnormal{\\textbf\{${strHostName}}} --- " .
+                    $self->processText($oChild->nodeGet('title')->textGet()) . "}]\n";
+            }

            foreach my $oExecute ($oChild->nodeList('execute'))
            {
                my $bExeShow = !$oExecute->paramTest('show', 'n');
-                my ($strCommand, $strOutput) = $self->execute($oSection,
-                                                              $self->{oManifest}->variableReplace($oChild->paramGet('host')),
-                                                              $oExecute, $iDepth + 3);
+                my ($strCommand, $strOutput) = $self->execute(
+                    $oSection, $self->{oManifest}->variableReplace($oChild->paramGet('host')), $oExecute,
+                    {iIndent => $iDepth + 3, bShow => $bShow && $bExeShow});

-                if ($bExeShow)
+                if ($bShow && $bExeShow)
                {
                    $strLatex .= "${strCommand}\n";

@@ -184,8 +188,11 @@ sub sectionProcess
                }
            }

-            $strLatex .=
-                "\\end{lstlisting}\n";
+            if ($bShow)
+            {
+                $strLatex .=
+                    "\\end{lstlisting}\n";
+            }
        }
        # Add code block
        elsif ($oChild->nameGet() eq 'code-block')
--- a/doc/xml/dtd/doc.dtd
+++ b/doc/xml/dtd/doc.dtd
@@ -57,6 +57,7 @@
 <!ELEMENT execute-list (title, execute+)>
    <!ATTLIST execute-list host CDATA #REQUIRED>
    <!ATTLIST execute-list keyword CDATA "">
+    <!ATTLIST execute-list show CDATA "">
 <!ELEMENT execute (exe-cmd, exe-cmd-extra?, exe-user?,  exe-var?, exe-retry?, exe-output?, (exe-highlight-type?, exe-highlight)?,
                   exe-err-expect?, exe-err-suppress?, exe-err-suppress-stderr?)>
    <!ATTLIST execute keyword CDATA "">
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -46,11 +46,11 @@
            <release-doc-list>
                <release-development-list>
                    <release-item>
-                        <p>Build containers from scratch for more accurate testing.</p>
+                        <p>Build containers from scratch for more accurate testing.  Use a prebuilt s3 server container.</p>
                    </release-item>

                    <release-item>
-                        <p>Document generator improvements.  Allow parameters to be passed when a container is created.  Allow <file>/etc/hosts</file> update to be skipped (for containers without bash).  Allow environment load to be skipped.  Allow bash wrapping to be skipped.  Allow forcing a command to run as a user without sudo.</p>
+                        <p>Document generator improvements.  Allow parameters to be passed when a container is created.  Allow <file>/etc/hosts</file> update to be skipped (for containers without bash).  Allow environment load to be skipped.  Allow bash wrapping to be skipped.  Allow forcing a command to run as a user without sudo.  Allow an entire execute list to be hidden.</p>
                    </release-item>
                </release-development-list>
            </release-doc-list>
--- a/doc/xml/user-guide.xml
+++ b/doc/xml/user-guide.xml
@@ -241,16 +241,16 @@

    <!-- ======================================================================================================================= -->
    <block-define id="host-install">
-        <execute-list keyword="default" host="{[host-install-host]}">
+        <execute-list keyword="default" host="{[host-install-host]}" show="n">
            <title>Update apt repository and install packages</title>

            <!-- Fix root tty -->
-            <execute user="root" user-force="y" load-env="n" show="n">
+            <execute user="root" user-force="y" load-env="n">
                <exe-cmd>sed -i 's/^mesg n/tty -s \&amp;\&amp; mesg n/g' /root/.profile</exe-cmd>
            </execute>

            <!-- Suppress dpkg interactive output -->
-            <execute user="root" user-force="y" show="n">
+            <execute user="root" user-force="y">
                <exe-cmd>rm /etc/apt/apt.conf.d/70debconf</exe-cmd>
            </execute>

@@ -260,7 +260,7 @@
            </execute>

            <!-- Create an ssh key for root so all hosts can ssh to each other as root -->
-            <execute user="root" user-force="y" show="n">
+            <execute user="root" user-force="y">
                <exe-cmd>{[ssh-key-install]}</exe-cmd>
            </execute>

@@ -276,19 +276,19 @@
            </execute>

            <!-- add doc user -->
-            <execute user="root" user-force="y" show="n">
+            <execute user="root" user-force="y">
                <exe-cmd>adduser --disabled-password --gecos "" {[host-user]}</exe-cmd>
            </execute>
-            <execute user="root" user-force="y" show="n">
+            <execute user="root" user-force="y">
                <exe-cmd>echo '%{[host-user]} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers</exe-cmd>
            </execute>
        </execute-list>

-        <execute-list keyword="co6" host="{[host-install-host]}">
+        <execute-list keyword="co6" host="{[host-install-host]}" show="n">
            <title>Update yum repository and install packages</title>

            <!-- Create an ssh key for root so all hosts can ssh to each other as root -->
-            <execute user="root" user-force="y" show="n">
+            <execute user="root" user-force="y">
                <exe-cmd>{[ssh-key-install]}</exe-cmd>
            </execute>

@@ -299,10 +299,10 @@
            </execute>

            <!-- add doc user -->
-            <execute user="root" user-force="y" show="n">
+            <execute user="root" user-force="y">
                <exe-cmd>adduser -n {[host-user]}</exe-cmd>
            </execute>
-            <execute user="root" user-force="y" show="n">
+            <execute user="root" user-force="y">
                <exe-cmd>echo '{[host-user]}        ALL=(ALL)       NOPASSWD: ALL' > /etc/sudoers.d/{[host-user]}</exe-cmd>
            </execute>

@@ -591,65 +591,10 @@
        <title>Installation</title>

        <!-- Create S3 server first to allow it time to boot before being used -->
-        <host-add id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="ubuntu:16.04" os="{[host-os]}">
-            <!-- Suppress dpkg interactive output -->
-            <execute user="root" user-force="y" load-env="n" show="n">
-                <exe-cmd>rm /etc/apt/apt.conf.d/70debconf</exe-cmd>
-            </execute>
-
-            <!-- Install OpenSSL -->
-            <execute user="root" user-force="y" load-env="n" show="n">
-                <exe-cmd>
-                    apt-get update &amp;&amp;
-                    apt-get -y install build-essential openssl wget git 2>&amp;1
-                </exe-cmd>
-            </execute>
-
-            <!-- Create self-signed cert -->
-            <execute user="root" user-force="y" load-env="n" show="n">
-                <exe-cmd>
-                    mkdir -p -m 755 /etc/fake-cert &amp;&amp;
-                    cd /etc/fake-cert &amp;&amp;
-                    openssl genrsa -out ca.key 2048 2>&amp;1 &amp;&amp;
-                    openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999
-                        -subj "/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org" 2>&amp;1 &amp;&amp;
-                    openssl genrsa -out server.key 2048 2>&amp;1 &amp;&amp;
-                    openssl req -new -key server.key -out server.csr
-                        -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org" 2>&amp;1 &amp;&amp;
-                    openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999
-                        -sha256 2>&amp;1 &amp;&amp;
-                    chmod 644 /etc/fake-cert/*
-                </exe-cmd>
-            </execute>
-
-            <!-- Install node.js -->
-            <execute user="root" user-force="y" load-env="n" show="n">
-                <exe-cmd>
-                    wget -q -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x &amp;&amp;
-                    bash /root/nodejs.sh  &amp;&amp;
-                    apt-get install -y nodejs
-                </exe-cmd>
-            </execute>
-
-            <!-- Install ScalityS3 -->
-            <execute user="root" user-force="y" load-env="n" show="n">
-                <exe-cmd>
-                    wget -q -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz &amp;&amp;
-                    mkdir /root/scalitys3 &amp;&amp;
-                    tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz &amp;&amp;
-                    cd /root/scalitys3 &amp;&amp;
-                    npm install  2>&amp;1 &amp;&amp;
-                    sed -i "0,/,/s//,\n    \"certFilePaths\":{\"key\":\"\/etc\/fake\-cert\/server.key\",\"cert\":\"\/etc\/fake\-cert\/server.crt\",\"ca\":\"\/etc\/fake\-cert\/ca.crt\"},/"
-                        ./config.json &amp;&amp;
-                    sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json
-                </exe-cmd>
-            </execute>
-
-            <!-- Run ScalityS3 -->
-            <execute user="root" user-force="y" load-env="n" bash-wrap="n" show="n">
-                <exe-cmd>
-                    nohup npm start --prefix /root/scalitys3 &amp;
-                </exe-cmd>
+        <host-add id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="pgbackrest/test:s3-server-20180612A" os="{[host-os]}">
+            <!-- Set host entries to redirect AWS to local s3 server -->
+            <execute user="root" user-force="y">
+                <exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
            </execute>
        </host-add>

@@ -1897,6 +1842,14 @@

        <p><backrest/> supports storing repositories in <proper>Amazon S3</proper>.  The bucket used to store the repository must be created in advance &amp;mdash; <backrest/> will not do it automatically.</p>

+        <execute-list host="{[host-s3]}" show="n">
+            <title>Create the bucket</title>
+
+            <execute show='n'>
+                <exe-cmd>aws s3 --no-verify-ssl mb s3://demo-bucket 2>&amp;1</exe-cmd>
+            </execute>
+        </execute-list>
+
        <backrest-config host="{[host-pg1]}" show="n" file="{[backrest-config-demo]}">
            <title>Clear the cipher settings</title>

@@ -1924,49 +1877,9 @@
        <execute-list host="{[host-pg1]}">
            <title>Create the stanza</title>

-            <!-- Install AWS CLI -->
-            <execute keyword="default" user="root" show='n'>
-                <exe-cmd>apt-get -y install python 2>&amp;1</exe-cmd>
-            </execute>
-            <execute keyword="co6" user="root" show='n'>
-                <exe-cmd>yum -y install epel-release 2>&amp;1</exe-cmd>
-            </execute>
-            <execute keyword="co6" user="root" show='n'>
-                <exe-cmd>yum -y install python-pip 2>&amp;1</exe-cmd>
-            </execute>
-            <execute keyword="default" user="root" show='n'>
-                <exe-cmd>
-                    wget -q --no-check-certificate -O /root/get-pip.py
-                    https://bootstrap.pypa.io/get-pip.py
-                </exe-cmd>
-            </execute>
-            <execute keyword="default" user="root" show='n'>
-                <exe-cmd>-H python /root/get-pip.py</exe-cmd>
-            </execute>
-            <execute keyword="co6" user="root" show='n'>
-                <exe-cmd>pip install {[dash]}-upgrade {[dash]}-no-cache-dir pip==9.0.3 2>&amp;1</exe-cmd>
-            </execute>
-            <execute user="root" show='n'>
-                <exe-cmd>-H pip install --upgrade awscli 2>&amp;1</exe-cmd>
-            </execute>
-            <execute show='n'>
-                <exe-cmd>aws configure set region us-east-1</exe-cmd>
-            </execute>
-            <execute show='n'>
-                <exe-cmd>aws configure set aws_access_key_id accessKey1</exe-cmd>
-            </execute>
-            <execute show='n'>
-                <exe-cmd>aws configure set aws_secret_access_key verySecretKey1</exe-cmd>
-            </execute>
-
            <!-- Set host entries to redirect AWS to local s3 server -->
-            <execute user="root" show='n'>
-                <exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" |
-                    sudo tee -a /etc/hosts</exe-cmd>
-            </execute>
-
-            <execute show='n'>
-                <exe-cmd>aws s3 --no-verify-ssl mb s3://demo-bucket 2>&amp;1</exe-cmd>
+            <execute user="root" user-force="y" show="n">
+                <exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
            </execute>

            <execute user="postgres" output="y">
--- a/test/docker/s3-server.docker
+++ b/test/docker/s3-server.docker
@@ -0,0 +1,50 @@
+# S3 Server used for testing and documentation
+# docker build -f s3-server.docker -t pgbackrest/test:s3-server-20180612A .
+FROM ubuntu:18.04
+
+# Suppress dpkg interactive output
+RUN rm /etc/apt/apt.conf.d/70debconf && \
+
+# Install base packages
+    apt-get update && \
+    apt-get -y install build-essential openssl wget git python-pip && \
+
+# Fix root tty
+    sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile && \
+
+# Generate fake certs
+    mkdir -p -m 755 /etc/fake-cert && \
+    cd /etc/fake-cert && \
+    openssl genrsa -out ca.key 2048 && \
+    openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 \
+        -subj "/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org" && \
+    openssl genrsa -out server.key 2048 && \
+    openssl req -new -key server.key -out server.csr \
+        -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org" && \
+    openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 \
+        -sha256 && \
+    chmod 644 /etc/fake-cert/* && \
+
+# Install AWS CLI
+    pip install --upgrade awscli && \
+    aws configure set region us-east-1 && \
+    aws configure set aws_access_key_id accessKey1 && \
+    aws configure set aws_secret_access_key verySecretKey1 && \
+
+# Install node.js
+    wget -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x && \
+    bash /root/nodejs.sh && \
+    apt-get install -y nodejs npm && \
+
+# Install Scality S3
+    wget -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz && \
+    mkdir /root/scalitys3 && \
+    tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz && \
+    cd /root/scalitys3 && \
+    npm install && \
+    sed -i "0,/,/s//,\n    \"certFilePaths\":{\"key\":\"\/etc\/fake\-cert\/server.key\",\"cert\":\"\/etc\/fake\-cert\/server.crt\",\"ca\":\"\/etc\/fake\-cert\/ca.crt\"},/" \
+        ./config.json && \
+    sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json
+
+# Start SSH when container starts
+ENTRYPOINT npm start --prefix /root/scalitys3
--- a/test/lib/pgBackRestTest/Common/ContainerTest.pm
+++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm
@@ -69,6 +69,11 @@ use constant CERT_FAKE_SERVER_KEY                                   => CERT_FAKE
 ####################################################################################################################################
 use constant CONTAINER_DEBUG                                        => false;

+####################################################################################################################################
+# Container Debug - speeds container debugging by splitting each section into a separate intermediate container
+####################################################################################################################################
+use constant CONTAINER_S3_SERVER_TAG                                => 's3-server-20180612A';
+
 ####################################################################################################################################
 # Generate Devel::Cover package name
 ####################################################################################################################################
@@ -631,19 +636,8 @@ sub containerBuild
            }
            else
            {
-                $strImageParent = $oVm->{&VM_U18}{&VM_IMAGE};
-
-                $strScript = sectionHeader() .
-                    "# Install required packages\n" .
-                    "    apt-get update && \\\n" .
-                    "    apt-get install -y wget git gnupg";
-
-                $strScript .= certSetup();
-                $strScript .= s3ServerSetup(VM_U18);
-
-                $strScript .= sectionHeader() .
-                    "# Fix root tty\n" .
-                    "    echo 'tty -s && mesg n || true' > /root/.profile";
+                $strImageParent = containerRepo() . ':' . CONTAINER_S3_SERVER_TAG;
+                $strScript = '';
            }

            $strScript .= "\n\nENTRYPOINT npm start --prefix /root/scalitys3";