From 1afea449e2f54e69f9f8f0d772a294fcf09e0d83 Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 2 Sep 2021 13:26:21 -0400 Subject: [PATCH] Build newer valgrind and move CA cert install in test containers. The newer version of valgrind helps with some arm64 issues that have been fixed since the architecture has become more popular. Also add the valgrind builds to the Vagrantfile and Dockerfile. Move the CA cert install from the base container to the test container. This means the CA cert can be changed without rebuilding all the base containers. --- test/Dockerfile | 10 +- test/Vagrantfile | 15 ++- test/container.yaml | 19 ++- .../pgBackRestTest/Common/ContainerTest.pm | 108 ++++++++++++------ 4 files changed, 97 insertions(+), 55 deletions(-) diff --git a/test/Dockerfile b/test/Dockerfile index 6fe39680b..8f7356633 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -9,7 +9,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ sudo vim htop jq rsync sysstat curl \ libdbd-pg-perl libxml-checker-perl libyaml-perl \ devscripts build-essential lintian git cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev libxml2-dev liblz4-dev \ - liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev bzip2 libbz2-dev pkg-config libyaml-dev + liblz4-tool libpq-dev lcov autoconf-archive zstd libzstd-dev bzip2 libbz2-dev pkg-config libyaml-dev wget # Install Docker RUN groupadd -g5000 docker @@ -18,6 +18,14 @@ RUN curl -fsSL https://get.docker.com | sh # Create docker user RUN adduser --ingroup=docker -uid=5000 --disabled-password --gecos \"\" docker +# Build valgrind +RUN export VALGRIND=valgrind-3.17.0 && \ + wget -q -O - https://sourceware.org/pub/valgrind/${VALGRIND?}.tar.bz2 | tar jx -C /root && \ + cd /root/${VALGRIND?} && \ + ./configure --silent && \ + make -s -j8 install && \ + rm -rf /root/${VALGRIND?} + # Configure sudo RUN echo '%docker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers diff --git a/test/Vagrantfile b/test/Vagrantfile index b37499d36..aa253df3a 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -75,8 +75,8 @@ Vagrant.configure(2) do |config| #----------------------------------------------------------------------------------------------------------------------- echo 'Install Build Tools' && date apt-get install -y devscripts build-essential lintian git cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \ - libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev bzip2 libbz2-dev \ - pkg-config libyaml-dev + libxml2-dev liblz4-dev liblz4-tool libpq-dev lcov autoconf-archive zstd libzstd-dev bzip2 libbz2-dev pkg-config \ + libyaml-dev #----------------------------------------------------------------------------------------------------------------------- echo 'Install Docker' && date @@ -88,8 +88,13 @@ Vagrant.configure(2) do |config| apt-get install -y vim htop jq #----------------------------------------------------------------------------------------------------------------------- - echo 'create link to repo in old location to make the transition less painful' && date - ln -s /home/vagrant/pgbackrest /backrest + echo 'Build valgrind' && date + export VALGRIND=valgrind-3.17.0 + wget -q -O - https://sourceware.org/pub/valgrind/${VALGRIND?}.tar.bz2 | tar jx -C /root + cd /root/${VALGRIND?} + ./configure + make -j8 install + rm -rf /root/${VALGRIND?} #----------------------------------------------------------------------------------------------------------------------- # echo 'Install TeX Live' && date @@ -99,7 +104,7 @@ Vagrant.configure(2) do |config| #----------------------------------------------------------------------------------------------------------------------- echo 'Build VM Images' && date - sudo su - vagrant -c '/backrest/test/test.pl --vm-build' + sudo su - vagrant -c '/home/vagrant/pgbackrest/test/test.pl --vm-build' #----------------------------------------------------------------------------------------------------------------------- echo 'Build End' && date diff --git a/test/container.yaml b/test/container.yaml index b136e8061..cdd93b8ef 100644 --- a/test/container.yaml +++ b/test/container.yaml @@ -12,17 +12,14 @@ # - docker login -u pgbackrest # - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?} # ********************************************************************************************************************************** -20210819A: +20210902A: x86_64: - co7: 03a73386264355a5d1750ccd13371db97d92ab28 - u16: 056645ea79dd0538fa35e91af2fe65dc79c68c1b - u20: cff054ccddd944f85e69bdf2c122b4890c60d6b7 + co7: c4d02428812374ab6a22efa437499592ae9c5f08 + f33: 04b3353c5fcc859c9df528e2fa78e6ed73c6390b + u16: 3c872cddfdf2534e32c074dfae1ec25295237b5c + u20: d81b5a7e496290710c4dbef7a7f07ae60ae9619f -20210818A: +20210901A: aarch64: - u20: cff054ccddd944f85e69bdf2c122b4890c60d6b7 - co7: 8726f3be088e4ba34eb45e32c1994a32ca0e5c1d - -20210717A: - x86_64: - f33: 80326dab5c7d0137062b8d181ca07f9b14142de7 + u20: d81b5a7e496290710c4dbef7a7f07ae60ae9619f + co7: a590bc38a94217b3bc9fe00a1051850c9deef145 diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index bab53adf8..073f68b81 100755 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -222,44 +222,70 @@ sub sshSetup } #################################################################################################################################### -# Cert Setup +# Copy text file into container. Note that this will not work if the file contains single quotes. #################################################################################################################################### -sub certSetup +sub fileCopy +{ + my $oStorage = shift; + my $strSourceFile = shift; + my $strDestFile = shift; + + my $strScript; + + foreach my $strLine (split("\n", ${$oStorage->get($strSourceFile)})) + { + $strScript .= " echo '${strLine}' " . (defined($strScript) ? '>>' : '>') . " ${strDestFile} && \\\n"; + } + + return $strScript; +} + +#################################################################################################################################### +# CA Setup +#################################################################################################################################### +sub caSetup { my $strOS = shift; + my $oStorage = shift; + my $strCaFile = shift; - my $strScript = - sectionHeader() . - "# Generate fake certs\n" . - " mkdir -p -m 755 /etc/fake-cert && \\\n" . - " cd /etc/fake-cert && \\\n" . - " openssl genrsa -out ca.key 2048 && \\\n" . - " openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 \\\n" . - " -subj \"/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org\" && \\\n" . - " openssl genrsa -out server.key 2048 && \\\n" . - " openssl req -new -key server.key -out server.csr \\\n" . - " -subj \"/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org\" && \\\n" . - " openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 \\\n" . - " -sha256 && \\\n" . - " chmod 644 /etc/fake-cert/* && \\\n"; + my $strOsBase = vmGet()->{$strOS}{&VM_OS_BASE}; - my $rhVm = vmGet(); + # Determine CA location + my $strCertFile = undef; - if ($rhVm->{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL) + if ($strOsBase eq VM_OS_BASE_RHEL) { - $strScript .= - " cp /etc/fake-cert/pgbackrest-test-ca.crt /etc/pki/ca-trust/source/anchors && \\\n" . - " update-ca-trust extract"; + $strCertFile = '/etc/pki/ca-trust/source/anchors'; } - elsif ($rhVm->{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) + elsif ($strOsBase eq VM_OS_BASE_DEBIAN) { - $strScript .= - " cp /etc/fake-cert/pgbackrest-test-ca.crt /usr/local/share/ca-certificates && \\\n" . - " update-ca-certificates"; + $strCertFile = '/usr/local/share/ca-certificates'; } else { - confess &log(ERROR, "unable to install certificate for $rhVm->{$strOS}{&VM_OS_BASE}"); + confess &log(ERROR, "unable to install CA for ${strOsBase}"); + } + + $strCertFile .= '/pgbackrest-test-ca.crt'; + + # Write CA + my $strScript = + sectionHeader() . + "# Install CA\n" . + fileCopy($oStorage, $strCaFile, $strCertFile) . + " chmod 644 ${strCertFile} && \\\n"; + + # Install CA + if ($strOsBase eq VM_OS_BASE_RHEL) + { + $strScript .= + " update-ca-trust extract"; + } + elsif ($strOsBase eq VM_OS_BASE_DEBIAN) + { + $strScript .= + " update-ca-certificates"; } return $strScript; @@ -370,6 +396,12 @@ sub containerBuild " libyaml-libyaml-perl tzdata devscripts lintian libxml-checker-perl txt2man debhelper \\\n" . " libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev pkg-config \\\n" . " libbz2-dev bzip2 libyaml-dev libjson-pp-perl liblz4-dev liblz4-tool gnupg"; + + # This package is required to build valgrind on 32-bit + if ($oVm->{$strOS}{&VM_ARCH} eq VM_ARCH_I386) + { + $strScript .= " g++-multilib"; + } } # Add zst command-line tool and development libs when available @@ -409,18 +441,15 @@ sub containerBuild } #--------------------------------------------------------------------------------------------------------------------------- - my $strCertPath = 'test/certificate'; - my $strCertName = 'pgbackrest-test'; + my $strValgrind = 'valgrind-3.17.0'; - $strCopy = '# Copy Test Certificates'; - - foreach my $strFile ('-ca.crt', '.crt', '.key') - { - $oStorageDocker->copy("${strCertPath}/${strCertName}${strFile}", "${strTempPath}/${strCertName}${strFile}"); - $strCopy .= "\nCOPY ${strCertName}${strFile} " . CERT_FAKE_PATH . "/${strCertName}${strFile}"; - } - - $strScript .= certSetup($strOS); + $strScript .= sectionHeader() . + "# Build valgrind\n" . + " wget -q -O - https://sourceware.org/pub/valgrind/${strValgrind}.tar.bz2 | tar jx -C /root && \\\n" . + " cd /root/${strValgrind} && \\\n" . + " ./configure --silent && \\\n" . + " make -s -j8 install && \\\n" . + " rm -rf /root/${strValgrind}"; #--------------------------------------------------------------------------------------------------------------------------- if (defined($oVm->{$strOS}{&VMDEF_LCOV_VERSION})) @@ -547,7 +576,10 @@ sub containerBuild $strCopy = undef; $strScript = ''; - #--------------------------------------------------------------------------------------------------------------------------- + #----------------------------------------------------------------------------------------------------------------------- + $strScript .= caSetup($strOS, $oStorageDocker, "test/certificate/pgbackrest-test-ca.crt"); + + #----------------------------------------------------------------------------------------------------------------------- $strScript .= sectionHeader() . "# Create banner to make sure pgBackRest ignores it\n" . " echo '***********************************************' > /etc/issue.net && \\\n" .