Basic multi-architecture support for test containers.

The tests worked fine on multiple architectures, but would only run "bare metal", i.e. tests that required containers could not be run.

Enable basic multi-architecture support by allowing containers to be built using whatever architecture the host supports. Also allow cached containers to be defined for multiple architectures in container.yaml.

Add a Dockerfile which can be used as a container for other containers to provide a consistent development environment.

The primary goal is to allow development on Mac M1 but other architectures should find these improvements useful.

test/Dockerfile

@@ -0,0 +1,40 @@
+# Force remove all containers:
+# docker rm -f $(docker ps -a -q)
+# Force remove all images:
+# docker rmi -f $(docker images -q)
+#
+# export PGB=pgbackrest;export PGB_IMAGE=$PGB/test:u20-main;PGB_REPO=~/Documents/Code/$PGB
+# docker build -f $PGB_REPO/test/Dockerfile -t $PGB_IMAGE .
+# docker run --privileged -itd --name $PGB-test -h $PGB-test -v $PGB_REPO:/home/docker/$PGB $PGB_IMAGE
+
+FROM ubuntu:focal
+
+# Suppress "dpkg-reconfigure: unable to re-open stdin: No file or directory" warning
+RUN export DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive && \
+
+# Install packages
+    apt-get update && \
+    apt-get install -y \
+        sudo vim htop jq rsync sysstat \
+        libdbd-pg-perl libxml-checker-perl libyaml-perl \
+        devscripts build-essential lintian git cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev libxml2-dev liblz4-dev \
+        liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev bzip2 libbz2-dev pkg-config && \
+
+# Install Docker
+    groupadd -g5000 docker && \
+    curl -fsSL https://get.docker.com | sh && \
+
+# Create docker user
+    adduser --ingroup=docker -uid=5000 --disabled-password --gecos \"\" docker && \
+
+# Configure sudo
+    echo '%docker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers && \
+
+# Mount tmpfs at /home/docker/test for faster testing
+    sudo -u docker mkdir -m 770 /home/docker/test && \
+    echo 'tmpfs /home/docker/test tmpfs size=4096M 0 1' >> /etc/fstab
+
+USER docker
+WORKDIR /home/docker
+
+ENTRYPOINT sudo rm -f /var/run/docker.pid && sudo mount -a && sudo dockerd

test/container.yaml

@@ -12,14 +12,23 @@
 # - docker login -u pgbackrest
 # - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
 # **********************************************************************************************************************************
+20210503A:
+  aarch64:
+    u20: 00a309971d5566ff3eb0db92c66b4f4ef0566d36
+    co7: 41b9ffa64e5914753708eb3cdde3892010683fd7
+
 20210113A:
-  co7: 2c1e13990f92635cc7d959dcbe497b765861c2b2
+  x86_64:
+    co7: 2c1e13990f92635cc7d959dcbe497b765861c2b2
 
 20200924A:
-  u18: d95d53e642fc1cea4a2b8e935ea7d9739f7d1c46
+  x86_64:
+    u18: d95d53e642fc1cea4a2b8e935ea7d9739f7d1c46
 
 20200507A:
-  f32: 61792779061d2a675509c65bfa64b61b8d4cea17
+  x86_64:
+    f32: 61792779061d2a675509c65bfa64b61b8d4cea17
 
 20200505A:
-  u12: 8a88ab44aace049d7da5ca1094375ff8b9aeb7ab
+  x86_64:
+    u12: 8a88ab44aace049d7da5ca1094375ff8b9aeb7ab

test/lib/pgBackRestTest/Common/ContainerTest.pm

@@ -108,7 +108,8 @@ sub containerWrite
 
         foreach my $strBuild (reverse(keys(%{$hContainerCache})))
         {
-            if (defined($hContainerCache->{$strBuild}{$strOS}) && $hContainerCache->{$strBuild}{$strOS} eq $strScriptSha1)
+            if (defined($hContainerCache->{$strBuild}{hostArch()}{$strOS}) &&
+                $hContainerCache->{$strBuild}{hostArch()}{$strOS} eq $strScriptSha1)
             {
                 &log(INFO, "Using cached ${strTag}-${strBuild} image (${strScriptSha1}) ...");
 
@@ -142,7 +143,7 @@ sub groupCreate
     my $strName = shift;
     my $iId = shift;
 
-    return "groupadd -g${iId} ${strName}";
+    return "groupadd -f -g${iId} ${strName}";
 }
 
 sub userCreate
@@ -464,14 +465,14 @@ sub containerBuild
                 {
                     $strScript .=
                         "    rpm -ivh \\\n" .
-                        "        https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/" .
+                        "        https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-" . hostArch() . "/" .
                             "pgdg-redhat-repo-latest.noarch.rpm && \\\n";
                 }
                 elsif ($strOS eq VM_F32)
                 {
                     $strScript .=
                         "    rpm -ivh \\\n" .
-                        "        https://download.postgresql.org/pub/repos/yum/reporpms/F-32-x86_64/" .
+                        "        https://download.postgresql.org/pub/repos/yum/reporpms/F-32-" . hostArch() . "/" .
                             "pgdg-fedora-repo-latest.noarch.rpm && \\\n";
                 }
 

test/lib/pgBackRestTest/Common/VmTest.pm

@@ -15,6 +15,7 @@ use Exporter qw(import);
 
 use pgBackRestDoc::Common::Exception;
 use pgBackRestDoc::Common::Log;
+use pgBackRestDoc::Common::String;
 
 use pgBackRestTest::Common::DbVersion;
 
@@ -582,6 +583,29 @@ sub vmArchBits
 
 push @EXPORT, qw(vmArchBits);
 
+####################################################################################################################################
+# Get host architecture
+####################################################################################################################################
+my $strHostArch = undef;
+
+sub hostArch
+{
+    if (!defined($strHostArch))
+    {
+        $strHostArch = trim(`uname -m`);
+
+        # Mac M1 reports arm64 but we generally need aarch64 (which Linux reports)
+        if ($strHostArch eq 'arm64')
+        {
+            $strHostArch = 'aarch64';
+        }
+    }
+
+    return $strHostArch;
+}
+
+push @EXPORT, qw(hostArch);
+
 ####################################################################################################################################
 # Does the VM support libbacktrace?
 ####################################################################################################################################

test/test.pl

@@ -334,7 +334,7 @@ eval
     }
 
     logLevelSet(uc($strLogLevel), uc($strLogLevel), OFF, !$bNoLogTimestamp);
-    &log(INFO, "test begin - log level ${strLogLevel}");
+    &log(INFO, 'test begin on ' . hostArch() . " - log level ${strLogLevel}");
 
     if (@stryModuleTest != 0 && @stryModule != 1)
     {