diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 384f086f3..03978d877 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -123,6 +123,24 @@ + + + + +

Containers now use a squid proxy for apt/yum to speed builds.

+ + + +

Obsolete containers are removed by the --vm-force option.

+ + + + + +

Greatly reduced the quantity of Docker containers built by default. Containers are only built for versions specified in db-minimal and those required to build documentation. Additional containers can be built with --db-version=all or by specifying a version, e.g. --db-version=9.4.

+ + + diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index 62f31d63d..f4e5ab70e 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -11,7 +11,7 @@ Debian/Ubuntu RHEL/CentOS 6 - 9.4 + 9.4 /usr/share/perl5 /usr/bin @@ -52,17 +52,17 @@ db-master {[host-user]} - {[image-user]}/{[host-os]}-db-doc + {[image-user]}/{[host-os]}-db-{[pg-version]}-doc-pre {[host-mount]} db-standby - {[host-user]} - {[image-user]}/{[host-os]}-db-doc + {[host-db-master-user]} + {[host-db-master-image]} {[host-mount]} backup {[host-user]} - {[image-user]}/{[host-os]}-backup-doc + {[image-user]}/{[host-os]}-backup-doc-pre {[host-mount]} diff --git a/test/Vagrantfile b/test/Vagrantfile index 874b6ad70..54152f780 100644 --- a/test/Vagrantfile +++ b/test/Vagrantfile @@ -24,7 +24,25 @@ Vagrant.configure(2) do |config| sudo usermod -aG docker vagrant # Install Perl modules - apt-get install -y libdbd-pg-perl libxml-checker-perl ghostscript libperl-critic-perl vim + apt-get install -y libdbd-pg-perl libxml-checker-perl libperl-critic-perl + + # Install additional modules + apt-get install -y vim htop + + # Install squid proxy (to speed container builds) + apt-get install -y squid3 + mkdir -m 700 /var/lib/squid + chown proxy /var/lib/squid + cp /etc/squid/squid.conf /etc/squid/squid.conf.old + echo 'http_port 172.17.0.1:3128' > /etc/squid/squid.conf + echo 'maximum_object_size 100 MB' >> /etc/squid/squid.conf + echo 'cache_replacement_policy heap LFUDA' >> /etc/squid/squid.conf + echo 'refresh_pattern . 0 20% 4320' >> /etc/squid/squid.conf + echo 'acl docker_network src 172.17.0.0/16' >> /etc/squid/squid.conf + echo 'http_access allow docker_network' >> /etc/squid/squid.conf + echo 'cache_dir ufs /var/lib/squid 4096 16 256' >> /etc/squid/squid.conf + echo 'cache_mem 16 MB' >> /etc/squid/squid.conf + service squid restart # Install Texlive mkdir /root/texlive @@ -40,8 +58,18 @@ Vagrant.configure(2) do |config| /usr/local/texlive/2016/bin/x86_64-linux/tlmgr install caption xcolor listings parskip helvetic ltablex titlesec \ epstopdf courier sectsty pgf ms + apt-get install -y ghostscript + + # Create backrest user and postgres group + groupadd -g5000 postgres + adduser --uid=5001 --ingroup=postgres --disabled-password --gecos "" backrest + + # Make postgres the primary group for vagrant user (and preserve vagrant group) + usermod -g postgres vagrant + usermod -a -G vagrant vagrant + # Build VM images - sudo -u vagrant /backrest/test/test.pl --vm-build + sudo -u vagrant /backrest/test/test.pl --vm-build --vm-force SHELL # Don't share the default vagrant folder diff --git a/test/lib/pgBackRestTest/Common/LogTest.pm b/test/lib/pgBackRestTest/Common/LogTest.pm index 26309575d..2a3449825 100644 --- a/test/lib/pgBackRestTest/Common/LogTest.pm +++ b/test/lib/pgBackRestTest/Common/LogTest.pm @@ -66,7 +66,7 @@ sub new {name => 'strComment', trace => true}, {name => 'strCommandMain', trace => true}, {name => 'strCommandRemote', trace => true}, - {name => 'strPgSqlBin', trace => true}, + {name => 'strPgSqlBin', required => false, trace => true}, {name => 'strTestPath', trace => true}, {name => 'strRepoPath', trace => true} ); @@ -333,9 +333,13 @@ sub regExpReplaceAll $strLine =~ s/$self->{strCommandMain}/[BACKREST_BIN]/g; $strLine =~ s/$self->{strCommandRemote}/[BACKREST_BIN]/g; - $strLine =~ s/$self->{strPgSqlBin}/[PGSQL_BIN_PATH]/g; $strLine =~ s/$self->{strTestPath}/[TEST_PATH]/g; + if (defined($self->{strPgSqlBin})) + { + $strLine =~ s/$self->{strPgSqlBin}/[PGSQL_BIN_PATH]/g; + } + $strLine = $self->regExpReplace($strLine, 'BACKREST_NAME_VERSION', '^' . BACKREST_NAME . ' ' . BACKREST_VERSION, undef, false); diff --git a/test/lib/pgBackRestTest/Common/VmTest.pm b/test/lib/pgBackRestTest/Common/VmTest.pm index 6fd7dd55c..9547ca63d 100644 --- a/test/lib/pgBackRestTest/Common/VmTest.pm +++ b/test/lib/pgBackRestTest/Common/VmTest.pm @@ -21,6 +21,12 @@ use Exporter qw(import); #################################################################################################################################### # VM hash keywords #################################################################################################################################### +use constant VM_DB => 'db'; + push @EXPORT, qw(VM_DB); +use constant VM_DB_DOC => 'db-doc'; + push @EXPORT, qw(VM_DB_DOC); +use constant VM_DB_MINIMAL => 'db-minimal'; + push @EXPORT, qw(VM_DB_MINIMAL); use constant VM_IMAGE => 'image'; push @EXPORT, qw(VM_IMAGE); use constant VM_OS => 'os'; @@ -29,6 +35,8 @@ use constant VM_OS_BASE => 'os-base' push @EXPORT, qw(VM_OS_BASE); use constant VM_OS_REPO => 'os-repo'; push @EXPORT, qw(VM_OS_REPO); +use constant VMDEF_PGSQL_BIN => 'pgsql-bin'; + push @EXPORT, qw(VMDEF_PGSQL_BIN); #################################################################################################################################### # Valid OS base List @@ -72,8 +80,9 @@ my $oyVm = &VM_OS_BASE => VM_OS_BASE_RHEL, &VM_OS => VM_OS_CENTOS, &VM_IMAGE => 'centos:6', + &VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin', - db => + &VM_DB => [ PG_VERSION_90, PG_VERSION_91, @@ -84,11 +93,16 @@ my $oyVm = PG_VERSION_96, ], - db_minimal => + &VM_DB_MINIMAL => [ PG_VERSION_90, PG_VERSION_91, ], + + &VM_DB_DOC => + [ + PG_VERSION_94, + ], }, # CentOS 7 @@ -97,8 +111,9 @@ my $oyVm = &VM_OS_BASE => VM_OS_BASE_RHEL, &VM_OS => VM_OS_CENTOS, &VM_IMAGE => 'centos:7', + &VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin', - db => + &VM_DB => [ PG_VERSION_93, PG_VERSION_94, @@ -106,7 +121,7 @@ my $oyVm = PG_VERSION_96, ], - db_minimal => + &VM_DB_MINIMAL => [ PG_VERSION_95, PG_VERSION_96, @@ -120,8 +135,9 @@ my $oyVm = &VM_OS => VM_OS_DEBIAN, &VM_OS_REPO => 'jessie', &VM_IMAGE => 'debian:8', + &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin', - db => + &VM_DB => [ PG_VERSION_84, PG_VERSION_90, @@ -133,7 +149,7 @@ my $oyVm = PG_VERSION_96, ], - db_minimal => + &VM_DB_MINIMAL => [ PG_VERSION_84, PG_VERSION_93, @@ -147,8 +163,9 @@ my $oyVm = &VM_OS => VM_OS_UBUNTU, &VM_OS_REPO => 'precise', &VM_IMAGE => 'ubuntu:12.04', + &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin', - db => + &VM_DB => [ PG_VERSION_83, PG_VERSION_84, @@ -161,7 +178,7 @@ my $oyVm = PG_VERSION_96, ], - db_minimal => + &VM_DB_MINIMAL => [ PG_VERSION_83, ], @@ -174,8 +191,9 @@ my $oyVm = &VM_OS => VM_OS_UBUNTU, &VM_OS_REPO => 'trusty', &VM_IMAGE => 'ubuntu:14.04', + &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin', - db => + &VM_DB => [ PG_VERSION_90, PG_VERSION_91, @@ -186,10 +204,15 @@ my $oyVm = PG_VERSION_96, ], - db_minimal => + &VM_DB_MINIMAL => [ PG_VERSION_92, ], + + &VM_DB_DOC => + [ + PG_VERSION_94, + ], }, # Ubuntu 16.04 @@ -199,8 +222,9 @@ my $oyVm = &VM_OS => VM_OS_UBUNTU, &VM_OS_REPO => 'xenial', &VM_IMAGE => 'ubuntu:16.04', + &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin', - db => + &VM_DB => [ PG_VERSION_91, PG_VERSION_92, @@ -210,7 +234,7 @@ my $oyVm = PG_VERSION_96, ], - db_minimal => + &VM_DB_MINIMAL => [ PG_VERSION_94, ], diff --git a/test/lib/pgBackRestTest/CommonTest.pm b/test/lib/pgBackRestTest/CommonTest.pm index ac537a4f0..2772d82a0 100755 --- a/test/lib/pgBackRestTest/CommonTest.pm +++ b/test/lib/pgBackRestTest/CommonTest.pm @@ -428,7 +428,6 @@ sub BackRestTestCommon_Setup $strCommonCommandRemote = defined($strExe) ? $strExe : "${strCommonBasePath}/bin/pgbackrest"; $strCommonCommandRemoteFull = "${strCommonCommandRemote} --stanza=${strCommonStanza}" . " --repo-path=${strCommonRepoPath} --no-config --command=test remote"; - $strCommonCommandPsql = "${strPgSqlBin}/psql -X %option% -h ${strCommonDbPath}"; $iCommonDbPort = 6543; $iModuleTestRunOnly = $iModuleTestRunOnlyParam; @@ -437,29 +436,34 @@ sub BackRestTestCommon_Setup $bLogForce = $bLogForceParam; # Get the Postgres version - my $strVersionRegExp = '(devel|((alpha|beta|rc)[0-9]+))$'; - my $strOutLog = executeTest($strPgSqlBin . '/postgres --version'); - - my @stryVersionToken = split(/ /, $strOutLog); - @stryVersionToken = split(/\./, $stryVersionToken[2]); - $strCommonDbVersion = $stryVersionToken[0] . '.' . trim($stryVersionToken[1]); - - # Warn if this is a devel/alpha/beta version - if ($strCommonDbVersion =~ /$strVersionRegExp/) + if (defined($strPgSqlBinParam)) { - my $strDevVersion = $strCommonDbVersion; - $strCommonDbVersion =~ s/$strVersionRegExp//; - $strDevVersion = substr($strDevVersion, length($strCommonDbVersion)); + my $strVersionRegExp = '(devel|((alpha|beta|rc)[0-9]+))$'; + my $strOutLog = executeTest($strPgSqlBin . '/postgres --version'); - &log(WARN, "Testing against ${strCommonDbVersion} ${strDevVersion} version"); - } + my @stryVersionToken = split(/ /, $strOutLog); + @stryVersionToken = split(/\./, $stryVersionToken[2]); + $strCommonDbVersion = $stryVersionToken[0] . '.' . trim($stryVersionToken[1]); - # Don't run unit tests for unsupported versions - my @stryVersionSupport = versionSupport(); + # Warn if this is a devel/alpha/beta version + if ($strCommonDbVersion =~ /$strVersionRegExp/) + { + my $strDevVersion = $strCommonDbVersion; + $strCommonDbVersion =~ s/$strVersionRegExp//; + $strDevVersion = substr($strDevVersion, length($strCommonDbVersion)); - if ($strCommonDbVersion < $stryVersionSupport[0]) - { - confess "currently only version $stryVersionSupport[0] and up are supported"; + &log(WARN, "Testing against ${strCommonDbVersion} ${strDevVersion} version"); + } + + # Don't run unit tests for unsupported versions + my @stryVersionSupport = versionSupport(); + + if ($strCommonDbVersion < $stryVersionSupport[0]) + { + confess "currently only version $stryVersionSupport[0] and up are supported"; + } + + $strCommonCommandPsql = "${strPgSqlBin}/psql -X %option% -h ${strCommonDbPath}"; } return true; diff --git a/test/lib/pgBackRestTest/Docker/ContainerTest.pm b/test/lib/pgBackRestTest/Docker/ContainerTest.pm index 30b2a7678..00462b813 100755 --- a/test/lib/pgBackRestTest/Docker/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Docker/ContainerTest.pm @@ -25,16 +25,19 @@ use pgBackRest::FileCommon; use pgBackRestTest::Common::ExecuteTest; use pgBackRestTest::Common::VmTest; -use constant TEST_GROUP => 'admin'; -use constant TEST_GROUP_ID => 1000; -use constant TEST_USER => 'vagrant'; -use constant TEST_USER_ID => TEST_GROUP_ID; - +#################################################################################################################################### +# User/group definitions +#################################################################################################################################### use constant POSTGRES_GROUP => 'postgres'; use constant POSTGRES_GROUP_ID => 5000; use constant POSTGRES_USER => POSTGRES_GROUP; use constant POSTGRES_USER_ID => POSTGRES_GROUP_ID; +use constant TEST_GROUP => POSTGRES_GROUP; +use constant TEST_GROUP_ID => POSTGRES_GROUP_ID; +use constant TEST_USER => 'vagrant'; +use constant TEST_USER_ID => 1000; + use constant BACKREST_GROUP => POSTGRES_GROUP; use constant BACKREST_GROUP_ID => POSTGRES_GROUP_ID; use constant BACKREST_USER => 'backrest'; @@ -118,14 +121,42 @@ sub backrestConfigCreate sub containerWrite { my $strTempPath = shift; - my $strImageName = shift; + my $strOS = shift; + my $strTitle = shift; + my $strImageParent = shift; my $strImage = shift; + my $strScript = shift; my $bForce = shift; + my $bPre = shift; + my $bPreOnly = shift; + + if (defined($bPre) && $bPre) + { + $strImage .= '-pre'; + $strImageParent .= '-pre'; + $strTitle .= ' (Pre-Installed)' + } + elsif (!defined($bPre)) + { + containerWrite($strTempPath, $strOS, $strTitle, $strImageParent, $strImage, $strScript, $bForce, true); + + if (defined($bPreOnly) && $bPreOnly) + { + return; + } + } + + &log(INFO, "Building ${strImage} image..."); + + $strScript = + "# ${strTitle} Container\n" . + "FROM ${strImageParent}\n\n" . + $strScript; # Write the image - fileStringWrite("${strTempPath}/${strImageName}", trim($strImage) . "\n", false); + fileStringWrite("${strTempPath}/${strImage}", trim($strScript) . "\n", false); executeTest('docker build' . (defined($bForce) && $bForce ? ' --no-cache' : '') . - " -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}", + " -f ${strTempPath}/${strImage} -t backrest/${strImage} ${strTempPath}", {bSuppressStdErr => true}); } @@ -143,18 +174,19 @@ sub sshSetup "RUN mkdir /home/${strUser}/.ssh\n" . "COPY id_rsa /home/${strUser}/.ssh/id_rsa\n" . "COPY id_rsa.pub /home/${strUser}/.ssh/authorized_keys\n" . - "RUN chown -R ${strUser}:${strGroup} /home/${strUser}/.ssh\n" . - "RUN chmod 700 /home/${strUser}/.ssh\n" . "RUN echo 'Host *' > /home/${strUser}/.ssh/config\n" . "RUN echo ' StrictHostKeyChecking no' >> /home/${strUser}/.ssh/config\n" . "RUN echo ' LogLevel quiet' >> /home/${strUser}/.ssh/config\n" . "RUN echo ' ControlMaster auto' >> /home/${strUser}/.ssh/config\n" . "RUN echo ' ControlPath /tmp/\%r\@\%h:\%p' >> /home/${strUser}/.ssh/config\n" . - "RUN echo ' ControlPersist 30' >> /home/${strUser}/.ssh/config"; + "RUN echo ' ControlPersist 30' >> /home/${strUser}/.ssh/config\n" . + "RUN chown -R ${strUser}:${strGroup} /home/${strUser}/.ssh\n" . + "RUN chmod 700 /home/${strUser}/.ssh\n" . + "RUN chmod 600 /home/${strUser}/.ssh/*"; } #################################################################################################################################### -# Repo Setup +# Repo setup #################################################################################################################################### sub repoSetup { @@ -168,6 +200,40 @@ sub repoSetup "RUN chmod 750 /var/lib/pgbackrest"; } + +#################################################################################################################################### +# Sudo setup +#################################################################################################################################### +sub sudoSetup +{ + my $strOS = shift; + my $strGroup = shift; + + my $strScript = + "# Setup sudo"; + + my $oVm = vmGet(); + + if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL) + { + $strScript .= + "\nRUN yum -y install sudo\n" . + "\nRUN echo '%${strGroup} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/${strGroup}" . + "\nRUN sed -i 's/^Defaults requiretty\$/\\# Defaults requiretty/' /etc/sudoers"; + } + elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) + { + $strScript .= + "\nRUN sed -i 's/^\\\%admin.*\$/\\\%${strGroup} ALL\\=\\(ALL\\) NOPASSWD\\: ALL/' /etc/sudoers"; + } + else + { + confess &log(ERROR, "unable to find base for ${strOS}"); + } + + return $strScript; +} + #################################################################################################################################### # Install Perl packages #################################################################################################################################### @@ -181,22 +247,22 @@ sub perlInstall if ($strOS eq VM_CO6) { return $strImage . - "RUN yum -y install perl perl-Time-HiRes perl-parent perl-JSON perl-Digest-SHA perl-DBD-Pg"; + "RUN yum install -y perl perl-Time-HiRes perl-parent perl-JSON perl-Digest-SHA perl-DBD-Pg"; } elsif ($strOS eq VM_CO7) { return $strImage . - "RUN yum -y install perl perl-Thread-Queue perl-JSON-PP perl-Digest-SHA perl-DBD-Pg"; + "RUN yum install -y perl perl-Thread-Queue perl-JSON-PP perl-Digest-SHA perl-DBD-Pg"; } elsif ($strOS eq VM_U12 || $strOS eq VM_U14) { return $strImage . - "RUN apt-get -y install libdbd-pg-perl libdbi-perl libnet-daemon-perl libplrpc-perl"; + "RUN apt-get install -y libdbd-pg-perl libdbi-perl libnet-daemon-perl libplrpc-perl"; } elsif ($strOS eq VM_U16 || $strOS eq VM_D8) { return $strImage . - "RUN apt-get -y install libdbd-pg-perl libdbi-perl"; + "RUN apt-get install -y libdbd-pg-perl libdbi-perl"; } confess &log(ERROR, "unable to install perl for os '${strOS}'"); @@ -209,11 +275,26 @@ sub containerBuild { my $strVm = shift; my $bVmForce = shift; + my $strDbVersionBuild = shift; # Create temp path my $strTempPath = dirname(abs_path($0)) . '/.vagrant/docker'; filePathCreate($strTempPath, '0770', true, true); + # Remove old images on force + if ($bVmForce) + { + my $strRegExp = '^backrest\/'; + + if ($strVm ne 'all') + { + $strRegExp .= "${strVm}-"; + } + + executeTest("rm -f ${strTempPath}/" . ($strVm eq 'all' ? '*' : "${strVm}-*")); + imageRemove("${strRegExp}.*"); + } + # Create SSH key (if it does not already exist) if (-e "${strTempPath}/id_rsa") { @@ -236,43 +317,48 @@ sub containerBuild } my $oOS = $$oVm{$strOS}; - my $strImage; - my $strImageName; + my $bDocBuild = false; # Base image ########################################################################################################################### - $strImageName = "${strOS}-base"; - &log(INFO, "Building ${strImageName} image..."); + my $strImageParent = "$$oVm{$strOS}{&VM_IMAGE}"; + my $strImage = "${strOS}-base"; - $strImage = "# Base Container\nFROM " . $$oVm{$strOS}{&VM_IMAGE}; - - # Install SSH - $strImage .= "\n\n# Install SSH\n"; + # Install package proxy + my $strScript = + "# Use squid proxy\n"; if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL) { - $strImage .= "RUN yum -y install openssh-server openssh-clients\n"; + $strScript .= + "RUN echo 'proxy=http://172.17.0.1:3128' >> /etc/yum.conf"; } elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) { - $strImage .= - "RUN apt-get update\n" . - "RUN apt-get -y install openssh-server\n"; + $strScript .= + "RUN echo 'Acquire {Retries \"0\"; HTTP {Proxy \"http://172.17.0.1:3128\";};};' > /etc/apt/apt.conf.d/vmproxy"; } - $strImage .= - "RUN rm -f /etc/ssh/ssh_host_rsa_key*\n" . - "RUN ssh-keygen -t rsa -b 1024 -f /etc/ssh/ssh_host_rsa_key"; + # Install base packages + $strScript .= "\n\n# Install base packages\n"; - # Create PostgreSQL Group - $strImage .= "\n\n" . postgresGroupCreate($strOS); + if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL) + { + $strScript .= "RUN yum -y install openssh-server openssh-clients\n"; + } + elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) + { + $strScript .= + "RUN apt-get update\n" . + "RUN apt-get -y install openssh-server wget\n"; + } # Add PostgreSQL packages - $strImage .= "\n\n# Add PostgreSQL packages\n"; + $strScript .= "\n# Add PostgreSQL packages\n"; if ($strOS eq VM_CO6) { - $strImage .= + $strScript .= "RUN rpm -ivh http://yum.postgresql.org/9.0/redhat/rhel-6-x86_64/pgdg-centos90-9.0-5.noarch.rpm\n" . "RUN rpm -ivh http://yum.postgresql.org/9.1/redhat/rhel-6-x86_64/pgdg-centos91-9.1-4.noarch.rpm\n" . "RUN rpm -ivh http://yum.postgresql.org/9.2/redhat/rhel-6-x86_64/pgdg-centos92-9.2-6.noarch.rpm\n" . @@ -283,7 +369,7 @@ sub containerBuild } elsif ($strOS eq VM_CO7) { - $strImage .= + $strScript .= "RUN rpm -ivh http://yum.postgresql.org/9.3/redhat/rhel-7-x86_64/pgdg-centos93-9.3-1.noarch.rpm\n" . "RUN rpm -ivh http://yum.postgresql.org/9.4/redhat/rhel-7-x86_64/pgdg-centos94-9.4-1.noarch.rpm\n" . "RUN rpm -ivh http://yum.postgresql.org/9.5/redhat/rhel-7-x86_64/pgdg-centos95-9.5-1.noarch.rpm\n" . @@ -291,211 +377,300 @@ sub containerBuild } elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) { - if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) - { - $strImage .= - "RUN apt-get install -y sudo wget\n"; - } - - $strImage .= + $strScript .= "RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ " . $$oVm{$strOS}{&VM_OS_REPO} . "-pgdg main 9.6' >> /etc/apt/sources.list.d/pgdg.list\n" . "RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -\n" . "RUN apt-get update"; } - # Create test group - $strImage .= - "\n\n# Create test group\n" . - groupCreate($strOS, TEST_GROUP, TEST_GROUP_ID) . "\n"; + $strScript .= + "\n\n# Regenerate SSH keys\n" . + "RUN rm -f /etc/ssh/ssh_host_rsa_key*\n" . + "RUN ssh-keygen -t rsa -b 1024 -f /etc/ssh/ssh_host_rsa_key"; - if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL) - { - $strImage .= - "RUN yum -y install sudo\n" . - "RUN echo '%" . TEST_GROUP . " ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/" . TEST_GROUP . "\n" . - "RUN sed -i 's/^Defaults requiretty\$/\\# Defaults requiretty/' /etc/sudoers"; - } - elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) - { - $strImage .= - "RUN sed -i 's/^\\\%admin.*\$/\\\%" . TEST_GROUP . " ALL\\=\\(ALL\\) NOPASSWD\\: ALL/' /etc/sudoers"; - } + # Create PostgreSQL Group + $strScript .= "\n\n" . postgresGroupCreate($strOS); # Create test user - $strImage .= + $strScript .= "\n\n# Create test user\n" . userCreate($strOS, TEST_USER, TEST_USER_ID, TEST_GROUP); # Suppress dpkg interactive output if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) { - $strImage .= + $strScript .= "\n\n# Suppress dpkg interactive output\n" . "RUN rm /etc/apt/apt.conf.d/70debconf"; } # Start SSH when container starts - $strImage .= + $strScript .= "\n\n# Start SSH when container starts\n"; # This is required in newer versions of u12 containers - seems like a bug in the container if ($strOS eq VM_U12) { - $strImage .= + $strScript .= "RUN mkdir /var/run/sshd\n"; } if ($strOS eq VM_U14 || $strOS eq VM_D8 || $strOS eq VM_U16) { - $strImage .= + $strScript .= "ENTRYPOINT service ssh restart && bash"; } else { - $strImage .= + $strScript .= "ENTRYPOINT /usr/sbin/sshd -D"; } # Write the image - containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); + containerWrite($strTempPath, $strOS, 'Base', $strImageParent, $strImage, $strScript, $bVmForce, false); + + # Base pre image + ########################################################################################################################### + $strImageParent = "backrest/${strOS}-base"; + $strImage = "${strOS}-base-pre"; + + # Install Perl packages + $strScript = + perlInstall($strOS) . "\n"; + + # Write the image + containerWrite($strTempPath, $strOS, 'Base (Pre-Installed)', $strImageParent, $strImage, $strScript, $bVmForce, false); # Db image ########################################################################################################################### - foreach my $strDbVersion (@{$$oOS{db}}) + my @stryDbBuild; + + # Get the list of db versions required for testing + if ($strDbVersionBuild eq 'all') + { + push(@stryDbBuild, @{$$oOS{&VM_DB}}); + } + elsif ($strDbVersionBuild eq 'minimal') + { + push(@stryDbBuild, @{$$oOS{&VM_DB_MINIMAL}}); + } + else + { + push(@stryDbBuild, $strDbVersionBuild); + } + + # Merge db versions required for docs + if (defined($$oOS{&VM_DB_DOC})) + { + @stryDbBuild = keys %{{map {($_ => 1)} ## no critic (BuiltinFunctions::ProhibitVoidMap) + (@stryDbBuild, @{$$oOS{&VM_DB_DOC}})}}; + $bDocBuild = true; + } + + foreach my $strDbVersion (@stryDbBuild) { my $strDbVersionNoDot = $strDbVersion; $strDbVersionNoDot =~ s/\.//; + my $strTitle = "Db ${strDbVersion}"; - $strImageName = "${strOS}-db-${strDbVersionNoDot}"; - &log(INFO, "Building ${strImageName} image..."); + my $bDocBuildVersion = ($bDocBuild && grep(/^$strDbVersion$/, @{$$oOS{&VM_DB_DOC}})); - $strImage = "# Database Container\nFROM backrest/${strOS}-base"; + $strImageParent = "backrest/${strOS}-base"; + $strImage = "${strOS}-db-${strDbVersion}"; # Create PostgreSQL User - $strImage .= "\n\n" . postgresUserCreate($strOS); - - # Install SSH key - $strImage .= - "\n\n" . sshSetup($strOS, POSTGRES_USER, POSTGRES_GROUP); + $strScript = postgresUserCreate($strOS); # Install PostgreSQL - $strImage .= + $strScript .= "\n\n# Install PostgreSQL"; if ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_DEBIAN) { - $strImage .= + $strScript .= "\nRUN apt-get install -y postgresql-${strDbVersion}" . "\nRUN pg_dropcluster --stop ${strDbVersion} main"; } elsif ($$oVm{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL) { - $strImage .= + $strScript .= "\nRUN yum -y install postgresql${strDbVersionNoDot}-server"; } # Write the image - containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); + containerWrite($strTempPath, $strOS, $strTitle, $strImageParent, $strImage, $strScript, $bVmForce, + $bDocBuildVersion ? undef : true, $bDocBuildVersion ? undef : true); + + # Db doc image + ######################################################################################################################## + if ($bDocBuildVersion) + { + $strImageParent = "backrest/${strOS}-db-${strDbVersion}"; + $strImage = "${strOS}-db-${strDbVersion}-doc"; + + # Install SSH key + $strScript = sshSetup($strOS, POSTGRES_USER, POSTGRES_GROUP); + + # Setup sudo + $strScript .= "\n\n" . sudoSetup($strOS, TEST_GROUP); + + # Write the image + containerWrite($strTempPath, $strOS, "${strTitle} Doc", $strImageParent, $strImage, $strScript, $bVmForce); + } + + # Db test image + ######################################################################################################################## + $strImageParent = "backrest/${strOS}-db-${strDbVersion}"; + $strImage = "${strOS}-db-${strDbVersion}-test"; + + # Create BackRest User + $strScript = backrestUserCreate($strOS); + + # Install SSH key + $strScript .= + "\n\n" . sshSetup($strOS, BACKREST_USER, BACKREST_GROUP); + + # Install SSH key + $strScript .= + "\n\n" . sshSetup($strOS, TEST_USER, TEST_GROUP); + + # Make test user home readable + $strScript .= + "\n\n# Make " . TEST_USER . " home dir readable\n" . + 'RUN chmod g+r,g+x /home/' . TEST_USER; + + # Write the image + containerWrite($strTempPath, $strOS, "${strTitle} Test", $strImageParent, $strImage, $strScript, $bVmForce, true, true); } - # Db Doc image - ########################################################################################################################### - $strImageName = "${strOS}-db-doc"; - &log(INFO, "Building ${strImageName} image..."); + # Db test image (for sythetic tests) + ######################################################################################################################## + $strImageParent = "backrest/${strOS}-base"; + $strImage = "${strOS}-db-test"; - $strImage = "# Database Doc Container\nFROM backrest/${strOS}-db-94"; - - # Create configuration file - $strImage .= - "\n\n" . backrestConfigCreate($strOS, POSTGRES_USER, POSTGRES_GROUP); + # Install SSH key + $strScript = sshSetup($strOS, TEST_USER, TEST_GROUP); # Write the image - containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); + containerWrite($strTempPath, $strOS, 'Db Synthetic Test', $strImageParent, $strImage, $strScript, $bVmForce, true, true); + + # Loop test image + ######################################################################################################################## + $strImageParent = "backrest/${strOS}-base"; + $strImage = "${strOS}-loop-test"; + + # Create BackRest User + $strScript = backrestUserCreate($strOS); + + # Install SSH key + $strScript .= + "\n\n" . sshSetup($strOS, BACKREST_USER, BACKREST_GROUP); + + # Install SSH key + $strScript .= + "\n\n" . sshSetup($strOS, TEST_USER, TEST_GROUP); + + # Make test user home readable + $strScript .= + "\n\n# Make " . TEST_USER . " home dir readable\n" . + 'RUN chmod g+r,g+x /home/' . TEST_USER; + + # Write the image + containerWrite($strTempPath, $strOS, 'Loop Test', $strImageParent, $strImage, $strScript, $bVmForce, true, true); # Backup image ########################################################################################################################### - $strImageName = "${strOS}-backup"; - &log(INFO, "Building ${strImageName} image..."); - - $strImage = "# Backup Container\nFROM backrest/${strOS}-base"; + $strImageParent = "backrest/${strOS}-base"; + $strImage = "${strOS}-backup"; + my $strTitle = "Backup"; # Create BackRest User - $strImage .= "\n\n" . backrestUserCreate($strOS); + $strScript = backrestUserCreate($strOS); # Install SSH key - $strImage .= + $strScript .= "\n\n" . sshSetup($strOS, BACKREST_USER, BACKREST_GROUP); # Write the image - containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); + containerWrite($strTempPath, $strOS, $strTitle, $strImageParent, $strImage, $strScript, $bVmForce, + $bDocBuild ? undef : true, $bDocBuild ? undef : true); # Backup Doc image ########################################################################################################################### - $strImageName = "${strOS}-backup-doc"; - &log(INFO, "Building ${strImageName} image..."); - - $strImage = "# Backup Doc Container\nFROM backrest/${strOS}-backup"; - - # Create configuration file - $strImage .= - "\n\n" . backrestConfigCreate($strOS, BACKREST_USER, BACKREST_GROUP); - - # Setup repository - $strImage .= - "\n\n" . repoSetup($strOS, BACKREST_USER, BACKREST_GROUP); - - # Install Perl packages - $strImage .= - "\n\n" . perlInstall($strOS) . "\n"; - - # Write the image - containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); - - # Test image - ########################################################################################################################### - foreach my $strDbVersion (@{$$oOS{db}}) + if ($bDocBuild) { - my $strDbVersionNoDot = $strDbVersion; - $strDbVersionNoDot =~ s/\.//; + $strImageParent = "backrest/${strOS}-backup"; + $strImage = "${strOS}-backup-doc"; - $strImageName = "${strOS}-test-${strDbVersionNoDot}"; - &log(INFO, "Building ${strImageName} image..."); + # Create configuration file + $strScript = backrestConfigCreate($strOS, BACKREST_USER, BACKREST_GROUP); - $strImage = "# Test Container\nFROM backrest/${strOS}-db-${strDbVersionNoDot}"; + # Setup repository + $strScript .= + "\n\n" . repoSetup($strOS, BACKREST_USER, BACKREST_GROUP); - # Create BackRest User - $strImage .= "\n\n" . backrestUserCreate($strOS); - - # Install SSH key - $strImage .= - "\n\n" . sshSetup($strOS, BACKREST_USER, BACKREST_GROUP); - - # Install SSH key for vagrant user - $strImage .= - "\n\n" . sshSetup($strOS, TEST_USER, TEST_GROUP); - - # Put vagrant user in postgres group so tests work properly (this will be removed in the future) - $strImage .= - "\n\n# Add postgres group to vagrant user\n" . - "RUN usermod -g " . BACKREST_GROUP . " -G " . TEST_GROUP . " " . TEST_USER; - - # Install Perl packages - $strImage .= - "\n\n" . perlInstall($strOS); - - # Make PostgreSQL home group readable - $strImage .= - "\n\n# Make vagrant home dir readable\n" . - "RUN chown -R vagrant:postgres /home/vagrant\n" . - "RUN chmod g+r,g+x /home/vagrant"; + # Setup sudo + $strScript .= "\n\n" . sudoSetup($strOS, TEST_GROUP); # Write the image - containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); + containerWrite($strTempPath, $strOS, "${strTitle} Doc", $strImageParent, $strImage, $strScript, $bVmForce, true, true); } + + # Backup Test image + ########################################################################################################################### + $strImageParent = "backrest/${strOS}-backup"; + $strImage = "${strOS}-backup-test"; + + # Make test user home readable + $strScript = + "# Make " . TEST_USER . " home dir readable\n" . + 'RUN chmod g+r,g+x /home/' . TEST_USER; + + # Write the image + containerWrite($strTempPath, $strOS, "${strTitle} Test", $strImageParent, $strImage, $strScript, $bVmForce, true, true); } } push @EXPORT, qw(containerBuild); +#################################################################################################################################### +# containerRemove +# +# Remove containers that match a regexp. +#################################################################################################################################### +sub containerRemove +{ + my $strRegExp = shift; + + foreach my $strContainer (sort(split("\n", trim(executeTest('docker ps -a --format "{{.Names}}"'))))) + { + if ($strContainer =~ $strRegExp) + { + executeTest("docker rm -f ${strContainer}", {bSuppressError => true}); + } + } +} + +#################################################################################################################################### +# imageRemove +# +# Remove images that match a regexp. +#################################################################################################################################### +sub imageRemove +{ + my $strRegExp = shift; + + foreach my $strImage (sort(split("\n", trim(executeTest('docker images --format "{{.Repository}}"'))))) + { + if ($strImage =~ $strRegExp) + { + &log(INFO, "Removing ${strImage} image..."); + executeTest("docker rmi -f ${strImage}", {bSuppressError => true}); + } + } +} + +push @EXPORT, qw(containerRemove); + 1; diff --git a/test/test.pl b/test/test.pl index 2ec3ffffa..3d693ce43 100755 --- a/test/test.pl +++ b/test/test.pl @@ -205,7 +205,7 @@ my $strBackRestBase = dirname(dirname(abs_path($0))); #################################################################################################################################### if ($bVmBuild) { - containerBuild($strVm, $bVmForce); + containerBuild($strVm, $bVmForce, $strDbVersion); exit 0; } @@ -422,7 +422,8 @@ eval # Run a reduced set of tests where each PG version is only tested on a single OS if ($strDbVersion eq 'minimal') { - $strDbVersionKey = 'db_minimal'; + $strDbVersionKey = 'db-minimal'; + $strDbVersionKey = 'db-minimal'; } if (defined($$oTest{db}) && $$oTest{db}) @@ -461,6 +462,17 @@ eval foreach my $iThreadTestMax (@{$iyThreadMax}) { + my $strDbVersion = + $iDbVersionIdx == -1 ? + undef : ${$$oyVm{$strTestOS}{$strDbVersionKey}}[$iDbVersionIdx]; + my $strPgSqlBin; + + if (defined($strDbVersion)) + { + $strPgSqlBin = $$oyVm{$strTestOS}{&VMDEF_PGSQL_BIN}; + $strPgSqlBin =~ s/\{\[version\]\}/$strDbVersion/g; + } + my $oTestRun = { os => $strTestOS, @@ -468,8 +480,9 @@ eval test => $$oTest{name}, run => $iTestRunIdx == -1 ? undef : $iTestRunIdx, thread => $iThreadTestMax, - db => $iDbVersionIdx == -1 ? undef : - ${$$oyVm{$strTestOS}{$strDbVersionKey}}[$iDbVersionIdx] + pgsql_bin => $strPgSqlBin, + db => $strDbVersion, + }; push(@{$oyTestRun}, $oTestRun); @@ -503,11 +516,11 @@ eval if (!$bDryRun || $bVmOut) { + containerRemove('test-[0-9]+'); + for (my $iProcessIdx = 0; $iProcessIdx < 8; $iProcessIdx++) { - # &log(INFO, "stop test-${iProcessIdx}"); push(@{$oyProcess}, undef); - executeTest("docker rm -f test-${iProcessIdx}", {bSuppressError => true}); } executeTest("sudo rm -rf ${strTestPath}/*"); @@ -569,7 +582,7 @@ eval my $strImage = 'test-' . $iProcessIdx; my $strHostTestPath = "${strTestPath}/${strImage}"; - executeTest("docker rm -f ${strImage}"); + containerRemove("test-${iProcessIdx}"); executeTest("sudo rm -rf ${strHostTestPath}"); } @@ -620,9 +633,11 @@ eval # Create host test directory filePathCreate($strHostTestPath, '0770'); - executeTest("docker run -itd -h $$oTest{os}-test --name=${strImage}" . - " -v ${strHostTestPath}:${strVmTestPath}" . - " -v ${strBackRestBase}:${strBackRestBase} backrest/$$oTest{os}-test-${strDbVersion}"); + executeTest( + "docker run -itd -h $$oTest{os}-test --name=${strImage}" . + " -v ${strHostTestPath}:${strVmTestPath}" . + " -v ${strBackRestBase}:${strBackRestBase}" . + " backrest/$$oTest{os}-" . (defined($$oTest{db}) ? "db-$$oTest{db}" : 'loop' ) . '-test-pre'); } # Build up command line for the individual test @@ -641,6 +656,7 @@ eval (defined($$oTest{run}) ? " --run=$$oTest{run}" : '') . (defined($$oTest{thread}) ? " --thread-max=$$oTest{thread}" : '') . (defined($$oTest{db}) ? " --db-version=$$oTest{db}" : '') . + (defined($$oTest{pgsql_bin}) ? ' --pgsql-bin=' . $$oTest{pgsql_bin} : '') . ($bDryRun ? " --dry-run" : '') . " --test-path=${strVmTestPath}" . " --no-cleanup --vm-out"; @@ -653,7 +669,7 @@ eval # Set permissions on the Docker test directory. This can be removed once users/groups are sync'd between # Docker and the host VM. - executeTest("docker exec ${strImage} chown vagrant:postgres -R ${strVmTestPath}"); + executeTest("docker exec ${strImage} chown vagrant:postgres -R ${strVmTestPath}"); my $oExec = new pgBackRestTest::Common::ExecuteTest( $strCommand, @@ -691,51 +707,6 @@ eval exit 0; } - ################################################################################################################################ - # Search for psql - ################################################################################################################################ - my @stryTestVersion; - my @stryVersionSupport = versionSupport(); - - if (!defined($strPgSqlBin)) - { - # Distribution-specific paths where the PostgreSQL binaries may be located - my @strySearchPath = - ( - '/usr/lib/postgresql/VERSION/bin', # Debian/Ubuntu - '/usr/pgsql-VERSION/bin', # CentOS/RHEL/Fedora - '/Library/PostgreSQL/VERSION/bin', # OSX - '/usr/local/bin' # BSD - ); - - foreach my $strSearchPath (@strySearchPath) - { - for (my $iVersionIdx = @stryVersionSupport - 1; $iVersionIdx >= 0; $iVersionIdx--) - { - if ($strDbVersion eq 'all' || $strDbVersion eq 'minimal' || $strDbVersion eq 'max' && @stryTestVersion == 0 || - $strDbVersion eq $stryVersionSupport[$iVersionIdx]) - { - my $strVersionPath = $strSearchPath; - $strVersionPath =~ s/VERSION/$stryVersionSupport[$iVersionIdx]/g; - - if (-e "${strVersionPath}/initdb") - { - &log(DEBUG, "FOUND pgsql-bin at ${strVersionPath}"); - push @stryTestVersion, $strVersionPath; - } - } - } - } - - # Make sure at least one version of postgres was found - @stryTestVersion > 0 - or confess 'pgsql-bin was not defined and postgres could not be located automatically'; - } - else - { - push @stryTestVersion, $strPgSqlBin; - } - ################################################################################################################################ # Clean whitespace only if test.pl is being run from the test directory in the backrest repo ################################################################################################################################ @@ -758,28 +729,23 @@ eval do { - if (BackRestTestCommon_Setup($strExe, $strTestPath, $stryTestVersion[0], $iModuleTestRun, - $bDryRun, $bNoCleanup, $bLogForce)) + if (BackRestTestCommon_Setup($strExe, $strTestPath, $strPgSqlBin, $iModuleTestRun, + $bDryRun, $bNoCleanup, $bLogForce)) { if (!$bVmOut && ($strModule eq 'all' || $strModule eq 'backup' && $strModuleTest eq 'all' || $strModule eq 'backup' && $strModuleTest eq 'full')) { - &log(INFO, "TESTING psql-bin = $stryTestVersion[0]\n"); + &log(INFO, "TESTING psql-bin = $strPgSqlBin\n"); } if ($bInfinite) { - $iRun++; - &log(INFO, "INFINITE - RUN ${iRun}\n"); + $iRun++; + &log(INFO, "INFINITE - RUN ${iRun}\n"); } - # if ($strModule eq 'all' || $strModule eq 'ini') - # { - # BackRestTestIni_Test($strModuleTest); - # } - if ($strModule eq 'all' || $strModule eq 'help') { BackRestTestHelp_Test($strModuleTest, undef, $bVmOut); @@ -806,22 +772,6 @@ eval { BackRestTestBackup_Test($strModuleTest, defined($iThreadMax) ? $iThreadMax : 4, $bVmOut); } - - if (@stryTestVersion > 1 && ($strModuleTest eq 'all' || $strModuleTest eq 'full')) - { - for (my $iVersionIdx = 1; $iVersionIdx < @stryTestVersion; $iVersionIdx++) - { - BackRestTestCommon_Setup($strExe, $strTestPath, $stryTestVersion[$iVersionIdx], - $iModuleTestRun, $bDryRun, $bNoCleanup); - &log(INFO, "TESTING psql-bin = $stryTestVersion[$iVersionIdx] for backup/full\n"); - BackRestTestBackup_Test('full', defined($iThreadMax) ? $iThreadMax : 4, $bVmOut); - } - } - } - - if ($strModule eq 'compare') - { - BackRestTestCompare_Test($strModuleTest); } } }