From 64b2858bb50c6b5f5f3cc6934c6dab671c203bbe Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 13 Apr 2016 18:47:29 -0400 Subject: [PATCH] Test VM build improvements. * Specific VMs can now be built by using --vm along with --vm-build. * Docker caching can be disabled with --vm-force. * ControlMaster is now used for al VMs to improve test speed. --- test/lib/BackRestTest/Docker/ContainerTest.pm | 98 ++++++++++--------- test/test.pl | 21 ++-- 2 files changed, 66 insertions(+), 53 deletions(-) diff --git a/test/lib/BackRestTest/Docker/ContainerTest.pm b/test/lib/BackRestTest/Docker/ContainerTest.pm index 66ba1df67..71bb2c8c0 100755 --- a/test/lib/BackRestTest/Docker/ContainerTest.pm +++ b/test/lib/BackRestTest/Docker/ContainerTest.pm @@ -109,6 +109,23 @@ sub backrestConfigCreate "RUN chown ${strUser}:${strGroup} /etc/pg_backrest.conf"; } +#################################################################################################################################### +# Write the Docker container +#################################################################################################################################### +sub containerWrite +{ + my $strTempPath = shift; + my $strImageName = shift; + my $strImage = shift; + my $bForce = shift; + + # Write the image + fileStringWrite("${strTempPath}/${strImageName}", "$strImage\n", false); + executeTest('docker build' . (defined($bForce) && $bForce ? ' --no-cache' : '') . + " -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}", + {bSuppressStdErr => true}); +} + #################################################################################################################################### # Setup SSH #################################################################################################################################### @@ -118,15 +135,19 @@ sub sshSetup my $strUser = shift; my $strGroup = shift; - return "# Setup SSH\n" . - "RUN mkdir /home/${strUser}/.ssh\n" . - "COPY id_rsa /home/${strUser}/.ssh/id_rsa\n" . - "COPY id_rsa.pub /home/${strUser}/.ssh/authorized_keys\n" . - "RUN chown -R ${strUser}:${strGroup} /home/${strUser}/.ssh\n" . - "RUN chmod 700 /home/${strUser}/.ssh\n" . - "RUN echo 'Host *' > /home/${strUser}/.ssh/config\n" . - "RUN echo ' StrictHostKeyChecking no' >> /home/${strUser}/.ssh/config\n" . - "RUN echo ' LogLevel quiet' >> /home/${strUser}/.ssh/config"; + return + "# Setup SSH\n" . + "RUN mkdir /home/${strUser}/.ssh\n" . + "COPY id_rsa /home/${strUser}/.ssh/id_rsa\n" . + "COPY id_rsa.pub /home/${strUser}/.ssh/authorized_keys\n" . + "RUN chown -R ${strUser}:${strGroup} /home/${strUser}/.ssh\n" . + "RUN chmod 700 /home/${strUser}/.ssh\n" . + "RUN echo 'Host *' > /home/${strUser}/.ssh/config\n" . + "RUN echo ' StrictHostKeyChecking no' >> /home/${strUser}/.ssh/config\n" . + "RUN echo ' LogLevel quiet' >> /home/${strUser}/.ssh/config\n" . + "RUN echo ' ControlMaster auto' >> /home/${strUser}/.ssh/config\n" . + "RUN echo ' ControlPath /tmp/\%r\@\%h:\%p' >> /home/${strUser}/.ssh/config\n" . + "RUN echo ' ControlPersist 30' >> /home/${strUser}/.ssh/config"; } #################################################################################################################################### @@ -178,6 +199,9 @@ sub perlInstall #################################################################################################################################### sub containerBuild { + my $strVm = shift; + my $bVmForce = shift; + # Create temp path my $strTempPath = dirname(abs_path($0)) . '/.vagrant/docker'; @@ -203,6 +227,11 @@ sub containerBuild foreach my $strOS (sort(keys(%$oyVm))) { + if ($strVm ne 'all' && $strVm ne $strOS) + { + next; + } + my $oOS = $$oyVm{$strOS}; my $strImage; my $strImageName; @@ -323,32 +352,26 @@ sub containerBuild $strImage .= "\n\n# Start SSH when container starts\n"; - if ($strOS eq OS_CO6) + # This is required in newer versions of u12 containers - seems like a bug in the caontainer + if ($strOS eq OS_U12) { $strImage .= - "ENTRYPOINT service sshd restart && bash"; + "RUN mkdir /var/run/sshd\n"; } - elsif ($strOS eq OS_CO7) - { - $strImage .= - # "ENTRYPOINT systemctl start sshd.service && bash"; - "ENTRYPOINT /usr/sbin/sshd -D && bash"; - } - elsif ($strOS eq OS_U12) - { - $strImage .= - "ENTRYPOINT /etc/init.d/ssh start && bash"; - } - elsif ($strOS eq OS_U14) + + if ($strOS eq OS_U14) { $strImage .= "ENTRYPOINT service ssh restart && bash"; } + else + { + $strImage .= + "ENTRYPOINT /usr/sbin/sshd -D"; + } # Write the image - fileStringWrite("${strTempPath}/${strImageName}", "$strImage\n", false); - executeTest("docker build -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}", - {bSuppressStdErr => true}); + containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); # Db image ########################################################################################################################### @@ -391,9 +414,7 @@ sub containerBuild } # Write the image - fileStringWrite("${strTempPath}/${strImageName}", "${strImage}\n", false); - executeTest("docker build -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}", - {bSuppressStdErr => true}); + containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); } # Db Doc image @@ -408,10 +429,7 @@ sub containerBuild "\n\n" . backrestConfigCreate($strOS, POSTGRES_USER, POSTGRES_GROUP); # Write the image - fileStringWrite("${strTempPath}/${strImageName}", "${strImage}\n", false); - executeTest("docker build -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}", - {bSuppressStdErr => true}); - + containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); # Backup image ########################################################################################################################### @@ -428,10 +446,7 @@ sub containerBuild "\n\n" . sshSetup($strOS, BACKREST_USER, BACKREST_GROUP); # Write the image - fileStringWrite("${strTempPath}/${strImageName}", "${strImage}\n", false); - executeTest("docker build -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}", - {bSuppressStdErr => true}); - + containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); # Backup Doc image ########################################################################################################################### @@ -453,10 +468,7 @@ sub containerBuild "\n\n" . perlInstall($strOS) . "\n"; # Write the image - fileStringWrite("${strTempPath}/${strImageName}", "${strImage}\n", false); - executeTest("docker build -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}", - {bSuppressStdErr => true}); - + containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); # Test image ########################################################################################################################### @@ -497,9 +509,7 @@ sub containerBuild "RUN chmod g+r,g+x /home/vagrant"; # Write the image - fileStringWrite("${strTempPath}/${strImageName}", "${strImage}\n", false); - executeTest("docker build -f ${strTempPath}/${strImageName} -t backrest/${strImageName} ${strTempPath}", - {bSuppressStdErr => true}); + containerWrite($strTempPath, $strImageName, $strImage, $bVmForce); } } } diff --git a/test/test.pl b/test/test.pl index 72aa96c39..88a52c5fb 100755 --- a/test/test.pl +++ b/test/test.pl @@ -71,8 +71,9 @@ test.pl [options] --quiet, -q equivalent to --log-level=off VM Options: + --vm docker container to build/test (u12, u14, co6, co7) --vm-build build Docker containers - --vm execute in a docker container (u12, u14, co6, co7) + --vm-force force a rebuild of Docker containers --vm-out Show VM output (default false) --process-max max VMs to run in parallel (default 1) @@ -85,7 +86,6 @@ test.pl [options] # Command line parameters #################################################################################################################################### my $strLogLevel = 'info'; -my $strOS = 'all'; my $bVmOut = false; my $strModule = 'all'; my $strModuleTest = 'all'; @@ -103,7 +103,9 @@ my $bQuiet = false; my $bInfinite = false; my $strDbVersion = 'all'; my $bLogForce = false; +my $strVm = 'all'; my $bVmBuild = false; +my $bVmForce = false; my $bNoLint = false; my $strCommandLine = join(' ', @ARGV); @@ -115,9 +117,10 @@ GetOptions ('q|quiet' => \$bQuiet, 'exes=s' => \$strExe, 'test-path=s' => \$strTestPath, 'log-level=s' => \$strLogLevel, - 'vm=s' => \$strOS, + 'vm=s' => \$strVm, 'vm-out' => \$bVmOut, 'vm-build' => \$bVmBuild, + 'vm-force' => \$bVmForce, 'module=s' => \$strModule, 'test=s' => \$strModuleTest, 'run=s' => \$iModuleTestRun, @@ -191,7 +194,7 @@ if (defined($iThreadMax) && ($iThreadMax < 1 || $iThreadMax > 32)) #################################################################################################################################### if ($bVmBuild) { - containerBuild(); + containerBuild($strVm, $bVmForce); exit 0; } @@ -333,7 +336,7 @@ eval ################################################################################################################################ # Start VM and run ################################################################################################################################ - if ($strOS ne 'none') + if ($strVm ne 'none') { if (!$bDryRun) { @@ -367,9 +370,9 @@ eval my $oyVm = vmGet(); - if ($strOS ne 'all' && !defined($${oyVm}{$strOS})) + if ($strVm ne 'all' && !defined($${oyVm}{$strVm})) { - confess &log(ERROR, "${strOS} is not a valid VM"); + confess &log(ERROR, "${strVm} is not a valid VM"); } # Determine which tests to run @@ -377,13 +380,13 @@ eval my $stryTestOS = []; - if ($strOS eq 'all') + if ($strVm eq 'all') { $stryTestOS = ['co6', 'u12', 'co7', 'u14']; } else { - $stryTestOS = [$strOS]; + $stryTestOS = [$strVm]; } foreach my $strTestOS (@{$stryTestOS})