Remove dependency on aws cli for testing.

This tool was only being used it a few places but was a pretty large dependency.

Rework the forceStorageMove() code using our storage layer and replace one aws cli cp with a storage put.

Also, remove the Dockerfile that was once used to build the Scality S3 test container.

test/Vagrantfile

@@ -70,16 +70,6 @@ Vagrant.configure(2) do |config|
         apt-get install -y devscripts build-essential lintian git lcov cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \
              libxml2-dev liblz4-dev libpq-dev valgrind
 
-        #---------------------------------------------------------------------------------------------------------------------------
-        echo 'Install AWS CLI' && date
-        apt-get install -y python-pip
-        pip install --upgrade awscli
-
-        # Configure AWS CLI
-        sudo -i -u vagrant aws configure set region us-east-1
-        sudo -i -u vagrant aws configure set aws_access_key_id accessKey1
-        sudo -i -u vagrant aws configure set aws_secret_access_key verySecretKey1
-
         #---------------------------------------------------------------------------------------------------------------------------
         echo 'Install Docker' && date
         curl -fsSL https://get.docker.com | sh

test/docker/s3-server.docker

@@ -1,50 +0,0 @@
-# S3 Server used for testing and documentation
-# docker build -f s3-server.docker -t pgbackrest/test:s3-server-20180612A .
-FROM ubuntu:18.04
-
-# Suppress dpkg interactive output
-RUN rm /etc/apt/apt.conf.d/70debconf && \
-
-# Install base packages
-    apt-get update && \
-    apt-get -y install build-essential openssl wget git python-pip && \
-
-# Fix root tty
-    sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile && \
-
-# Generate fake certs
-    mkdir -p -m 755 /etc/fake-cert && \
-    cd /etc/fake-cert && \
-    openssl genrsa -out ca.key 2048 && \
-    openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 \
-        -subj "/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org" && \
-    openssl genrsa -out server.key 2048 && \
-    openssl req -new -key server.key -out server.csr \
-        -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org" && \
-    openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 \
-        -sha256 && \
-    chmod 644 /etc/fake-cert/* && \
-
-# Install AWS CLI
-    pip install --upgrade awscli && \
-    aws configure set region us-east-1 && \
-    aws configure set aws_access_key_id accessKey1 && \
-    aws configure set aws_secret_access_key verySecretKey1 && \
-
-# Install node.js
-    wget -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x && \
-    bash /root/nodejs.sh && \
-    apt-get install -y nodejs npm && \
-
-# Install Scality S3
-    wget -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz && \
-    mkdir /root/scalitys3 && \
-    tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz && \
-    cd /root/scalitys3 && \
-    npm install && \
-    sed -i "0,/,/s//,\n    \"certFilePaths\":{\"key\":\"\/etc\/fake\-cert\/server.key\",\"cert\":\"\/etc\/fake\-cert\/server.crt\",\"ca\":\"\/etc\/fake\-cert\/ca.crt\"},/" \
-        ./config.json && \
-    sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json
-
-# Start SSH when container starts
-ENTRYPOINT npm start --prefix /root/scalitys3

test/lib/pgBackRestTest/Common/FileTest.pm

@@ -32,6 +32,7 @@ use pgBackRest::Storage::Base;
 use pgBackRestTest::Common::ExecuteTest;
 use pgBackRestTest::Common::HostGroupTest;
 use pgBackRestTest::Common::LogTest;
+use pgBackRestTest::Common::RunTest;
 use pgBackRestTest::Common::VmTest;
 use pgBackRestTest::Env::Host::HostBaseTest;
 use pgBackRestTest::Env::Host::HostBackupTest;
@@ -204,9 +205,34 @@ sub forceStorageMove
     # If S3 then use storage commands to remove
     if ($oStorage->type() eq STORAGE_S3)
     {
-        hostGroupGet()->hostGet(HOST_S3)->executeS3(
+        if ($bRecurse)
-            'mv' . ($bRecurse ? ' --recursive' : '') . ' s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strSourcePathExp) .
+        {
-                ' s3://' . HOST_S3_BUCKET . $oStorage->pathGet($strDestinationPathExp));
+            my $rhManifest = $oStorage->manifest($strSourcePathExp);
+
+            foreach my $strName (sort(keys(%{$rhManifest})))
+            {
+                if ($rhManifest->{$strName}{type} eq 'f')
+                {
+                    $oStorage->put(
+                        new pgBackRest::Storage::StorageWrite(
+                            $oStorage,
+                            pgBackRest::LibC::StorageWrite->new(
+                                $oStorage->{oStorageC}, "${strDestinationPathExp}/${strName}", 0, undef, undef, 0, true, false)),
+                        ${$oStorage->get(
+                            new pgBackRest::Storage::StorageRead(
+                                $oStorage,
+                                pgBackRest::LibC::StorageRead->new(
+                                    $oStorage->{oStorageC}, "${strSourcePathExp}/${strName}", false)))});
+
+                    $oStorage->remove("${strSourcePathExp}/${strName}");
+                }
+            }
+        }
+        else
+        {
+            $oStorage->put($strDestinationPathExp, ${$oStorage->get($strSourcePathExp)});
+            $oStorage->remove($strSourcePathExp);
+        }
     }
     # Else remove using filesystem commands
     else

test/lib/pgBackRestTest/Env/Host/HostS3Test.pm

@@ -85,48 +85,4 @@ sub new
     );
 }
 
-####################################################################################################################################
-# executeS3
-####################################################################################################################################
-sub executeS3
-{
-    my $self = shift;
-
-    # Assign function parameters, defaults, and log debug info
-    my
-    (
-        $strOperation,
-        $strCommand
-    ) =
-        logDebugParam
-        (
-            __PACKAGE__ . '->executeS3', \@_,
-            {name => 'strCommand', trace => true},
-        );
-
-    # Retry the command until timeout
-    my $oWait = waitInit(60);
-    my $bSuccess = false;
-    my $strTotalCommand =
-        'export PYTHONWARNINGS="ignore" && aws --endpoint-url=https://' . $self->ipGet() . ' s3 --no-verify-ssl ' . $strCommand;
-
-    do
-    {
-        my $oExec = new pgBackRestTest::Common::ExecuteTest($strTotalCommand, {bSuppressError => true, bSuppressStdErr => true});
-        $oExec->begin();
-
-        $bSuccess = $oExec->end() == 0;
-    }
-    while (!$bSuccess && waitMore($oWait));
-
-    # If no success run again to display the error
-    if (!$bSuccess)
-    {
-        executeTest($strTotalCommand);
-    }
-
-    # Return from function and log return values if any
-    return logDebugReturn($strOperation);
-}
-
 1;

test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm

@@ -540,7 +540,7 @@ sub run
 
         if ($bS3)
         {
-            $oHostS3->executeS3('cp /etc/hosts s3://' . HOST_S3_BUCKET . "${strTempFile}");
+            storageRepo()->put($strTempFile, "TEMP");
         }
         else
         {

test/travis.pl

@@ -168,7 +168,7 @@ eval
         }
         else
         {
-            $strPackage .= " python-pip libdbd-pg-perl";
+            $strPackage .= " libdbd-pg-perl";
         }
 
         processBegin('install test packages');
@@ -191,13 +191,6 @@ eval
             processExec("sudo adduser --ingroup=\${USER?} --uid=5001 --disabled-password --gecos \"\" " . BACKREST_USER);
             processEnd();
 
-            processBegin("install and configure aws cli");
-            processExec('pip install --upgrade --user awscli', {bSuppressStdErr => true});
-            processExec('aws configure set region us-east-1');
-            processExec('aws configure set aws_access_key_id accessKey1');
-            processExec('aws configure set aws_secret_access_key verySecretKey1');
-            processEnd();
-
             # Build the container
             processBegin("${strVm} build");
             processExec("${strTestExe} --vm-build --vm=${strVm}", {bShowOutputAsync => true, bOutLogOnError => false});