From b27f9e886b4f21ac199ededde3a189355eac9f88 Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 26 May 2020 09:16:57 -0400 Subject: [PATCH] Refactor TLS server test harness for ease of use. The prior harness required a separate function to contain the server behavior but this made keeping the client/server code in sync very difficult and in general meant test writing took longer. Now, commands to define server behavior are inline with the client code, which should greatly simplify test writing. --- test/src/common/harnessTls.c | 417 +++++++--- test/src/common/harnessTls.h | 46 +- test/src/module/common/ioHttpTest.c | 658 +++++++--------- test/src/module/common/ioTlsTest.c | 255 +++--- test/src/module/storage/s3Test.c | 1120 ++++++++++++++------------- 5 files changed, 1323 insertions(+), 1173 deletions(-) diff --git a/test/src/common/harnessTls.c b/test/src/common/harnessTls.c index 176a6e721..f4d7bde1d 100644 --- a/test/src/common/harnessTls.c +++ b/test/src/common/harnessTls.c @@ -1,5 +1,5 @@ /*********************************************************************************************************************************** -Tls Test Harness +TLS Test Harness ***********************************************************************************************************************************/ #include #include @@ -13,27 +13,208 @@ Tls Test Harness #include "common/error.h" #include "common/io/socket/session.h" #include "common/io/tls/session.intern.h" +#include "common/log.h" #include "common/type/buffer.h" +#include "common/type/json.h" #include "common/wait.h" +#include "common/harnessDebug.h" #include "common/harnessTest.h" #include "common/harnessTls.h" /*********************************************************************************************************************************** -Test defaults +Command enum ***********************************************************************************************************************************/ -#define TLS_TEST_HOST "tls.test.pgbackrest.org" - -static int testServerSocket = 0; -static SSL_CTX *testServerContext = NULL; -static TlsSession *testServerSession = NULL; +typedef enum +{ + hrnTlsCmdAbort, + hrnTlsCmdAccept, + hrnTlsCmdClose, + hrnTlsCmdDone, + hrnTlsCmdExpect, + hrnTlsCmdReply, + hrnTlsCmdSleep, +} HrnTlsCmd; /*********************************************************************************************************************************** -Initialize TLS and listen on the specified port for TLS connections +Constants ***********************************************************************************************************************************/ -void -harnessTlsServerInit(unsigned int port, const char *serverCert, const char *serverKey) +#define TLS_TEST_HOST "tls.test.pgbackrest.org" +#define TLS_CERT_TEST_KEY TLS_CERT_FAKE_PATH "/pgbackrest-test.key" + +/*********************************************************************************************************************************** +Local variables +***********************************************************************************************************************************/ +static struct { + IoWrite *clientWrite; // Write interface for client to send commands to the server +} hrnTlsLocal; + +/*********************************************************************************************************************************** +Send commands to the server +***********************************************************************************************************************************/ +static void +hrnTlsServerCommand(HrnTlsCmd cmd, const Variant *data) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(ENUM, cmd); + FUNCTION_HARNESS_PARAM(VARIANT, data); + FUNCTION_HARNESS_END(); + + ASSERT(hrnTlsLocal.clientWrite != NULL); + + ioWriteStrLine(hrnTlsLocal.clientWrite, jsonFromUInt(cmd)); + ioWriteStrLine(hrnTlsLocal.clientWrite, jsonFromVar(data)); + ioWriteFlush(hrnTlsLocal.clientWrite); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +/**********************************************************************************************************************************/ +void hrnTlsClientBegin(IoWrite *write) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(IO_WRITE, write); + FUNCTION_HARNESS_END(); + + ASSERT(hrnTlsLocal.clientWrite == NULL); + ASSERT(write != NULL); + + hrnTlsLocal.clientWrite = write; + ioWriteOpen(write); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +void hrnTlsClientEnd(void) +{ + FUNCTION_HARNESS_VOID(); + + ASSERT(hrnTlsLocal.clientWrite != NULL); + + hrnTlsServerCommand(hrnTlsCmdDone, NULL); + hrnTlsLocal.clientWrite = NULL; + + FUNCTION_HARNESS_RESULT_VOID(); +} + +/**********************************************************************************************************************************/ +void +hrnTlsServerAbort(void) +{ + FUNCTION_HARNESS_VOID(); + + hrnTlsServerCommand(hrnTlsCmdAbort, NULL); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +void +hrnTlsServerAccept(void) +{ + FUNCTION_HARNESS_VOID(); + + hrnTlsServerCommand(hrnTlsCmdAccept, NULL); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +void +hrnTlsServerClose() +{ + FUNCTION_HARNESS_VOID(); + + hrnTlsServerCommand(hrnTlsCmdClose, NULL); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +void +hrnTlsServerExpect(const String *data) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STRING, data); + FUNCTION_HARNESS_END(); + + ASSERT(data != NULL); + + hrnTlsServerCommand(hrnTlsCmdExpect, VARSTR(data)); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +void +hrnTlsServerExpectZ(const char *data) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STRINGZ, data); + FUNCTION_HARNESS_END(); + + ASSERT(data != NULL); + + hrnTlsServerCommand(hrnTlsCmdExpect, VARSTRZ(data)); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +void +hrnTlsServerReply(const String *data) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STRING, data); + FUNCTION_HARNESS_END(); + + ASSERT(data != NULL); + + hrnTlsServerCommand(hrnTlsCmdReply, VARSTR(data)); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +void +hrnTlsServerReplyZ(const char *data) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(STRINGZ, data); + FUNCTION_HARNESS_END(); + + ASSERT(data != NULL); + + hrnTlsServerCommand(hrnTlsCmdReply, VARSTRZ(data)); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +void +hrnTlsServerSleep(TimeMSec sleepMs) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(TIME_MSEC, sleepMs); + FUNCTION_HARNESS_END(); + + ASSERT(sleepMs > 0); + + hrnTlsServerCommand(hrnTlsCmdSleep, VARUINT64(sleepMs)); + + FUNCTION_HARNESS_RESULT_VOID(); +} + +/**********************************************************************************************************************************/ +void hrnTlsServerRunParam(IoRead *read, const String *certificate, const String *key) +{ + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(IO_READ, read); + FUNCTION_HARNESS_PARAM(STRING, certificate); + FUNCTION_HARNESS_PARAM(STRING, key); + FUNCTION_HARNESS_END(); + + ASSERT(read != NULL); + ASSERT(certificate != NULL); + ASSERT(key != NULL); + + // Open read connection to client + ioReadOpen(read); + // Add test hosts if (testContainer()) { @@ -47,28 +228,31 @@ harnessTlsServerInit(unsigned int port, const char *serverCert, const char *serv const SSL_METHOD *method = SSLv23_method(); cryptoError(method == NULL, "unable to load TLS method"); - testServerContext = SSL_CTX_new(method); - cryptoError(testServerContext == NULL, "unable to create TLS context"); + SSL_CTX *serverContext = SSL_CTX_new(method); + cryptoError(serverContext == NULL, "unable to create TLS context"); // Configure the context by setting key and cert cryptoError( - SSL_CTX_use_certificate_file(testServerContext, serverCert, SSL_FILETYPE_PEM) <= 0, "unable to load server certificate"); + SSL_CTX_use_certificate_file(serverContext, strPtr(certificate), SSL_FILETYPE_PEM) <= 0, + "unable to load server certificate"); cryptoError( - SSL_CTX_use_PrivateKey_file(testServerContext, serverKey, SSL_FILETYPE_PEM) <= 0, "unable to load server private key"); + SSL_CTX_use_PrivateKey_file(serverContext, strPtr(key), SSL_FILETYPE_PEM) <= 0, "unable to load server private key"); // Create the socket + int serverSocket; + struct sockaddr_in address; address.sin_family = AF_INET; - address.sin_port = htons((uint16_t)port); + address.sin_port = htons((uint16_t)hrnTlsServerPort()); address.sin_addr.s_addr = htonl(INADDR_ANY); - if ((testServerSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) + if ((serverSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) THROW_SYS_ERROR(AssertError, "unable to create socket"); // Set the address as reusable so we can bind again in the same process for testing int reuseAddr = 1; - setsockopt(testServerSocket, SOL_SOCKET, SO_REUSEADDR, &reuseAddr, sizeof(reuseAddr)); + setsockopt(serverSocket, SOL_SOCKET, SO_REUSEADDR, &reuseAddr, sizeof(reuseAddr)); // Bind the address. It might take a bit to bind if another process was recently using it so retry a few times. Wait *wait = waitNew(2000); @@ -76,7 +260,7 @@ harnessTlsServerInit(unsigned int port, const char *serverCert, const char *serv do { - result = bind(testServerSocket, (struct sockaddr *)&address, sizeof(address)); + result = bind(serverSocket, (struct sockaddr *)&address, sizeof(address)); } while (result < 0 && waitMore(wait)); @@ -84,107 +268,136 @@ harnessTlsServerInit(unsigned int port, const char *serverCert, const char *serv THROW_SYS_ERROR(AssertError, "unable to bind socket"); // Listen for client connections - if (listen(testServerSocket, 1) < 0) + if (listen(serverSocket, 1) < 0) THROW_SYS_ERROR(AssertError, "unable to listen on socket"); + + // Loop until no more commands + TlsSession *serverSession = NULL; + bool done = false; + + do + { + HrnTlsCmd cmd = jsonToUInt(ioReadLine(read)); + const Variant *data = jsonToVar(ioReadLine(read)); + + switch (cmd) + { + case hrnTlsCmdAbort: + { + tlsSessionClose(serverSession, false); + tlsSessionFree(serverSession); + serverSession = NULL; + + break; + } + + case hrnTlsCmdAccept: + { + struct sockaddr_in addr; + unsigned int len = sizeof(addr); + + int testClientSocket = accept(serverSocket, (struct sockaddr *)&addr, &len); + + if (testClientSocket < 0) + THROW_SYS_ERROR(AssertError, "unable to accept socket"); + + SSL *testClientSSL = SSL_new(serverContext); + + serverSession = tlsSessionNew( + testClientSSL, sckSessionNew(sckSessionTypeServer, testClientSocket, STRDEF("client"), 0, 5000), 5000); + + break; + } + + case hrnTlsCmdClose: + { + tlsSessionClose(serverSession, true); + tlsSessionFree(serverSession); + serverSession = NULL; + + break; + } + + case hrnTlsCmdDone: + { + done = true; + + break; + } + + case hrnTlsCmdExpect: + { + const String *expected = varStr(data); + Buffer *buffer = bufNew(strSize(expected)); + + ioRead(tlsSessionIoRead(serverSession), buffer); + + // Treat any ? characters as wildcards so variable elements (e.g. auth hashes) can be ignored + String *actual = strNewBuf(buffer); + + for (unsigned int actualIdx = 0; actualIdx < strSize(actual); actualIdx++) + { + if (strPtr(expected)[actualIdx] == '?') + ((char *)strPtr(actual))[actualIdx] = '?'; + } + + // Error if actual does not match expected + if (!strEq(actual, expected)) + THROW_FMT(AssertError, "server expected '%s' but got '%s'", strPtr(expected), strPtr(actual)); + + break; + } + + case hrnTlsCmdReply: + { + ioWrite(tlsSessionIoWrite(serverSession), BUFSTR(varStr(data))); + ioWriteFlush(tlsSessionIoWrite(serverSession)); + + break; + } + + case hrnTlsCmdSleep: + { + sleepMSec(varUInt64Force(data)); + + break; + } + } + } + while (!done); + + // Free TLS context + SSL_CTX_free(serverContext); + + FUNCTION_HARNESS_RESULT_VOID(); } -/**********************************************************************************************************************************/ -void -harnessTlsServerInitDefault(void) +void hrnTlsServerRun(IoRead *read) { + FUNCTION_HARNESS_BEGIN(); + FUNCTION_HARNESS_PARAM(IO_READ, read); + FUNCTION_HARNESS_END(); + if (testContainer()) - harnessTlsServerInit(harnessTlsTestPort(), TLS_CERT_TEST_CERT, TLS_CERT_TEST_KEY); + hrnTlsServerRunParam(read, STRDEF(TLS_CERT_TEST_CERT), STRDEF(TLS_CERT_TEST_KEY)); else { - harnessTlsServerInit( - harnessTlsTestPort(), - strPtr(strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".crt", testRepoPath())), - strPtr(strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".key", testRepoPath()))); - } -} - -/*********************************************************************************************************************************** -Expect an exact string from the client -***********************************************************************************************************************************/ -void -harnessTlsServerExpect(const char *expected) -{ - Buffer *buffer = bufNew(strlen(expected)); - - ioRead(tlsSessionIoRead(testServerSession), buffer); - - // Treat any ? characters as wildcards so variable elements (e.g. auth hashes) can be ignored - String *actual = strNewBuf(buffer); - - for (unsigned int actualIdx = 0; actualIdx < strSize(actual); actualIdx++) - { - if (expected[actualIdx] == '?') - ((char *)strPtr(actual))[actualIdx] = '?'; + hrnTlsServerRunParam( + read, strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".crt", testRepoPath()), + strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".key", testRepoPath())); } - // Error if actual does not match expected - if (!strEqZ(actual, expected)) - THROW_FMT(AssertError, "server expected '%s' but got '%s'", expected, strPtr(actual)); -} - -/*********************************************************************************************************************************** -Send a reply to the client -***********************************************************************************************************************************/ -void -harnessTlsServerReply(const char *reply) -{ - ioWrite(tlsSessionIoWrite(testServerSession), BUF((unsigned char *)reply, strlen(reply))); - ioWriteFlush(tlsSessionIoWrite(testServerSession)); -} - -/*********************************************************************************************************************************** -Accept a TLS connection from the client -***********************************************************************************************************************************/ -void -harnessTlsServerAccept(void) -{ - struct sockaddr_in addr; - unsigned int len = sizeof(addr); - - int testClientSocket = accept(testServerSocket, (struct sockaddr *)&addr, &len); - - if (testClientSocket < 0) - THROW_SYS_ERROR(AssertError, "unable to accept socket"); - - SSL *testClientSSL = SSL_new(testServerContext); - - testServerSession = tlsSessionNew( - testClientSSL, sckSessionNew(sckSessionTypeServer, testClientSocket, STRDEF("client"), 0, 5000), 5000); -} - -/*********************************************************************************************************************************** -Close the connection -***********************************************************************************************************************************/ -void -harnessTlsServerClose(void) -{ - tlsSessionClose(testServerSession, true); - tlsSessionFree(testServerSession); - testServerSession = NULL; + FUNCTION_HARNESS_RESULT_VOID(); } /**********************************************************************************************************************************/ -void -harnessTlsServerAbort(void) -{ - tlsSessionClose(testServerSession, false); - tlsSessionFree(testServerSession); - testServerSession = NULL; -} - -/**********************************************************************************************************************************/ -const String *harnessTlsTestHost(void) +const String *hrnTlsServerHost(void) { return strNew(testContainer() ? TLS_TEST_HOST : "127.0.0.1"); } /**********************************************************************************************************************************/ -unsigned int harnessTlsTestPort(void) +unsigned int hrnTlsServerPort(void) { return 44443 + testIdx(); } diff --git a/test/src/common/harnessTls.h b/test/src/common/harnessTls.h index cdd24ee41..0e6cec47a 100644 --- a/test/src/common/harnessTls.h +++ b/test/src/common/harnessTls.h @@ -1,46 +1,62 @@ /*********************************************************************************************************************************** -Tls Test Harness +TLS Test Harness Simple TLS server for testing TLS client functionality. ***********************************************************************************************************************************/ #ifndef TEST_COMMON_HARNESS_TLS_H #define TEST_COMMON_HARNESS_TLS_H +#include "common/debug.h" +#include "common/io/tls/session.h" + /*********************************************************************************************************************************** Path and prefix for test certificates ***********************************************************************************************************************************/ #define TEST_CERTIFICATE_PREFIX "test/certificate/pgbackrest-test" /*********************************************************************************************************************************** -Tls test defaults +TLS test defaults ***********************************************************************************************************************************/ #define TLS_CERT_FAKE_PATH "/etc/fake-cert" #define TLS_CERT_TEST_CERT TLS_CERT_FAKE_PATH "/pgbackrest-test.crt" -#define TLS_CERT_TEST_KEY TLS_CERT_FAKE_PATH "/pgbackrest-test.key" /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ -void harnessTlsServerInit(unsigned int port, const char *serverCert, const char *serverKey); +// Begin/end client +void hrnTlsClientBegin(IoWrite *write); +void hrnTlsClientEnd(void); -// Initialize TLS with default parameters -void harnessTlsServerInitDefault(void); - -void harnessTlsServerAccept(void); -void harnessTlsServerExpect(const char *expected); -void harnessTlsServerReply(const char *reply); -void harnessTlsServerClose(void); +// Run server +void hrnTlsServerRun(IoRead *read); // Abort the server session (i.e. don't perform proper TLS shutdown) -void harnessTlsServerAbort(void); +void hrnTlsServerAbort(void); + +// Accept new TLS connection +void hrnTlsServerAccept(void); + +// Close the TLS connection +void hrnTlsServerClose(void); + +// Expect the specfified string +void hrnTlsServerExpect(const String *data); +void hrnTlsServerExpectZ(const char *data); + +// Reply with the specfified string +void hrnTlsServerReply(const String *data); +void hrnTlsServerReplyZ(const char *data); + +// Sleep specfified milliseconds +void hrnTlsServerSleep(TimeMSec sleepMs); /*********************************************************************************************************************************** Getters/Setters ***********************************************************************************************************************************/ // Hostname to use for testing -- this will vary based on whether the test is running in a container -const String *harnessTlsTestHost(void); +const String *hrnTlsServerHost(void); -// Port to use for testing. This will be unique for each test running in parallel to avoid conflicts -unsigned int harnessTlsTestPort(void); +// Port to use for testing. This will be unique for each test running in parallel to avoid conflicts +unsigned int hrnTlsServerPort(void); #endif diff --git a/test/src/module/common/ioHttpTest.c b/test/src/module/common/ioHttpTest.c index ebdc56f87..8907ae68f 100644 --- a/test/src/module/common/ioHttpTest.c +++ b/test/src/module/common/ioHttpTest.c @@ -3,320 +3,12 @@ Test Http ***********************************************************************************************************************************/ #include -#include "common/time.h" +#include "common/io/handleRead.h" +#include "common/io/handleWrite.h" #include "common/harnessFork.h" #include "common/harnessTls.h" -/*********************************************************************************************************************************** -Test server -***********************************************************************************************************************************/ -static void -testHttpServer(void) -{ - FUNCTION_HARNESS_VOID(); - - harnessTlsServerInitDefault(); - - // Test no output from server - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - sleepMSec(600); - harnessTlsServerClose(); - - // Test invalid http version - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.0 200 OK\r\n"); - - harnessTlsServerClose(); - - // Test no space in status - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200OK\r\n"); - - harnessTlsServerClose(); - - // Test unexpected end of headers - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n"); - - harnessTlsServerClose(); - - // Test missing colon in header - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "header-value\r\n"); - - harnessTlsServerClose(); - - // Test invalid transfer encoding - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "transfer-encoding:bogus\r\n"); - - harnessTlsServerClose(); - - // Test content length and transfer encoding both set - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "transfer-encoding:chunked\r\n" - "content-length:777\r\n" - "\r\n"); - - harnessTlsServerClose(); - - // Test 5xx error with no retry - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 503 Slow Down\r\n" - "\r\n"); - - harnessTlsServerClose(); - - // Request with no content (with an internal error) - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\n" - "host:myhost.com\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 500 Internal Error\r\n" - "Connection:close\r\n" - "\r\n"); - - harnessTlsServerClose(); - - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\n" - "host:myhost.com\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "key1:0\r\n" - " key2 : value2\r\n" - "Connection:ack\r\n" - "\r\n"); - - // Head request with content-length but no content - harnessTlsServerExpect( - "HEAD / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "content-length:380\r\n" - "\r\n"); - - // Head request with transfer encoding but no content - harnessTlsServerExpect( - "HEAD / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "Transfer-Encoding: chunked\r\n" - "\r\n"); - - // Head request with connection close but no content - harnessTlsServerExpect( - "HEAD / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "Connection:close\r\n" - "\r\n"); - - harnessTlsServerClose(); - - harnessTlsServerAccept(); - - // Error with content (with a few slow down errors) - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 503 Slow Down\r\n" - "content-length:3\r\n" - "Connection:close\r\n" - "\r\n" - "123"); - - harnessTlsServerClose(); - - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 503 Slow Down\r\n" - "Transfer-Encoding:chunked\r\n" - "Connection:close\r\n" - "\r\n" - "0\r\n" - "\r\n"); - - harnessTlsServerClose(); - - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 404 Not Found\r\n" - "content-length:0\r\n" - "\r\n"); - - // Error with content - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 403 \r\n" - "content-length:7\r\n" - "\r\n" - "CONTENT"); - - // Request with content - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET /path/file%201.txt HTTP/1.1\r\n" - "content-length:30\r\n" - "\r\n" - "012345678901234567890123456789"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "Connection:close\r\n" - "\r\n" - "01234567890123456789012345678901"); - - harnessTlsServerClose(); - - // Request with eof before content complete with retry - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET /path/file%201.txt HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "content-length:32\r\n" - "\r\n" - "0123456789012345678901234567890"); - - harnessTlsServerClose(); - - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET /path/file%201.txt HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "content-length:32\r\n" - "\r\n" - "01234567890123456789012345678901"); - - // Request with eof before content complete - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET /path/file%201.txt HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "content-length:32\r\n" - "\r\n" - "0123456789012345678901234567890"); - - harnessTlsServerClose(); - - // Request with chunked content - harnessTlsServerAccept(); - - harnessTlsServerExpect( - "GET / HTTP/1.1\r\n" - "\r\n"); - - harnessTlsServerReply( - "HTTP/1.1 200 OK\r\n" - "Transfer-Encoding:chunked\r\n" - "\r\n" - "20\r\n" - "01234567890123456789012345678901\r\n" - "10\r\n" - "0123456789012345\r\n" - "0\r\n" - "\r\n"); - - harnessTlsServerClose(); - - FUNCTION_HARNESS_RESULT_VOID(); -} - /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -436,7 +128,6 @@ testRun(void) if (testBegin("HttpClient")) { HttpClient *client = NULL; - ioBufferSizeSet(35); // Reset statistics httpClientStatLocal = (HttpClientStat){0}; @@ -444,71 +135,164 @@ testRun(void) TEST_RESULT_PTR(httpClientStatStr(), NULL, "no stats yet"); TEST_ASSIGN( - client, httpClientNew(strNew("localhost"), harnessTlsTestPort(), 500, testContainer(), NULL, NULL), + client, httpClientNew(strNew("localhost"), hrnTlsServerPort(), 500, testContainer(), NULL, NULL), "new client"); TEST_ERROR_FMT( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), HostConnectError, - "unable to connect to 'localhost:%u': [111] Connection refused", harnessTlsTestPort()); + "unable to connect to 'localhost:%u': [111] Connection refused", hrnTlsServerPort()); HARNESS_FORK_BEGIN() { - HARNESS_FORK_CHILD_BEGIN(0, false) + HARNESS_FORK_CHILD_BEGIN(0, true) { // Start http test server - TEST_RESULT_VOID(testHttpServer(), "http server begin"); + TEST_RESULT_VOID( + hrnTlsServerRun(ioHandleReadNew(strNew("test server read"), HARNESS_FORK_CHILD_READ(), 5000)), + "http server begin"); } HARNESS_FORK_CHILD_END(); HARNESS_FORK_PARENT_BEGIN() { - // Test no output from server + hrnTlsClientBegin(ioHandleWriteNew(strNew("test client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0))); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("create client"); + + ioBufferSizeSet(35); + TEST_ASSIGN( - client, httpClientNew(harnessTlsTestHost(), harnessTlsTestPort(), 5000, testContainer(), NULL, NULL), + client, httpClientNew(hrnTlsServerHost(), hrnTlsServerPort(), 5000, testContainer(), NULL, NULL), "new client"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("no output from server"); + client->timeout = 0; + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerSleep(600); + + hrnTlsServerClose(); + TEST_ERROR( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FileReadError, "unexpected eof while reading line"); - // Test invalid http version + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("invalid http version"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.0 200 OK\r\n"); + + hrnTlsServerClose(); + TEST_ERROR( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "http version of response 'HTTP/1.0 200 OK' must be HTTP/1.1"); - // Test no space in status + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("no space in status"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200OK\r\n"); + + hrnTlsServerClose(); + TEST_ERROR( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "response status '200OK' must have a space"); - // Test unexpected end of headers + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("unexpected end of headers"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\n"); + + hrnTlsServerClose(); + TEST_ERROR( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FileReadError, "unexpected eof while reading line"); - // Test missing colon in header + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("missing colon in header"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\nheader-value\r\n"); + + hrnTlsServerClose(); + TEST_ERROR( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "header 'header-value' missing colon"); - // Test invalid transfer encoding + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("invalid transfer encoding"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\ntransfer-encoding:bogus\r\n"); + + hrnTlsServerClose(); + TEST_ERROR( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "only 'chunked' is supported for 'transfer-encoding' header"); - // Test content length and transfer encoding both set + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("content length and transfer encoding both set"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\ntransfer-encoding:chunked\r\ncontent-length:777\r\n\r\n"); + + hrnTlsServerClose(); + TEST_ERROR( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FormatError, "'transfer-encoding' and 'content-length' headers are both set"); - // Test 5xx error with no retry + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("5xx error with no retry"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 503 Slow Down\r\n\r\n"); + + hrnTlsServerClose(); + TEST_ERROR( httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), ServiceError, "[503] Slow Down"); - // Request with no content - client->timeout = 5000; + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("request with no content (with an internal error)"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\nhost:myhost.com\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 500 Internal Error\r\nConnection:close\r\n\r\n"); + + hrnTlsServerClose(); + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET /?name=%2Fpath%2FA%20Z.txt&type=test HTTP/1.1\r\nhost:myhost.com\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\nkey1:0\r\n key2 : value2\r\nConnection:ack\r\n\r\n"); HttpHeader *headerRequest = httpHeaderNew(NULL); httpHeaderAdd(headerRequest, strNew("host"), strNew("myhost.com")); @@ -517,71 +301,113 @@ testRun(void) httpQueryAdd(query, strNew("name"), strNew("/path/A Z.txt")); httpQueryAdd(query, strNew("type"), strNew("test")); + client->timeout = 5000; + TEST_RESULT_VOID( - httpClientRequest(client, strNew("GET"), strNew("/"), query, headerRequest, NULL, false), - "request with no content"); - TEST_RESULT_UINT(httpClientResponseCode(client), 200, " check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", " check response message"); - TEST_RESULT_UINT(httpClientEof(client), true, " io is eof"); + httpClientRequest(client, strNew("GET"), strNew("/"), query, headerRequest, NULL, false), "request"); + TEST_RESULT_UINT(httpClientResponseCode(client), 200, "check response code"); + TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", "check response message"); + TEST_RESULT_UINT(httpClientEof(client), true, "io is eof"); TEST_RESULT_STR_Z( httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'ack', key1: '0', key2: 'value2'}", - " check response headers"); + "check response headers"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("head request with content-length but no content"); + + hrnTlsServerExpectZ("HEAD / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\ncontent-length:380\r\n\r\n"); - // Head request with content-length but no content TEST_RESULT_VOID( - httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), - "head request with content-length"); - TEST_RESULT_UINT(httpClientResponseCode(client), 200, " check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", " check response message"); - TEST_RESULT_BOOL(httpClientEof(client), true, " io is eof"); - TEST_RESULT_BOOL(httpClientBusy(client), false, " client is not busy"); + httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), "request"); + TEST_RESULT_UINT(httpClientResponseCode(client), 200, "check response code"); + TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", "check response message"); + TEST_RESULT_BOOL(httpClientEof(client), true, "io is eof"); + TEST_RESULT_BOOL(httpClientBusy(client), false, "client is not busy"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{content-length: '380'}", " check response headers"); + httpHeaderToLog(httpClientResponseHeader(client)), "{content-length: '380'}", "check response headers"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("head request with transfer encoding but no content"); + + hrnTlsServerExpectZ("HEAD / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"); - // Head request with transfer encoding but no content TEST_RESULT_VOID( - httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), - "head request with transfer encoding"); - TEST_RESULT_UINT(httpClientResponseCode(client), 200, " check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", " check response message"); - TEST_RESULT_BOOL(httpClientEof(client), true, " io is eof"); - TEST_RESULT_BOOL(httpClientBusy(client), false, " client is not busy"); + httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), "request"); + TEST_RESULT_UINT(httpClientResponseCode(client), 200, "check response code"); + TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", "check response message"); + TEST_RESULT_BOOL(httpClientEof(client), true, "io is eof"); + TEST_RESULT_BOOL(httpClientBusy(client), false, "client is not busy"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{transfer-encoding: 'chunked'}", - " check response headers"); + httpHeaderToLog(httpClientResponseHeader(client)), "{transfer-encoding: 'chunked'}", "check response headers"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("head request with connection close but no content"); + + hrnTlsServerExpectZ("HEAD / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\nConnection:close\r\n\r\n"); + + hrnTlsServerClose(); - // Head request with connection close but no content TEST_RESULT_VOID( - httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), - "head request with connection close"); - TEST_RESULT_UINT(httpClientResponseCode(client), 200, " check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", " check response message"); - TEST_RESULT_BOOL(httpClientEof(client), true, " io is eof"); - TEST_RESULT_BOOL(httpClientBusy(client), false, " client is not busy"); + httpClientRequest(client, strNew("HEAD"), strNew("/"), NULL, httpHeaderNew(NULL), NULL, true), "request"); + TEST_RESULT_UINT(httpClientResponseCode(client), 200, "check response code"); + TEST_RESULT_STR_Z(httpClientResponseMessage(client), "OK", "check response message"); + TEST_RESULT_BOOL(httpClientEof(client), true, "io is eof"); + TEST_RESULT_BOOL(httpClientBusy(client), false, "client is not busy"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}", " check response headers"); + httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}", "check response headers"); - // Error with content length 0 - TEST_RESULT_VOID( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "error with content length 0"); - TEST_RESULT_UINT(httpClientResponseCode(client), 404, " check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "Not Found", " check response message"); + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error with content (with a few slow down errors)"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 503 Slow Down\r\ncontent-length:3\r\nConnection:close\r\n\r\n123"); + + hrnTlsServerClose(); + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 503 Slow Down\r\nTransfer-Encoding:chunked\r\nConnection:close\r\n\r\n0\r\n\r\n"); + + hrnTlsServerClose(); + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 404 Not Found\r\ncontent-length:0\r\n\r\n"); + + TEST_RESULT_VOID(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "request"); + TEST_RESULT_UINT(httpClientResponseCode(client), 404, "check response code"); + TEST_RESULT_STR_Z(httpClientResponseMessage(client), "Not Found", "check response message"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{content-length: '0'}", " check response headers"); + httpHeaderToLog(httpClientResponseHeader(client)), "{content-length: '0'}", "check response headers"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error with content"); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 403 \r\ncontent-length:7\r\n\r\nCONTENT"); - // Error with content Buffer *buffer = NULL; - TEST_ASSIGN( - buffer, httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), - "error with content length"); - TEST_RESULT_UINT(httpClientResponseCode(client), 403, " check response code"); - TEST_RESULT_STR_Z(httpClientResponseMessage(client), "", " check empty response message"); + TEST_ASSIGN(buffer, httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "request"); + TEST_RESULT_UINT(httpClientResponseCode(client), 403, "check response code"); + TEST_RESULT_STR_Z(httpClientResponseMessage(client), "", "check empty response message"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{content-length: '7'}", " check response headers"); - TEST_RESULT_STR_Z(strNewBuf(buffer), "CONTENT", " check response"); + httpHeaderToLog(httpClientResponseHeader(client)), "{content-length: '7'}", "check response headers"); + TEST_RESULT_STR_Z(strNewBuf(buffer), "CONTENT", "check response"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("request with content using content-length"); + + hrnTlsServerExpectZ("GET /path/file%201.txt HTTP/1.1\r\ncontent-length:30\r\n\r\n012345678901234567890123456789"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\nConnection:close\r\n\r\n01234567890123456789012345678901"); + + hrnTlsServerClose(); - // Request with content using content-length ioBufferSizeSet(30); TEST_ASSIGN( @@ -590,48 +416,86 @@ testRun(void) client, strNew("GET"), strNew("/path/file 1.txt"), NULL, httpHeaderAdd(httpHeaderNew(NULL), strNew("content-length"), strNew("30")), BUFSTRDEF("012345678901234567890123456789"), true), - "request with content length"); + "request"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}", - " check response headers"); - TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901", " check response"); - TEST_RESULT_UINT(httpClientRead(client, bufNew(1), true), 0, " call internal read to check eof"); + httpHeaderToLog(httpClientResponseHeader(client)), "{connection: 'close'}", "check response headers"); + TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901", "check response"); + TEST_RESULT_UINT(httpClientRead(client, bufNew(1), true), 0, "call internal read to check eof"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("request with eof before content complete with retry"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET /path/file%201.txt HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\ncontent-length:32\r\n\r\n0123456789012345678901234567890"); + + hrnTlsServerClose(); + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET /path/file%201.txt HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\ncontent-length:32\r\n\r\n01234567890123456789012345678901"); - // Request with eof before content complete with retry TEST_ASSIGN( buffer, httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, NULL, true), - "request with content length retry"); - TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901", " check response"); - TEST_RESULT_UINT(httpClientRead(client, bufNew(1), true), 0, " call internal read to check eof"); + "request"); + TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901", "check response"); + TEST_RESULT_UINT(httpClientRead(client, bufNew(1), true), 0, "call internal read to check eof"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("request with eof before content complete"); + + hrnTlsServerExpectZ("GET /path/file%201.txt HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ("HTTP/1.1 200 OK\r\ncontent-length:32\r\n\r\n0123456789012345678901234567890"); + + hrnTlsServerClose(); - // Request with eof before content and error buffer = bufNew(32); - TEST_RESULT_VOID( - httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, NULL, false), - "request with content length error"); - TEST_RESULT_BOOL(httpClientBusy(client), true, " client is busy"); - TEST_ERROR( - ioRead(httpClientIoRead(client), buffer), FileReadError, "unexpected EOF reading HTTP content"); - // Request with content using chunked encoding TEST_RESULT_VOID( - httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), - "request with chunked encoding"); + httpClientRequest(client, strNew("GET"), strNew("/path/file 1.txt"), NULL, NULL, NULL, false), "request"); + TEST_RESULT_BOOL(httpClientBusy(client), true, "client is busy"); + TEST_ERROR(ioRead(httpClientIoRead(client), buffer), FileReadError, "unexpected EOF reading HTTP content"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("request with chunked content"); + + hrnTlsServerAccept(); + + hrnTlsServerExpectZ("GET / HTTP/1.1\r\n\r\n"); + hrnTlsServerReplyZ( + "HTTP/1.1 200 OK\r\nTransfer-Encoding:chunked\r\n\r\n" + "20\r\n01234567890123456789012345678901\r\n" + "10\r\n0123456789012345\r\n" + "0\r\n\r\n"); + + TEST_RESULT_VOID(httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), "request"); TEST_RESULT_STR_Z( - httpHeaderToLog(httpClientResponseHeader(client)), "{transfer-encoding: 'chunked'}", - " check response headers"); + httpHeaderToLog(httpClientResponseHeader(client)), "{transfer-encoding: 'chunked'}", "check response headers"); buffer = bufNew(35); - TEST_RESULT_VOID(ioRead(httpClientIoRead(client), buffer), " read response"); - TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901012", " check response"); - TEST_RESULT_BOOL(httpClientStatStr() != NULL, true, "check statistics exist"); + TEST_RESULT_VOID(ioRead(httpClientIoRead(client), buffer), "read response"); + TEST_RESULT_STR_Z(strNewBuf(buffer), "01234567890123456789012345678901012", "check response"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("close connection"); + + hrnTlsServerClose(); TEST_RESULT_VOID(httpClientFree(client), "free client"); + + // ----------------------------------------------------------------------------------------------------------------- + hrnTlsClientEnd(); } HARNESS_FORK_PARENT_END(); } HARNESS_FORK_END(); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("statistics exist"); + + TEST_RESULT_BOOL(httpClientStatStr() != NULL, true, "check"); } // ***************************************************************************************************************************** @@ -642,7 +506,7 @@ testRun(void) HttpClient *client2 = NULL; TEST_ASSIGN( - cache, httpClientCacheNew(strNew("localhost"), harnessTlsTestPort(), 5000, true, NULL, NULL), "new http client cache"); + cache, httpClientCacheNew(strNew("localhost"), hrnTlsServerPort(), 5000, true, NULL, NULL), "new http client cache"); TEST_ASSIGN(client1, httpClientCacheGet(cache), "get http client"); TEST_RESULT_PTR(client1, *(HttpClient **)lstGet(cache->clientList, 0), " check http client"); TEST_RESULT_PTR(httpClientCacheGet(cache), *(HttpClient **)lstGet(cache->clientList, 0), " get same http client"); diff --git a/test/src/module/common/ioTlsTest.c b/test/src/module/common/ioTlsTest.c index ca3dc5ea7..5632e7377 100644 --- a/test/src/module/common/ioTlsTest.c +++ b/test/src/module/common/ioTlsTest.c @@ -4,92 +4,16 @@ Test Tls Client #include #include -#include "common/time.h" +#include "common/io/handleRead.h" +#include "common/io/handleWrite.h" #include "common/harnessFork.h" #include "common/harnessTls.h" /*********************************************************************************************************************************** -Test server with subject alternate names +Version that allows custom certs ***********************************************************************************************************************************/ -#ifdef TEST_CONTAINER_REQUIRED - -static void -testTlsServerAltName(void) -{ - FUNCTION_HARNESS_VOID(); - - harnessTlsServerInit( - harnessTlsTestPort(), - strPtr(strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-alt-name.crt", testRepoPath())), - strPtr(strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".key", testRepoPath()))); - - // Certificate error on invalid ca path - harnessTlsServerAccept(); - harnessTlsServerClose(); - - // Success on valid ca file and match common name - harnessTlsServerAccept(); - harnessTlsServerClose(); - - // Success on valid ca file and match alt name - harnessTlsServerAccept(); - harnessTlsServerClose(); - - // Unable to find matching hostname in certificate - harnessTlsServerAccept(); - harnessTlsServerClose(); - - // Certificate error - harnessTlsServerAccept(); - harnessTlsServerClose(); - - // Certificate ignored - harnessTlsServerAccept(); - harnessTlsServerClose(); - - FUNCTION_HARNESS_RESULT_VOID(); -} - -#endif // TEST_CONTAINER_REQUIRED - -/*********************************************************************************************************************************** -Test server -***********************************************************************************************************************************/ -static void -testTlsServer(void) -{ - FUNCTION_HARNESS_VOID(); - - harnessTlsServerInitDefault(); - - // First protocol exchange - harnessTlsServerAccept(); - - harnessTlsServerExpect("some protocol info"); - harnessTlsServerReply("something:0\n"); - - sleepMSec(100); - harnessTlsServerReply("some "); - - sleepMSec(100); - harnessTlsServerReply("contentAND MORE"); - - // This will cause the client to disconnect - sleepMSec(500); - - // Second protocol exchange - harnessTlsServerExpect("more protocol info"); - harnessTlsServerReply("0123456789AB"); - harnessTlsServerClose(); - - // Test aborted connection before read complete - harnessTlsServerAccept(); - harnessTlsServerReply("0123456789AB"); - harnessTlsServerAbort(); - - FUNCTION_HARNESS_RESULT_VOID(); -} +void hrnTlsServerRunParam(IoRead *read, const String *certificate, const String *key); /*********************************************************************************************************************************** Test Run @@ -257,14 +181,14 @@ testRun(void) { SocketClient *client = NULL; - TEST_ASSIGN(client, sckClientNew(strNew("localhost"), harnessTlsTestPort(), 100), "new client"); + TEST_ASSIGN(client, sckClientNew(strNew("localhost"), hrnTlsServerPort(), 100), "new client"); TEST_ERROR_FMT( sckClientOpen(client), HostConnectError, "unable to connect to 'localhost:%u': [111] Connection refused", - harnessTlsTestPort()); + hrnTlsServerPort()); // This address should not be in use in a test environment -- if it is the test will fail - TEST_ASSIGN(client, sckClientNew(strNew("172.31.255.255"), harnessTlsTestPort(), 100), "new client"); - TEST_ERROR_FMT(sckClientOpen(client), HostConnectError, "timeout connecting to '172.31.255.255:%u'", harnessTlsTestPort()); + TEST_ASSIGN(client, sckClientNew(strNew("172.31.255.255"), hrnTlsServerPort(), 100), "new client"); + TEST_ERROR_FMT(sckClientOpen(client), HostConnectError, "timeout connecting to '172.31.255.255:%u'", hrnTlsServerPort()); } // Additional coverage not provided by testing with actual certificates @@ -292,17 +216,26 @@ testRun(void) // Connection errors // ------------------------------------------------------------------------------------------------------------------------- TEST_ASSIGN( - client, tlsClientNew(sckClientNew(strNew("99.99.99.99.99"), harnessTlsTestPort(), 0), 0, true, NULL, NULL), + client, tlsClientNew(sckClientNew(strNew("99.99.99.99.99"), hrnTlsServerPort(), 0), 0, true, NULL, NULL), "new client"); TEST_ERROR( tlsClientOpen(client), HostConnectError, "unable to get address for '99.99.99.99.99': [-2] Name or service not known"); TEST_ASSIGN( - client, tlsClientNew(sckClientNew(strNew("localhost"), harnessTlsTestPort(), 100), 100, true, NULL, NULL), + client, tlsClientNew(sckClientNew(strNew("localhost"), hrnTlsServerPort(), 100), 100, true, NULL, NULL), "new client"); TEST_ERROR_FMT( tlsClientOpen(client), HostConnectError, "unable to connect to 'localhost:%u': [111] Connection refused", - harnessTlsTestPort()); + hrnTlsServerPort()); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("bogus client cert/path"); + + TEST_ERROR( + tlsClientOpen( + tlsClientNew( + sckClientNew(strNew("localhost"), hrnTlsServerPort(), 5000), 0, true, strNew("bogus.crt"), strNew("/bogus"))), + CryptoError, "unable to set user-defined CA certificate location: [33558530] No such file or directory"); // Certificate location and validation errors // ------------------------------------------------------------------------------------------------------------------------- @@ -317,64 +250,105 @@ testRun(void) HARNESS_FORK_BEGIN() { - HARNESS_FORK_CHILD_BEGIN(0, false) + HARNESS_FORK_CHILD_BEGIN(0, true) { // Start server to test various certificate errors - TEST_RESULT_VOID(testTlsServerAltName(), "tls alt name server begin"); + TEST_RESULT_VOID( + hrnTlsServerRunParam( + ioHandleReadNew(strNew("test server read"), HARNESS_FORK_CHILD_READ(), 5000), + strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-alt-name.crt", testRepoPath()), + strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".key", testRepoPath())), + "tls alt name server begin"); } HARNESS_FORK_CHILD_END(); HARNESS_FORK_PARENT_BEGIN() { - TEST_ERROR( - tlsClientOpen( - tlsClientNew( - sckClientNew(strNew("localhost"), harnessTlsTestPort(), 5000), 0, true, strNew("bogus.crt"), - strNew("/bogus"))), - CryptoError, "unable to set user-defined CA certificate location: [33558530] No such file or directory"); + hrnTlsClientBegin(ioHandleWriteNew(strNew("test client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0))); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("certificate error on invalid ca path"); + + hrnTlsServerAccept(); + hrnTlsServerClose(); TEST_ERROR_FMT( tlsClientOpen( tlsClientNew( - sckClientNew(strNew("localhost"), harnessTlsTestPort(), 5000), 0, true, NULL, strNew("/bogus"))), + sckClientNew(strNew("localhost"), hrnTlsServerPort(), 5000), 0, true, NULL, strNew("/bogus"))), CryptoError, "unable to verify certificate presented by 'localhost:%u': [20] unable to get local issuer certificate", - harnessTlsTestPort()); + hrnTlsServerPort()); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("valid ca file and match common name"); + + hrnTlsServerAccept(); + hrnTlsServerClose(); TEST_RESULT_VOID( tlsClientOpen( tlsClientNew( - sckClientNew(strNew("test.pgbackrest.org"), harnessTlsTestPort(), 5000), 0, true, + sckClientNew(strNew("test.pgbackrest.org"), hrnTlsServerPort(), 5000), 0, true, strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-ca.crt", testRepoPath()), NULL)), - "success on valid ca file and match common name"); + "open connection"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("valid ca file and match alt name"); + + hrnTlsServerAccept(); + hrnTlsServerClose(); + TEST_RESULT_VOID( tlsClientOpen( tlsClientNew( - sckClientNew(strNew("host.test2.pgbackrest.org"), harnessTlsTestPort(), 5000), 0, true, + sckClientNew(strNew("host.test2.pgbackrest.org"), hrnTlsServerPort(), 5000), 0, true, strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-ca.crt", testRepoPath()), NULL)), - "success on valid ca file and match alt name"); + "open connection"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("unable to find matching hostname in certificate"); + + hrnTlsServerAccept(); + hrnTlsServerClose(); + TEST_ERROR( tlsClientOpen( tlsClientNew( - sckClientNew(strNew("test3.pgbackrest.org"), harnessTlsTestPort(), 5000), 0, true, + sckClientNew(strNew("test3.pgbackrest.org"), hrnTlsServerPort(), 5000), 0, true, strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-ca.crt", testRepoPath()), NULL)), CryptoError, "unable to find hostname 'test3.pgbackrest.org' in certificate common name or subject alternative names"); + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("certificate error"); + + hrnTlsServerAccept(); + hrnTlsServerClose(); + TEST_ERROR_FMT( tlsClientOpen( tlsClientNew( - sckClientNew(strNew("localhost"), harnessTlsTestPort(), 5000), 0, true, + sckClientNew(strNew("localhost"), hrnTlsServerPort(), 5000), 0, true, strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".crt", testRepoPath()), NULL)), CryptoError, "unable to verify certificate presented by 'localhost:%u': [20] unable to get local issuer certificate", - harnessTlsTestPort()); + hrnTlsServerPort()); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("no certificate verify"); + + hrnTlsServerAccept(); + hrnTlsServerClose(); TEST_RESULT_VOID( tlsClientOpen( - tlsClientNew(sckClientNew(strNew("localhost"), harnessTlsTestPort(), 5000), 0, false, NULL, NULL)), - "success on no verify"); + tlsClientNew(sckClientNew(strNew("localhost"), hrnTlsServerPort(), 5000), 0, false, NULL, NULL)), + "open connection"); + + // ----------------------------------------------------------------------------------------------------------------- + hrnTlsClientEnd(); } HARNESS_FORK_PARENT_END(); } @@ -396,20 +370,26 @@ testRun(void) HARNESS_FORK_BEGIN() { - HARNESS_FORK_CHILD_BEGIN(0, false) + HARNESS_FORK_CHILD_BEGIN(0, true) { - TEST_RESULT_VOID(testTlsServer(), "tls server begin"); + TEST_RESULT_VOID( + hrnTlsServerRun(ioHandleReadNew(strNew("test server read"), HARNESS_FORK_CHILD_READ(), 5000)), + "tls server begin"); } HARNESS_FORK_CHILD_END(); HARNESS_FORK_PARENT_BEGIN() { + hrnTlsClientBegin(ioHandleWriteNew(strNew("test client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0))); ioBufferSizeSet(12); TEST_ASSIGN( client, - tlsClientNew(sckClientNew(harnessTlsTestHost(), harnessTlsTestPort(), 5000), 0, testContainer(), NULL, NULL), + tlsClientNew(sckClientNew(hrnTlsServerHost(), hrnTlsServerPort(), 5000), 0, testContainer(), NULL, NULL), "new client"); + + hrnTlsServerAccept(); + TEST_ASSIGN(session, tlsClientOpen(client), "open client"); // ----------------------------------------------------------------------------------------------------------------- @@ -439,49 +419,76 @@ testRun(void) TEST_ERROR(tlsSessionResultProcess(session, SSL_ERROR_ZERO_RETURN, 0, false), ProtocolError, "unexpected TLS eof"); // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("first protocol exchange"); + + hrnTlsServerExpectZ("some protocol info"); + hrnTlsServerReplyZ("something:0\n"); + const Buffer *input = BUFSTRDEF("some protocol info"); TEST_RESULT_VOID(ioWrite(tlsSessionIoWrite(session), input), "write input"); ioWriteFlush(tlsSessionIoWrite(session)); TEST_RESULT_STR_Z(ioReadLine(tlsSessionIoRead(session)), "something:0", "read line"); - TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), false, " check eof = false"); + TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), false, "check eof = false"); + + hrnTlsServerSleep(100); + hrnTlsServerReplyZ("some "); + + hrnTlsServerSleep(100); + hrnTlsServerReplyZ("contentAND MORE"); Buffer *output = bufNew(12); TEST_RESULT_UINT(ioRead(tlsSessionIoRead(session), output), 12, "read output"); - TEST_RESULT_STR_Z(strNewBuf(output), "some content", " check output"); - TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), false, " check eof = false"); + TEST_RESULT_STR_Z(strNewBuf(output), "some content", "check output"); + TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), false, "check eof = false"); output = bufNew(8); TEST_RESULT_UINT(ioRead(tlsSessionIoRead(session), output), 8, "read output"); - TEST_RESULT_STR_Z(strNewBuf(output), "AND MORE", " check output"); - TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), false, " check eof = false"); + TEST_RESULT_STR_Z(strNewBuf(output), "AND MORE", "check output"); + TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), false, "check eof = false"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("read eof"); + + hrnTlsServerSleep(500); output = bufNew(12); session->socketSession->timeout = 100; TEST_ERROR_FMT( ioRead(tlsSessionIoRead(session), output), ProtocolError, - "timeout after 100ms waiting for read from '%s:%u'", strPtr(harnessTlsTestHost()), harnessTlsTestPort()); + "timeout after 100ms waiting for read from '%s:%u'", strPtr(hrnTlsServerHost()), hrnTlsServerPort()); session->socketSession->timeout = 5000; // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("second protocol exchange"); + + hrnTlsServerExpectZ("more protocol info"); + hrnTlsServerReplyZ("0123456789AB"); + + hrnTlsServerClose(); + input = BUFSTRDEF("more protocol info"); TEST_RESULT_VOID(ioWrite(tlsSessionIoWrite(session), input), "write input"); ioWriteFlush(tlsSessionIoWrite(session)); output = bufNew(12); TEST_RESULT_UINT(ioRead(tlsSessionIoRead(session), output), 12, "read output"); - TEST_RESULT_STR_Z(strNewBuf(output), "0123456789AB", " check output"); - TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), false, " check eof = false"); + TEST_RESULT_STR_Z(strNewBuf(output), "0123456789AB", "check output"); + TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), false, "check eof = false"); output = bufNew(12); TEST_RESULT_UINT(ioRead(tlsSessionIoRead(session), output), 0, "read no output after eof"); - TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), true, " check eof = true"); + TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), true, "check eof = true"); TEST_RESULT_VOID(tlsSessionClose(session, false), "close again"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("aborted connection before read complete (blocking socket)"); + hrnTlsServerAccept(); + hrnTlsServerReplyZ("0123456789AB"); + hrnTlsServerAbort(); + socketLocal.block = true; TEST_ASSIGN(session, tlsClientOpen(client), "open client again (was closed by server)"); socketLocal.block = false; @@ -490,14 +497,22 @@ testRun(void) TEST_ERROR(ioRead(tlsSessionIoRead(session), output), KernelError, "TLS syscall error"); // ----------------------------------------------------------------------------------------------------------------- - TEST_RESULT_BOOL(sckClientStatStr() != NULL, true, "check statistics exist"); - TEST_RESULT_BOOL(tlsClientStatStr() != NULL, true, "check statistics exist"); + TEST_TITLE("close connection"); TEST_RESULT_VOID(tlsClientFree(client), "free client"); + + // ----------------------------------------------------------------------------------------------------------------- + hrnTlsClientEnd(); } HARNESS_FORK_PARENT_END(); } HARNESS_FORK_END(); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("stastistics exist"); + + TEST_RESULT_BOOL(sckClientStatStr() != NULL, true, "check socket"); + TEST_RESULT_BOOL(tlsClientStatStr() != NULL, true, "check tls"); } FUNCTION_HARNESS_RESULT_VOID(); diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c index c7f9bbd66..6eb8aec37 100644 --- a/test/src/module/storage/s3Test.c +++ b/test/src/module/storage/s3Test.c @@ -3,521 +3,140 @@ Test S3 Storage ***********************************************************************************************************************************/ #include +#include "common/io/handleRead.h" +#include "common/io/handleWrite.h" + #include "common/harnessConfig.h" #include "common/harnessFork.h" #include "common/harnessStorage.h" #include "common/harnessTls.h" /*********************************************************************************************************************************** -Test server +Constants ***********************************************************************************************************************************/ #define S3_TEST_HOST "s3.amazonaws.com" -#define DATE_REPLACE "????????" -#define DATETIME_REPLACE "????????T??????Z" -#define SHA256_REPLACE \ - "????????????????????????????????????????????????????????????????" -static const char * -testS3ServerRequest(const char *verb, const char *uri, const char *content, StorageS3UriStyle uriStyle) +/*********************************************************************************************************************************** +Helper to build test requests +***********************************************************************************************************************************/ +typedef struct TestRequestParam { + VAR_PARAM_HEADER; + const char *content; +} TestRequestParam; + +#define testRequestP(s3, verb, uri, ...) \ + testRequest(s3, verb, uri, (TestRequestParam){VAR_PARAM_INIT, __VA_ARGS__}) + +void +testRequest(Storage *s3, const char *verb, const char *uri, TestRequestParam param) +{ + // Add authorization string String *request = strNewFmt( "%s %s HTTP/1.1\r\n" - "authorization:AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/" DATE_REPLACE "/us-east-1/s3/aws4_request," + "authorization:AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request," "SignedHeaders=content-length;", verb, uri); - if (content != NULL) + if (param.content != NULL) strCat(request, "content-md5;"); strCatFmt( request, - "host;x-amz-content-sha256;x-amz-date,Signature=" SHA256_REPLACE "\r\n" + "host;x-amz-content-sha256;x-amz-date,Signature=????????????????????????????????????????????????????????????????\r\n" "content-length:%zu\r\n", - content == NULL ? 0 : strlen(content)); + param.content == NULL ? 0 : strlen(param.content)); - if (content != NULL) + // Add md5 + if (param.content != NULL) { char md5Hash[HASH_TYPE_MD5_SIZE_HEX]; - encodeToStr(encodeBase64, bufPtr(cryptoHashOne(HASH_TYPE_MD5_STR, BUFSTRZ(content))), HASH_TYPE_M5_SIZE, md5Hash); + encodeToStr(encodeBase64, bufPtr(cryptoHashOne(HASH_TYPE_MD5_STR, BUFSTRZ(param.content))), HASH_TYPE_M5_SIZE, md5Hash); strCatFmt(request, "content-md5:%s\r\n", md5Hash); } - if (uriStyle == storageS3UriStyleHost) + // Add host + if (((StorageS3 *)storageDriver(s3))->uriStyle == storageS3UriStyleHost) strCatFmt(request, "host:bucket." S3_TEST_HOST "\r\n"); else strCatFmt(request, "host:" S3_TEST_HOST "\r\n"); + // Add content sha256 and date strCatFmt( request, "x-amz-content-sha256:%s\r\n" - "x-amz-date:" DATETIME_REPLACE "\r\n" + "x-amz-date:????????T??????Z" "\r\n" "\r\n", - content == NULL ? HASH_TYPE_SHA256_ZERO : strPtr(bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR, BUFSTRZ(content))))); + param.content == NULL ? HASH_TYPE_SHA256_ZERO : strPtr(bufHex(cryptoHashOne(HASH_TYPE_SHA256_STR, + BUFSTRZ(param.content))))); - if (content != NULL) - strCat(request, content); + // Add content + if (param.content != NULL) + strCat(request, param.content); - return strPtr(request); + hrnTlsServerExpect(request); } -static const char * -testS3ServerResponse(unsigned int code, const char *message, const char *header, const char *content) +/*********************************************************************************************************************************** +Helper to build test responses +***********************************************************************************************************************************/ +typedef struct TestResponseParam { - String *response = strNewFmt("HTTP/1.1 %u %s\r\n", code, message); + VAR_PARAM_HEADER; + unsigned int code; + const char *header; + const char *content; +} TestResponseParam; - if (header != NULL) - strCatFmt(response, "%s\r\n", header); +#define testResponseP(...) \ + testResponse((TestResponseParam){VAR_PARAM_INIT, __VA_ARGS__}) - if (content != NULL) +void +testResponse(TestResponseParam param) +{ + // Set code to 200 if not specified + param.code = param.code == 0 ? 200 : param.code; + + // Output header and code + String *response = strNewFmt("HTTP/1.1 %u ", param.code); + + // Add reason for some codes + switch (param.code) + { + case 200: + { + strCat(response, "OK"); + break; + } + + case 403: + { + strCat(response, "Forbidden"); + break; + } + } + + // End header + strCat(response, "\r\n"); + + // Headers + if (param.header != NULL) + strCatFmt(response, "%s\r\n", param.header); + + // Content + if (param.content != NULL) { strCatFmt( response, "content-length:%zu\r\n" "\r\n" "%s", - strlen(content), content); + strlen(param.content), param.content); } else strCat(response, "\r\n"); - return strPtr(response); -} - -static void -testS3Server(void) -{ - FUNCTION_HARNESS_VOID(); - - harnessTlsServerInitDefault(); - harnessTlsServerAccept(); - - // storageS3NewRead() and StorageS3FileRead - // ------------------------------------------------------------------------------------------------------------------------- - // Ignore missing file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/fi%26le.txt", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL)); - - // Error on missing file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL)); - - // Get file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, "this is a sample file")); - - // Get zero-length file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file0.txt", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); - - // Throw non-404 error - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/file.txt", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(303, "Some bad status", NULL, "CONTENT")); - - // storageS3NewWrite() and StorageWriteS3 - // ------------------------------------------------------------------------------------------------------------------------- - // File is written all at once - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD", storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 403, "Forbidden", NULL, - "" - "" - "RequestTimeTooSkewed" - "The difference between the request time and the current time is too large." - "20190726T221748Z" - "2019-07-26T22:33:27Z" - "900000" - "601AA1A7F7E37AE9" - "KYMys77PoloZrGCkiQRyOIl0biqdHsk4T2EdTkhzkH1l8x00D4lvv/py5uUuHwQXG9qz6NRuldQ=" - "")); - - harnessTlsServerAccept(); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "ABCD", storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); - - // Zero-length file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_PUT, "/file.txt", "", storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); - - // File is written in chunks with nothing left over on close - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 200, "OK", NULL, - "" - "" - "bucket" - "file.txt" - "WxRt" - "")); - - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", "1234567890123456", storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", "etag:WxRt1", NULL)); - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", "7890123456789012", storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", "eTag:WxRt2", NULL)); - - harnessTlsServerExpect(testS3ServerRequest( - HTTP_VERB_POST, "/file.txt?uploadId=WxRt", - "\n" - "" - "1WxRt1" - "2WxRt2" - "\n", - storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); - - // File is written in chunks with something left over on close - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_POST, "/file.txt?uploads=", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 200, "OK", NULL, - "" - "" - "bucket" - "file.txt" - "RR55" - "")); - - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", "1234567890123456", storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", "etag:RR551", NULL)); - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", "7890", storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", "eTag:RR552", NULL)); - - harnessTlsServerExpect(testS3ServerRequest( - HTTP_VERB_POST, "/file.txt?uploadId=RR55", - "\n" - "" - "1RR551" - "2RR552" - "\n", - storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); - - // storageDriverExists() - // ------------------------------------------------------------------------------------------------------------------------- - // File missing - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL)); - - // File exists - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 200, "OK", - "content-length:999\r\n" - "Last-Modified: Wed, 21 Oct 2015 07:28:00 GMT", - NULL)); - - // Info() - // ------------------------------------------------------------------------------------------------------------------------- - // File missing - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL)); - - // File exists - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 200, "OK", - "content-length:9999\r\n" - "Last-Modified: Wed, 21 Oct 2015 07:28:00 GMT", - NULL)); - - // File exists and only checking existence - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file2.txt", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 200, "OK", - "content-length:777\r\n" - "Last-Modified: Wed, 22 Oct 2015 07:28:00 GMT", - NULL)); - - // InfoList() - // ------------------------------------------------------------------------------------------------------------------------- - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL, storageS3UriStyleHost)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " " - " path/to/test_file" - " 2009-10-12T17:50:30.000Z" - " 787" - " " - " " - " path/to/test_path/" - " " - "")); - - // storageDriverList() - // ------------------------------------------------------------------------------------------------------------------------- - // Throw errors - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( 344, "Another bad status", NULL, NULL)); - - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 344, "Another bad status with xml", NULL, - "" - "" - "SomeOtherCode" - "")); - - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 403, "Forbidden", NULL, - "" - "" - "RequestTimeTooSkewed" - "The difference between the request time and the current time is too large." - "")); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 403, "Forbidden", NULL, - "" - "" - "RequestTimeTooSkewed" - "The difference between the request time and the current time is too large." - "")); - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse( - 403, "Forbidden", NULL, - "" - "" - "RequestTimeTooSkewed" - "The difference between the request time and the current time is too large." - "")); - - // list a file/path in root - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2", NULL, storageS3UriStyleHost)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " " - " test1.txt" - " " - " " - " path1/" - " " - "")); - - // list a file in root with expression - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test", NULL, storageS3UriStyleHost)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " " - " test1.txt" - " " - "")); - - // list files with continuation - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL, storageS3UriStyleHost)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " 1ueGcxLPRx1Tr/XYExHnhbYLgveDs2J/wm36Hy4vbOwM=" - " " - " path/to/test1.txt" - " " - " " - " path/to/test2.txt" - " " - " " - " path/to/path1/" - " " - "")); - - harnessTlsServerExpect( - testS3ServerRequest( - HTTP_VERB_GET, - "/?continuation-token=1ueGcxLPRx1Tr%2FXYExHnhbYLgveDs2J%2Fwm36Hy4vbOwM%3D&delimiter=%2F&list-type=2" - "&prefix=path%2Fto%2F", - NULL, storageS3UriStyleHost)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " " - " path/to/test3.txt" - " " - " " - " path/to/path2/" - " " - "")); - - // list files with expression - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest", NULL, storageS3UriStyleHost)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " " - " path/to/test1.txt" - " " - " " - " path/to/test2.txt" - " " - " " - " path/to/test3.txt" - " " - " " - " path/to/test1.path/" - " " - " " - " path/to/test2.path/" - " " - "")); - - // storageDriverPathRemove() - // ------------------------------------------------------------------------------------------------------------------------- - // Switch to path-style URIs - harnessTlsServerClose(); - harnessTlsServerAccept(); - - // delete files from root - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2", NULL, storageS3UriStylePath)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " " - " test1.txt" - " " - " " - " path1/xxx.zzz" - " " - "")); - - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=", - "\n" - "true" - "test1.txt" - "path1/xxx.zzz" - "\n", - storageS3UriStylePath)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, "")); - - // nothing to do in empty subpath - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F", NULL, storageS3UriStylePath)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - "")); - - // delete with continuation - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2Fto%2F", NULL, storageS3UriStylePath)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " continue" - " " - " path/to/test3/" - " " - " " - " path/to/test1.txt" - " " - "")); - - harnessTlsServerExpect( - testS3ServerRequest( - HTTP_VERB_GET, "/bucket/?continuation-token=continue&list-type=2&prefix=path%2Fto%2F", NULL, - storageS3UriStylePath)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " " - " path/to/test3.txt" - " " - " " - " path/to/test2.txt" - " " - "")); - - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=", - "\n" - "true" - "path/to/test1.txt" - "path/to/test3.txt" - "\n", - storageS3UriStylePath)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); - - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=", - "\n" - "true" - "path/to/test2.txt" - "\n", - storageS3UriStylePath)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", NULL, NULL)); - - // delete error - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F", NULL, storageS3UriStylePath)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - " " - " path/sample.txt" - " " - " " - " path/sample2.txt" - " " - "")); - - harnessTlsServerExpect( - testS3ServerRequest(HTTP_VERB_POST, "/bucket/?delete=", - "\n" - "true" - "path/sample.txt" - "path/sample2.txt" - "\n", - storageS3UriStylePath)); - harnessTlsServerReply( - testS3ServerResponse( - 200, "OK", NULL, - "" - "" - "sample2.txtAccessDeniedAccess Denied" - "")); - - // storageDriverRemove() - // ------------------------------------------------------------------------------------------------------------------------- - // remove file - harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_DELETE, "/bucket/path/to/test.txt", NULL, storageS3UriStylePath)); - harnessTlsServerReply(testS3ServerResponse(204, "No Content", NULL, NULL)); - - harnessTlsServerClose(); - - FUNCTION_HARNESS_RESULT_VOID(); + hrnTlsServerReply(response); } /*********************************************************************************************************************************** @@ -533,8 +152,8 @@ testRun(void) const String *bucket = strNew("bucket"); const String *region = strNew("us-east-1"); const String *endPoint = strNew("s3.amazonaws.com"); - const String *host = harnessTlsTestHost(); - const unsigned int port = harnessTlsTestPort(); + const String *host = hrnTlsServerHost(); + const unsigned int port = hrnTlsServerPort(); const String *accessKey = strNew("AKIAIOSFODNN7EXAMPLE"); const String *secretAccessKey = strNew("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); const String *securityToken = strNew( @@ -582,7 +201,7 @@ testRun(void) strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s", strPtr(endPoint))); strLstAdd(argList, strNewFmt("--repo1-s3-host=%s", strPtr(host))); strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH); - strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_FAKE_PATH "/pgbackrest-test.crt"); + strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_TEST_CERT); setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true); @@ -608,7 +227,7 @@ testRun(void) strLstAdd(argList, strNewFmt("--repo1-s3-region=%s", strPtr(region))); strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s:999", strPtr(endPoint))); strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH); - strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_FAKE_PATH "/pgbackrest-test.crt"); + strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_TEST_CERT); setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true); @@ -635,7 +254,7 @@ testRun(void) strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s:999", strPtr(endPoint))); strLstAdd(argList, strNewFmt("--repo1-s3-host=%s:7777", strPtr(host))); strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH); - strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_FAKE_PATH "/pgbackrest-test.crt"); + strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_TEST_CERT); setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true); @@ -663,7 +282,7 @@ testRun(void) strLstAdd(argList, strNewFmt("--repo1-s3-host=%s:7777", strPtr(host))); strLstAddZ(argList, "--repo1-s3-port=9001"); strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH); - strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_FAKE_PATH "/pgbackrest-test.crt"); + strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_TEST_CERT); setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true); setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true); setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true); @@ -755,14 +374,18 @@ testRun(void) { HARNESS_FORK_BEGIN() { - HARNESS_FORK_CHILD_BEGIN(0, false) + HARNESS_FORK_CHILD_BEGIN(0, true) { - TEST_RESULT_VOID(testS3Server(), "s3 server begin"); + TEST_RESULT_VOID( + hrnTlsServerRun(ioHandleReadNew(strNew("test server read"), HARNESS_FORK_CHILD_READ(), 5000)), + "s3 server begin"); } HARNESS_FORK_CHILD_END(); HARNESS_FORK_PARENT_BEGIN() { + hrnTlsClientBegin(ioHandleWriteNew(strNew("test client write"), HARNESS_FORK_PARENT_WRITE_PROCESS(0))); + Storage *s3 = storageS3New( path, true, NULL, bucket, endPoint, storageS3UriStyleHost, region, accessKey, secretAccessKey, NULL, 16, 2, host, port, 5000, testContainer(), NULL, NULL); @@ -771,17 +394,48 @@ testRun(void) // ----------------------------------------------------------------------------------------------------------------- TEST_RESULT_VOID(storagePathSyncP(s3, strNew("path")), "path sync is a noop"); - // storageS3NewRead() and StorageS3FileRead // ----------------------------------------------------------------------------------------------------------------- - TEST_RESULT_PTR( - storageGetP(storageNewReadP(s3, strNew("fi&le.txt"), .ignoreMissing = true)), NULL, "ignore missing file"); + TEST_TITLE("ignore missing file"); + + hrnTlsServerAccept(); + testRequestP(s3, HTTP_VERB_GET, "/fi%26le.txt"); + testResponseP(.code = 404); + + TEST_RESULT_PTR(storageGetP(storageNewReadP(s3, strNew("fi&le.txt"), .ignoreMissing = true)), NULL, "get file"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on missing file"); + + testRequestP(s3, HTTP_VERB_GET, "/file.txt"); + testResponseP(.code = 404); + TEST_ERROR( storageGetP(storageNewReadP(s3, strNew("file.txt"))), FileMissingError, "unable to open '/file.txt': No such file or directory"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get file"); + + testRequestP(s3, HTTP_VERB_GET, "/file.txt"); + testResponseP(.content = "this is a sample file"); + TEST_RESULT_STR_Z( strNewBuf(storageGetP(storageNewReadP(s3, strNew("file.txt")))), "this is a sample file", "get file"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get zero-length file"); + + testRequestP(s3, HTTP_VERB_GET, "/file0.txt"); + testResponseP(); + TEST_RESULT_STR_Z(strNewBuf(storageGetP(storageNewReadP(s3, strNew("file0.txt")))), "", "get zero-length file"); + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("non-404 error"); + + testRequestP(s3, HTTP_VERB_GET, "/file.txt"); + testResponseP(.code = 303, .content = "CONTENT"); + StorageRead *read = NULL; TEST_ASSIGN(read, storageNewReadP(s3, strNew("file.txt"), .ignoreMissing = true), "new read file"); TEST_RESULT_BOOL(storageReadIgnoreMissing(read), true, " check ignore missing"); @@ -789,7 +443,7 @@ testRun(void) TEST_ERROR( ioReadOpen(storageReadIo(read)), ProtocolError, - "S3 request failed with 303: Some bad status\n" + "S3 request failed with 303: \n" "*** URI/Query ***:\n" "/file.txt\n" "*** Request Headers ***:\n" @@ -803,12 +457,31 @@ testRun(void) "*** Response Content ***:\n" "CONTENT") - // storageS3NewWrite() and StorageWriteS3 // ----------------------------------------------------------------------------------------------------------------- - // File is written all at once + TEST_TITLE("write file in one part"); + + testRequestP(s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD"); + testResponseP( + .code = 403, + .content = + "" + "" + "RequestTimeTooSkewed" + "The difference between the request time and the current time is too large." + "20190726T221748Z" + "2019-07-26T22:33:27Z" + "900000" + "601AA1A7F7E37AE9" + "KYMys77PoloZrGCkiQRyOIl0biqdHsk4T2EdTkhzkH1l8x00D4lvv/py5uUuHwQXG9qz6NRuldQ=" + ""); + hrnTlsServerClose(); + hrnTlsServerAccept(); + testRequestP(s3, HTTP_VERB_PUT, "/file.txt", .content = "ABCD"); + testResponseP(); + StorageWrite *write = NULL; - TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write file"); - TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("ABCD")), "put file all at once"); + TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write"); + TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("ABCD")), "write"); TEST_RESULT_BOOL(storageWriteAtomic(write), true, "write is atomic"); TEST_RESULT_BOOL(storageWriteCreatePath(write), true, "path will be created"); @@ -820,31 +493,102 @@ testRun(void) TEST_RESULT_VOID(storageWriteS3Close(write->driver), "close file again"); - // Zero-length file - TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write file"); - TEST_RESULT_VOID(storagePutP(write, NULL), "write zero-length file"); - - // File is written in chunks with nothing left over on close - TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write file"); - TEST_RESULT_VOID( - storagePutP(write, BUFSTRDEF("12345678901234567890123456789012")), - "write file in chunks -- nothing left on close"); - - // File is written in chunks with something left over on close - TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write file"); - TEST_RESULT_VOID( - storagePutP(write, BUFSTRDEF("12345678901234567890")), - "write file in chunks -- something left on close"); - - // storageDriverExists() // ----------------------------------------------------------------------------------------------------------------- - TEST_RESULT_BOOL(storageExistsP(s3, strNew("BOGUS")), false, "file does not exist"); - TEST_RESULT_BOOL(storageExistsP(s3, strNew("subdir/file1.txt")), true, "file exists"); + TEST_TITLE("write zero-length file"); + + testRequestP(s3, HTTP_VERB_PUT, "/file.txt", .content = ""); + testResponseP(); + + TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write"); + TEST_RESULT_VOID(storagePutP(write, NULL), "write"); - // Info() // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("write file in chunks with nothing left over on close"); + + testRequestP(s3, HTTP_VERB_POST, "/file.txt?uploads="); + testResponseP( + .content = + "" + "" + "bucket" + "file.txt" + "WxRt" + ""); + + testRequestP(s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=WxRt", .content = "1234567890123456"); + testResponseP(.header = "etag:WxRt1"); + + testRequestP(s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=WxRt", .content = "7890123456789012"); + testResponseP(.header = "eTag:WxRt2"); + + testRequestP( + s3, HTTP_VERB_POST, "/file.txt?uploadId=WxRt", + .content = + "\n" + "" + "1WxRt1" + "2WxRt2" + "\n"); + testResponseP(); + + TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write"); + TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("12345678901234567890123456789012")), "write"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("write file in chunks with something left over on close"); + + testRequestP(s3, HTTP_VERB_POST, "/file.txt?uploads="); + testResponseP( + .content = + "" + "" + "bucket" + "file.txt" + "RR55" + ""); + + testRequestP(s3, HTTP_VERB_PUT, "/file.txt?partNumber=1&uploadId=RR55", .content = "1234567890123456"); + testResponseP(.header = "etag:RR551"); + + testRequestP(s3, HTTP_VERB_PUT, "/file.txt?partNumber=2&uploadId=RR55", .content = "7890"); + testResponseP(.header = "eTag:RR552"); + + testRequestP( + s3, HTTP_VERB_POST, "/file.txt?uploadId=RR55", + .content = + "\n" + "" + "1RR551" + "2RR552" + "\n"); + testResponseP(); + + TEST_ASSIGN(write, storageNewWriteP(s3, strNew("file.txt")), "new write"); + TEST_RESULT_VOID(storagePutP(write, BUFSTRDEF("12345678901234567890")), "write"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("file missing"); + + testRequestP(s3, HTTP_VERB_HEAD, "/BOGUS"); + testResponseP(.code = 404); + + TEST_RESULT_BOOL(storageExistsP(s3, strNew("BOGUS")), false, "check"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("info for missing file"); + + // File missing + testRequestP(s3, HTTP_VERB_HEAD, "/BOGUS"); + testResponseP(.code = 404); + TEST_RESULT_BOOL(storageInfoP(s3, strNew("BOGUS"), .ignoreMissing = true).exists, false, "file does not exist"); + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("info for file"); + + testRequestP(s3, HTTP_VERB_HEAD, "/subdir/file1.txt"); + testResponseP(.header = "content-length:9999\r\nLast-Modified: Wed, 21 Oct 2015 07:28:00 GMT"); + StorageInfo info; TEST_ASSIGN(info, storageInfoP(s3, strNew("subdir/file1.txt")), "file exists"); TEST_RESULT_BOOL(info.exists, true, " check exists"); @@ -852,7 +596,11 @@ testRun(void) TEST_RESULT_UINT(info.size, 9999, " check exists"); TEST_RESULT_INT(info.timeModified, 1445412480, " check time"); - TEST_TITLE("file exists and only checking existence"); + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("info check existence only"); + + testRequestP(s3, HTTP_VERB_HEAD, "/subdir/file2.txt"); + testResponseP(.header = "content-length:777\r\nLast-Modified: Wed, 22 Oct 2015 07:28:00 GMT"); TEST_ASSIGN(info, storageInfoP(s3, strNew("subdir/file2.txt"), .level = storageInfoLevelExists), "file exists"); TEST_RESULT_BOOL(info.exists, true, " check exists"); @@ -861,33 +609,20 @@ testRun(void) TEST_RESULT_INT(info.timeModified, 0, " check time"); // ----------------------------------------------------------------------------------------------------------------- - TEST_TITLE("list basic level"); - - HarnessStorageInfoListCallbackData callbackData = - { - .content = strNew(""), - }; - - TEST_ERROR( - storageInfoListP(s3, strNew("/"), hrnStorageInfoListCallback, NULL, .errorOnMissing = true), - AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed"); - - TEST_RESULT_VOID( - storageInfoListP(s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData), "info list files"); - TEST_RESULT_STR_Z( - callbackData.content, - "test_path {path}\n" - "test_file {file, s=787, t=1255369830}\n", - " check content"); - - // ----------------------------------------------------------------------------------------------------------------- - TEST_TITLE("various errors"); + TEST_TITLE("errorOnMissing invalid because there are no paths"); TEST_ERROR( storageListP(s3, strNew("/"), .errorOnMissing = true), AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error without xml"); + + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"); + testResponseP(.code = 344); + TEST_ERROR(storageListP(s3, strNew("/")), ProtocolError, - "S3 request failed with 344: Another bad status\n" + "S3 request failed with 344: \n" "*** URI/Query ***:\n" "/?delimiter=%2F&list-type=2\n" "*** Request Headers ***:\n" @@ -896,8 +631,21 @@ testRun(void) "host: bucket." S3_TEST_HOST "\n" "x-amz-content-sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" "x-amz-date: "); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error with xml"); + + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"); + testResponseP( + .code = 344, + .content = + "" + "" + "SomeOtherCode" + ""); + TEST_ERROR(storageListP(s3, strNew("/")), ProtocolError, - "S3 request failed with 344: Another bad status with xml\n" + "S3 request failed with 344: \n" "*** URI/Query ***:\n" "/?delimiter=%2F&list-type=2\n" "*** Request Headers ***:\n" @@ -910,6 +658,40 @@ testRun(void) "content-length: 79\n" "*** Response Content ***:\n" "SomeOtherCode"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("time skewed error after retries"); + + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"); + testResponseP( + .code = 403, + .content = + "" + "" + "RequestTimeTooSkewed" + "The difference between the request time and the current time is too large." + ""); + + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"); + testResponseP( + .code = 403, + .content = + "" + "" + "RequestTimeTooSkewed" + "The difference between the request time and the current time is too large." + ""); + + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"); + testResponseP( + .code = 403, + .content = + "" + "" + "RequestTimeTooSkewed" + "The difference between the request time and the current time is too large." + ""); + TEST_ERROR(storageListP(s3, strNew("/")), ProtocolError, "S3 request failed with 403: Forbidden\n" "*** URI/Query ***:\n" @@ -926,35 +708,135 @@ testRun(void) "RequestTimeTooSkewed" "The difference between the request time and the current time is too large."); + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("list basic level"); + + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F"); + testResponseP( + .content = + "" + "" + " " + " path/to/test_file" + " 2009-10-12T17:50:30.000Z" + " 787" + " " + " " + " path/to/test_path/" + " " + ""); + + HarnessStorageInfoListCallbackData callbackData = + { + .content = strNew(""), + }; + + TEST_ERROR( + storageInfoListP(s3, strNew("/"), hrnStorageInfoListCallback, NULL, .errorOnMissing = true), + AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed"); + + TEST_RESULT_VOID( + storageInfoListP(s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData), "list"); + TEST_RESULT_STR_Z( + callbackData.content, + "test_path {path}\n" + "test_file {file, s=787, t=1255369830}\n", + "check"); + // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("list exists level"); + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2"); + testResponseP( + .content = + "" + "" + " " + " test1.txt" + " " + " " + " path1/" + " " + ""); + callbackData.content = strNew(""); + TEST_RESULT_VOID( storageInfoListP(s3, strNew("/"), hrnStorageInfoListCallback, &callbackData, .level = storageInfoLevelExists), - "list a file/path in root"); + "list"); TEST_RESULT_STR_Z( callbackData.content, "path1 {}\n" "test1.txt {}\n", - " check content"); + "check"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("list a file in root with expression"); + + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=test"); + testResponseP( + .content = + "" + "" + " " + " test1.txt" + " " + ""); callbackData.content = strNew(""); + TEST_RESULT_VOID( storageInfoListP( s3, strNew("/"), hrnStorageInfoListCallback, &callbackData, .expression = strNew("^test.*$"), .level = storageInfoLevelExists), - "list a file in root with expression"); + "list"); TEST_RESULT_STR_Z( callbackData.content, "test1.txt {}\n", - " check content"); + "check"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("list files with continuation"); + + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F"); + testResponseP( + .content = + "" + "" + " 1ueGcxLPRx1Tr/XYExHnhbYLgveDs2J/wm36Hy4vbOwM=" + " " + " path/to/test1.txt" + " " + " " + " path/to/test2.txt" + " " + " " + " path/to/path1/" + " " + ""); + + testRequestP( + s3, HTTP_VERB_GET, + "/?continuation-token=1ueGcxLPRx1Tr%2FXYExHnhbYLgveDs2J%2Fwm36Hy4vbOwM%3D&delimiter=%2F&list-type=2" + "&prefix=path%2Fto%2F"); + testResponseP( + .content = + "" + "" + " " + " path/to/test3.txt" + " " + " " + " path/to/path2/" + " " + ""); callbackData.content = strNew(""); + TEST_RESULT_VOID( storageInfoListP( s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData, .level = storageInfoLevelExists), - "list files with continuation"); + "list"); TEST_RESULT_STR_Z( callbackData.content, "path1 {}\n" @@ -962,41 +844,201 @@ testRun(void) "test2.txt {}\n" "path2 {}\n" "test3.txt {}\n", - " check content"); + "check"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("list files with expression"); + + testRequestP(s3, HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2Ftest"); + testResponseP( + .content = + "" + "" + " " + " path/to/test1.txt" + " " + " " + " path/to/test2.txt" + " " + " " + " path/to/test3.txt" + " " + " " + " path/to/test1.path/" + " " + " " + " path/to/test2.path/" + " " + ""); callbackData.content = strNew(""); + TEST_RESULT_VOID( storageInfoListP( s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData, .expression = strNew("^test(1|3)"), .level = storageInfoLevelExists), - "list files with expression"); + "list"); TEST_RESULT_STR_Z( callbackData.content, "test1.path {}\n" "test1.txt {}\n" "test3.txt {}\n", - " check content"); + "check"); - // storageDriverPathRemove() // ----------------------------------------------------------------------------------------------------------------- - // Switch to path-style URIs + TEST_TITLE("switch to path-style URIs"); + + hrnTlsServerClose(); + s3 = storageS3New( path, true, NULL, bucket, endPoint, storageS3UriStylePath, region, accessKey, secretAccessKey, NULL, 16, 2, host, port, 5000, testContainer(), NULL, NULL); + hrnTlsServerAccept(); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error when no recurse because there are no paths"); + TEST_ERROR( storagePathRemoveP(s3, strNew("/")), AssertError, "assertion 'param.recurse || storageFeature(this, storageFeaturePath)' failed"); - TEST_RESULT_VOID(storagePathRemoveP(s3, strNew("/"), .recurse = true), "remove root path"); - TEST_RESULT_VOID(storagePathRemoveP(s3, strNew("/path"), .recurse = true), "nothing to do in empty subpath"); - TEST_RESULT_VOID(storagePathRemoveP(s3, strNew("/path/to"), .recurse = true), "delete with continuation"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("remove files from root"); + + testRequestP(s3, HTTP_VERB_GET, "/bucket/?list-type=2"); + testResponseP( + .content = + "" + "" + " " + " test1.txt" + " " + " " + " path1/xxx.zzz" + " " + ""); + + testRequestP( + s3, HTTP_VERB_POST, "/bucket/?delete=", + .content = + "\n" + "true" + "test1.txt" + "path1/xxx.zzz" + "\n"); + testResponseP(.content = ""); + + TEST_RESULT_VOID(storagePathRemoveP(s3, strNew("/"), .recurse = true), "remove"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("remove files in empty subpath (nothing to do)"); + + testRequestP(s3, HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F"); + testResponseP( + .content = + "" + "" + ""); + + TEST_RESULT_VOID(storagePathRemoveP(s3, strNew("/path"), .recurse = true), "remove"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("remove files with continuation"); + + testRequestP(s3, HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2Fto%2F"); + testResponseP( + .content = + "" + "" + " continue" + " " + " path/to/test3/" + " " + " " + " path/to/test1.txt" + " " + ""); + + testRequestP(s3, HTTP_VERB_GET, "/bucket/?continuation-token=continue&list-type=2&prefix=path%2Fto%2F"); + testResponseP( + .content = + "" + "" + " " + " path/to/test3.txt" + " " + " " + " path/to/test2.txt" + " " + ""); + + testRequestP( + s3, HTTP_VERB_POST, "/bucket/?delete=", + .content = + "\n" + "true" + "path/to/test1.txt" + "path/to/test3.txt" + "\n"); + testResponseP(); + + testRequestP( + s3, HTTP_VERB_POST, "/bucket/?delete=", + .content = + "\n" + "true" + "path/to/test2.txt" + "\n"); + testResponseP(); + + TEST_RESULT_VOID(storagePathRemoveP(s3, strNew("/path/to"), .recurse = true), "remove"); + + // ----------------------------------------------------------------------------------------------------------------- + TEST_TITLE("remove error"); + + testRequestP(s3, HTTP_VERB_GET, "/bucket/?list-type=2&prefix=path%2F"); + testResponseP( + .content = + "" + "" + " " + " path/sample.txt" + " " + " " + " path/sample2.txt" + " " + ""); + + testRequestP( + s3, HTTP_VERB_POST, "/bucket/?delete=", + .content = + "\n" + "true" + "path/sample.txt" + "path/sample2.txt" + "\n"); + testResponseP( + .content = + "" + "" + "sample2.txtAccessDeniedAccess Denied" + ""); + TEST_ERROR( storagePathRemoveP(s3, strNew("/path"), .recurse = true), FileRemoveError, "unable to remove file 'sample2.txt': [AccessDenied] Access Denied"); - // storageDriverRemove() // ----------------------------------------------------------------------------------------------------------------- - TEST_RESULT_VOID(storageRemoveP(s3, strNew("/path/to/test.txt")), "remove file"); + TEST_TITLE("remove file"); + + testRequestP(s3, HTTP_VERB_DELETE, "/bucket/path/to/test.txt"); + testResponseP(.code = 204); + + TEST_RESULT_VOID(storageRemoveP(s3, strNew("/path/to/test.txt")), "remove"); + + // ----------------------------------------------------------------------------------------------------------------- + hrnTlsClientEnd(); } HARNESS_FORK_PARENT_END(); }