Non-blocking TLS implementation.

The prior blocking implementation seemed to be prone to locking up on some (especially recent) kernel versions. Since we were unable to reproduce the issue in a development environment we can only speculate as to the cause, but there is a good chance that blocking sockets were the issue or contributed to the issue.

So move to a non-blocking implementation to hopefully clear up these issues. Testing in production environments that were prone to locking shows that the approach is promising and at the very least not a regression.

The main differences from the blocking version are the non-blocking connect() implementation and handling of WANT_READ/WANT_WRITE retries for all SSL*() functions.

Timeouts in the tests needed to be increased because socket connect() and TLS SSL_connect() were not included in the timeout before. The tests don't run any slower, though. In fact, all platforms but Ubuntu 12.04 worked fine with the shorter timeouts.

doc/xml/release.xml

@@ -27,6 +27,17 @@
                 </release-bug-list>
 
                 <release-improvement-list>
+                    <release-item>
+                        <release-item-contributor-list>
+                             <release-item-reviewer id="cynthia.shang"/>
+                             <release-item-reviewer id="stephen.frost"/>
+                             <!-- Actually tester, but we don't have a tag for that yet -->
+                             <release-item-reviewer id="slava.pagerduty"/>
+                        </release-item-contributor-list>
+
+                        <p>Non-blocking TLS implementation.</p>
+                    </release-item>
+
                     <release-item>
                         <release-item-contributor-list>
                              <release-item-reviewer id="cynthia.shang"/>
@@ -8375,6 +8386,11 @@
             <contributor-id type="github">slardiere</contributor-id>
         </contributor>
 
+        <contributor id="slava.pagerduty">
+            <contributor-name-display>slava-pagerduty</contributor-name-display>
+            <contributor-id type="github">slava-pagerduty</contributor-id>
+        </contributor>
+
         <contributor id="stefan.fercot">
             <contributor-name-display>Stefan Fercot</contributor-name-display>
             <contributor-id type="github">pgstef</contributor-id>

src/common/io/socket/client.c

@@ -3,10 +3,8 @@ Socket Client
 ***********************************************************************************************************************************/
 #include "build.auto.h"
 
-#include <arpa/inet.h>
-#include <netdb.h>
 #include <netinet/in.h>
-#include <sys/select.h>
+#include <sys/types.h>
 #include <sys/socket.h>
 #include <unistd.h>
 
@@ -129,9 +127,7 @@ sckClientOpen(SocketClient *this)
                     THROW_ON_SYS_ERROR(fd == -1, HostConnectError, "unable to create socket");
 
                     sckOptionSet(fd);
-
+                    sckConnect(fd, this->host, this->port, hostAddress, waitRemaining(wait));
-                    if (connect(fd, hostAddress->ai_addr, hostAddress->ai_addrlen) == -1)
-                        THROW_SYS_ERROR_FMT(HostConnectError, "unable to connect to '%s:%u'", strPtr(this->host), this->port);
                 }
                 FINALLY()
                 {

src/common/io/socket/common.c

@@ -22,6 +22,8 @@ static struct SocketLocal
 {
     bool init;                                                      // sckInit() has been called
 
+    bool block;                                                     // Use blocking mode socket
+
     bool keepAlive;                                                 // Are socket keep alives enabled?
     int tcpKeepAliveCount;                                          // TCP keep alive count (0 disables)
     int tcpKeepAliveIdle;                                           // TCP keep alive idle (0 disables)
@@ -44,6 +46,7 @@ sckInit(bool keepAlive, int tcpKeepAliveCount, int tcpKeepAliveIdle, int tcpKeep
     ASSERT(tcpKeepAliveInterval >= 0);
 
     socketLocal.init = true;
+    socketLocal.block = false;
     socketLocal.keepAlive = keepAlive;
     socketLocal.tcpKeepAliveCount = tcpKeepAliveCount;
     socketLocal.tcpKeepAliveIdle = tcpKeepAliveIdle;
@@ -71,6 +74,15 @@ sckOptionSet(int fd)
         setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &socketValue, sizeof(int)) == -1, ProtocolError, "unable set TCP_NODELAY");
 #endif
 
+    // Put the socket in non-blocking mode
+    if (!socketLocal.block)
+    {
+        int flags;
+
+        THROW_ON_SYS_ERROR((flags = fcntl(fd, F_GETFL)) == -1, ProtocolError, "unable to get flags");
+        THROW_ON_SYS_ERROR(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1, ProtocolError, "unable to set O_NONBLOCK");
+    }
+
     // Automatically close the socket (in the child process) on a successful execve() call. Connections are never shared between
     // processes so there is no reason to leave them open.
 #ifdef F_SETFD
@@ -124,6 +136,55 @@ sckOptionSet(int fd)
     FUNCTION_TEST_RETURN_VOID();
 }
 
+/**********************************************************************************************************************************/
+static bool
+sckConnectInProgress(int errNo)
+{
+    return errNo == EINPROGRESS || errNo == EINTR;
+}
+
+void
+sckConnect(int fd, const String *host, unsigned int port, const struct addrinfo *hostAddress, TimeMSec timeout)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(INT, fd);
+        FUNCTION_LOG_PARAM(STRING, host);
+        FUNCTION_LOG_PARAM(UINT, port);
+        FUNCTION_LOG_PARAM_P(VOID, hostAddress);
+        FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
+    FUNCTION_LOG_END();
+
+    ASSERT(host != NULL);
+    ASSERT(hostAddress != NULL);
+
+    // Attempt connection
+    if (connect(fd, hostAddress->ai_addr, hostAddress->ai_addrlen) == -1)
+    {
+        // Save the error
+        int errNo = errno;
+
+        // The connection has started but since we are in non-blocking mode it has not completed yet
+        if (sckConnectInProgress(errNo))
+        {
+            // Wait for write-ready
+            if (!sckReadyWrite(fd, timeout))
+                THROW_FMT(HostConnectError, "timeout connecting to '%s:%u'", strPtr(host), port);
+
+            // Check for success or error. If the connection was successful this will set errNo to 0.
+            socklen_t errNoLen = sizeof(errNo);
+
+            THROW_ON_SYS_ERROR(
+                getsockopt(fd, SOL_SOCKET, SO_ERROR, &errNo, &errNoLen) == -1, HostConnectError, "unable to get socket error");
+        }
+
+        // Throw error if it is still set
+        if (errNo != 0)
+            THROW_SYS_ERROR_CODE_FMT(errNo, HostConnectError, "unable to connect to '%s:%u'", strPtr(host), port);
+    }
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
 /***********************************************************************************************************************************
 Use poll() to determine when data is ready to read/write on a socket. Retry after EINTR with whatever time is left on the timer.
 ***********************************************************************************************************************************/

src/common/io/socket/common.h

@@ -4,7 +4,10 @@ Socket Common Functions
 #ifndef COMMON_IO_SOCKET_COMMON_H
 #define COMMON_IO_SOCKET_COMMON_H
 
+#include <netdb.h>
+
 #include "common/time.h"
+#include "common/type/string.h"
 
 /***********************************************************************************************************************************
 Functions
@@ -15,6 +18,9 @@ void sckInit(bool keepAlive, int tcpKeepAliveCount, int tcpKeepAliveIdle, int tc
 // Set options on a socket
 void sckOptionSet(int fd);
 
+// Connect socket to an IP address
+void sckConnect(int fd, const String *host, unsigned int port, const struct addrinfo *hostAddress, TimeMSec timeout);
+
 // Wait until the socket is ready to read/write or timeout
 bool sckReady(int fd, bool read, bool write, TimeMSec timeout);
 bool sckReadyRead(int fd, TimeMSec timeout);

src/common/io/tls/session.c

@@ -75,52 +75,92 @@ tlsSessionClose(TlsSession *this, bool shutdown)
 }
 
 /***********************************************************************************************************************************
-Report TLS errors.  Returns true if the command should continue and false if it should exit.
+Process result from SSL_read(), SSL_write(), SSL_connect(), and SSL_accept().
+
+Returns:
+0 if the function should be tried again with the same parameters
+-1 if the connection was closed gracefully
+> 0 with the read/write size if SSL_read()/SSL_write() was called
 ***********************************************************************************************************************************/
-static bool
+// Helper to process error conditions
-tlsSessionError(TlsSession *this, int code)
+static int
+tlsSessionResultProcess(TlsSession *this, int errorTls, int errorSys, bool closeOk)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(TLS_SESSION, this);
-        FUNCTION_LOG_PARAM(INT, code);
+        FUNCTION_LOG_PARAM(INT, errorTls);
+        FUNCTION_LOG_PARAM(INT, errorSys);
+        FUNCTION_LOG_PARAM(BOOL, closeOk);
     FUNCTION_LOG_END();
 
-    bool result = false;
+    ASSERT(this != NULL);
+    ASSERT(this->session != NULL);
 
-    switch (code)
+    int result = -1;
+
+    switch (errorTls)
     {
         // The connection was closed
         case SSL_ERROR_ZERO_RETURN:
         {
+            if (!closeOk)
+                THROW(ProtocolError, "unexpected TLS eof");
+
             tlsSessionClose(this, false);
             break;
         }
 
-        // Try the read/write again
+        // Try again after waiting for read ready
         case SSL_ERROR_WANT_READ:
+        {
+            sckSessionReadyRead(this->socketSession);
+            result = 0;
+            break;
+        }
+
+        // Try again after waiting for write ready
         case SSL_ERROR_WANT_WRITE:
         {
-            result = true;
+            sckSessionReadyWrite(this->socketSession);
+            result = 0;
             break;
         }
 
-        // A syscall failed (usually indicates unexpected eof)
+        // A syscall failed (this usually indicates unexpected eof)
         case SSL_ERROR_SYSCALL:
-        {
+            THROW_SYS_ERROR_CODE(errorSys, KernelError, "TLS syscall error");
-            // Get the error before closing so it is not cleared
-            int errNo = errno;
-            tlsSessionClose(this, false);
 
-            // Throw the sys error
+        // Any other error that we cannot handle
-            THROW_SYS_ERROR_CODE(errNo, KernelError, "tls failed syscall");
-        }
-
-        // Some other tls error that cannot be handled
         default:
-            THROW_FMT(ServiceError, "tls error [%d]", code);
+            THROW_FMT(ServiceError, "TLS error [%d]", errorTls);
     }
 
-    FUNCTION_LOG_RETURN(BOOL, result);
+    FUNCTION_LOG_RETURN(INT, result);
+}
+
+static int
+tlsSessionResult(TlsSession *this, int result, bool closeOk)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(TLS_SESSION, this);
+        FUNCTION_LOG_PARAM(INT, result);
+        FUNCTION_LOG_PARAM(BOOL, closeOk);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(this->session != NULL);
+
+    // Process errors
+    if (result <= 0)
+    {
+        // Get TLS error and store errno in case of syscall error
+        int errorTls = SSL_get_error(this->session, result);
+        int errorSys = errno;
+
+        result = tlsSessionResultProcess(this, errorTls, errorSys, closeOk);
+    }
+
+    FUNCTION_LOG_RETURN(INT, result);
 }
 
 /***********************************************************************************************************************************
@@ -142,27 +182,26 @@ tlsSessionRead(THIS_VOID, Buffer *buffer, bool block)
     ASSERT(buffer != NULL);
     ASSERT(!bufFull(buffer));
 
-    ssize_t result = 0;
+    int result = 0;
 
     // If blocking read keep reading until buffer is full
     do
     {
-        // If no tls data pending then check the socket
+        // If no TLS data pending then check the socket to reduce blocking
         if (!SSL_pending(this->session))
             sckSessionReadyRead(this->socketSession);
 
         // Read and handle errors
-        result = SSL_read(this->session, bufRemainsPtr(buffer), (int)bufRemains(buffer));
+        result = tlsSessionResult(this, SSL_read(this->session, bufRemainsPtr(buffer), (int)bufRemains(buffer)), true);
 
-        if (result <= 0)
-        {
-            // Break if the error indicates that we should not continue trying
-            if (!tlsSessionError(this, SSL_get_error(this->session, (int)result)))
-                break;
-        }
         // Update amount of buffer used
-        else
+        if (result > 0)
+        {
             bufUsedInc(buffer, (size_t)result);
+        }
+        // If the connection was closed then we are at eof. It is up to the layer above TLS to decide if this is an error.
+        else if (result == -1)
+            break;
     }
     while (block && bufRemains(buffer) > 0);
 
@@ -170,51 +209,8 @@ tlsSessionRead(THIS_VOID, Buffer *buffer, bool block)
 }
 
 /***********************************************************************************************************************************
-Write to the tls session
+Write to the TLS session
 ***********************************************************************************************************************************/
-static bool
-tlsSessionWriteContinue(TlsSession *this, int writeResult, int writeError, size_t writeSize)
-{
-    FUNCTION_LOG_BEGIN(logLevelTrace);
-        FUNCTION_LOG_PARAM(TLS_SESSION, this);
-        FUNCTION_LOG_PARAM(INT, writeResult);
-        FUNCTION_LOG_PARAM(INT, writeError);
-        FUNCTION_LOG_PARAM(SIZE, writeSize);
-    FUNCTION_LOG_END();
-
-    ASSERT(this != NULL);
-    ASSERT(writeSize > 0);
-
-    bool result = true;
-
-    // Handle errors
-    if (writeResult <= 0)
-    {
-        // If error = SSL_ERROR_NONE then this is the first write attempt so continue
-        if (writeError != SSL_ERROR_NONE)
-        {
-            // Error if the error indicates that we should not continue trying
-            if (!tlsSessionError(this, writeError))
-                THROW_FMT(FileWriteError, "unable to write to tls [%d]", writeError);
-
-            // Wait for the socket to be readable for tls renegotiation
-            sckSessionReadyRead(this->socketSession);
-        }
-    }
-    else
-    {
-        if ((size_t)writeResult != writeSize)
-        {
-            THROW_FMT(
-                FileWriteError, "unable to write to tls, write size %d does not match expected size %zu", writeResult, writeSize);
-        }
-
-        result = false;
-    }
-
-    FUNCTION_LOG_RETURN(BOOL, result);
-}
-
 static void
 tlsSessionWrite(THIS_VOID, const Buffer *buffer)
 {
@@ -230,12 +226,13 @@ tlsSessionWrite(THIS_VOID, const Buffer *buffer)
     ASSERT(buffer != NULL);
 
     int result = 0;
-    int error = SSL_ERROR_NONE;
 
-    while (tlsSessionWriteContinue(this, result, error, bufUsed(buffer)))
+    while (result == 0)
     {
-        result = SSL_write(this->session, bufPtrConst(buffer), (int)bufUsed(buffer));
+        result = tlsSessionResult(this, SSL_write(this->session, bufPtrConst(buffer), (int)bufUsed(buffer)), false);
-        error = SSL_get_error(this->session, result);
+
+        // Either a retry or all data was written
+        CHECK(result == 0 || (size_t)result == bufUsed(buffer));
     }
 
     FUNCTION_LOG_RETURN_VOID();
@@ -288,14 +285,20 @@ tlsSessionNew(SSL *session, SocketSession *socketSession, TimeMSec timeout)
         // Ensure session is freed
         memContextCallbackSet(this->memContext, tlsSessionFreeResource, this);
 
-        // Negotiate TLS session
+        // Assign socket to TLS session
         cryptoError(
             SSL_set_fd(this->session, sckSessionFd(this->socketSession)) != 1, "unable to add socket to TLS session");
 
-        if (sckSessionType(this->socketSession) == sckSessionTypeClient)
+        // Negotiate TLS session
-            cryptoError(SSL_connect(this->session) != 1, "unable to negotiate client TLS session");
+        int result = 0;
-        else
+
-            cryptoError(SSL_accept(this->session) != 1, "unable to negotiate server TLS session");
+        while (result == 0)
+        {
+            if (sckSessionType(this->socketSession) == sckSessionTypeClient)
+                result = tlsSessionResult(this, SSL_connect(this->session), false);
+            else
+                result = tlsSessionResult(this, SSL_accept(this->session), false);
+        }
 
         // Create read and write interfaces
         this->write = ioWriteNewP(this, .write = tlsSessionWrite);

test/define.yaml

@@ -239,7 +239,7 @@ unit:
 
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: io-tls
-        total: 4
+        total: 5
         containerReq: true
 
         coverage:

test/src/module/common/ioHttpTest.c

@@ -464,13 +464,13 @@ testRun(void)
             {
                 // Test no output from server
                 TEST_ASSIGN(
-                    client, httpClientNew(harnessTlsTestHost(), harnessTlsTestPort(), 500, testContainer(), NULL, NULL),
+                    client, httpClientNew(harnessTlsTestHost(), harnessTlsTestPort(), 5000, testContainer(), NULL, NULL),
                     "new client");
                 client->timeout = 0;
 
-                TEST_ERROR_FMT(
+                TEST_ERROR(
-                    httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), ProtocolError,
+                    httpClientRequest(client, strNew("GET"), strNew("/"), NULL, NULL, NULL, false), FileReadError,
-                    "timeout after 500ms waiting for read from '%s:%u'", strPtr(harnessTlsTestHost()), harnessTlsTestPort());
+                    "unexpected eof while reading line");
 
                 // Test invalid http version
                 TEST_ERROR(
@@ -508,7 +508,7 @@ testRun(void)
                     "[503] Slow Down");
 
                 // Request with no content
-                client->timeout = 2000;
+                client->timeout = 5000;
 
                 HttpHeader *headerRequest = httpHeaderNew(NULL);
                 httpHeaderAdd(headerRequest, strNew("host"), strNew("myhost.com"));
@@ -642,7 +642,7 @@ testRun(void)
         HttpClient *client2 = NULL;
 
         TEST_ASSIGN(
-            cache, httpClientCacheNew(strNew("localhost"), harnessTlsTestPort(), 500, true, NULL, NULL), "new http client cache");
+            cache, httpClientCacheNew(strNew("localhost"), harnessTlsTestPort(), 5000, true, NULL, NULL), "new http client cache");
         TEST_ASSIGN(client1, httpClientCacheGet(cache), "get http client");
         TEST_RESULT_PTR(client1, *(HttpClient **)lstGet(cache->clientList, 0), "    check http client");
         TEST_RESULT_PTR(httpClientCacheGet(cache), *(HttpClient **)lstGet(cache->clientList, 0), "    get same http client");

test/src/module/common/ioTlsTest.c

@@ -87,11 +87,6 @@ testTlsServer(void)
     harnessTlsServerReply("0123456789AB");
     harnessTlsServerAbort();
 
-    // Need data in read buffer to test tlsWriteContinue()
-    harnessTlsServerAccept();
-    harnessTlsServerReply("0123456789AB");
-    harnessTlsServerClose();
-
     FUNCTION_HARNESS_RESULT_VOID();
 }
 
@@ -119,6 +114,15 @@ testRun(void)
         int result;
         const char *port = "7777";
 
+        const char *hostLocal = "127.0.0.1";
+        struct addrinfo *hostLocalAddress;
+
+        if ((result = getaddrinfo(hostLocal, port, &hints, &hostLocalAddress)) != 0)
+        {
+            THROW_FMT(                                              // {uncoverable - lookup on IP should never fail}
+                HostConnectError, "unable to get address for '%s': [%d] %s", hostLocal, result, gai_strerror(result));
+        }
+
         const char *hostBad = "172.31.255.255";
         struct addrinfo *hostBadAddress;
 
@@ -207,12 +211,6 @@ testRun(void)
             // ---------------------------------------------------------------------------------------------------------------------
             TEST_TITLE("connect to non-blocking socket to test write ready");
 
-            // Put the socket in non-blocking mode
-            int flags;
-
-            THROW_ON_SYS_ERROR((flags = fcntl(fd, F_GETFL)) == -1, ProtocolError, "unable to get flags");
-            THROW_ON_SYS_ERROR(fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1, ProtocolError, "unable to set O_NONBLOCK");
-
             // Attempt connection
             CHECK(connect(fd, hostBadAddress->ai_addr, hostBadAddress->ai_addrlen) == -1);
 
@@ -224,10 +222,25 @@ testRun(void)
                 sckSessionReadyWrite(session), ProtocolError, "timeout after 100ms waiting for write to '172.31.255.255:7777'");
 
             TEST_RESULT_VOID(sckSessionFree(session), "free socket session");
+
+            // ---------------------------------------------------------------------------------------------------------------------
+            TEST_TITLE("unable to connect to blocking socket");
+
+            socketLocal.block = true;
+            TEST_ERROR(
+                sckClientOpen(sckClientNew(STR(hostLocal), 7777, 0)), HostConnectError,
+                "unable to connect to '127.0.0.1:7777': [111] Connection refused");
+            socketLocal.block = false;
+
+            // ---------------------------------------------------------------------------------------------------------------------
+            TEST_TITLE("uncovered conditions for sckConnect()");
+
+            TEST_RESULT_BOOL(sckConnectInProgress(EINTR), true, "connection in progress (EINTR)");
         }
         FINALLY()
         {
-            // This needs to be freed or valgrind will complain
+            // These need to be freed or valgrind will complain
+            freeaddrinfo(hostLocalAddress);
             freeaddrinfo(hostBadAddress);
         }
         TRY_END();
@@ -236,6 +249,21 @@ testRun(void)
         socketLocal = socketLocalSave;
     }
 
+    // *****************************************************************************************************************************
+    if (testBegin("SocketClient"))
+    {
+        SocketClient *client = NULL;
+
+        TEST_ASSIGN(client, sckClientNew(strNew("localhost"), harnessTlsTestPort(), 100), "new client");
+        TEST_ERROR_FMT(
+            sckClientOpen(client), HostConnectError, "unable to connect to 'localhost:%u': [111] Connection refused",
+            harnessTlsTestPort());
+
+        // This address should not be in use in a test environment -- if it is the test will fail
+        TEST_ASSIGN(client, sckClientNew(strNew("172.31.255.255"), harnessTlsTestPort(), 100), "new client");
+        TEST_ERROR_FMT(sckClientOpen(client), HostConnectError, "timeout connecting to '172.31.255.255:%u'", harnessTlsTestPort());
+    }
+
     // Additional coverage not provided by testing with actual certificates
     // *****************************************************************************************************************************
     if (testBegin("asn1ToStr(), tlsClientHostVerify(), and tlsClientHostVerifyName()"))
@@ -300,13 +328,13 @@ testRun(void)
                 TEST_ERROR(
                     tlsClientOpen(
                         tlsClientNew(
-                            sckClientNew(strNew("localhost"), harnessTlsTestPort(), 500), 500, true, strNew("bogus.crt"),
+                            sckClientNew(strNew("localhost"), harnessTlsTestPort(), 5000), 0, true, strNew("bogus.crt"),
                             strNew("/bogus"))),
                     CryptoError, "unable to set user-defined CA certificate location: [33558530] No such file or directory");
                 TEST_ERROR_FMT(
                     tlsClientOpen(
                         tlsClientNew(
-                            sckClientNew(strNew("localhost"), harnessTlsTestPort(), 500), 500, true, NULL, strNew("/bogus"))),
+                            sckClientNew(strNew("localhost"), harnessTlsTestPort(), 5000), 0, true, NULL, strNew("/bogus"))),
                     CryptoError,
                     "unable to verify certificate presented by 'localhost:%u': [20] unable to get local issuer certificate",
                     harnessTlsTestPort());
@@ -316,19 +344,19 @@ testRun(void)
                     TEST_RESULT_VOID(
                         tlsClientOpen(
                             tlsClientNew(
-                                sckClientNew(strNew("test.pgbackrest.org"), harnessTlsTestPort(), 500), 500, true,
+                                sckClientNew(strNew("test.pgbackrest.org"), harnessTlsTestPort(), 5000), 0, true,
                                 strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-ca.crt", testRepoPath()), NULL)),
                         "success on valid ca file and match common name");
                     TEST_RESULT_VOID(
                         tlsClientOpen(
                             tlsClientNew(
-                                sckClientNew(strNew("host.test2.pgbackrest.org"), harnessTlsTestPort(), 500), 500, true,
+                                sckClientNew(strNew("host.test2.pgbackrest.org"), harnessTlsTestPort(), 5000), 0, true,
                                 strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-ca.crt", testRepoPath()), NULL)),
                         "success on valid ca file and match alt name");
                     TEST_ERROR(
                         tlsClientOpen(
                             tlsClientNew(
-                                sckClientNew(strNew("test3.pgbackrest.org"), harnessTlsTestPort(), 500), 500, true,
+                                sckClientNew(strNew("test3.pgbackrest.org"), harnessTlsTestPort(), 5000), 0, true,
                                 strNewFmt("%s/" TEST_CERTIFICATE_PREFIX "-ca.crt", testRepoPath()), NULL)),
                         CryptoError,
                         "unable to find hostname 'test3.pgbackrest.org' in certificate common name or subject alternative names");
@@ -337,7 +365,7 @@ testRun(void)
                 TEST_ERROR_FMT(
                     tlsClientOpen(
                         tlsClientNew(
-                            sckClientNew(strNew("localhost"), harnessTlsTestPort(), 500), 500, true,
+                            sckClientNew(strNew("localhost"), harnessTlsTestPort(), 5000), 0, true,
                             strNewFmt("%s/" TEST_CERTIFICATE_PREFIX ".crt", testRepoPath()),
                         NULL)),
                     CryptoError,
@@ -346,7 +374,7 @@ testRun(void)
 
                 TEST_RESULT_VOID(
                     tlsClientOpen(
-                        tlsClientNew(sckClientNew(strNew("localhost"), harnessTlsTestPort(), 500), 500, false, NULL, NULL)),
+                        tlsClientNew(sckClientNew(strNew("localhost"), harnessTlsTestPort(), 5000), 0, false, NULL, NULL)),
                         "success on no verify");
             }
             HARNESS_FORK_PARENT_END();
@@ -380,7 +408,7 @@ testRun(void)
 
                 TEST_ASSIGN(
                     client,
-                    tlsClientNew(sckClientNew(harnessTlsTestHost(), harnessTlsTestPort(), 500), 500, testContainer(), NULL, NULL),
+                    tlsClientNew(sckClientNew(harnessTlsTestHost(), harnessTlsTestPort(), 5000), 0, testContainer(), NULL, NULL),
                     "new client");
                 TEST_ASSIGN(session, tlsClientOpen(client), "open client");
 
@@ -403,6 +431,13 @@ testRun(void)
                 TEST_RESULT_BOOL(sckReadyWrite(session->socketSession->fd, 100), true, "socket is write ready");
                 TEST_RESULT_VOID(sckSessionReadyWrite(session->socketSession), "socket session is write ready");
 
+                // -----------------------------------------------------------------------------------------------------------------
+                TEST_TITLE("uncovered errors");
+
+                TEST_RESULT_INT(tlsSessionResultProcess(session, SSL_ERROR_WANT_WRITE, 0, false), 0, "write ready");
+                TEST_ERROR(tlsSessionResultProcess(session, SSL_ERROR_WANT_X509_LOOKUP, 0, false), ServiceError, "TLS error [4]");
+                TEST_ERROR(tlsSessionResultProcess(session, SSL_ERROR_ZERO_RETURN, 0, false), ProtocolError, "unexpected TLS eof");
+
                 // -----------------------------------------------------------------------------------------------------------------
                 const Buffer *input = BUFSTRDEF("some protocol info");
                 TEST_RESULT_VOID(ioWrite(tlsSessionIoWrite(session), input), "write input");
@@ -422,9 +457,11 @@ testRun(void)
                 TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), false, "    check eof = false");
 
                 output = bufNew(12);
+                session->socketSession->timeout = 100;
                 TEST_ERROR_FMT(
                     ioRead(tlsSessionIoRead(session), output), ProtocolError,
-                    "timeout after 500ms waiting for read from '%s:%u'", strPtr(harnessTlsTestHost()), harnessTlsTestPort());
+                    "timeout after 100ms waiting for read from '%s:%u'", strPtr(harnessTlsTestHost()), harnessTlsTestPort());
+                session->socketSession->timeout = 5000;
 
                 // -----------------------------------------------------------------------------------------------------------------
                 input = BUFSTRDEF("more protocol info");
@@ -441,26 +478,16 @@ testRun(void)
                 TEST_RESULT_BOOL(ioReadEof(tlsSessionIoRead(session)), true, "    check eof = true");
 
                 TEST_RESULT_VOID(tlsSessionClose(session, false), "close again");
-                TEST_ERROR(tlsSessionError(session, SSL_ERROR_WANT_X509_LOOKUP), ServiceError, "tls error [4]");
 
                 // -----------------------------------------------------------------------------------------------------------------
-                TEST_TITLE("aborted connection before read complete");
+                TEST_TITLE("aborted connection before read complete (blocking socket)");
 
+                socketLocal.block = true;
                 TEST_ASSIGN(session, tlsClientOpen(client), "open client again (was closed by server)");
+                socketLocal.block = false;
 
                 output = bufNew(13);
-                TEST_ERROR(ioRead(tlsSessionIoRead(session), output), KernelError, "tls failed syscall");
+                TEST_ERROR(ioRead(tlsSessionIoRead(session), output), KernelError, "TLS syscall error");
-
-                // -----------------------------------------------------------------------------------------------------------------
-                TEST_ASSIGN(session, tlsClientOpen(client), "open client again (was closed by server)");
-
-                TEST_RESULT_BOOL(tlsSessionWriteContinue(session, -1, SSL_ERROR_WANT_READ, 1), true, "continue on WANT_READ");
-                TEST_RESULT_BOOL(tlsSessionWriteContinue(session, 0, SSL_ERROR_NONE, 1), true, "continue on WANT_READ");
-                TEST_ERROR(
-                    tlsSessionWriteContinue(session, 77, 0, 88), FileWriteError,
-                    "unable to write to tls, write size 77 does not match expected size 88");
-                TEST_ERROR(
-                    tlsSessionWriteContinue(session, 0, SSL_ERROR_ZERO_RETURN, 1), FileWriteError, "unable to write to tls [6]");
 
                 // -----------------------------------------------------------------------------------------------------------------
                 TEST_RESULT_BOOL(sckClientStatStr() != NULL, true, "check statistics exist");

test/src/module/storage/s3Test.c

@@ -765,7 +765,7 @@ testRun(void)
             {
                 Storage *s3 = storageS3New(
                     path, true, NULL, bucket, endPoint, storageS3UriStyleHost, region, accessKey, secretAccessKey, NULL, 16, 2,
-                    host, port, 1000, testContainer(), NULL, NULL);
+                    host, port, 5000, testContainer(), NULL, NULL);
 
                 // Coverage for noop functions
                 // -----------------------------------------------------------------------------------------------------------------
@@ -982,7 +982,7 @@ testRun(void)
                 // Switch to path-style URIs
                 s3 = storageS3New(
                     path, true, NULL, bucket, endPoint, storageS3UriStylePath, region, accessKey, secretAccessKey, NULL, 16, 2,
-                    host, port, 1000, testContainer(), NULL, NULL);
+                    host, port, 5000, testContainer(), NULL, NULL);
 
                 TEST_ERROR(
                     storagePathRemoveP(s3, strNew("/")), AssertError,