Add HTTP retries for 408 and 429 errors.

HTTP client errors 408 and 429 were not being retried but there seems to be some benefit to doing so.

408 has only been seen once and in that case the server was returning 400 (with Request Timeout in the response body) but it seems worth doing since this could happen during times of high congestion. Requests are not sent until request content is ready so there is not much else to be done to handle this error.

429 has been seen occasionally in the past but now seems to be common on Cloudflare R2. Ideally we would have a large initial back off here but it is not clear that is worth it at this time. The existing Fibonacci back off should be enough to allow operations to proceed if possible during the configured timeout.

doc/xml/release/2025/2.57.0.xml

@@ -1,2 +1,16 @@
 <release date="XXXX-XX-XX" version="2.57.0dev" title="Under Development">
+    <release-core-list>
+        <release-improvement-list>
+            <release-item>
+                <github-pull-request id="2661"/>
+
+                <release-item-contributor-list>
+                    <release-item-contributor id="david.steele"/>
+                    <release-item-reviewer id="david.christensen"/>
+                </release-item-contributor-list>
+
+                <p>Add HTTP retries for 408 and 429 errors.</p>
+            </release-item>
+        </release-improvement-list>
+    </release-core-list>
 </release>

src/common/io/http/request.c

@@ -181,9 +181,7 @@ httpRequestProcess(HttpRequest *const this, const bool waitForResponse, const bo
                     {
                         result = httpResponseNew(session, httpRequestVerb(this), contentCache);
 
-                        // Retry when response code is 5xx. These errors generally represent a server error for a request that looks
-                        // valid. There are a few errors that might be permanently fatal but they are rare and it seems best not to
-                        // try and pick and choose errors in this class to retry.
+                        // Error on response codes that should be retried to engage the retry logic
                         if (httpResponseCodeRetry(result))
                             THROW_FMT(ServiceError, "[%u] %s", httpResponseCode(result), strZ(httpResponseReason(result)));
 

src/common/io/http/response.c

@@ -451,6 +451,16 @@ httpResponseContent(HttpResponse *const this)
     FUNCTION_TEST_RETURN(BUFFER, this->content);
 }
 
+/**********************************************************************************************************************************/
+FN_EXTERN bool
+httpResponseCodeRetry(const HttpResponse *const this)
+{
+    return
+        httpResponseCode(this) / 100 == HTTP_RESPONSE_CODE_CLASS_RETRY ||
+        httpResponseCode(this) == HTTP_RESPONSE_CODE_REQUEST_TIMEOUT ||
+        httpResponseCode(this) == HTTP_RESPONSE_CODE_TOO_MANY_REQUESTS;
+}
+
 /**********************************************************************************************************************************/
 FN_EXTERN HttpResponseMulti *
 httpResponseMultiNew(const Buffer *const content, const String *const contentType)

src/common/io/http/response.h

@@ -24,6 +24,8 @@ HTTP Response Constants
 #define HTTP_RESPONSE_CODE_PERMANENT_REDIRECT                       308
 #define HTTP_RESPONSE_CODE_FORBIDDEN                                403
 #define HTTP_RESPONSE_CODE_NOT_FOUND                                404
+#define HTTP_RESPONSE_CODE_REQUEST_TIMEOUT                          408
+#define HTTP_RESPONSE_CODE_TOO_MANY_REQUESTS                        429
 
 // 2xx indicates success
 #define HTTP_RESPONSE_CODE_CLASS_OK                                 2
@@ -87,11 +89,7 @@ httpResponseCodeOk(const HttpResponse *const this)
 }
 
 // Should the request be retried?
-FN_INLINE_ALWAYS bool
-httpResponseCodeRetry(const HttpResponse *const this)
-{
-    return httpResponseCode(this) / 100 == HTTP_RESPONSE_CODE_CLASS_RETRY;
-}
+FN_EXTERN bool httpResponseCodeRetry(const HttpResponse *this);
 
 // Fetch all response content. Content will be cached so it can be retrieved again without additional cost.
 FN_EXTERN const Buffer *httpResponseContent(HttpResponse *this);

test/src/module/common/ioHttpTest.c

@@ -616,7 +616,7 @@ testRun(void)
                 TEST_RESULT_Z(logBuf, "{connection: 'close'}", "check response headers");
 
                 // -----------------------------------------------------------------------------------------------------------------
-                TEST_TITLE("error with content (with a few slow down errors)");
+                TEST_TITLE("error with content (with a few errors to be retried)");
 
                 hrnServerScriptAccept(http);
 
@@ -626,6 +626,18 @@ testRun(void)
                 hrnServerScriptClose(http);
                 hrnServerScriptAccept(http);
 
+                hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
+                hrnServerScriptReplyZ(http, "HTTP/1.1 408 Request Timeout\r\ncontent-length:0\r\nConnection:close\r\n\r\n");
+
+                hrnServerScriptClose(http);
+                hrnServerScriptAccept(http);
+
+                hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
+                hrnServerScriptReplyZ(http, "HTTP/1.1 429 Too Many Requests\r\ncontent-length:0\r\nConnection:close\r\n\r\n");
+
+                hrnServerScriptClose(http);
+                hrnServerScriptAccept(http);
+
                 hrnServerScriptExpectZ(http, "GET / HTTP/1.1\r\n" TEST_USER_AGENT "\r\n");
                 hrnServerScriptReplyZ(
                     http, "HTTP/1.1 503 Slow Down\r\nTransfer-Encoding:chunked\r\nConnection:close\r\n\r\n0\r\n\r\n");