Fix PostgreSQL query performance for large datasets.

The asynchronous logic used to implement the query timeout was misusing PQisBusy(), which caused the wait handler to throttle the consumption of command results. It could introduce a large delay on a query up to `db-timeout` because of the back-off sequence.

Following the recommendation of libpq, fix by polling the client socket for data availability and then continue consuming results and checking for command busyness.

doc/xml/release/2024/2.54.xml

@@ -1,5 +1,28 @@
 <release date="XXXX-XX-XX" version="2.54dev" title="UNDER DEVELOPMENT">
     <release-core-list>
+        <release-bug-list>
+            <release-item>
+                <commit subject="Move fd module shim to io module test."/>
+                <commit subject="Allow fdReady() shim to be run only once with a specified return value."/>
+                <commit subject="Refactor Wait object to expose remaining wait as waitRemains()."/>
+                <commit subject="Fix PostgreSQL query performance for large datasets.">
+                    <github-issue id="2458"/>
+                    <github-pull-request id="2459"/>
+                </commit>
+
+                <release-item-contributor-list>
+                    <release-item-ideator id="antoine.millet"/>
+                    <release-item-contributor id="thibault.vincent"/>
+                    <release-item-contributor id="david.steele"/>
+                    <release-item-reviewer id="david.christensen"/>
+                    <!-- Actually tester, but we don't have a tag for that yet -->
+                    <release-item-reviewer id="antoine.millet"/>
+                </release-item-contributor-list>
+
+                <p>Fix <postgres/> query performance for large datasets.</p>
+            </release-item>
+        </release-bug-list>
+
         <release-feature-list>
             <release-item>
                 <github-pull-request id="2404"/>

doc/xml/release/contributor.xml

@@ -85,6 +85,11 @@
     <contributor-id type="github">andr-sokolov</contributor-id>
 </contributor>
 
+<contributor id="antoine.millet">
+    <contributor-name-display>Antoine Millet</contributor-name-display>
+    <contributor-id type="github">NaPs</contributor-id>
+</contributor>
+
 <contributor id="anton.glushakov">
     <contributor-name-display>Anton Glushakov</contributor-name-display>
     <contributor-id type="github">glushakov</contributor-id>
@@ -985,6 +990,11 @@
     <contributor-id type="github">tanelsuurhans</contributor-id>
 </contributor>
 
+<contributor id="thibault.vincent">
+    <contributor-name-display>Thibault Vincent</contributor-name-display>
+    <contributor-id type="github">npdgm</contributor-id>
+</contributor>
+
 <contributor id="tim.garton">
     <contributor-name-display>Tim Garton</contributor-name-display>
     <contributor-id type="github">ralfthewise</contributor-id>

src/postgres/client.c

@@ -6,6 +6,7 @@ Postgres Client
 #include <libpq-fe.h>
 
 #include "common/debug.h"
+#include "common/io/fd.h"
 #include "common/log.h"
 #include "common/wait.h"
 #include "postgres/client.h"
@@ -198,7 +199,7 @@ pgClientQuery(PgClient *const this, const String *const query, const PgClientQue
             PQconsumeInput(this->connection);
             busy = PQisBusy(this->connection);
         }
-        while (busy && waitMore(wait));
+        while (busy && fdReadyRead(PQsocket(this->connection), waitRemains(wait)));
 
         // If the query is still busy after the timeout attempt to cancel
         if (busy)

test/src/common/harnessPq.c

@@ -13,6 +13,7 @@ Pq Test Harness
 #include "common/type/string.h"
 #include "common/type/variantList.h"
 
+#include "common/harnessFd.h"
 #include "common/harnessPq.h"
 #include "common/harnessTest.h"
 
@@ -269,6 +270,19 @@ PQisBusy(PGconn *conn)
     return hrnPqScriptRun(HRN_PQ_ISBUSY, NULL, (HrnPqScript *)conn)->resultInt;
 }
 
+/***********************************************************************************************************************************
+Shim for PQsocket()
+***********************************************************************************************************************************/
+int
+PQsocket(const PGconn *conn)
+{
+    int result = hrnPqScriptRun(HRN_PQ_SOCKET, NULL, (HrnPqScript *)conn)->resultInt;
+
+    hrnFdReadyShimOne(result == true);
+
+    return result;
+}
+
 /***********************************************************************************************************************************
 Shim for PQgetCancel()
 ***********************************************************************************************************************************/

test/src/common/harnessPq.h

@@ -36,6 +36,7 @@ Function constants
 #define HRN_PQ_RESULTERRORMESSAGE                                   "PQresultErrorMessage"
 #define HRN_PQ_RESULTSTATUS                                         "PQresultStatus"
 #define HRN_PQ_SENDQUERY                                            "PQsendQuery"
+#define HRN_PQ_SOCKET                                               "PQsocket"
 #define HRN_PQ_STATUS                                               "PQstatus"
 
 /***********************************************************************************************************************************

test/src/module/postgres/clientTest.c

@@ -153,10 +153,13 @@ testRun(void)
             {.function = HRN_PQ_SENDQUERY, .param = "[\"" TEST_QUERY "\"]", .resultInt = 1},
             {.function = HRN_PQ_CONSUMEINPUT, .sleep = 600},
             {.function = HRN_PQ_ISBUSY, .resultInt = 1},
+            {.function = HRN_PQ_SOCKET, .resultInt = 1},
             {.function = HRN_PQ_CONSUMEINPUT},
             {.function = HRN_PQ_ISBUSY, .resultInt = 1},
+            {.function = HRN_PQ_SOCKET, .resultInt = 1},
             {.function = HRN_PQ_CONSUMEINPUT},
             {.function = HRN_PQ_ISBUSY, .resultInt = 1},
+            {.function = HRN_PQ_SOCKET},
             {.function = HRN_PQ_GETCANCEL},
             {.function = HRN_PQ_CANCEL, .resultInt = 1},
             {.function = HRN_PQ_FREECANCEL},
@@ -182,10 +185,13 @@ testRun(void)
             {.function = HRN_PQ_SENDQUERY, .param = "[\"" TEST_QUERY "\"]", .resultInt = 1},
             {.function = HRN_PQ_CONSUMEINPUT, .sleep = 300},
             {.function = HRN_PQ_ISBUSY, .resultInt = 1},
+            {.function = HRN_PQ_SOCKET, .resultInt = 1},
             {.function = HRN_PQ_CONSUMEINPUT, .sleep = 300},
             {.function = HRN_PQ_ISBUSY, .resultInt = 1},
+            {.function = HRN_PQ_SOCKET, .resultInt = 1},
             {.function = HRN_PQ_CONSUMEINPUT},
             {.function = HRN_PQ_ISBUSY, .resultInt = 1},
+            {.function = HRN_PQ_SOCKET},
             {.function = HRN_PQ_GETCANCEL},
             {.function = HRN_PQ_CANCEL, .resultInt = 0, .resultZ = TEST_PQ_ERROR},
             {.function = HRN_PQ_FREECANCEL});
@@ -208,10 +214,13 @@ testRun(void)
             {.function = HRN_PQ_SENDQUERY, .param = "[\"" TEST_QUERY "\"]", .resultInt = 1},
             {.function = HRN_PQ_CONSUMEINPUT, .sleep = 600},
             {.function = HRN_PQ_ISBUSY, .resultInt = 1},
+            {.function = HRN_PQ_SOCKET, .resultInt = 1},
             {.function = HRN_PQ_CONSUMEINPUT},
             {.function = HRN_PQ_ISBUSY, .resultInt = 1},
+            {.function = HRN_PQ_SOCKET, .resultInt = 1},
             {.function = HRN_PQ_CONSUMEINPUT},
             {.function = HRN_PQ_ISBUSY, .resultInt = 1},
+            {.function = HRN_PQ_SOCKET},
             {.function = HRN_PQ_GETCANCEL, .resultNull = true});
 
         TEST_ERROR(