From 9eaeb33c882b79a1cade3ce65b5d1b51b6978b05 Mon Sep 17 00:00:00 2001
From: David Steele <david@pgmasters.net>
Date: Mon, 19 Aug 2019 21:36:01 -0400
Subject: [PATCH] Fix slow manifest build for very large quantities of
 tables/segments.

storagePosixInfoList() processed each directory in a single memory context.  If the directory contained hundreds of thousands of files processing became very slow due to the number of allocations.

Instead, reset the memory context every thousand files to minimize the number of allocations active at once, improving both speed and memory consumption.

Reported by Jens Wilke.
---
 doc/xml/release.xml         | 10 ++++++++++
 src/storage/posix/storage.c |  5 ++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/doc/xml/release.xml b/doc/xml/release.xml
index aa9b8717d..07126c411 100644
--- a/doc/xml/release.xml
+++ b/doc/xml/release.xml
@@ -14,6 +14,16 @@
     <release-list>
         <release date="XXXX-XX-XX" version="2.17dev" title="UNDER DEVELOPMENT">
             <release-core-list>
+                <release-bug-list>
+                    <release-item>
+                        <release-item-contributor-list>
+                            <release-item-ideator id="jens.wilke"/>
+                        </release-item-contributor-list>
+
+                        <p>Fix slow manifest build for very large quantities of tables/segments.</p>
+                    </release-item>
+                </release-bug-list>
+
                 <release-improvement-list>
                     <release-item>
                         <release-item-contributor-list>
diff --git a/src/storage/posix/storage.c b/src/storage/posix/storage.c
index 35eadc83a..3f5209a07 100644
--- a/src/storage/posix/storage.c
+++ b/src/storage/posix/storage.c
@@ -223,7 +223,7 @@ storagePosixInfoList(THIS_VOID, const String *path, StorageInfoListCallback call
 
         TRY_BEGIN()
         {
-            MEM_CONTEXT_TEMP_BEGIN()
+            MEM_CONTEXT_TEMP_RESET_BEGIN()
             {
                 // Read the directory entries
                 struct dirent *dirEntry = readdir(dir);
@@ -235,6 +235,9 @@ storagePosixInfoList(THIS_VOID, const String *path, StorageInfoListCallback call
 
                     // Get next entry
                     dirEntry = readdir(dir);
+
+                    // Reset the memory context occasionally so we don't use too much memory or slow down processing
+                    MEM_CONTEXT_TEMP_RESET(1000);
                 }
             }
             MEM_CONTEXT_TEMP_END();