Migrate walIsPartial(), walIsSegment(), and walSegmentFind() from Perl to C.

Also refactor regular expression defines to make them more reusable.

doc/xml/release.xml

@@ -38,6 +38,10 @@
                     <release-item>
                         <p>Add helper for repository storage.</p>
                     </release-item>
+
+                    <release-item>
+                        <p>Migrate <code>walIsPartial()</code>, <code>walIsSegment()</code>, and <code>walSegmentFind()</code> from Perl to C.</p>
+                    </release-item>
                 </release-development-list>
             </release-core-list>
         </release>

src/Makefile

@@ -140,7 +140,7 @@ install: pgbackrest
 ####################################################################################################################################
 # Compile rules
 ####################################################################################################################################
-command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
+command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h protocol/storage/helper.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
 	$(CC) $(CFLAGS) -c command/archive/common.c -o command/archive/common.o
 
 command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/info.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h

src/command/archive/common.c

@@ -10,8 +10,10 @@ Archive Push Command
 #include "common/debug.h"
 #include "common/log.h"
 #include "common/memContext.h"
+#include "common/regExp.h"
 #include "common/wait.h"
 #include "postgres/version.h"
+#include "protocol/storage/helper.h"
 #include "storage/helper.h"
 
 /***********************************************************************************************************************************
@@ -111,6 +113,102 @@ archiveAsyncStatus(ArchiveMode archiveMode, const String *walSegment, bool confe
     FUNCTION_DEBUG_RESULT(BOOL, result);
 }
 
+/***********************************************************************************************************************************
+Is the segment partial?
+***********************************************************************************************************************************/
+bool
+walIsPartial(const String *walSegment)
+{
+    FUNCTION_DEBUG_BEGIN(logLevelTrace);
+        FUNCTION_DEBUG_PARAM(STRING, walSegment);
+
+        FUNCTION_DEBUG_ASSERT(walSegment != NULL);
+        FUNCTION_DEBUG_ASSERT(walIsSegment(walSegment));
+    FUNCTION_DEBUG_END();
+
+    FUNCTION_DEBUG_RESULT(BOOL, strEndsWithZ(walSegment, WAL_SEGMENT_PARTIAL_EXT));
+}
+
+/***********************************************************************************************************************************
+Is the file a segment or some other file (e.g. .history, .backup, etc)
+***********************************************************************************************************************************/
+bool
+walIsSegment(const String *walSegment)
+{
+    FUNCTION_DEBUG_BEGIN(logLevelTrace);
+        FUNCTION_DEBUG_PARAM(STRING, walSegment);
+
+        FUNCTION_DEBUG_ASSERT(walSegment != NULL);
+    FUNCTION_DEBUG_END();
+
+    // Create the regular expression to identify WAL segments if it does not already exist
+    static RegExp *regExpSegment = NULL;
+
+    if (regExpSegment == NULL)
+    {
+        MEM_CONTEXT_BEGIN(memContextTop())
+        {
+            regExpSegment = regExpNew(strNew(WAL_SEGMENT_PARTIAL_REGEXP));
+        }
+        MEM_CONTEXT_END();
+    }
+
+    FUNCTION_DEBUG_RESULT(BOOL, regExpMatch(regExpSegment, walSegment));
+}
+
+/***********************************************************************************************************************************
+Find a WAL segment in the repository
+
+The file name can have several things appended such as a hash, compression extension, and partial extension so it is possible to
+have multiple files that match the segment, though more than one match is not a good thing.
+***********************************************************************************************************************************/
+String *
+walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment)
+{
+    FUNCTION_DEBUG_BEGIN(logLevelDebug);
+        FUNCTION_DEBUG_PARAM(STORAGE, storage);
+        FUNCTION_DEBUG_PARAM(STRING, archiveId);
+        FUNCTION_DEBUG_PARAM(STRING, walSegment);
+
+        FUNCTION_DEBUG_ASSERT(storage != NULL);
+        FUNCTION_DEBUG_ASSERT(archiveId != NULL);
+        FUNCTION_DEBUG_ASSERT(walSegment != NULL);
+        FUNCTION_DEBUG_ASSERT(walIsSegment(walSegment));
+    FUNCTION_DEBUG_END();
+
+    String *result = NULL;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        // Get a list of all WAL segments that match
+        StringList *list = storageListP(
+            storage, strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(strSubN(walSegment, 0, 16))),
+            .expression = strNewFmt("^%s%s-[0-f]{40}(\\.gz){0,1}$", strPtr(strSubN(walSegment, 0, 24)),
+                walIsPartial(walSegment) ? WAL_SEGMENT_PARTIAL_EXT : ""));
+
+        // If there are results
+        if (list != NULL && strLstSize(list) > 0)
+        {
+            // Error if there is more than one match
+            if (strLstSize(list) > 1)
+            {
+                THROW_FMT(
+                    ArchiveDuplicateError,
+                    "duplicates found in archive for WAL segment %s: %s\n"
+                        "HINT: are multiple primaries archiving to this stanza?",
+                    strPtr(walSegment), strPtr(strLstJoin(strLstSort(list, sortOrderAsc), ", ")));
+            }
+
+            // Copy file name of WAL segment found into the calling context
+            memContextSwitch(MEM_CONTEXT_OLD());
+            result = strDup(strLstGet(list, 0));
+        }
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_DEBUG_RESULT(STRING, result);
+}
+
 /***********************************************************************************************************************************
 Get the next WAL segment given a WAL segment and WAL segment size
 ***********************************************************************************************************************************/

src/command/archive/common.h

@@ -18,12 +18,22 @@ typedef enum
 } ArchiveMode;
 
 #include "common/type/stringList.h"
+#include "storage/storage.h"
 
 /***********************************************************************************************************************************
 WAL segment constants
 ***********************************************************************************************************************************/
-// Only match on a WAL segment without checksum appended
+// Extension for partial segments
-#define WAL_SEGMENT_REGEXP                                          "^[0-F]{24}$"
+#define WAL_SEGMENT_PARTIAL_EXT                                     ".partial"
+
+// Match when the first 24 characters match a WAL segment
+#define WAL_SEGMENT_PREFIX_REGEXP                                   "^[0-F]{24}"
+
+// Match on a WAL segment without checksum appended
+#define WAL_SEGMENT_REGEXP                                          WAL_SEGMENT_PREFIX_REGEXP "$"
+
+// Match on a WAL segment with partial allowed
+#define WAL_SEGMENT_PARTIAL_REGEXP                                  WAL_SEGMENT_PREFIX_REGEXP "(\\.partial){0,1}$"
 
 // Defines the size of standard WAL segment name -- this should never changed
 #define WAL_SEGMENT_NAME_SIZE                                       ((unsigned int)24)
@@ -36,6 +46,9 @@ Functions
 ***********************************************************************************************************************************/
 bool archiveAsyncStatus(ArchiveMode archiveMode, const String *walSegment, bool confessOnError);
 
+bool walIsPartial(const String *walSegment);
+bool walIsSegment(const String *walSegment);
+String *walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment);
 String *walSegmentNext(const String *walSegment, size_t walSegmentSize, unsigned int pgVersion);
 StringList *walSegmentRange(const String *walSegmentBegin, size_t walSegmentSize, unsigned int pgVersion, unsigned int range);
 

test/define.yaml

@@ -593,7 +593,7 @@ unit:
     test:
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: common
-        total: 3
+        total: 6
 
         coverage:
           command/archive/common: full

test/src/module/archive/commonTest.c

@@ -5,6 +5,8 @@ Test Archive Common
 
 #include "common/harnessConfig.h"
 
+#include "protocol/storage/helper.h"
+
 /***********************************************************************************************************************************
 Test Run
 ***********************************************************************************************************************************/
@@ -13,6 +15,9 @@ testRun(void)
 {
     FUNCTION_HARNESS_VOID();
 
+    // Create default storage object for testing
+    Storage *storageTest = storageNewP(strNew(testPath()), .write = true);
+
     // *****************************************************************************************************************************
     if (testBegin("archiveAsyncStatus()"))
     {
@@ -98,6 +103,62 @@ testRun(void)
         unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment)))));
     }
 
+    // *****************************************************************************************************************************
+    if (testBegin("walIsPartial()"))
+    {
+        TEST_RESULT_BOOL(walIsPartial(strNew("000000010000000100000001")), false, "not partial");
+        TEST_RESULT_BOOL(walIsPartial(strNew("FFFFFFFFFFFFFFFFFFFFFFFF.partial")), true, "partial");
+    }
+
+    // *****************************************************************************************************************************
+    if (testBegin("walIsSegment()"))
+    {
+        TEST_RESULT_BOOL(walIsSegment(strNew("000000010000000100000001")), true, "wal segment");
+        TEST_RESULT_BOOL(walIsSegment(strNew("FFFFFFFFFFFFFFFFFFFFFFFF.partial")), true, "partial wal segment");
+        TEST_RESULT_BOOL(walIsSegment(strNew("0000001A.history")), false, "history file");
+    }
+
+    // *****************************************************************************************************************************
+    if (testBegin("walSegmentFind()"))
+    {
+        // Load configuration to set repo-path and stanza
+        StringList *argList = strLstNew();
+        strLstAddZ(argList, "pgbackrest");
+        strLstAddZ(argList, "--stanza=db");
+        strLstAdd(argList, strNewFmt("--repo-path=%s", testPath()));
+        strLstAddZ(argList, "archive-get");
+        harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
+
+        TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no path");
+
+        storagePathCreateNP(storageTest, strNew("archive/db/9.6-2/1234567812345678"));
+        TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no segment");
+
+        storagePutNP(
+            storageNewWriteNP(
+                storageTest,
+                strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
+            NULL);
+
+        TEST_RESULT_STR(
+            strPtr(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"))),
+            "123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "found segment");
+
+        storagePutNP(
+            storageNewWriteNP(
+                storageTest,
+                strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz")),
+            NULL);
+
+        TEST_ERROR(
+            walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")),
+            ArchiveDuplicateError,
+            "duplicates found in archive for WAL segment 123456781234567812345678:"
+                " 123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+                ", 123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz"
+                "\nHINT: are multiple primaries archiving to this stanza?");
+    }
+
     // *****************************************************************************************************************************
     if (testBegin("walSegmentNext()"))
     {