Add sorting, filters, and recursion to storageInfoList().

These are needed for the ls command and are also useful for testing.

src/Makefile.in

@@ -597,7 +597,7 @@ storage/s3/storage.o: storage/s3/storage.c build.auto.h common/assert.h common/c
 storage/s3/write.o: storage/s3/write.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/type/xml.h storage/info.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/s3/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/s3/write.c -o storage/s3/write.o
 
-storage/storage.o: storage/storage.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h storage/info.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
+storage/storage.o: storage/storage.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h storage/info.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/storage.c -o storage/storage.o
 
 storage/write.o: storage/write.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/write.h storage/write.intern.h version.h

src/storage/storage.c

@@ -8,9 +8,11 @@ Storage Interface
 
 #include "common/debug.h"
 #include "common/io/io.h"
+#include "common/type/list.h"
 #include "common/log.h"
 #include "common/memContext.h"
 #include "common/object.h"
+#include "common/regExp.h"
 #include "common/wait.h"
 #include "storage/storage.intern.h"
 
@@ -272,6 +274,157 @@ storageInfo(const Storage *this, const String *fileExp, StorageInfoParam param)
 /***********************************************************************************************************************************
 Info for all files/paths in a path
 ***********************************************************************************************************************************/
+typedef struct StorageInfoListSortData
+{
+    MemContext *memContext;                                         // Mem context to use for allocating data in this struct
+    StringList *ownerList;                                          // List of users and groups to reduce memory usage
+    List *infoList;                                                 // List of info
+} StorageInfoListSortData;
+
+static void
+storageInfoListSortCallback(void *data, const StorageInfo *info)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_LOG_PARAM_P(VOID, data);
+        FUNCTION_LOG_PARAM(STORAGE_INFO, info);
+    FUNCTION_TEST_END();
+
+    StorageInfoListSortData *infoData = data;
+
+    MEM_CONTEXT_BEGIN(infoData->memContext)
+    {
+        // Copy info and dup strings
+        StorageInfo infoCopy = *info;
+        infoCopy.name = strDup(info->name);
+        infoCopy.linkDestination = strDup(info->linkDestination);
+        infoCopy.user = strLstAddIfMissing(infoData->ownerList, info->user);
+        infoCopy.group = strLstAddIfMissing(infoData->ownerList, info->group);
+
+        lstAdd(infoData->infoList, &infoCopy);
+    }
+    MEM_CONTEXT_END();
+
+    FUNCTION_TEST_RETURN_VOID();
+}
+
+static bool
+storageInfoListSort(
+    const Storage *this, const String *path, SortOrder sortOrder, StorageInfoListCallback callback, void *callbackData)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(STORAGE, this);
+        FUNCTION_LOG_PARAM(STRING, path);
+        FUNCTION_LOG_PARAM(ENUM, sortOrder);
+        FUNCTION_LOG_PARAM(FUNCTIONP, callback);
+        FUNCTION_LOG_PARAM_P(VOID, callbackData);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(callback != NULL);
+
+    bool result = false;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        // If no sorting then use the callback directly
+        if (sortOrder == sortOrderNone)
+        {
+            result = this->interface.infoList(this->driver, path, callback, callbackData);
+        }
+        // Else sort the info before sending it to the callback
+        else
+        {
+            StorageInfoListSortData data =
+            {
+                .memContext = MEM_CONTEXT_TEMP(),
+                .ownerList = strLstNew(),
+                .infoList = lstNewP(sizeof(StorageInfo), .comparator = lstComparatorStr),
+            };
+
+            result = this->interface.infoList(this->driver, path, storageInfoListSortCallback, &data);
+            lstSort(data.infoList, sortOrder);
+
+            MEM_CONTEXT_TEMP_RESET_BEGIN()
+            {
+                for (unsigned int infoIdx = 0; infoIdx < lstSize(data.infoList); infoIdx++)
+                {
+                    // Pass info to the caller
+                    callback(callbackData, lstGet(data.infoList, infoIdx));
+
+                    // Reset the memory context occasionally
+                    MEM_CONTEXT_TEMP_RESET(1000);
+                }
+            }
+            MEM_CONTEXT_TEMP_END();
+        }
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    FUNCTION_LOG_RETURN(BOOL, result);
+}
+
+typedef struct StorageInfoListData
+{
+    const Storage *storage;                                         // Storage object;
+    StorageInfoListCallback callbackFunction;                       // Original callback function
+    void *callbackData;                                             // Original callback data
+    RegExp *expression;                                             // Filter for names
+    bool recurse;                                                   // Should we recurse?
+    SortOrder sortOrder;                                            // Sort order
+    const String *path;                                             // Top-level path for info
+    const String *subPath;                                          // Path below the top-level path (starts as NULL)
+} StorageInfoListData;
+
+static void
+storageInfoListCallback(void *data, const StorageInfo *info)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_LOG_PARAM_P(VOID, data);
+        FUNCTION_LOG_PARAM(STORAGE_INFO, info);
+    FUNCTION_TEST_END();
+
+    StorageInfoListData *listData = data;
+
+    // Is this the . path?
+    bool dotPath = info->type == storageTypePath && strEq(info->name, DOT_STR);
+
+    // Skip . paths when getting info for subpaths (since info was already reported in the parent path)
+    if (dotPath && listData->subPath != NULL)
+    {
+        FUNCTION_TEST_RETURN_VOID();
+        return;
+    }
+
+    // Update the name in info with the subpath
+    StorageInfo infoUpdate = *info;
+
+    if (listData->subPath != NULL)
+        infoUpdate.name = strNewFmt("%s/%s", strPtr(listData->subPath), strPtr(infoUpdate.name));
+
+    // Only continue if there is no expression or the expression matches
+    if (listData->expression == NULL || regExpMatch(listData->expression, infoUpdate.name))
+    {
+        if (listData->sortOrder != sortOrderDesc)
+            listData->callbackFunction(listData->callbackData, &infoUpdate);
+
+        // Recurse into paths
+        if (infoUpdate.type == storageTypePath && listData->recurse && !dotPath)
+        {
+            StorageInfoListData data = *listData;
+            data.subPath = infoUpdate.name;
+
+            storageInfoListSort(
+                data.storage, strNewFmt("%s/%s", strPtr(data.path), strPtr(data.subPath)), data.sortOrder, storageInfoListCallback,
+                &data);
+        }
+
+        if (listData->sortOrder == sortOrderDesc)
+            listData->callbackFunction(listData->callbackData, &infoUpdate);
+    }
+
+    FUNCTION_TEST_RETURN_VOID();
+}
+
 bool
 storageInfoList(
     const Storage *this, const String *pathExp, StorageInfoListCallback callback, void *callbackData, StorageInfoListParam param)
@@ -282,6 +435,9 @@ storageInfoList(
         FUNCTION_LOG_PARAM(FUNCTIONP, callback);
         FUNCTION_LOG_PARAM_P(VOID, callbackData);
         FUNCTION_LOG_PARAM(BOOL, param.errorOnMissing);
+        FUNCTION_LOG_PARAM(ENUM, param.sortOrder);
+        FUNCTION_LOG_PARAM(STRING, param.expression);
+        FUNCTION_LOG_PARAM(BOOL, param.recurse);
     FUNCTION_LOG_END();
 
     ASSERT(this != NULL);
@@ -296,8 +452,26 @@ storageInfoList(
         // Build the path
         String *path = storagePathNP(this, pathExp);
 
-        // Call driver function
+        // If there is an expression or recursion then the info will need to be filtered through a local callback
-        result = this->interface.infoList(this->driver, path, callback, callbackData);
+        if (param.expression != NULL || param.recurse)
+        {
+            StorageInfoListData data =
+            {
+                .storage = this,
+                .callbackFunction = callback,
+                .callbackData = callbackData,
+                .sortOrder = param.sortOrder,
+                .recurse = param.recurse,
+                .path = path,
+            };
+
+            if (param.expression != NULL)
+                data.expression = regExpNew(param.expression);
+
+            result = storageInfoListSort(this, path, param.sortOrder, storageInfoListCallback, &data);
+        }
+        else
+            result = storageInfoListSort(this, path, param.sortOrder, callback, callbackData);
 
         if (!result && param.errorOnMissing)
             THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, strPtr(path));

src/storage/storage.h

@@ -109,6 +109,9 @@ typedef void (*StorageInfoListCallback)(void *callbackData, const StorageInfo *i
 typedef struct StorageInfoListParam
 {
     bool errorOnMissing;
+    SortOrder sortOrder;
+    const String *expression;
+    bool recurse;
 } StorageInfoListParam;
 
 #define storageInfoListP(this, fileExp, callback, callbackData, ...)                                                               \

test/src/common/harnessStorage.c

@@ -0,0 +1,94 @@
+/***********************************************************************************************************************************
+Storage Test Harness
+***********************************************************************************************************************************/
+#include <inttypes.h>
+
+#include "common/user.h"
+#include "storage/storage.h"
+
+#include "common/harnessStorage.h"
+
+/**********************************************************************************************************************************/
+void
+hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info)
+{
+    HarnessStorageInfoListCallbackData *data = callbackData;
+
+    // Add LF for after the first item
+    if (strSize(data->content) != 0)
+        strCat(data->content, "\n");
+
+    strCatFmt(data->content, "%s {", strPtr(info->name));
+
+    switch (info->type)
+    {
+        case storageTypeFile:
+        {
+            strCat(data->content, "file");
+
+            if (!data->sizeOmit)
+                strCatFmt(data->content, ", s=%" PRIu64, info->size);
+
+            break;
+        }
+
+        case storageTypeLink:
+        {
+            strCatFmt(data->content, "link, d=%s", strPtr(info->linkDestination));
+            break;
+        }
+
+        case storageTypePath:
+        {
+            strCat(data->content, "path");
+            break;
+        }
+
+        case storageTypeSpecial:
+        {
+            strCat(data->content, "special");
+            break;
+        }
+    }
+
+    if (info->type != storageTypeSpecial)
+    {
+        if (info->type != storageTypeLink)
+        {
+            if (!data->modeOmit || (info->type == storageTypePath && data->modePath != info->mode) ||
+                (info->type == storageTypeFile && data->modeFile != info->mode))
+            {
+                strCatFmt(data->content, ", m=%04o", info->mode);
+            }
+
+            if (!data->timestampOmit)
+                strCatFmt(data->content, ", t=%" PRIu64, (uint64_t)info->timeModified);
+        }
+
+        if (!data->userOmit || userId() != info->userId)
+        {
+            if (info->user != NULL)
+            {
+                strCatFmt(data->content, ", u=%s", strPtr(info->user));
+            }
+            else
+            {
+                strCatFmt(data->content, ", u=%d", (int)info->userId);
+            }
+        }
+
+        if (!data->groupOmit || groupId() != info->groupId)
+        {
+            if (info->group != NULL)
+            {
+                strCatFmt(data->content, ", g=%s", strPtr(info->group));
+            }
+            else
+            {
+                strCatFmt(data->content, ", g=%d", (int)info->groupId);
+            }
+        }
+    }
+
+    strCat(data->content, "}");
+}

test/src/common/harnessStorage.h

@@ -0,0 +1,26 @@
+/***********************************************************************************************************************************
+Storage Test Harness
+
+Helper functions for testing storage and related functions.
+***********************************************************************************************************************************/
+#ifndef TEST_COMMON_HARNESS_STORAGE_H
+#define TEST_COMMON_HARNESS_STORAGE_H
+
+/***********************************************************************************************************************************
+Callback for formatting info list results
+***********************************************************************************************************************************/
+typedef struct HarnessStorageInfoListCallbackData
+{
+    String *content;                                                // String where content should be added
+    bool modeOmit;                                                  // Should the specified mode be ommitted?
+    mode_t modeFile;                                                // File to omit if modeOmit is true
+    mode_t modePath;                                                // Path mode to omit if modeOmit is true
+    bool timestampOmit;                                             // Should the timestamp be ommitted?
+    bool userOmit;                                                  // Should the current user be ommitted?
+    bool groupOmit;                                                 // Should the current group be ommitted?
+    bool sizeOmit;                                                  // Should the size be ommitted
+} HarnessStorageInfoListCallbackData;
+
+void hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info);
+
+#endif

test/src/module/storage/posixTest.c

@@ -11,6 +11,7 @@ Test Posix Storage
 
 #include "common/harnessConfig.h"
 #include "common/harnessFork.h"
+#include "common/harnessStorage.h"
 
 /***********************************************************************************************************************************
 Test function for path expression
@@ -38,24 +39,6 @@ Macro to create a path and file that cannot be accessed
             strPtr(fileNoPerm)))),                                                                                                 \
         0, "create no perm path/file");
 
-/***********************************************************************************************************************************
-Callback and data for storageInfoList() tests
-***********************************************************************************************************************************/
-unsigned int testStorageInfoListSize = 0;
-StorageInfo testStorageInfoList[256];
-
-void
-testStorageInfoListCallback(void *callbackData, const StorageInfo *info)
-{
-    MEM_CONTEXT_BEGIN((MemContext *)callbackData)
-    {
-        testStorageInfoList[testStorageInfoListSize] = *info;
-        testStorageInfoList[testStorageInfoListSize].name = strDup(info->name);
-        testStorageInfoListSize++;
-    }
-    MEM_CONTEXT_END();
-}
-
 /***********************************************************************************************************************************
 Test Run
 ***********************************************************************************************************************************/
@@ -283,24 +266,100 @@ testRun(void)
             STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathNoPerm));
 
         // -------------------------------------------------------------------------------------------------------------------------
-        testStorageInfoListSize = 0;
+        HarnessStorageInfoListCallbackData callbackData =
+        {
+            .content = strNew(""),
+        };
+
         TEST_RESULT_VOID(
-            storagePosixInfoListEntry((StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"),
+            storagePosixInfoListEntry(
-                testStorageInfoListCallback, (void *)memContextCurrent()),
+                (StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"),
-            "missing file");
+                hrnStorageInfoListCallback, &callbackData),
-        TEST_RESULT_UINT(testStorageInfoListSize, 0, "    no file found");
+            "missing path");
+        TEST_RESULT_STR_Z(callbackData.content, "", "    check content");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        storagePathCreateP(storageTest, strNew("pg/.include"), .mode = 0766);
+        storagePathCreateP(storageTest, strNew("pg"), .mode = 0766);
+
+        struct utimbuf utimeTest = {.actime = 1000000000, .modtime = 1555160777};
+        THROW_ON_SYS_ERROR_FMT(
+            utime(strPtr(strNewFmt("%s/pg", testPath())), &utimeTest) != 0, FileWriteError, "unable to set time for '%s'/pg",
+            testPath());
+
+        callbackData.content = strNew("");
 
         TEST_RESULT_VOID(
-            storageInfoListNP(storageTest, strNew("pg"), testStorageInfoListCallback, (void *)memContextCurrent()),
+            storageInfoListP(storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData),
-            "empty directory");
+            "directory with one dot file sorted");
-        TEST_RESULT_UINT(testStorageInfoListSize, 2, "    two paths returned");
+        TEST_RESULT_STR_Z(
-        TEST_RESULT_STR(
+            callbackData.content, strPtr(strNewFmt(". {path, m=0766, t=1555160777, u=%s, g=%s}", testUser(), testGroup())),
-            strPtr(testStorageInfoList[0].name), strEqZ(testStorageInfoList[1].name, ".") ? ".include" : ".", "    check name");
+            "    check content");
-        TEST_RESULT_STR(
+
-            strPtr(testStorageInfoList[1].name), strEqZ(testStorageInfoList[0].name, ".") ? ".include" : ".", "    check name");
+        // -------------------------------------------------------------------------------------------------------------------------
+        storagePathCreateP(storageTest, strNew("pg/.include"), .mode = 0755);
+        ASSERT(system(strPtr(strNewFmt("sudo chown 77777:77777 %s/pg/.include", testPath()))) == 0);
+
+        storagePutNP(storageNewWriteP(storageTest, strNew("pg/file"), .modeFile = 0660), BUFSTRDEF("TESTDATA"));
+
+        ASSERT(system(strPtr(strNewFmt("ln -s ../file %s/pg/link", testPath()))) == 0);
+        ASSERT(system(strPtr(strNewFmt("mkfifo -m 777 %s/pg/pipe", testPath()))) == 0);
+
+        callbackData = (HarnessStorageInfoListCallbackData)
+        {
+            .content = strNew(""),
+            .timestampOmit = true,
+            .modeOmit = true,
+            .modePath = 0766,
+            .modeFile = 0600,
+            .userOmit = true,
+            .groupOmit = true,
+        };
+
+        TEST_RESULT_VOID(
+            storageInfoListP(storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderAsc),
+            "directory with one dot file sorted");
+        TEST_RESULT_STR_Z(
+            callbackData.content,
+            ". {path}\n"
+            ".include {path, m=0755, u=77777, g=77777}\n"
+            "file {file, s=8, m=0660}\n"
+            "link {link, d=../file}\n"
+            "pipe {special}",
+            "    check content");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        ASSERT(system(strPtr(strNewFmt("sudo rmdir %s/pg/.include", testPath()))) == 0);
+        storagePathCreateP(storageTest, strNew("pg/path"), .mode = 0700);
+        storagePutNP(storageNewWriteP(storageTest, strNew("pg/path/file"), .modeFile = 0600), BUFSTRDEF("TESTDATA"));
+
+        callbackData.content = strNew("");
+
+        TEST_RESULT_VOID(
+            storageInfoListP(
+                storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderDesc, .recurse = true),
+            "recurse descending");
+        TEST_RESULT_STR_Z(
+            callbackData.content,
+            "pipe {special}\n"
+            "path/file {file, s=8}\n"
+            "path {path, m=0700}\n"
+            "link {link, d=../file}\n"
+            "file {file, s=8, m=0660}\n"
+            ". {path}",
+            "    check content");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        callbackData.content = strNew("");
+
+        TEST_RESULT_VOID(
+            storageInfoListP(
+                storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderAsc,
+                .expression = STRDEF("^path")),
+            "filter");
+        TEST_RESULT_STR_Z(
+            callbackData.content,
+            "path {path, m=0700}",
+            "    check content");
     }
 
     // *****************************************************************************************************************************