From e45baa1830ac5251fc4284d094c3e12920e50862 Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 12 Sep 2019 16:03:05 -0400 Subject: [PATCH] Add sorting, filters, and recursion to storageInfoList(). These are needed for the ls command and are also useful for testing. --- src/Makefile.in | 2 +- src/storage/storage.c | 178 +++++++++++++++++++++++++++- src/storage/storage.h | 3 + test/src/common/harnessStorage.c | 94 +++++++++++++++ test/src/common/harnessStorage.h | 26 ++++ test/src/module/storage/posixTest.c | 121 ++++++++++++++----- 6 files changed, 390 insertions(+), 34 deletions(-) create mode 100644 test/src/common/harnessStorage.c create mode 100644 test/src/common/harnessStorage.h diff --git a/src/Makefile.in b/src/Makefile.in index d4c62d6be..554a6203e 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -597,7 +597,7 @@ storage/s3/storage.o: storage/s3/storage.c build.auto.h common/assert.h common/c storage/s3/write.o: storage/s3/write.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/type/xml.h storage/info.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/s3/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/s3/write.c -o storage/s3/write.o -storage/storage.o: storage/storage.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h storage/info.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h +storage/storage.o: storage/storage.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h storage/info.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h $(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/storage.c -o storage/storage.o storage/write.o: storage/write.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/write.h storage/write.intern.h version.h diff --git a/src/storage/storage.c b/src/storage/storage.c index 80e456fb2..2778db079 100644 --- a/src/storage/storage.c +++ b/src/storage/storage.c @@ -8,9 +8,11 @@ Storage Interface #include "common/debug.h" #include "common/io/io.h" +#include "common/type/list.h" #include "common/log.h" #include "common/memContext.h" #include "common/object.h" +#include "common/regExp.h" #include "common/wait.h" #include "storage/storage.intern.h" @@ -272,6 +274,157 @@ storageInfo(const Storage *this, const String *fileExp, StorageInfoParam param) /*********************************************************************************************************************************** Info for all files/paths in a path ***********************************************************************************************************************************/ +typedef struct StorageInfoListSortData +{ + MemContext *memContext; // Mem context to use for allocating data in this struct + StringList *ownerList; // List of users and groups to reduce memory usage + List *infoList; // List of info +} StorageInfoListSortData; + +static void +storageInfoListSortCallback(void *data, const StorageInfo *info) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_LOG_PARAM_P(VOID, data); + FUNCTION_LOG_PARAM(STORAGE_INFO, info); + FUNCTION_TEST_END(); + + StorageInfoListSortData *infoData = data; + + MEM_CONTEXT_BEGIN(infoData->memContext) + { + // Copy info and dup strings + StorageInfo infoCopy = *info; + infoCopy.name = strDup(info->name); + infoCopy.linkDestination = strDup(info->linkDestination); + infoCopy.user = strLstAddIfMissing(infoData->ownerList, info->user); + infoCopy.group = strLstAddIfMissing(infoData->ownerList, info->group); + + lstAdd(infoData->infoList, &infoCopy); + } + MEM_CONTEXT_END(); + + FUNCTION_TEST_RETURN_VOID(); +} + +static bool +storageInfoListSort( + const Storage *this, const String *path, SortOrder sortOrder, StorageInfoListCallback callback, void *callbackData) +{ + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(STORAGE, this); + FUNCTION_LOG_PARAM(STRING, path); + FUNCTION_LOG_PARAM(ENUM, sortOrder); + FUNCTION_LOG_PARAM(FUNCTIONP, callback); + FUNCTION_LOG_PARAM_P(VOID, callbackData); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(callback != NULL); + + bool result = false; + + MEM_CONTEXT_TEMP_BEGIN() + { + // If no sorting then use the callback directly + if (sortOrder == sortOrderNone) + { + result = this->interface.infoList(this->driver, path, callback, callbackData); + } + // Else sort the info before sending it to the callback + else + { + StorageInfoListSortData data = + { + .memContext = MEM_CONTEXT_TEMP(), + .ownerList = strLstNew(), + .infoList = lstNewP(sizeof(StorageInfo), .comparator = lstComparatorStr), + }; + + result = this->interface.infoList(this->driver, path, storageInfoListSortCallback, &data); + lstSort(data.infoList, sortOrder); + + MEM_CONTEXT_TEMP_RESET_BEGIN() + { + for (unsigned int infoIdx = 0; infoIdx < lstSize(data.infoList); infoIdx++) + { + // Pass info to the caller + callback(callbackData, lstGet(data.infoList, infoIdx)); + + // Reset the memory context occasionally + MEM_CONTEXT_TEMP_RESET(1000); + } + } + MEM_CONTEXT_TEMP_END(); + } + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(BOOL, result); +} + +typedef struct StorageInfoListData +{ + const Storage *storage; // Storage object; + StorageInfoListCallback callbackFunction; // Original callback function + void *callbackData; // Original callback data + RegExp *expression; // Filter for names + bool recurse; // Should we recurse? + SortOrder sortOrder; // Sort order + const String *path; // Top-level path for info + const String *subPath; // Path below the top-level path (starts as NULL) +} StorageInfoListData; + +static void +storageInfoListCallback(void *data, const StorageInfo *info) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_LOG_PARAM_P(VOID, data); + FUNCTION_LOG_PARAM(STORAGE_INFO, info); + FUNCTION_TEST_END(); + + StorageInfoListData *listData = data; + + // Is this the . path? + bool dotPath = info->type == storageTypePath && strEq(info->name, DOT_STR); + + // Skip . paths when getting info for subpaths (since info was already reported in the parent path) + if (dotPath && listData->subPath != NULL) + { + FUNCTION_TEST_RETURN_VOID(); + return; + } + + // Update the name in info with the subpath + StorageInfo infoUpdate = *info; + + if (listData->subPath != NULL) + infoUpdate.name = strNewFmt("%s/%s", strPtr(listData->subPath), strPtr(infoUpdate.name)); + + // Only continue if there is no expression or the expression matches + if (listData->expression == NULL || regExpMatch(listData->expression, infoUpdate.name)) + { + if (listData->sortOrder != sortOrderDesc) + listData->callbackFunction(listData->callbackData, &infoUpdate); + + // Recurse into paths + if (infoUpdate.type == storageTypePath && listData->recurse && !dotPath) + { + StorageInfoListData data = *listData; + data.subPath = infoUpdate.name; + + storageInfoListSort( + data.storage, strNewFmt("%s/%s", strPtr(data.path), strPtr(data.subPath)), data.sortOrder, storageInfoListCallback, + &data); + } + + if (listData->sortOrder == sortOrderDesc) + listData->callbackFunction(listData->callbackData, &infoUpdate); + } + + FUNCTION_TEST_RETURN_VOID(); +} + bool storageInfoList( const Storage *this, const String *pathExp, StorageInfoListCallback callback, void *callbackData, StorageInfoListParam param) @@ -282,6 +435,9 @@ storageInfoList( FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM_P(VOID, callbackData); FUNCTION_LOG_PARAM(BOOL, param.errorOnMissing); + FUNCTION_LOG_PARAM(ENUM, param.sortOrder); + FUNCTION_LOG_PARAM(STRING, param.expression); + FUNCTION_LOG_PARAM(BOOL, param.recurse); FUNCTION_LOG_END(); ASSERT(this != NULL); @@ -296,8 +452,26 @@ storageInfoList( // Build the path String *path = storagePathNP(this, pathExp); - // Call driver function - result = this->interface.infoList(this->driver, path, callback, callbackData); + // If there is an expression or recursion then the info will need to be filtered through a local callback + if (param.expression != NULL || param.recurse) + { + StorageInfoListData data = + { + .storage = this, + .callbackFunction = callback, + .callbackData = callbackData, + .sortOrder = param.sortOrder, + .recurse = param.recurse, + .path = path, + }; + + if (param.expression != NULL) + data.expression = regExpNew(param.expression); + + result = storageInfoListSort(this, path, param.sortOrder, storageInfoListCallback, &data); + } + else + result = storageInfoListSort(this, path, param.sortOrder, callback, callbackData); if (!result && param.errorOnMissing) THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, strPtr(path)); diff --git a/src/storage/storage.h b/src/storage/storage.h index 7339ec3c4..880f11f94 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -109,6 +109,9 @@ typedef void (*StorageInfoListCallback)(void *callbackData, const StorageInfo *i typedef struct StorageInfoListParam { bool errorOnMissing; + SortOrder sortOrder; + const String *expression; + bool recurse; } StorageInfoListParam; #define storageInfoListP(this, fileExp, callback, callbackData, ...) \ diff --git a/test/src/common/harnessStorage.c b/test/src/common/harnessStorage.c new file mode 100644 index 000000000..eb15d649b --- /dev/null +++ b/test/src/common/harnessStorage.c @@ -0,0 +1,94 @@ +/*********************************************************************************************************************************** +Storage Test Harness +***********************************************************************************************************************************/ +#include + +#include "common/user.h" +#include "storage/storage.h" + +#include "common/harnessStorage.h" + +/**********************************************************************************************************************************/ +void +hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info) +{ + HarnessStorageInfoListCallbackData *data = callbackData; + + // Add LF for after the first item + if (strSize(data->content) != 0) + strCat(data->content, "\n"); + + strCatFmt(data->content, "%s {", strPtr(info->name)); + + switch (info->type) + { + case storageTypeFile: + { + strCat(data->content, "file"); + + if (!data->sizeOmit) + strCatFmt(data->content, ", s=%" PRIu64, info->size); + + break; + } + + case storageTypeLink: + { + strCatFmt(data->content, "link, d=%s", strPtr(info->linkDestination)); + break; + } + + case storageTypePath: + { + strCat(data->content, "path"); + break; + } + + case storageTypeSpecial: + { + strCat(data->content, "special"); + break; + } + } + + if (info->type != storageTypeSpecial) + { + if (info->type != storageTypeLink) + { + if (!data->modeOmit || (info->type == storageTypePath && data->modePath != info->mode) || + (info->type == storageTypeFile && data->modeFile != info->mode)) + { + strCatFmt(data->content, ", m=%04o", info->mode); + } + + if (!data->timestampOmit) + strCatFmt(data->content, ", t=%" PRIu64, (uint64_t)info->timeModified); + } + + if (!data->userOmit || userId() != info->userId) + { + if (info->user != NULL) + { + strCatFmt(data->content, ", u=%s", strPtr(info->user)); + } + else + { + strCatFmt(data->content, ", u=%d", (int)info->userId); + } + } + + if (!data->groupOmit || groupId() != info->groupId) + { + if (info->group != NULL) + { + strCatFmt(data->content, ", g=%s", strPtr(info->group)); + } + else + { + strCatFmt(data->content, ", g=%d", (int)info->groupId); + } + } + } + + strCat(data->content, "}"); +} diff --git a/test/src/common/harnessStorage.h b/test/src/common/harnessStorage.h new file mode 100644 index 000000000..65a4743a8 --- /dev/null +++ b/test/src/common/harnessStorage.h @@ -0,0 +1,26 @@ +/*********************************************************************************************************************************** +Storage Test Harness + +Helper functions for testing storage and related functions. +***********************************************************************************************************************************/ +#ifndef TEST_COMMON_HARNESS_STORAGE_H +#define TEST_COMMON_HARNESS_STORAGE_H + +/*********************************************************************************************************************************** +Callback for formatting info list results +***********************************************************************************************************************************/ +typedef struct HarnessStorageInfoListCallbackData +{ + String *content; // String where content should be added + bool modeOmit; // Should the specified mode be ommitted? + mode_t modeFile; // File to omit if modeOmit is true + mode_t modePath; // Path mode to omit if modeOmit is true + bool timestampOmit; // Should the timestamp be ommitted? + bool userOmit; // Should the current user be ommitted? + bool groupOmit; // Should the current group be ommitted? + bool sizeOmit; // Should the size be ommitted +} HarnessStorageInfoListCallbackData; + +void hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info); + +#endif diff --git a/test/src/module/storage/posixTest.c b/test/src/module/storage/posixTest.c index 54a2889b5..1a336fb2a 100644 --- a/test/src/module/storage/posixTest.c +++ b/test/src/module/storage/posixTest.c @@ -11,6 +11,7 @@ Test Posix Storage #include "common/harnessConfig.h" #include "common/harnessFork.h" +#include "common/harnessStorage.h" /*********************************************************************************************************************************** Test function for path expression @@ -38,24 +39,6 @@ Macro to create a path and file that cannot be accessed strPtr(fileNoPerm)))), \ 0, "create no perm path/file"); -/*********************************************************************************************************************************** -Callback and data for storageInfoList() tests -***********************************************************************************************************************************/ -unsigned int testStorageInfoListSize = 0; -StorageInfo testStorageInfoList[256]; - -void -testStorageInfoListCallback(void *callbackData, const StorageInfo *info) -{ - MEM_CONTEXT_BEGIN((MemContext *)callbackData) - { - testStorageInfoList[testStorageInfoListSize] = *info; - testStorageInfoList[testStorageInfoListSize].name = strDup(info->name); - testStorageInfoListSize++; - } - MEM_CONTEXT_END(); -} - /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -283,24 +266,100 @@ testRun(void) STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathNoPerm)); // ------------------------------------------------------------------------------------------------------------------------- - testStorageInfoListSize = 0; + HarnessStorageInfoListCallbackData callbackData = + { + .content = strNew(""), + }; + TEST_RESULT_VOID( - storagePosixInfoListEntry((StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"), - testStorageInfoListCallback, (void *)memContextCurrent()), - "missing file"); - TEST_RESULT_UINT(testStorageInfoListSize, 0, " no file found"); + storagePosixInfoListEntry( + (StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"), + hrnStorageInfoListCallback, &callbackData), + "missing path"); + TEST_RESULT_STR_Z(callbackData.content, "", " check content"); // ------------------------------------------------------------------------------------------------------------------------- - storagePathCreateP(storageTest, strNew("pg/.include"), .mode = 0766); + storagePathCreateP(storageTest, strNew("pg"), .mode = 0766); + + struct utimbuf utimeTest = {.actime = 1000000000, .modtime = 1555160777}; + THROW_ON_SYS_ERROR_FMT( + utime(strPtr(strNewFmt("%s/pg", testPath())), &utimeTest) != 0, FileWriteError, "unable to set time for '%s'/pg", + testPath()); + + callbackData.content = strNew(""); TEST_RESULT_VOID( - storageInfoListNP(storageTest, strNew("pg"), testStorageInfoListCallback, (void *)memContextCurrent()), - "empty directory"); - TEST_RESULT_UINT(testStorageInfoListSize, 2, " two paths returned"); - TEST_RESULT_STR( - strPtr(testStorageInfoList[0].name), strEqZ(testStorageInfoList[1].name, ".") ? ".include" : ".", " check name"); - TEST_RESULT_STR( - strPtr(testStorageInfoList[1].name), strEqZ(testStorageInfoList[0].name, ".") ? ".include" : ".", " check name"); + storageInfoListP(storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData), + "directory with one dot file sorted"); + TEST_RESULT_STR_Z( + callbackData.content, strPtr(strNewFmt(". {path, m=0766, t=1555160777, u=%s, g=%s}", testUser(), testGroup())), + " check content"); + + // ------------------------------------------------------------------------------------------------------------------------- + storagePathCreateP(storageTest, strNew("pg/.include"), .mode = 0755); + ASSERT(system(strPtr(strNewFmt("sudo chown 77777:77777 %s/pg/.include", testPath()))) == 0); + + storagePutNP(storageNewWriteP(storageTest, strNew("pg/file"), .modeFile = 0660), BUFSTRDEF("TESTDATA")); + + ASSERT(system(strPtr(strNewFmt("ln -s ../file %s/pg/link", testPath()))) == 0); + ASSERT(system(strPtr(strNewFmt("mkfifo -m 777 %s/pg/pipe", testPath()))) == 0); + + callbackData = (HarnessStorageInfoListCallbackData) + { + .content = strNew(""), + .timestampOmit = true, + .modeOmit = true, + .modePath = 0766, + .modeFile = 0600, + .userOmit = true, + .groupOmit = true, + }; + + TEST_RESULT_VOID( + storageInfoListP(storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderAsc), + "directory with one dot file sorted"); + TEST_RESULT_STR_Z( + callbackData.content, + ". {path}\n" + ".include {path, m=0755, u=77777, g=77777}\n" + "file {file, s=8, m=0660}\n" + "link {link, d=../file}\n" + "pipe {special}", + " check content"); + + // ------------------------------------------------------------------------------------------------------------------------- + ASSERT(system(strPtr(strNewFmt("sudo rmdir %s/pg/.include", testPath()))) == 0); + storagePathCreateP(storageTest, strNew("pg/path"), .mode = 0700); + storagePutNP(storageNewWriteP(storageTest, strNew("pg/path/file"), .modeFile = 0600), BUFSTRDEF("TESTDATA")); + + callbackData.content = strNew(""); + + TEST_RESULT_VOID( + storageInfoListP( + storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderDesc, .recurse = true), + "recurse descending"); + TEST_RESULT_STR_Z( + callbackData.content, + "pipe {special}\n" + "path/file {file, s=8}\n" + "path {path, m=0700}\n" + "link {link, d=../file}\n" + "file {file, s=8, m=0660}\n" + ". {path}", + " check content"); + + // ------------------------------------------------------------------------------------------------------------------------- + callbackData.content = strNew(""); + + TEST_RESULT_VOID( + storageInfoListP( + storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderAsc, + .expression = STRDEF("^path")), + "filter"); + TEST_RESULT_STR_Z( + callbackData.content, + "path {path, m=0700}", + " check content"); } // *****************************************************************************************************************************