From 5e55d588507e7948dbb48bb8d9427ab4d4503c20 Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 6 Apr 2020 16:09:18 -0400 Subject: [PATCH] Simplify storage driver info and list functions. The storage driver requires two list functions to be implemented, list and infoList. But the former is a subset of the latter so implementing both in every driver is wasteful. The reason both exist is that in Posix it is cheaper to get a list of names than it is to stat files to get size, time, etc. In S3 these operations are equivalent. Introduce storageInfoLevelType to determine the amount of information required by the caller. That way Posix can work efficiently and all drivers can return only the data required which saves some bandwidth. The storageList() and storageInfoList() functions remain in the storage interface since they are useful -- the only change is simplifying the drivers with no external impact. Note that since list() accepted an expression infoList() must now do so. Checking the expression is optional for the driver but can be used to limit results or save IO costs. Similarly, exists() and pathExists() are just specialized forms of info() so adapt them to call info() instead. --- doc/xml/release.xml | 10 + src/storage/info.h | 25 ++ src/storage/posix/storage.c | 303 ++++++++-------------- src/storage/remote/protocol.c | 51 ++-- src/storage/remote/protocol.h | 6 - src/storage/remote/storage.c | 122 ++------- src/storage/s3/storage.c | 131 +++------- src/storage/storage.c | 112 +++++--- src/storage/storage.h | 5 + src/storage/storage.intern.h | 77 ++---- test/define.yaml | 4 +- test/src/common/harnessStorage.c | 132 +++++----- test/src/module/command/archiveGetTest.c | 2 +- test/src/module/command/controlTest.c | 2 +- test/src/module/config/parseTest.c | 2 +- test/src/module/performance/storageTest.c | 28 +- test/src/module/storage/posixTest.c | 25 +- test/src/module/storage/remoteTest.c | 122 +++------ test/src/module/storage/s3Test.c | 121 ++++++--- 19 files changed, 522 insertions(+), 758 deletions(-) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index b7f4330eb..5b6d80c1e 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -24,6 +24,16 @@

TCP keep-alive options are configurable.

+ + + + + + + +

Simplify storage driver info and list functions.

+ + diff --git a/src/storage/info.h b/src/storage/info.h index c90087b5b..614728856 100644 --- a/src/storage/info.h +++ b/src/storage/info.h @@ -6,6 +6,25 @@ Storage Info #include +/*********************************************************************************************************************************** +Specify the level of information required when calling functions that return StorageInfo +***********************************************************************************************************************************/ +typedef enum +{ + // The info type is determined by driver capabilities. This mimics the prior behavior where drivers would always return as + // much information as they could. + storageInfoLevelDefault = 0, + + // Only test for existence. All drivers support this type. + storageInfoLevelExists, + + // Basic information. All drivers support this type. + storageInfoLevelBasic, + + // Detailed information that is generally only available from filesystems such as Posix + storageInfoLevelDetail, +} StorageInfoLevel; + /*********************************************************************************************************************************** Storage type ***********************************************************************************************************************************/ @@ -22,11 +41,17 @@ Object type ***********************************************************************************************************************************/ typedef struct StorageInfo { + // Set when info type >= storageInfoLevelExists const String *name; // Name of path/file/link + StorageInfoLevel level; // Level of information provided bool exists; // Does the path/file/link exist? + + // Set when info type >= storageInfoLevelBasic (undefined at lower levels) StorageType type; // Type file/path/link) uint64_t size; // Size (path/link is 0) time_t timeModified; // Time file was last modified + + // Set when info type >= storageInfoLevelDetail (undefined at lower levels) mode_t mode; // Mode of path/file/link uid_t userId; // User that owns the file uid_t groupId; // Group that owns the file diff --git a/src/storage/posix/storage.c b/src/storage/posix/storage.c index 4e82a6465..a40253ab4 100644 --- a/src/storage/posix/storage.c +++ b/src/storage/posix/storage.c @@ -42,57 +42,25 @@ struct StoragePosix MemContext *memContext; // Object memory context }; -/**********************************************************************************************************************************/ -static bool -storagePosixExists(THIS_VOID, const String *file, StorageInterfaceExistsParam param) -{ - THIS(StoragePosix); - - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(STORAGE_POSIX, this); - FUNCTION_LOG_PARAM(STRING, file); - (void)param; // No parameters are used - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(file != NULL); - - bool result = false; - - // Attempt to stat the file to determine if it exists - struct stat statFile; - - // Any error other than entry not found should be reported - if (stat(strPtr(file), &statFile) == -1) - { - if (errno != ENOENT) - THROW_SYS_ERROR_FMT(FileOpenError, "unable to stat '%s'", strPtr(file)); - } - // Else found - else - result = !S_ISDIR(statFile.st_mode); - - FUNCTION_LOG_RETURN(BOOL, result); -} - /**********************************************************************************************************************************/ static StorageInfo -storagePosixInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param) +storagePosixInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param) { THIS(StoragePosix); FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(STORAGE_POSIX, this); FUNCTION_LOG_PARAM(STRING, file); + FUNCTION_LOG_PARAM(ENUM, level); FUNCTION_LOG_PARAM(BOOL, param.followLink); FUNCTION_LOG_END(); ASSERT(this != NULL); ASSERT(file != NULL); - StorageInfo result = {0}; + StorageInfo result = {.level = level}; - // Attempt to stat the file + // Stat the file to check if it exists struct stat statFile; if ((param.followLink ? stat(strPtr(file), &statFile) : lstat(strPtr(file), &statFile)) == -1) @@ -100,40 +68,50 @@ storagePosixInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param) if (errno != ENOENT) THROW_SYS_ERROR_FMT(FileOpenError, STORAGE_ERROR_INFO, strPtr(file)); } - // On success load info into a structure + // On success the file exists else { result.exists = true; - result.groupId = statFile.st_gid; - result.group = groupNameFromId(result.groupId); - result.userId = statFile.st_uid; - result.user = userNameFromId(result.userId); - result.timeModified = statFile.st_mtime; - if (S_ISREG(statFile.st_mode)) + // Add basic level info + if (result.level >= storageInfoLevelBasic) { - result.type = storageTypeFile; - result.size = (uint64_t)statFile.st_size; + result.timeModified = statFile.st_mtime; + + if (S_ISREG(statFile.st_mode)) + { + result.type = storageTypeFile; + result.size = (uint64_t)statFile.st_size; + } + else if (S_ISDIR(statFile.st_mode)) + result.type = storageTypePath; + else if (S_ISLNK(statFile.st_mode)) + result.type = storageTypeLink; + else + result.type = storageTypeSpecial; } - else if (S_ISDIR(statFile.st_mode)) - result.type = storageTypePath; - else if (S_ISLNK(statFile.st_mode)) + + // Add detail level info + if (result.level >= storageInfoLevelDetail) { - result.type = storageTypeLink; + result.groupId = statFile.st_gid; + result.group = groupNameFromId(result.groupId); + result.userId = statFile.st_uid; + result.user = userNameFromId(result.userId); + result.mode = statFile.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO); - char linkDestination[PATH_MAX]; - ssize_t linkDestinationSize = 0; + if (result.type == storageTypeLink) + { + char linkDestination[PATH_MAX]; + ssize_t linkDestinationSize = 0; - THROW_ON_SYS_ERROR_FMT( - (linkDestinationSize = readlink(strPtr(file), linkDestination, sizeof(linkDestination) - 1)) == -1, - FileReadError, "unable to get destination for link '%s'", strPtr(file)); + THROW_ON_SYS_ERROR_FMT( + (linkDestinationSize = readlink(strPtr(file), linkDestination, sizeof(linkDestination) - 1)) == -1, + FileReadError, "unable to get destination for link '%s'", strPtr(file)); - result.linkDestination = strNewN(linkDestination, (size_t)linkDestinationSize); + result.linkDestination = strNewN(linkDestination, (size_t)linkDestinationSize); + } } - else - result.type = storageTypeSpecial; - - result.mode = statFile.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO); } FUNCTION_LOG_RETURN(STORAGE_INFO, result); @@ -145,12 +123,14 @@ storagePosixInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param) // get complete test coverage this function must be split out. static void storagePosixInfoListEntry( - StoragePosix *this, const String *path, const String *name, StorageInfoListCallback callback, void *callbackData) + StoragePosix *this, const String *path, const String *name, StorageInfoLevel level, StorageInfoListCallback callback, + void *callbackData) { FUNCTION_TEST_BEGIN(); FUNCTION_TEST_PARAM(STORAGE_POSIX, this); FUNCTION_TEST_PARAM(STRING, path); FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_PARAM(ENUM, level); FUNCTION_TEST_PARAM(FUNCTIONP, callback); FUNCTION_TEST_PARAM_P(VOID, callbackData); FUNCTION_TEST_END(); @@ -160,17 +140,13 @@ storagePosixInfoListEntry( ASSERT(name != NULL); ASSERT(callback != NULL); - if (!strEqZ(name, "..")) + StorageInfo storageInfo = storageInterfaceInfoP( + this, strEq(name, DOT_STR) ? strDup(path) : strNewFmt("%s/%s", strPtr(path), strPtr(name)), level); + + if (storageInfo.exists) { - String *pathInfo = strEqZ(name, ".") ? strDup(path) : strNewFmt("%s/%s", strPtr(path), strPtr(name)); - - StorageInfo storageInfo = storageInterfaceInfoP(this, pathInfo); - - if (storageInfo.exists) - { - storageInfo.name = name; - callback(callbackData, &storageInfo); - } + storageInfo.name = name; + callback(callbackData, &storageInfo); } FUNCTION_TEST_RETURN_VOID(); @@ -178,13 +154,15 @@ storagePosixInfoListEntry( static bool storagePosixInfoList( - THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) + THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData, + StorageInterfaceInfoListParam param) { THIS(StoragePosix); FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(STORAGE_POSIX, this); FUNCTION_LOG_PARAM(STRING, path); + FUNCTION_LOG_PARAM(ENUM, level); FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM_P(VOID, callbackData); (void)param; // No parameters are used @@ -219,8 +197,21 @@ storagePosixInfoList( while (dirEntry != NULL) { - // Get info and perform callback - storagePosixInfoListEntry(this, path, STR(dirEntry->d_name), callback, callbackData); + const String *name = STR(dirEntry->d_name); + + // Always skip .. + if (!strEq(name, DOTDOT_STR)) + { + // If only making a list of files that exist then no need to go get detailed info which requires calling + // stat() and is therefore relatively slow + if (level == storageInfoLevelExists) + { + callback(callbackData, &(StorageInfo){.name = name, .level = storageInfoLevelExists, .exists = true}); + } + // Else more info is required which requires a call to stat() + else + storagePosixInfoListEntry(this, path, name, level, callback, callbackData); + } // Get next entry dirEntry = readdir(dir); @@ -241,78 +232,9 @@ storagePosixInfoList( FUNCTION_LOG_RETURN(BOOL, result); } -/**********************************************************************************************************************************/ -static StringList * -storagePosixList(THIS_VOID, const String *path, StorageInterfaceListParam param) -{ - THIS(StoragePosix); - - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(STORAGE_POSIX, this); - FUNCTION_LOG_PARAM(STRING, path); - FUNCTION_LOG_PARAM(STRING, param.expression); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(path != NULL); - - StringList *result = NULL; - DIR *dir = NULL; - - TRY_BEGIN() - { - // Open the directory for read - dir = opendir(strPtr(path)); - - // If the directory could not be opened process errors but ignore missing directories when specified - if (!dir) - { - if (errno != ENOENT) - THROW_SYS_ERROR_FMT(PathOpenError, STORAGE_ERROR_LIST, strPtr(path)); - } - else - { - MEM_CONTEXT_TEMP_BEGIN() - { - // Prepare regexp if an expression was passed - RegExp *regExp = param.expression == NULL ? NULL : regExpNew(param.expression); - - // Create the string list now that we know the directory is valid - result = strLstNew(); - - // Read the directory entries - struct dirent *dirEntry = readdir(dir); - - while (dirEntry != NULL) - { - const String *entry = STR(dirEntry->d_name); - - // Exclude current/parent directory and apply the expression if specified - if (!strEqZ(entry, ".") && !strEqZ(entry, "..") && (regExp == NULL || regExpMatch(regExp, entry))) - strLstAdd(result, entry); - - dirEntry = readdir(dir); - } - - // Move finished list up to the old context - strLstMove(result, memContextPrior()); - } - MEM_CONTEXT_TEMP_END(); - } - } - FINALLY() - { - if (dir != NULL) - closedir(dir); - } - TRY_END(); - - FUNCTION_LOG_RETURN(STRING_LIST, result); -} - /**********************************************************************************************************************************/ static bool -storagePosixMove(THIS_VOID, StorageRead *source, StorageWrite *destination, StorageInterfaceMoveParam param) +storagePosixMove(THIS_VOID, StorageRead *source, StorageWrite *destination, StorageInterfaceMoveParam param) { THIS(StoragePosix); @@ -341,8 +263,9 @@ storagePosixMove(THIS_VOID, StorageRead *source, StorageWrite *destination, Sto // Determine which file/path is missing if (errno == ENOENT) { - if (!storageInterfaceExistsP(this, sourceFile)) - THROW_SYS_ERROR_FMT(FileMissingError, "unable to move missing file '%s'", strPtr(sourceFile)); + // Check if the source is missing. Rename does not follow links so there is no need to set followLink. + if (!storageInterfaceInfoP(this, sourceFile, storageInfoLevelExists).exists) + THROW_SYS_ERROR_FMT(FileMissingError, "unable to move missing source '%s'", strPtr(sourceFile)); if (!storageWriteCreatePath(destination)) { @@ -465,39 +388,45 @@ storagePosixPathCreate( } /**********************************************************************************************************************************/ -static bool -storagePosixPathExists(THIS_VOID, const String *path, StorageInterfacePathExistsParam param) +typedef struct StoragePosixPathRemoveData { - THIS(StoragePosix); + StoragePosix *driver; // Driver + const String *path; // Path +} StoragePosixPathRemoveData; - FUNCTION_LOG_BEGIN(logLevelTrace); - FUNCTION_LOG_PARAM(STORAGE_POSIX, this); - FUNCTION_LOG_PARAM(STRING, path); - (void)param; // No parameters are used - FUNCTION_LOG_END(); +static void +storagePosixPathRemoveCallback(void *callbackData, const StorageInfo *info) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM_P(VOID, callbackData); + FUNCTION_TEST_PARAM_P(STORAGE_INFO, info); + FUNCTION_TEST_END(); - ASSERT(this != NULL); - ASSERT(path != NULL); + ASSERT(callbackData != NULL); + ASSERT(info != NULL); - bool result = false; - - // Attempt to stat the file to determine if it exists - struct stat statPath; - - // Any error other than entry not found should be reported - if (stat(strPtr(path), &statPath) == -1) + if (!strEqZ(info->name, ".")) { - if (errno != ENOENT) - THROW_SYS_ERROR_FMT(PathOpenError, "unable to stat '%s'", strPtr(path)); - } - // Else found - else - result = S_ISDIR(statPath.st_mode); + StoragePosixPathRemoveData *data = callbackData; + String *file = strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)); - FUNCTION_LOG_RETURN(BOOL, result); + // Rather than stat the file to discover what type it is, just try to unlink it and see what happens + if (unlink(strPtr(file)) == -1) + { + // These errors indicate that the entry is actually a path so we'll try to delete it that way + if (errno == EPERM || errno == EISDIR) // {uncovered_branch - no EPERM on tested systems} + { + storageInterfacePathRemoveP(data->driver, file, true); + } + // Else error + else + THROW_SYS_ERROR_FMT(PathRemoveError, STORAGE_ERROR_PATH_REMOVE_FILE, strPtr(file)); + } + } + + FUNCTION_TEST_RETURN_VOID(); } -/**********************************************************************************************************************************/ static bool storagePosixPathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfacePathRemoveParam param) { @@ -520,29 +449,14 @@ storagePosixPathRemove(THIS_VOID, const String *path, bool recurse, StorageInter // Recurse if requested if (recurse) { - // Get a list of files in this path - StringList *fileList = storageInterfaceListP(this, path); - - // Only continue if the path exists - if (fileList != NULL) + StoragePosixPathRemoveData data = { - // Delete all paths and files - for (unsigned int fileIdx = 0; fileIdx < strLstSize(fileList); fileIdx++) - { - String *file = strNewFmt("%s/%s", strPtr(path), strPtr(strLstGet(fileList, fileIdx))); + .driver = this, + .path = path, + }; - // Rather than stat the file to discover what type it is, just try to unlink it and see what happens - if (unlink(strPtr(file)) == -1) - { - // These errors indicate that the entry is actually a path so we'll try to delete it that way - if (errno == EPERM || errno == EISDIR) // {uncovered_branch - no EPERM on tested systems} - storageInterfacePathRemoveP(this, file, true); - // Else error - else - THROW_SYS_ERROR_FMT(PathRemoveError, STORAGE_ERROR_PATH_REMOVE_FILE, strPtr(file)); - } - } - } + // Remove all sub paths/files + storageInterfaceInfoListP(this, path, storageInfoLevelExists, storagePosixPathRemoveCallback, &data); } // Delete the path @@ -635,15 +549,12 @@ static const StorageInterface storageInterfacePosix = { .feature = 1 << storageFeaturePath | 1 << storageFeatureCompress | 1 << storageFeatureLimitRead, - .exists = storagePosixExists, .info = storagePosixInfo, .infoList = storagePosixInfoList, - .list = storagePosixList, .move = storagePosixMove, .newRead = storagePosixNewRead, .newWrite = storagePosixNewWrite, .pathCreate = storagePosixPathCreate, - .pathExists = storagePosixPathExists, .pathRemove = storagePosixPathRemove, .pathSync = storagePosixPathSync, .remove = storagePosixRemove, @@ -691,7 +602,9 @@ storagePosixNewInternal( // If this is a posix driver then add link features if (strEq(type, STORAGE_POSIX_TYPE_STR)) - driver->interface.feature |= (1 << storageFeatureHardLink | 1 << storageFeatureSymLink | 1 << storageFeaturePathSync); + driver->interface.feature |= + 1 << storageFeatureHardLink | 1 << storageFeatureSymLink | 1 << storageFeaturePathSync | + 1 << storageFeatureInfoDetail; this = storageNew(type, path, modeFile, modePath, write, pathExpressionFunction, driver, driver->interface); } diff --git a/src/storage/remote/protocol.c b/src/storage/remote/protocol.c index a7cd0c677..09b6e37b0 100644 --- a/src/storage/remote/protocol.c +++ b/src/storage/remote/protocol.c @@ -24,15 +24,12 @@ Remote Storage Protocol Handler /*********************************************************************************************************************************** Constants ***********************************************************************************************************************************/ -STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_EXISTS_STR, PROTOCOL_COMMAND_STORAGE_EXISTS); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_FEATURE_STR, PROTOCOL_COMMAND_STORAGE_FEATURE); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_INFO_STR, PROTOCOL_COMMAND_STORAGE_INFO); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR, PROTOCOL_COMMAND_STORAGE_INFO_LIST); -STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_LIST_STR, PROTOCOL_COMMAND_STORAGE_LIST); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR, PROTOCOL_COMMAND_STORAGE_OPEN_READ); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE_STR, PROTOCOL_COMMAND_STORAGE_OPEN_WRITE); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_CREATE_STR, PROTOCOL_COMMAND_STORAGE_PATH_CREATE); -STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR, PROTOCOL_COMMAND_STORAGE_PATH_EXISTS); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE_STR, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_SYNC_STR, PROTOCOL_COMMAND_STORAGE_PATH_SYNC); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_REMOVE_STR, PROTOCOL_COMMAND_STORAGE_REMOVE); @@ -134,6 +131,8 @@ storageRemoteInfoWriteType(ProtocolServer *server, StorageType type) FUNCTION_TEST_RETURN_VOID(); } +// Helper to write storage info into the protocol. This function is not called unless the info exists so no need to write exists +// or check for level == storageInfoLevelExists. static void storageRemoteInfoWrite(ProtocolServer *server, const StorageInfo *info) { @@ -143,18 +142,22 @@ storageRemoteInfoWrite(ProtocolServer *server, const StorageInfo *info) FUNCTION_TEST_END(); storageRemoteInfoWriteType(server, info->type); - protocolServerWriteLine(server, jsonFromUInt(info->userId)); - protocolServerWriteLine(server, jsonFromStr(info->user)); - protocolServerWriteLine(server, jsonFromUInt(info->groupId)); - protocolServerWriteLine(server, jsonFromStr(info->group)); - protocolServerWriteLine(server, jsonFromUInt(info->mode)); protocolServerWriteLine(server, jsonFromInt64(info->timeModified)); if (info->type == storageTypeFile) protocolServerWriteLine(server, jsonFromUInt64(info->size)); - if (info->type == storageTypeLink) - protocolServerWriteLine(server, jsonFromStr(info->linkDestination)); + if (info->level >= storageInfoLevelDetail) + { + protocolServerWriteLine(server, jsonFromUInt(info->userId)); + protocolServerWriteLine(server, jsonFromStr(info->user)); + protocolServerWriteLine(server, jsonFromUInt(info->groupId)); + protocolServerWriteLine(server, jsonFromStr(info->group)); + protocolServerWriteLine(server, jsonFromUInt(info->mode)); + + if (info->type == storageTypeLink) + protocolServerWriteLine(server, jsonFromStr(info->linkDestination)); + } FUNCTION_TEST_RETURN_VOID(); } @@ -199,11 +202,7 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto MEM_CONTEXT_TEMP_BEGIN() { - if (strEq(command, PROTOCOL_COMMAND_STORAGE_EXISTS_STR)) - { - protocolServerResponse(server, VARBOOL(storageInterfaceExistsP(driver, varStr(varLstGet(paramList, 0))))); - } - else if (strEq(command, PROTOCOL_COMMAND_STORAGE_FEATURE_STR)) + if (strEq(command, PROTOCOL_COMMAND_STORAGE_FEATURE_STR)) { protocolServerWriteLine(server, jsonFromStr(storagePathP(storage, NULL))); protocolServerWriteLine(server, jsonFromUInt64(interface.feature)); @@ -213,7 +212,8 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto else if (strEq(command, PROTOCOL_COMMAND_STORAGE_INFO_STR)) { StorageInfo info = storageInterfaceInfoP( - driver, varStr(varLstGet(paramList, 0)), .followLink = varBool(varLstGet(paramList, 1))); + driver, varStr(varLstGet(paramList, 0)), (StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)), + .followLink = varBool(varLstGet(paramList, 2))); protocolServerResponse(server, VARBOOL(info.exists)); @@ -226,20 +226,12 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto else if (strEq(command, PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR)) { bool result = storageInterfaceInfoListP( - driver, varStr(varLstGet(paramList, 0)), storageRemoteProtocolInfoListCallback, server); + driver, varStr(varLstGet(paramList, 0)), (StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)), + storageRemoteProtocolInfoListCallback, server); protocolServerWriteLine(server, NULL); protocolServerResponse(server, VARBOOL(result)); } - else if (strEq(command, PROTOCOL_COMMAND_STORAGE_LIST_STR)) - { - protocolServerResponse( - server, - varNewVarLst( - varLstNewStrLst( - storageInterfaceListP( - driver, varStr(varLstGet(paramList, 0)), .expression = varStr(varLstGet(paramList, 1)))))); - } else if (strEq(command, PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR)) { // Create the read object @@ -356,13 +348,6 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto protocolServerResponse(server, NULL); } - else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR)) - { - // Not all drivers implement pathExists() - CHECK(interface.pathExists != NULL); - - protocolServerResponse(server, VARBOOL(storageInterfacePathExistsP(driver, varStr(varLstGet(paramList, 0))))); - } else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE_STR)) { protocolServerResponse(server, diff --git a/src/storage/remote/protocol.h b/src/storage/remote/protocol.h index b5449ca5a..01262db98 100644 --- a/src/storage/remote/protocol.h +++ b/src/storage/remote/protocol.h @@ -13,24 +13,18 @@ Constants ***********************************************************************************************************************************/ #define PROTOCOL_BLOCK_HEADER "BRBLOCK" -#define PROTOCOL_COMMAND_STORAGE_EXISTS "storageExists" - STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_EXISTS_STR); #define PROTOCOL_COMMAND_STORAGE_FEATURE "storageFeature" STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_FEATURE_STR); #define PROTOCOL_COMMAND_STORAGE_INFO "storageInfo" STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_INFO_STR); #define PROTOCOL_COMMAND_STORAGE_INFO_LIST "storageInfoList" STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR); -#define PROTOCOL_COMMAND_STORAGE_LIST "storageList" - STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_LIST_STR); #define PROTOCOL_COMMAND_STORAGE_OPEN_READ "storageOpenRead" STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR); #define PROTOCOL_COMMAND_STORAGE_OPEN_WRITE "storageOpenWrite" STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE_STR); #define PROTOCOL_COMMAND_STORAGE_PATH_CREATE "storagePathCreate" STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_PATH_CREATE_STR); -#define PROTOCOL_COMMAND_STORAGE_PATH_EXISTS "storagePathExists" - STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR); #define PROTOCOL_COMMAND_STORAGE_REMOVE "storageRemove" STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_REMOVE_STR); #define PROTOCOL_COMMAND_STORAGE_PATH_REMOVE "storagePathRemove" diff --git a/src/storage/remote/storage.c b/src/storage/remote/storage.c index 8890105a0..fb5ed5329 100644 --- a/src/storage/remote/storage.c +++ b/src/storage/remote/storage.c @@ -29,35 +29,6 @@ struct StorageRemote unsigned int compressLevel; // Protocol compression level }; -/**********************************************************************************************************************************/ -static bool -storageRemoteExists(THIS_VOID, const String *file, StorageInterfaceExistsParam param) -{ - THIS(StorageRemote); - - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STORAGE_REMOTE, this); - FUNCTION_LOG_PARAM(STRING, file); - (void)param; // No parameters are used - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(file != NULL); - - bool result = false; - - MEM_CONTEXT_TEMP_BEGIN() - { - ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_EXISTS_STR); - protocolCommandParamAdd(command, VARSTR(file)); - - result = varBool(protocolClientExecute(this->client, command, true)); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(BOOL, result); -} - /**********************************************************************************************************************************/ // Helper to convert protocol storage type to an enum static StorageType @@ -95,41 +66,47 @@ storageRemoteInfoParse(ProtocolClient *client, StorageInfo *info) FUNCTION_TEST_END(); info->type = storageRemoteInfoParseType(strPtr(protocolClientReadLine(client))[0]); - info->userId = jsonToUInt(protocolClientReadLine(client)); - info->user = jsonToStr(protocolClientReadLine(client)); - info->groupId = jsonToUInt(protocolClientReadLine(client)); - info->group = jsonToStr(protocolClientReadLine(client)); - info->mode = jsonToUInt(protocolClientReadLine(client)); info->timeModified = (time_t)jsonToUInt64(protocolClientReadLine(client)); if (info->type == storageTypeFile) info->size = jsonToUInt64(protocolClientReadLine(client)); - if (info->type == storageTypeLink) - info->linkDestination = jsonToStr(protocolClientReadLine(client)); + if (info->level >= storageInfoLevelDetail) + { + info->userId = jsonToUInt(protocolClientReadLine(client)); + info->user = jsonToStr(protocolClientReadLine(client)); + info->groupId = jsonToUInt(protocolClientReadLine(client)); + info->group = jsonToStr(protocolClientReadLine(client)); + info->mode = jsonToUInt(protocolClientReadLine(client)); + + if (info->type == storageTypeLink) + info->linkDestination = jsonToStr(protocolClientReadLine(client)); + } FUNCTION_TEST_RETURN_VOID(); } static StorageInfo -storageRemoteInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param) +storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param) { THIS(StorageRemote); FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STORAGE_REMOTE, this); FUNCTION_LOG_PARAM(STRING, file); + FUNCTION_LOG_PARAM(ENUM, level); FUNCTION_LOG_PARAM(BOOL, param.followLink); FUNCTION_LOG_END(); ASSERT(this != NULL); - StorageInfo result = {.exists = false}; + StorageInfo result = {.level = level}; MEM_CONTEXT_TEMP_BEGIN() { ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_STR); protocolCommandParamAdd(command, VARSTR(file)); + protocolCommandParamAdd(command, VARUINT(level)); protocolCommandParamAdd(command, VARBOOL(param.followLink)); result.exists = varBool(protocolClientExecute(this->client, command, true)); @@ -161,13 +138,15 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param /**********************************************************************************************************************************/ static bool storageRemoteInfoList( - THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) + THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData, + StorageInterfaceInfoListParam param) { THIS(StorageRemote); FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(STORAGE_REMOTE, this); FUNCTION_LOG_PARAM(STRING, path); + FUNCTION_LOG_PARAM(ENUM, level); FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM_P(VOID, callbackData); (void)param; // No parameters are used @@ -183,6 +162,7 @@ storageRemoteInfoList( { ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR); protocolCommandParamAdd(command, VARSTR(path)); + protocolCommandParamAdd(command, VARUINT(level)); // Send command protocolClientWriteCommand(this->client, command); @@ -195,7 +175,7 @@ storageRemoteInfoList( while (strSize(name) != 0) { - StorageInfo info = {.exists = true, .name = jsonToStr(name)}; + StorageInfo info = {.exists = true, .level = level, .name = jsonToStr(name)}; storageRemoteInfoParse(this->client, &info); callback(callbackData, &info); @@ -217,36 +197,6 @@ storageRemoteInfoList( FUNCTION_LOG_RETURN(BOOL, result); } -/**********************************************************************************************************************************/ -static StringList * -storageRemoteList(THIS_VOID, const String *path, StorageInterfaceListParam param) -{ - THIS(StorageRemote); - - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STORAGE_REMOTE, this); - FUNCTION_LOG_PARAM(STRING, path); - FUNCTION_LOG_PARAM(STRING, param.expression); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(path != NULL); - - StringList *result = NULL; - - MEM_CONTEXT_TEMP_BEGIN() - { - ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_LIST_STR); - protocolCommandParamAdd(command, VARSTR(path)); - protocolCommandParamAdd(command, VARSTR(param.expression)); - - result = strLstMove(strLstNewVarLst(varVarLst(protocolClientExecute(this->client, command, true))), memContextPrior()); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(STRING_LIST, result); -} - /**********************************************************************************************************************************/ static StorageRead * storageRemoteNewRead(THIS_VOID, const String *file, bool ignoreMissing, StorageInterfaceNewReadParam param) @@ -338,35 +288,6 @@ storageRemotePathCreate( FUNCTION_LOG_RETURN_VOID(); } -/**********************************************************************************************************************************/ -static bool -storageRemotePathExists(THIS_VOID, const String *path, StorageInterfacePathExistsParam param) -{ - THIS(StorageRemote); - - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STORAGE_REMOTE, this); - FUNCTION_LOG_PARAM(STRING, path); - (void)param; // No parameters are used - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(path != NULL); - - bool result = false; - - MEM_CONTEXT_TEMP_BEGIN() - { - ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR); - protocolCommandParamAdd(command, VARSTR(path)); - - result = varBool(protocolClientExecute(this->client, command, true)); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(BOOL, result); -} - /**********************************************************************************************************************************/ static bool storageRemotePathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfacePathRemoveParam param) @@ -456,14 +377,11 @@ storageRemoteRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam p /**********************************************************************************************************************************/ static const StorageInterface storageInterfaceRemote = { - .exists = storageRemoteExists, .info = storageRemoteInfo, .infoList = storageRemoteInfoList, - .list = storageRemoteList, .newRead = storageRemoteNewRead, .newWrite = storageRemoteNewWrite, .pathCreate = storageRemotePathCreate, - .pathExists = storageRemotePathExists, .pathRemove = storageRemotePathRemove, .pathSync = storageRemotePathSync, .remove = storageRemoteRemove, diff --git a/src/storage/s3/storage.c b/src/storage/s3/storage.c index 11507630d..69573ddf2 100644 --- a/src/storage/s3/storage.c +++ b/src/storage/s3/storage.c @@ -422,9 +422,6 @@ storageS3ListInternal( { const String *continuationToken = NULL; - // Prepare regexp if an expression was passed - RegExp *regExp = expression == NULL ? NULL : regExpNew(expression); - // Build the base prefix by stripping off the initial / const String *basePrefix; @@ -490,9 +487,8 @@ storageS3ListInternal( // Strip off base prefix and final / subPath = strSubN(subPath, strSize(basePrefix), strSize(subPath) - strSize(basePrefix) - 1); - // Add to list after checking expression if present - if (regExp == NULL || regExpMatch(regExp, subPath)) - callback(this, callbackData, subPath, storageTypePath, subPathNode); + // Add to list + callback(this, callbackData, subPath, storageTypePath, subPathNode); } // Get file list @@ -508,9 +504,8 @@ storageS3ListInternal( // Strip off the base prefix when present file = strEmpty(basePrefix) ? file : strSub(file, strSize(basePrefix)); - // Add to list after checking expression if present - if (regExp == NULL || regExpMatch(regExp, file)) - callback(this, callbackData, file, storageTypeFile, fileNode); + // Add to list + callback(this, callbackData, file, storageTypeFile, fileNode); } // Get the continuation token and store it in the outer temp context @@ -529,56 +524,31 @@ storageS3ListInternal( FUNCTION_LOG_RETURN_VOID(); } -/**********************************************************************************************************************************/ -static bool -storageS3Exists(THIS_VOID, const String *file, StorageInterfaceExistsParam param) -{ - THIS(StorageS3); - - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STORAGE_S3, this); - FUNCTION_LOG_PARAM(STRING, file); - (void)param; // No parameters are used - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(file != NULL); - - bool result = false; - - MEM_CONTEXT_TEMP_BEGIN() - { - result = httpClientResponseCodeOk(storageS3Request(this, HTTP_VERB_HEAD_STR, file, NULL, NULL, true, true).httpClient); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(BOOL, result); -} - /**********************************************************************************************************************************/ static StorageInfo -storageS3Info(THIS_VOID, const String *file, StorageInterfaceInfoParam param) +storageS3Info(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param) { THIS(StorageS3); FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(STORAGE_S3, this); FUNCTION_LOG_PARAM(STRING, file); + FUNCTION_LOG_PARAM(ENUM, level); (void)param; // No parameters are used FUNCTION_LOG_END(); ASSERT(this != NULL); ASSERT(file != NULL); - StorageInfo result = {0}; - // Attempt to get file info StorageS3RequestResult httpResult = storageS3Request(this, HTTP_VERB_HEAD_STR, file, NULL, NULL, true, true); - // On success load info into a structure - if (httpClientResponseCodeOk(httpResult.httpClient)) + // Does the file exist? + StorageInfo result = {.level = level, .exists = httpClientResponseCodeOk(httpResult.httpClient)}; + + // Add basic level info if requested and the file exists + if (result.level >= storageInfoLevelBasic && result.exists) { - result.exists = true; result.type = storageTypeFile; result.size = cvtZToUInt64(strPtr(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_CONTENT_LENGTH_STR))); result.timeModified = httpLastModifiedToTime(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_LAST_MODIFIED_STR)); @@ -590,6 +560,7 @@ storageS3Info(THIS_VOID, const String *file, StorageInterfaceInfoParam param) /**********************************************************************************************************************************/ typedef struct StorageS3InfoListData { + StorageInfoLevel level; // Level of info to set StorageInfoListCallback callback; // User-supplied callback function void *callbackData; // User-supplied callback data } StorageS3InfoListData; @@ -630,13 +601,20 @@ storageS3InfoListCallback(StorageS3 *this, void *callbackData, const String *nam StorageInfo info = { - .type = type, .name = name, - .size = type == storageTypeFile ? cvtZToUInt64(strPtr(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_SIZE_STR, true)))) : 0, - .timeModified = type == storageTypeFile ? - storageS3CvtTime(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_LAST_MODIFIED_STR, true))) : 0, + .level = data->level, + .exists = true, }; + if (data->level >= storageInfoLevelBasic) + { + info.type = type; + info.size = type == storageTypeFile ? + cvtZToUInt64(strPtr(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_SIZE_STR, true)))) : 0; + info.timeModified = type == storageTypeFile ? + storageS3CvtTime(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_LAST_MODIFIED_STR, true))) : 0; + } + data->callback(data->callbackData, &info); FUNCTION_TEST_RETURN_VOID(); @@ -644,16 +622,18 @@ storageS3InfoListCallback(StorageS3 *this, void *callbackData, const String *nam static bool storageS3InfoList( - THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) + THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData, + StorageInterfaceInfoListParam param) { THIS(StorageS3); FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(STORAGE_S3, this); FUNCTION_LOG_PARAM(STRING, path); + FUNCTION_LOG_PARAM(ENUM, level); FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM_P(VOID, callbackData); - (void)param; // No parameters are used + FUNCTION_LOG_PARAM(STRING, param.expression); FUNCTION_LOG_END(); ASSERT(this != NULL); @@ -662,65 +642,14 @@ storageS3InfoList( MEM_CONTEXT_TEMP_BEGIN() { - StorageS3InfoListData data = {.callback = callback, .callbackData = callbackData}; - storageS3ListInternal(this, path, false, false, storageS3InfoListCallback, &data); + StorageS3InfoListData data = {.level = level, .callback = callback, .callbackData = callbackData}; + storageS3ListInternal(this, path, param.expression, false, storageS3InfoListCallback, &data); } MEM_CONTEXT_TEMP_END(); FUNCTION_LOG_RETURN(BOOL, true); } -/**********************************************************************************************************************************/ -static void -storageS3ListCallback(StorageS3 *this, void *callbackData, const String *name, StorageType type, const XmlNode *xml) -{ - FUNCTION_TEST_BEGIN(); - FUNCTION_TEST_PARAM(STORAGE_S3, this); - FUNCTION_TEST_PARAM_P(VOID, callbackData); - FUNCTION_TEST_PARAM(STRING, name); - FUNCTION_TEST_PARAM(ENUM, type); - FUNCTION_TEST_PARAM(XML_NODE, xml); - FUNCTION_TEST_END(); - - (void)this; - ASSERT(callbackData != NULL); - ASSERT(name != NULL); - (void)type; - (void)xml; - - strLstAdd((StringList *)callbackData, name); - - FUNCTION_TEST_RETURN_VOID(); -} - -static StringList * -storageS3List(THIS_VOID, const String *path, StorageInterfaceListParam param) -{ - THIS(StorageS3); - - FUNCTION_LOG_BEGIN(logLevelDebug); - FUNCTION_LOG_PARAM(STORAGE_S3, this); - FUNCTION_LOG_PARAM(STRING, path); - FUNCTION_LOG_PARAM(STRING, param.expression); - FUNCTION_LOG_END(); - - ASSERT(this != NULL); - ASSERT(path != NULL); - - StringList *result = NULL; - - MEM_CONTEXT_TEMP_BEGIN() - { - result = strLstNew(); - - storageS3ListInternal(this, path, param.expression, false, storageS3ListCallback, result); - strLstMove(result, memContextPrior()); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(STRING_LIST, result); -} - /**********************************************************************************************************************************/ static StorageRead * storageS3NewRead(THIS_VOID, const String *file, bool ignoreMissing, StorageInterfaceNewReadParam param) @@ -909,10 +838,8 @@ storageS3Remove(THIS_VOID, const String *file, StorageInterfaceRemoveParam param /**********************************************************************************************************************************/ static const StorageInterface storageInterfaceS3 = { - .exists = storageS3Exists, .info = storageS3Info, .infoList = storageS3InfoList, - .list = storageS3List, .newRead = storageS3NewRead, .newWrite = storageS3NewWrite, .pathRemove = storageS3PathRemove, diff --git a/src/storage/storage.c b/src/storage/storage.c index 7698b82dd..c06e10be3 100644 --- a/src/storage/storage.c +++ b/src/storage/storage.c @@ -55,9 +55,7 @@ storageNew( ASSERT(type != NULL); ASSERT(strSize(path) >= 1 && strPtr(path)[0] == '/'); ASSERT(driver != NULL); - ASSERT(interface.exists != NULL); ASSERT(interface.info != NULL); - ASSERT(interface.list != NULL); ASSERT(interface.infoList != NULL); ASSERT(interface.newRead != NULL); ASSERT(interface.newWrite != NULL); @@ -153,17 +151,19 @@ storageExists(const Storage *this, const String *pathExp, StorageExistsParam par MEM_CONTEXT_TEMP_BEGIN() { - // Build the path to the file - String *path = storagePathP(this, pathExp); - // Create Wait object of timeout > 0 Wait *wait = param.timeout != 0 ? waitNew(param.timeout) : NULL; // Loop until file exists or timeout do { - // Call driver function - result = storageInterfaceExistsP(this->driver, path); + // storageInfoLevelBasic is required here because storageInfoLevelExists will not return the type and this function + // specifically wants to test existence of a *file*, not just the existence of anything with the specified name. + StorageInfo info = storageInfoP( + this, pathExp, .level = storageInfoLevelBasic, .ignoreMissing = true, .followLink = true); + + // Only exists if it is a file + result = info.exists && info.type == storageTypeFile; } while (!result && wait != NULL && waitMore(wait)); } @@ -239,6 +239,7 @@ storageInfo(const Storage *this, const String *fileExp, StorageInfoParam param) FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STORAGE, this); FUNCTION_LOG_PARAM(STRING, fileExp); + FUNCTION_LOG_PARAM(ENUM, param.level); FUNCTION_LOG_PARAM(BOOL, param.ignoreMissing); FUNCTION_LOG_PARAM(BOOL, param.followLink); FUNCTION_LOG_PARAM(BOOL, param.noPathEnforce); @@ -255,7 +256,11 @@ storageInfo(const Storage *this, const String *fileExp, StorageInfoParam param) String *file = storagePathP(this, fileExp, .noEnforce = param.noPathEnforce); // Call driver function - result = storageInterfaceInfoP(this->driver, file, .followLink = param.followLink); + if (param.level == storageInfoLevelDefault) + param.level = storageFeature(this, storageFeatureInfoDetail) ? storageInfoLevelDetail : storageInfoLevelBasic; + + result = storageInterfaceInfoP( + this->driver, file, param.level, .followLink = param.followLink); // Error if the file missing and not ignoring if (!result.exists && !param.ignoreMissing) @@ -311,11 +316,14 @@ storageInfoListSortCallback(void *data, const StorageInfo *info) static bool storageInfoListSort( - const Storage *this, const String *path, SortOrder sortOrder, StorageInfoListCallback callback, void *callbackData) + const Storage *this, const String *path, StorageInfoLevel level, const String *expression, SortOrder sortOrder, + StorageInfoListCallback callback, void *callbackData) { FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_PARAM(STORAGE, this); FUNCTION_LOG_PARAM(STRING, path); + FUNCTION_LOG_PARAM(ENUM, level); + FUNCTION_LOG_PARAM(STRING, expression); FUNCTION_LOG_PARAM(ENUM, sortOrder); FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM_P(VOID, callbackData); @@ -331,7 +339,7 @@ storageInfoListSort( // If no sorting then use the callback directly if (sortOrder == sortOrderNone) { - result = storageInterfaceInfoListP(this->driver, path, callback, callbackData); + result = storageInterfaceInfoListP(this->driver, path, level, callback, callbackData, .expression = expression); } // Else sort the info before sending it to the callback else @@ -343,7 +351,8 @@ storageInfoListSort( .infoList = lstNewP(sizeof(StorageInfo), .comparator = lstComparatorStr), }; - result = storageInterfaceInfoListP(this->driver, path, storageInfoListSortCallback, &data); + result = storageInterfaceInfoListP( + this->driver, path, level, storageInfoListSortCallback, &data, .expression = expression); lstSort(data.infoList, sortOrder); MEM_CONTEXT_TEMP_RESET_BEGIN() @@ -370,7 +379,8 @@ typedef struct StorageInfoListData const Storage *storage; // Storage object; StorageInfoListCallback callbackFunction; // Original callback function void *callbackData; // Original callback data - RegExp *expression; // Filter for names + const String *expression; // Filter for names + RegExp *regExp; // Compiled filter for names bool recurse; // Should we recurse? SortOrder sortOrder; // Sort order const String *path; // Top-level path for info @@ -404,7 +414,7 @@ storageInfoListCallback(void *data, const StorageInfo *info) infoUpdate.name = strNewFmt("%s/%s", strPtr(listData->subPath), strPtr(infoUpdate.name)); // Only continue if there is no expression or the expression matches - if (listData->expression == NULL || regExpMatch(listData->expression, infoUpdate.name)) + if (listData->expression == NULL || regExpMatch(listData->regExp, infoUpdate.name)) { if (listData->sortOrder != sortOrderDesc) listData->callbackFunction(listData->callbackData, &infoUpdate); @@ -416,8 +426,8 @@ storageInfoListCallback(void *data, const StorageInfo *info) data.subPath = infoUpdate.name; storageInfoListSort( - data.storage, strNewFmt("%s/%s", strPtr(data.path), strPtr(data.subPath)), data.sortOrder, storageInfoListCallback, - &data); + data.storage, strNewFmt("%s/%s", strPtr(data.path), strPtr(data.subPath)), infoUpdate.level, data.expression, + data.sortOrder, storageInfoListCallback, &data); } if (listData->sortOrder == sortOrderDesc) @@ -436,6 +446,7 @@ storageInfoList( FUNCTION_LOG_PARAM(STRING, pathExp); FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM_P(VOID, callbackData); + FUNCTION_LOG_PARAM(ENUM, param.level); FUNCTION_LOG_PARAM(BOOL, param.errorOnMissing); FUNCTION_LOG_PARAM(ENUM, param.sortOrder); FUNCTION_LOG_PARAM(STRING, param.expression); @@ -451,6 +462,10 @@ storageInfoList( MEM_CONTEXT_TEMP_BEGIN() { + // Info level + if (param.level == storageInfoLevelDefault) + param.level = storageFeature(this, storageFeatureInfoDetail) ? storageInfoLevelDetail : storageInfoLevelBasic; + // Build the path String *path = storagePathP(this, pathExp); @@ -462,18 +477,20 @@ storageInfoList( .storage = this, .callbackFunction = callback, .callbackData = callbackData, + .expression = param.expression, .sortOrder = param.sortOrder, .recurse = param.recurse, .path = path, }; - if (param.expression != NULL) - data.expression = regExpNew(param.expression); + if (data.expression != NULL) + data.regExp = regExpNew(param.expression); - result = storageInfoListSort(this, path, param.sortOrder, storageInfoListCallback, &data); + result = storageInfoListSort( + this, path, param.level, param.expression, param.sortOrder, storageInfoListCallback, &data); } else - result = storageInfoListSort(this, path, param.sortOrder, callback, callbackData); + result = storageInfoListSort(this, path, param.level, NULL, param.sortOrder, callback, callbackData); if (!result && param.errorOnMissing) THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, strPtr(path)); @@ -484,6 +501,26 @@ storageInfoList( } /**********************************************************************************************************************************/ +static void +storageListCallback(void *data, const StorageInfo *info) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_LOG_PARAM_P(VOID, data); + FUNCTION_LOG_PARAM(STORAGE_INFO, info); + FUNCTION_TEST_END(); + + // Skip . path + if (strEq(info->name, DOT_STR)) + { + FUNCTION_TEST_RETURN_VOID(); + return; + } + + strLstAdd((StringList *)data, info->name); + + FUNCTION_TEST_RETURN_VOID(); +} + StringList * storageList(const Storage *this, const String *pathExp, StorageListParam param) { @@ -503,23 +540,16 @@ storageList(const Storage *this, const String *pathExp, StorageListParam param) MEM_CONTEXT_TEMP_BEGIN() { - // Build the path - String *path = storagePathP(this, pathExp); + result = strLstNew(); - // Get the list - result = storageInterfaceListP(this->driver, path, .expression = param.expression); - - // If the path does not exist - if (result == NULL) + // Build an empty list if the directory does not exist by default. This makes the logic in calling functions simpler when + // the caller doesn't care if the path is missing. + if (!storageInfoListP( + this, pathExp, storageListCallback, result, .level = storageInfoLevelExists, .errorOnMissing = param.errorOnMissing, + .expression = param.expression)) { - // Error if requested - if (param.errorOnMissing) - THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_MISSING, strPtr(path)); - - // Build an empty list if the directory does not exist by default. This makes the logic in calling functions simpler - // when they don't care if the path is missing. - if (!param.nullOnMissing) - result = strLstNew(); + if (param.nullOnMissing) + result = NULL; } // Move list up to the old context @@ -781,17 +811,13 @@ storagePathExists(const Storage *this, const String *pathExp) FUNCTION_LOG_END(); ASSERT(this != NULL); - ASSERT(this->interface.pathExists != NULL); + ASSERT(storageFeature(this, storageFeaturePath)); - bool result = false; + // storageInfoLevelBasic is required here because storageInfoLevelExists will not return the type and this function specifically + // wants to test existence of a *path*, not just the existence of anything with the specified name. + StorageInfo info = storageInfoP(this, pathExp, .level = storageInfoLevelBasic, .ignoreMissing = true, .followLink = true); - MEM_CONTEXT_TEMP_BEGIN() - { - result = storageInterfacePathExistsP(this->driver, storagePathP(this, pathExp)); - } - MEM_CONTEXT_TEMP_END(); - - FUNCTION_LOG_RETURN(BOOL, result); + FUNCTION_LOG_RETURN(BOOL, info.exists && info.type == storageTypePath); } /**********************************************************************************************************************************/ diff --git a/src/storage/storage.h b/src/storage/storage.h index c89f7ded0..474cb352b 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -50,6 +50,9 @@ typedef enum // Does the storage support symlinks? Symlinks allow paths/files/links to be accessed from another path. storageFeatureSymLink, + + // Does the storage support detailed info, i.e. user, group, mode, link destination, etc. + storageFeatureInfoDetail, } StorageFeature; /*********************************************************************************************************************************** @@ -89,6 +92,7 @@ Buffer *storageGet(StorageRead *file, StorageGetParam param); typedef struct StorageInfoParam { VAR_PARAM_HEADER; + StorageInfoLevel level; bool ignoreMissing; bool followLink; bool noPathEnforce; @@ -105,6 +109,7 @@ typedef void (*StorageInfoListCallback)(void *callbackData, const StorageInfo *i typedef struct StorageInfoListParam { VAR_PARAM_HEADER; + StorageInfoLevel level; bool errorOnMissing; bool recurse; SortOrder sortOrder; diff --git a/src/storage/storage.intern.h b/src/storage/storage.intern.h index f77e30d66..610e27ab3 100644 --- a/src/storage/storage.intern.h +++ b/src/storage/storage.intern.h @@ -25,9 +25,6 @@ Error messages #define STORAGE_ERROR_INFO "unable to get info for path/file '%s'" #define STORAGE_ERROR_INFO_MISSING "unable to get info for missing path/file '%s'" -#define STORAGE_ERROR_LIST "unable to list files for path '%s'" -#define STORAGE_ERROR_LIST_MISSING "unable to list files for missing path '%s'" - #define STORAGE_ERROR_LIST_INFO "unable to list file info for path '%s'" #define STORAGE_ERROR_LIST_INFO_MISSING "unable to list file info for missing path '%s'" @@ -53,19 +50,12 @@ typedef String *StoragePathExpressionCallback(const String *expression, const St /*********************************************************************************************************************************** Required interface functions ***********************************************************************************************************************************/ -// Does a file exist? This function is only for files, not paths. -typedef struct StorageInterfaceExistsParam -{ - VAR_PARAM_HEADER; -} StorageInterfaceExistsParam; - -typedef bool StorageInterfaceExists(void *thisVoid, const String *file, StorageInterfaceExistsParam param); - -#define storageInterfaceExistsP(thisVoid, file, ...) \ - STORAGE_COMMON_INTERFACE(thisVoid).exists(thisVoid, file, (StorageInterfaceExistsParam){VAR_PARAM_INIT, __VA_ARGS__}) - -// --------------------------------------------------------------------------------------------------------------------------------- -// Get information about a file +// Get information about a file/link/path +// +// The level parameter controls the amount of information that will be returned. See the StorageInfo type and StorageInfoLevel enum +// for details about information that must be provided at each level. The driver should only return the amount of information +// requested even if more is available. All drivers must implement the storageInfoLevelExists and storageInfoLevelBasic levels. Only +// drivers with the storageFeatureInfoDetail feature must implement the storageInfoLevelDetail level. typedef struct StorageInterfaceInfoParam { VAR_PARAM_HEADER; @@ -74,25 +64,11 @@ typedef struct StorageInterfaceInfoParam bool followLink; } StorageInterfaceInfoParam; -typedef StorageInfo StorageInterfaceInfo(void *thisVoid, const String *file, StorageInterfaceInfoParam param); +typedef StorageInfo StorageInterfaceInfo( + void *thisVoid, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param); -#define storageInterfaceInfoP(thisVoid, file, ...) \ - STORAGE_COMMON_INTERFACE(thisVoid).info(thisVoid, file, (StorageInterfaceInfoParam){VAR_PARAM_INIT, __VA_ARGS__}) - -// --------------------------------------------------------------------------------------------------------------------------------- -// Get a list of files -typedef struct StorageInterfaceListParam -{ - VAR_PARAM_HEADER; - - // Regular expression used to filter the results - const String *expression; -} StorageInterfaceListParam; - -typedef StringList *StorageInterfaceList(void *thisVoid, const String *path, StorageInterfaceListParam param); - -#define storageInterfaceListP(thisVoid, path, ...) \ - STORAGE_COMMON_INTERFACE(thisVoid).list(thisVoid, path, (StorageInterfaceListParam){VAR_PARAM_INIT, __VA_ARGS__}) +#define storageInterfaceInfoP(thisVoid, file, level, ...) \ + STORAGE_COMMON_INTERFACE(thisVoid).info(thisVoid, file, level, (StorageInterfaceInfoParam){VAR_PARAM_INIT, __VA_ARGS__}) // --------------------------------------------------------------------------------------------------------------------------------- // Create a file read object. The file should not be opened immediately -- open() will be called on the IoRead interface when the @@ -157,17 +133,29 @@ typedef StorageWrite *StorageInterfaceNewWrite(void *thisVoid, const String *fil // --------------------------------------------------------------------------------------------------------------------------------- // Get info for a path and all paths/files in the path (does not recurse) +// +// See storageInterfaceInfoP() for usage of the level parameter. typedef struct StorageInterfaceInfoListParam { VAR_PARAM_HEADER; + + // Regular expression used to filter the results. The expression is always checked in the callback passed to + // storageInterfaceInfoListP() so checking the expression in the driver is entirely optional. The driver should only use the + // expression if it can improve performance or limit network transfer. + // + // Partial matching of the expression is fine as long as nothing that should match is excluded, e.g. it is OK to prefix match + // using the prefix returned from regExpPrefix(). This may cause extra results to be sent to the callback but won't exclude + // anything that matches the expression exactly. + const String *expression; } StorageInterfaceInfoListParam; typedef bool StorageInterfaceInfoList( - void *thisVoid, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param); + void *thisVoid, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData, + StorageInterfaceInfoListParam param); -#define storageInterfaceInfoListP(thisVoid, path, callback, callbackData, ...) \ +#define storageInterfaceInfoListP(thisVoid, path, level, callback, callbackData, ...) \ STORAGE_COMMON_INTERFACE(thisVoid).infoList( \ - thisVoid, path, callback, callbackData, (StorageInterfaceInfoListParam){VAR_PARAM_INIT, __VA_ARGS__}) + thisVoid, path, level, callback, callbackData, (StorageInterfaceInfoListParam){VAR_PARAM_INIT, __VA_ARGS__}) // --------------------------------------------------------------------------------------------------------------------------------- // Remove a path (and optionally recurse) @@ -227,18 +215,6 @@ typedef void StorageInterfacePathCreate( STORAGE_COMMON_INTERFACE(thisVoid).pathCreate( \ thisVoid, path, errorOnExists, noParentCreate, mode, (StorageInterfacePathCreateParam){VAR_PARAM_INIT, __VA_ARGS__}) -// --------------------------------------------------------------------------------------------------------------------------------- -// Does a path exist? -typedef struct StorageInterfacePathExistsParam -{ - VAR_PARAM_HEADER; -} StorageInterfacePathExistsParam; - -typedef bool StorageInterfacePathExists(void *thisVoid, const String *path, StorageInterfacePathExistsParam param); - -#define storageInterfacePathExistsP(thisVoid, path, ...) \ - STORAGE_COMMON_INTERFACE(thisVoid).pathExists(thisVoid, path, (StorageInterfacePathExistsParam){VAR_PARAM_INIT, __VA_ARGS__}) - // --------------------------------------------------------------------------------------------------------------------------------- // Sync a path typedef struct StorageInterfacePathSyncParam @@ -260,10 +236,8 @@ typedef struct StorageInterface uint64_t feature; // Required functions - StorageInterfaceExists *exists; StorageInterfaceInfo *info; StorageInterfaceInfoList *infoList; - StorageInterfaceList *list; StorageInterfaceNewRead *newRead; StorageInterfaceNewWrite *newWrite; StorageInterfacePathRemove *pathRemove; @@ -272,7 +246,6 @@ typedef struct StorageInterface // Optional functions StorageInterfaceMove *move; StorageInterfacePathCreate *pathCreate; - StorageInterfacePathExists *pathExists; StorageInterfacePathSync *pathSync; } StorageInterface; diff --git a/test/define.yaml b/test/define.yaml index bec11fdce..17ef832d7 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -409,6 +409,8 @@ unit: storage/posix/read: full storage/posix/storage: full storage/posix/write: full + + # Provide as much coverage as possible for these modules but some coverage needs to be provided by other driver tests storage/helper: full storage/read: full storage/storage: full @@ -416,7 +418,7 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: remote - total: 12 + total: 9 containerReq: true binReq: true diff --git a/test/src/common/harnessStorage.c b/test/src/common/harnessStorage.c index 920bfea66..95d3d0f79 100644 --- a/test/src/common/harnessStorage.c +++ b/test/src/common/harnessStorage.c @@ -21,92 +21,96 @@ hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info) strCatFmt(data->content, "%s {", strPtr(info->name)); - switch (info->type) + if (info->level > storageInfoLevelExists) { - case storageTypeFile: + switch (info->type) { - strCat(data->content, "file"); - - if (!data->sizeOmit) + case storageTypeFile: { - uint64_t size = info->size; + strCat(data->content, "file"); - // If the file is compressed then decompress to get the real size. Note that only gz is used in unit tests since - // it is the only compression type guaranteed to be present. - if (data->fileCompressed) + if (info->level >= storageInfoLevelBasic && !data->sizeOmit) { - ASSERT(data->storage != NULL); + uint64_t size = info->size; - StorageRead *read = storageNewReadP( - data->storage, - data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name); - ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), decompressFilter(compressTypeGz)); - size = bufUsed(storageGetP(read)); + // If the file is compressed then decompress to get the real size. Note that only gz is used in unit tests since + // it is the only compression type guaranteed to be present. + if (data->fileCompressed) + { + ASSERT(data->storage != NULL); + + StorageRead *read = storageNewReadP( + data->storage, + data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name); + ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), decompressFilter(compressTypeGz)); + size = bufUsed(storageGetP(read)); + } + + strCatFmt(data->content, ", s=%" PRIu64, size); } - strCatFmt(data->content, ", s=%" PRIu64, size); + break; } - break; - } - - case storageTypeLink: - { - strCatFmt(data->content, "link, d=%s", strPtr(info->linkDestination)); - break; - } - - case storageTypePath: - { - strCat(data->content, "path"); - break; - } - - case storageTypeSpecial: - { - strCat(data->content, "special"); - break; - } - } - - if (info->type != storageTypeSpecial) - { - if (info->type != storageTypeLink) - { - if (!data->modeOmit || (info->type == storageTypePath && data->modePath != info->mode) || - (info->type == storageTypeFile && data->modeFile != info->mode)) + case storageTypeLink: { - strCatFmt(data->content, ", m=%04o", info->mode); + strCatFmt(data->content, "link, d=%s", strPtr(info->linkDestination)); + break; + } + + case storageTypePath: + { + strCat(data->content, "path"); + break; + } + + case storageTypeSpecial: + { + strCat(data->content, "special"); + break; } } - if (info->type == storageTypeFile) + if (info->type != storageTypeSpecial) { - if (!data->timestampOmit) - strCatFmt(data->content, ", t=%" PRIu64, (uint64_t)info->timeModified); - } + if (info->type != storageTypeLink) + { + if (info->level >= storageInfoLevelDetail && + (!data->modeOmit || (info->type == storageTypePath && data->modePath != info->mode) || + (info->type == storageTypeFile && data->modeFile != info->mode))) + { + strCatFmt(data->content, ", m=%04o", info->mode); + } + } - if (!data->userOmit || userId() != info->userId) - { - if (info->user != NULL) + if (info->type == storageTypeFile && info->level >= storageInfoLevelBasic) { - strCatFmt(data->content, ", u=%s", strPtr(info->user)); + if (!data->timestampOmit) + strCatFmt(data->content, ", t=%" PRIu64, (uint64_t)info->timeModified); } - else - { - strCatFmt(data->content, ", u=%d", (int)info->userId); - } - } - if (!data->groupOmit || groupId() != info->groupId) - { - if (info->group != NULL) + if (info->level >= storageInfoLevelDetail && (!data->userOmit || userId() != info->userId)) { - strCatFmt(data->content, ", g=%s", strPtr(info->group)); + if (info->user != NULL) + { + strCatFmt(data->content, ", u=%s", strPtr(info->user)); + } + else + { + strCatFmt(data->content, ", u=%d", (int)info->userId); + } } - else + + if (info->level >= storageInfoLevelDetail && (!data->groupOmit || groupId() != info->groupId)) { - strCatFmt(data->content, ", g=%d", (int)info->groupId); + if (info->group != NULL) + { + strCatFmt(data->content, ", g=%s", strPtr(info->group)); + } + else + { + strCatFmt(data->content, ", g=%d", (int)info->groupId); + } } } } diff --git a/test/src/module/command/archiveGetTest.c b/test/src/module/command/archiveGetTest.c index 4d3fb9982..ebe190d94 100644 --- a/test/src/module/command/archiveGetTest.c +++ b/test/src/module/command/archiveGetTest.c @@ -258,7 +258,7 @@ testRun(void) TEST_ERROR_FMT( queueNeed(strNew("000000010000000100000001"), false, queueSize, walSegmentSize, PG_VERSION_92), - PathMissingError, "unable to list files for missing path '%s/spool/archive/test1/in'", testPath()); + PathMissingError, "unable to list file info for missing path '%s/spool/archive/test1/in'", testPath()); // ------------------------------------------------------------------------------------------------------------------------- storagePathCreateP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN)); diff --git a/test/src/module/command/controlTest.c b/test/src/module/command/controlTest.c index 43d206423..28bdbf935 100644 --- a/test/src/module/command/controlTest.c +++ b/test/src/module/command/controlTest.c @@ -92,7 +92,7 @@ testRun(void) TEST_RESULT_VOID(storageRemoveP(storageData, strNew("lockpath/all" STOP_FILE_EXT)), "remove stop file"); TEST_RESULT_INT(system(strPtr(strNewFmt("chmod 444 %s", strPtr(lockPath)))), 0, "change perms"); TEST_ERROR_FMT( - cmdStop(), FileOpenError, "unable to stat '%s/all.stop': [13] Permission denied", strPtr(lockPath)); + cmdStop(), FileOpenError, "unable to get info for path/file '%s/all.stop': [13] Permission denied", strPtr(lockPath)); TEST_RESULT_INT(system(strPtr(strNewFmt("chmod 700 %s", strPtr(lockPath)))), 0, "change perms"); TEST_RESULT_VOID( storagePathRemoveP(storageData, lockPath, .recurse = true, .errorOnMissing = true), " remove the lock path"); diff --git a/test/src/module/config/parseTest.c b/test/src/module/config/parseTest.c index c770d9b12..84a9ecbf7 100644 --- a/test/src/module/config/parseTest.c +++ b/test/src/module/config/parseTest.c @@ -179,7 +179,7 @@ testRun(void) TEST_ERROR( cfgFileLoad(parseOptionList, backupCmdDefConfigValue, backupCmdDefConfigInclPathValue, oldConfigDefault), PathMissingError, - "unable to list files for missing path '/BOGUS'"); + "unable to list file info for missing path '/BOGUS'"); // --config-include-path valid, --config invalid (does not exist) value = strLstNew(); diff --git a/test/src/module/performance/storageTest.c b/test/src/module/performance/storageTest.c index 3b12b17f9..5f0809e26 100644 --- a/test/src/module/performance/storageTest.c +++ b/test/src/module/performance/storageTest.c @@ -29,29 +29,18 @@ stress testing as needed. /*********************************************************************************************************************************** Dummy functions and interface for constructing test drivers ***********************************************************************************************************************************/ -static bool -storageTestDummyExists(THIS_VOID, const String *file, StorageInterfaceExistsParam param) -{ - (void)thisVoid; (void)file; (void)param; return false; -} - static StorageInfo -storageTestDummyInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param) +storageTestDummyInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param) { - (void)thisVoid; (void)file; (void)param; return (StorageInfo){.exists = false}; + (void)thisVoid; (void)file; (void)level; (void)param; return (StorageInfo){.exists = false}; } static bool storageTestDummyInfoList( - THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) + THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData, + StorageInterfaceInfoListParam param) { - (void)thisVoid; (void)path; (void)callback; (void)callbackData; (void)param; return false; -} - -static StringList * -storageTestDummyList(THIS_VOID, const String *path, StorageInterfaceListParam param) -{ - (void)thisVoid; (void)path; (void)param; return NULL; + (void)thisVoid; (void)path; (void)level; (void)callback; (void)callbackData; (void)param; return false; } static StorageRead * @@ -80,10 +69,8 @@ storageTestDummyRemove(THIS_VOID, const String *file, StorageInterfaceRemovePara static const StorageInterface storageInterfaceTestDummy = { - .exists = storageTestDummyExists, .info = storageTestDummyInfo, .infoList = storageTestDummyInfoList, - .list = storageTestDummyList, .newRead = storageTestDummyNewRead, .newWrite = storageTestDummyNewWrite, .pathRemove = storageTestDummyPathRemove, @@ -114,10 +101,11 @@ typedef struct static bool storageTestPerfInfoList( - THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) + THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData, + StorageInterfaceInfoListParam param) { THIS(StorageTestPerfInfoList); - (void)path; (void)param; + (void)path; (void)level; (void)param; MEM_CONTEXT_TEMP_BEGIN() { diff --git a/test/src/module/storage/posixTest.c b/test/src/module/storage/posixTest.c index 369ab9f89..d284aa8fd 100644 --- a/test/src/module/storage/posixTest.c +++ b/test/src/module/storage/posixTest.c @@ -102,7 +102,7 @@ testRun(void) TEST_RESULT_BOOL(storageTest->write, true, " check write"); TEST_RESULT_BOOL(storageTest->pathExpressionFunction != NULL, true, " check expression function is set"); - TEST_RESULT_PTR(storageInterface(storageTest).exists, storageTest->interface.exists, " check interface"); + TEST_RESULT_PTR(storageInterface(storageTest).info, storageTest->interface.info, " check interface"); TEST_RESULT_PTR(storageDriver(storageTest), storageTest->driver, " check driver"); TEST_RESULT_PTR(storageType(storageTest), storageTest->type, " check type"); TEST_RESULT_BOOL(storageFeature(storageTest, storageFeaturePath), true, " check path feature"); @@ -127,16 +127,19 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_ERROR_FMT( storageExistsP(storageTest, fileNoPerm), FileOpenError, - "unable to stat '%s': [13] Permission denied", strPtr(fileNoPerm)); + "unable to get info for path/file '%s': [13] Permission denied", strPtr(fileNoPerm)); TEST_ERROR_FMT( - storagePathExistsP(storageTest, fileNoPerm), PathOpenError, - "unable to stat '%s': [13] Permission denied", strPtr(fileNoPerm)); + storagePathExistsP(storageTest, fileNoPerm), FileOpenError, + "unable to get info for path/file '%s': [13] Permission denied", strPtr(fileNoPerm)); // ------------------------------------------------------------------------------------------------------------------------- String *fileExists = strNewFmt("%s/exists", testPath()); + String *pathExists = strNewFmt("%s/pathExists", testPath()); TEST_RESULT_INT(system(strPtr(strNewFmt("touch %s", strPtr(fileExists)))), 0, "create exists file"); + TEST_SYSTEM_FMT("mkdir %s", strPtr(pathExists)); TEST_RESULT_BOOL(storageExistsP(storageTest, fileExists), true, "file exists"); + TEST_RESULT_BOOL(storageExistsP(storageTest, pathExists), false, "not a file"); TEST_RESULT_BOOL(storagePathExistsP(storageTest, fileExists), false, "not a path"); TEST_RESULT_INT(system(strPtr(strNewFmt("sudo rm %s", strPtr(fileExists)))), 0, "remove exists file"); @@ -304,7 +307,7 @@ testRun(void) TEST_RESULT_VOID( storagePosixInfoListEntry( - (StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"), + (StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"), storageInfoLevelBasic, hrnStorageInfoListCallback, &callbackData), "missing path"); TEST_RESULT_STR_Z(callbackData.content, "", " check content"); @@ -395,7 +398,7 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_ERROR_FMT( - storageListP(storageTest, strNew(BOGUS_STR), .errorOnMissing = true), PathMissingError, STORAGE_ERROR_LIST_MISSING, + storageListP(storageTest, strNew(BOGUS_STR), .errorOnMissing = true), PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, strPtr(strNewFmt("%s/BOGUS", testPath()))); TEST_RESULT_PTR(storageListP(storageTest, strNew(BOGUS_STR), .nullOnMissing = true), NULL, "null for missing dir"); @@ -404,12 +407,12 @@ testRun(void) // ------------------------------------------------------------------------------------------------------------------------- TEST_ERROR_FMT( storageListP(storageTest, pathNoPerm), PathOpenError, - STORAGE_ERROR_LIST ": [13] Permission denied", strPtr(pathNoPerm)); + STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathNoPerm)); // Should still error even when ignore missing TEST_ERROR_FMT( storageListP(storageTest, pathNoPerm), PathOpenError, - STORAGE_ERROR_LIST ": [13] Permission denied", strPtr(pathNoPerm)); + STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathNoPerm)); // ------------------------------------------------------------------------------------------------------------------------- TEST_RESULT_VOID( @@ -464,7 +467,7 @@ testRun(void) TEST_ERROR_FMT( storageMoveP(storageTest, source, destination), FileMissingError, - "unable to move missing file '%s': [2] No such file or directory", strPtr(sourceFile)); + "unable to move missing source '%s': [2] No such file or directory", strPtr(sourceFile)); // ------------------------------------------------------------------------------------------------------------------------- source = storageNewReadP(storageTest, fileNoPerm); @@ -623,14 +626,14 @@ testRun(void) strPtr(pathRemove2)); TEST_ERROR_FMT( storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError, - STORAGE_ERROR_LIST ": [13] Permission denied", strPtr(pathRemove2)); + STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathRemove2)); // ------------------------------------------------------------------------------------------------------------------------- TEST_RESULT_INT(system(strPtr(strNewFmt("sudo chmod 777 %s", strPtr(pathRemove1)))), 0, "top path can be removed"); TEST_ERROR_FMT( storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError, - STORAGE_ERROR_LIST ": [13] Permission denied", strPtr(pathRemove2)); + STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathRemove2)); // ------------------------------------------------------------------------------------------------------------------------- String *fileRemove = strNewFmt("%s/remove.txt", strPtr(pathRemove2)); diff --git a/test/src/module/storage/remoteTest.c b/test/src/module/storage/remoteTest.c index 4f655c750..b138515b9 100644 --- a/test/src/module/storage/remoteTest.c +++ b/test/src/module/storage/remoteTest.c @@ -83,34 +83,6 @@ testRun(void) TEST_RESULT_BOOL(storageRemoteProtocol(strNew(BOGUS_STR), varLstNew(), server), false, "invalid function"); } - // Do these tests against a pg remote for coverage - // ***************************************************************************************************************************** - if (testBegin("storageExists()")) - { - Storage *storageRemote = NULL; - TEST_ASSIGN(storageRemote, storagePgGet(1, false), "get remote pg storage"); - storagePathCreateP(storageTest, strNew("pg")); - - TEST_RESULT_BOOL(storageExistsP(storageRemote, strNew("test.txt")), false, "file does not exist"); - - storagePutP(storageNewWriteP(storageTest, strNew("repo/test.txt")), BUFSTRDEF("TEST")); - TEST_RESULT_BOOL(storageExistsP(storageRemote, strNew("test.txt")), true, "file exists"); - - // Check protocol function directly - // ------------------------------------------------------------------------------------------------------------------------- - cfgOptionSet(cfgOptRemoteType, cfgSourceParam, VARSTRDEF("pg")); - cfgOptionValidSet(cfgOptRemoteType, true); - - VariantList *paramList = varLstNew(); - varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test.txt", testPath()))); - - TEST_RESULT_BOOL( - storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_EXISTS_STR, paramList, server), true, "protocol exists"); - TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":true}\n", "check result"); - - bufUsedSet(serverWrite, 0); - } - // ***************************************************************************************************************************** if (testBegin("storageInfo()")) { @@ -232,13 +204,13 @@ testRun(void) bufUsedSet(serverWrite, 0); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("protocol output that is not tested elsewhere"); + TEST_TITLE("protocol output that is not tested elsewhere (basic)"); - info = (StorageInfo){.type = storageTypeLink, .linkDestination = STRDEF("../")}; + info = (StorageInfo){.level = storageInfoLevelDetail, .type = storageTypeLink, .linkDestination = STRDEF("../")}; TEST_RESULT_VOID(storageRemoteInfoWrite(server, &info), "write link info"); ioWriteFlush(serverWriteIo); - TEST_RESULT_STR_Z(strNewBuf(serverWrite), ".l\n.0\n.null\n.0\n.null\n.0\n.0\n.\"../\"\n", "check result"); + TEST_RESULT_STR_Z(strNewBuf(serverWrite), ".l\n.0\n.0\n.null\n.0\n.null\n.0\n.\"../\"\n", "check result"); bufUsedSet(serverWrite, 0); @@ -247,6 +219,7 @@ testRun(void) VariantList *paramList = varLstNew(); varLstAdd(paramList, varNewStrZ(BOGUS_STR)); + varLstAdd(paramList, varNewUInt(storageInfoLevelBasic)); varLstAdd(paramList, varNewBool(false)); TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_STR, paramList, server), true, "protocol list"); @@ -255,10 +228,16 @@ testRun(void) bufUsedSet(serverWrite, 0); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("check protocol function directly with a file"); + TEST_TITLE("check protocol function directly with a file (basic level)"); + + // Do these tests against pg for coverage. We're not really going to get a pg remote here because the remote for this host + // id has already been created. This will just test that the pg storage is selected to provide coverage. + cfgOptionSet(cfgOptRemoteType, cfgSourceParam, VARSTRDEF("pg")); + cfgOptionValidSet(cfgOptRemoteType, true); paramList = varLstNew(); varLstAdd(paramList, varNewStrZ(hrnReplaceKey("{[path]}/repo/test"))); + varLstAdd(paramList, varNewUInt(storageInfoLevelBasic)); varLstAdd(paramList, varNewBool(false)); TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_STR, paramList, server), true, "protocol list"); @@ -266,7 +245,26 @@ testRun(void) strNewBuf(serverWrite), hrnReplaceKey( "{\"out\":true}\n" - ".f\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.416\n.1555160001\n.6\n" + ".f\n.1555160001\n.6\n" + "{}\n"), + "check result"); + + bufUsedSet(serverWrite, 0); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("check protocol function directly with a file (detail level)"); + + paramList = varLstNew(); + varLstAdd(paramList, varNewStrZ(hrnReplaceKey("{[path]}/repo/test"))); + varLstAdd(paramList, varNewUInt(storageInfoLevelDetail)); + varLstAdd(paramList, varNewBool(false)); + + TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_STR, paramList, server), true, "protocol list"); + TEST_RESULT_STR_Z( + strNewBuf(serverWrite), + hrnReplaceKey( + "{\"out\":true}\n" + ".f\n.1555160001\n.6\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.416\n" "{}\n"), "check result"); @@ -319,13 +317,14 @@ testRun(void) VariantList *paramList = varLstNew(); varLstAdd(paramList, varNewStrZ(hrnReplaceKey("{[path]}/repo"))); + varLstAdd(paramList, varNewUInt(storageInfoLevelDetail)); TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR, paramList, server), true, "call protocol"); TEST_RESULT_STR_Z( strNewBuf(serverWrite), hrnReplaceKey( - ".\".\"\n.p\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.488\n.1555160000\n" - ".\"test\"\n.f\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.416\n.1555160001\n.6\n" + ".\".\"\n.p\n.1555160000\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.488\n" + ".\"test\"\n.f\n.1555160001\n.6\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.416\n" ".\n" "{\"out\":true}\n"), "check result"); @@ -333,37 +332,6 @@ testRun(void) bufUsedSet(serverWrite, 0); } - // ***************************************************************************************************************************** - if (testBegin("storageList()")) - { - Storage *storageRemote = NULL; - TEST_ASSIGN(storageRemote, storageRepoGet(strNew(STORAGE_TYPE_POSIX), false), "get remote repo storage"); - storagePathCreateP(storageTest, strNew("repo")); - - TEST_RESULT_PTR(storageListP(storageRemote, strNew(BOGUS_STR), .nullOnMissing = true), NULL, "null for missing dir"); - TEST_RESULT_UINT(strLstSize(storageListP(storageRemote, NULL)), 0, "empty list for missing dir"); - - // ------------------------------------------------------------------------------------------------------------------------- - storagePathCreateP(storageTest, strNew("repo/testy")); - TEST_RESULT_STR_Z(strLstJoin(storageListP(storageRemote, NULL), ","), "testy" , "list path"); - - storagePathCreateP(storageTest, strNew("repo/testy2\"")); - TEST_RESULT_STR_Z( - strLstJoin(strLstSort(storageListP(storageRemote, strNewFmt("%s/repo", testPath())), sortOrderAsc), ","), - "testy,testy2\"" , "list 2 paths"); - - // Check protocol function directly - // ------------------------------------------------------------------------------------------------------------------------- - VariantList *paramList = varLstNew(); - varLstAdd(paramList, varNewStr(strNewFmt("%s/repo", testPath()))); - varLstAdd(paramList, varNewStr(strNew("^testy$"))); - - TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_LIST_STR, paramList, server), true, "protocol list"); - TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":[\"testy\"]}\n", "check result"); - - bufUsedSet(serverWrite, 0); - } - // ***************************************************************************************************************************** if (testBegin("storageNewRead()")) { @@ -650,28 +618,6 @@ testRun(void) strNewBuf(storageGetP(storageNewReadP(storageTest, strNew("repo/test4.txt.pgbackrest.tmp")))), "", "check file"); } - // ***************************************************************************************************************************** - if (testBegin("storagePathExists()")) - { - Storage *storageRemote = NULL; - TEST_ASSIGN(storageRemote, storageRepoGet(strNew(STORAGE_TYPE_POSIX), false), "get remote repo storage"); - storagePathCreateP(storageTest, strNew("repo")); - - TEST_RESULT_BOOL(storagePathExistsP(storageRemote, strNew("missing")), false, "path does not exist"); - TEST_RESULT_BOOL(storagePathExistsP(storageRemote, NULL), true, "path exists"); - - // Check protocol function directly - // ------------------------------------------------------------------------------------------------------------------------- - VariantList *paramList = varLstNew(); - varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test.txt", testPath()))); - - TEST_RESULT_BOOL( - storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR, paramList, server), true, "protocol path exists"); - TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":false}\n", "check result"); - - bufUsedSet(serverWrite, 0); - } - // ***************************************************************************************************************************** if (testBegin("storagePathCreate()")) { diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c index 85e039309..37db19ed1 100644 --- a/test/src/module/storage/s3Test.c +++ b/test/src/module/storage/s3Test.c @@ -4,6 +4,7 @@ Test S3 Storage #include #include "common/harnessConfig.h" +#include "common/harnessStorage.h" #include "common/harnessTls.h" /*********************************************************************************************************************************** @@ -200,7 +201,11 @@ testS3Server(void) // File exists harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL, storageS3UriStyleHost)); - harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:999", NULL)); + harnessTlsServerReply(testS3ServerResponse( + 200, "OK", + "content-length:999\r\n" + "Last-Modified: Wed, 21 Oct 2015 07:28:00 GMT", + NULL)); // Info() // ------------------------------------------------------------------------------------------------------------------------- @@ -216,6 +221,14 @@ testS3Server(void) "Last-Modified: Wed, 21 Oct 2015 07:28:00 GMT", NULL)); + // File exists and only checking existence + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file2.txt", NULL, storageS3UriStyleHost)); + harnessTlsServerReply(testS3ServerResponse( + 200, "OK", + "content-length:777\r\n" + "Last-Modified: Wed, 22 Oct 2015 07:28:00 GMT", + NULL)); + // InfoList() // ------------------------------------------------------------------------------------------------------------------------- harnessTlsServerExpect( @@ -506,24 +519,6 @@ testS3Server(void) } } -/*********************************************************************************************************************************** -Callback and data for storageInfoList() tests -***********************************************************************************************************************************/ -unsigned int testStorageInfoListSize = 0; -StorageInfo testStorageInfoList[256]; - -void -testStorageInfoListCallback(void *callbackData, const StorageInfo *info) -{ - MEM_CONTEXT_BEGIN((MemContext *)callbackData) - { - testStorageInfoList[testStorageInfoListSize] = *info; - testStorageInfoList[testStorageInfoListSize].name = strDup(info->name); - testStorageInfoListSize++; - } - MEM_CONTEXT_END(); -} - /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -847,26 +842,37 @@ testRun(void) TEST_RESULT_UINT(info.size, 9999, " check exists"); TEST_RESULT_INT(info.timeModified, 1445412480, " check time"); - // InfoList() + TEST_TITLE("file exists and only checking existence"); + + TEST_ASSIGN(info, storageInfoP(s3, strNew("subdir/file2.txt"), .level = storageInfoLevelExists), "file exists"); + TEST_RESULT_BOOL(info.exists, true, " check exists"); + TEST_RESULT_UINT(info.type, storageTypeFile, " check type"); + TEST_RESULT_UINT(info.size, 0, " check exists"); + TEST_RESULT_INT(info.timeModified, 0, " check time"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("list basic level"); + + HarnessStorageInfoListCallbackData callbackData = + { + .content = strNew(""), + }; + TEST_ERROR( - storageInfoListP(s3, strNew("/"), testStorageInfoListCallback, NULL, .errorOnMissing = true), + storageInfoListP(s3, strNew("/"), hrnStorageInfoListCallback, NULL, .errorOnMissing = true), AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed"); TEST_RESULT_VOID( - storageInfoListP(s3, strNew("/path/to"), testStorageInfoListCallback, (void *)memContextCurrent()), "info list files"); + storageInfoListP(s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData), "info list files"); + TEST_RESULT_STR_Z( + callbackData.content, + "test_path {path}\n" + "test_file {file, s=787, t=1255369830}\n", + " check content"); - TEST_RESULT_UINT(testStorageInfoListSize, 2, " file and path returned"); - TEST_RESULT_STR_Z(testStorageInfoList[0].name, "test_path", " check name"); - TEST_RESULT_UINT(testStorageInfoList[0].size, 0, " check size"); - TEST_RESULT_UINT(testStorageInfoList[0].type, storageTypePath, " check type"); - TEST_RESULT_STR_Z(testStorageInfoList[1].name, "test_file", " check name"); - TEST_RESULT_UINT(testStorageInfoList[1].size, 787, " check size"); - TEST_RESULT_INT(testStorageInfoList[1].timeModified, 1255369830, " check time"); - TEST_RESULT_UINT(testStorageInfoList[1].type, storageTypeFile, " check type"); - - // storageDriverList() // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("various errors"); + TEST_ERROR( storageListP(s3, strNew("/"), .errorOnMissing = true), AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed"); @@ -910,16 +916,55 @@ testRun(void) "RequestTimeTooSkewed" "The difference between the request time and the current time is too large."); - TEST_RESULT_STR_Z(strLstJoin(storageListP(s3, strNew("/")), ","), "path1,test1.txt", "list a file/path in root"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("list exists level"); + + callbackData.content = strNew(""); + TEST_RESULT_VOID( + storageInfoListP(s3, strNew("/"), hrnStorageInfoListCallback, &callbackData, .level = storageInfoLevelExists), + "list a file/path in root"); TEST_RESULT_STR_Z( - strLstJoin(storageListP(s3, strNew("/"), .expression = strNew("^test.*$")), ","), "test1.txt", + callbackData.content, + "path1 {}\n" + "test1.txt {}\n", + " check content"); + + callbackData.content = strNew(""); + TEST_RESULT_VOID( + storageInfoListP( + s3, strNew("/"), hrnStorageInfoListCallback, &callbackData, .expression = strNew("^test.*$"), + .level = storageInfoLevelExists), "list a file in root with expression"); TEST_RESULT_STR_Z( - strLstJoin(storageListP(s3, strNew("/path/to")), ","), - "path1,test1.txt,test2.txt,path2,test3.txt", "list files with continuation"); + callbackData.content, + "test1.txt {}\n", + " check content"); + + callbackData.content = strNew(""); + TEST_RESULT_VOID( + storageInfoListP(s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData, .level = storageInfoLevelExists), + "list files with continuation"); TEST_RESULT_STR_Z( - strLstJoin(storageListP(s3, strNew("/path/to"), .expression = strNew("^test(1|3)")), ","), - "test1.path,test1.txt,test3.txt", "list files with expression"); + callbackData.content, + "path1 {}\n" + "test1.txt {}\n" + "test2.txt {}\n" + "path2 {}\n" + "test3.txt {}\n", + " check content"); + + callbackData.content = strNew(""); + TEST_RESULT_VOID( + storageInfoListP( + s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData, .expression = strNew("^test(1|3)"), + .level = storageInfoLevelExists), + "list files with expression"); + TEST_RESULT_STR_Z( + callbackData.content, + "test1.path {}\n" + "test1.txt {}\n" + "test3.txt {}\n", + " check content"); // storageDriverPathRemove() // -------------------------------------------------------------------------------------------------------------------------