db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-09 00:45:49 +02:00
Code Issues Releases Activity

Simplify storage driver info and list functions.

Browse Source 
The storage driver requires two list functions to be implemented, list and infoList. But the former is a subset of the latter so implementing both in every driver is wasteful. The reason both exist is that in Posix it is cheaper to get a list of names than it is to stat files to get size, time, etc. In S3 these operations are equivalent.

Introduce storageInfoLevelType to determine the amount of information required by the caller. That way Posix can work efficiently and all drivers can return only the data required which saves some bandwidth. The storageList() and storageInfoList() functions remain in the storage interface since they are useful -- the only change is simplifying the drivers with no external impact.

Note that since list() accepted an expression infoList() must now do so. Checking the expression is optional for the driver but can be used to limit results or save IO costs.

Similarly, exists() and pathExists() are just specialized forms of info() so adapt them to call info() instead.
This commit is contained in:
David Steele
2020-04-06 16:09:18 -04:00
parent 7679f8f886
commit 5e55d58850
19 changed files with 522 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
doc/xml/release.xml
View File

@ -24,6 +24,16 @@
<p><proper>TCP</proper> keep-alive options are configurable.</p> <p><proper>TCP</proper> keep-alive options are configurable.</p>
</release-item> </release-item>
</release-improvement-list> </release-improvement-list>
<release-development-list>
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>Simplify storage driver info and list functions.</p>
</release-item>
</release-development-list>
</release-core-list> </release-core-list>
</release> </release>

25
src/storage/info.h
View File

@ -6,6 +6,25 @@ Storage Info
#include <sys/types.h> #include <sys/types.h>
/***********************************************************************************************************************************
Specify the level of information required when calling functions that return StorageInfo
***********************************************************************************************************************************/
typedef enum
{
// The info type is determined by driver capabilities. This mimics the prior behavior where drivers would always return as
// much information as they could.
storageInfoLevelDefault = 0,
// Only test for existence. All drivers support this type.
storageInfoLevelExists,
// Basic information. All drivers support this type.
storageInfoLevelBasic,
// Detailed information that is generally only available from filesystems such as Posix
storageInfoLevelDetail,
} StorageInfoLevel;
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Storage type Storage type
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
@ -22,11 +41,17 @@ Object type
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
typedef struct StorageInfo typedef struct StorageInfo
{ {
// Set when info type >= storageInfoLevelExists
const String *name; // Name of path/file/link const String *name; // Name of path/file/link
StorageInfoLevel level; // Level of information provided
bool exists; // Does the path/file/link exist? bool exists; // Does the path/file/link exist?
// Set when info type >= storageInfoLevelBasic (undefined at lower levels)
StorageType type; // Type file/path/link) StorageType type; // Type file/path/link)
uint64_t size; // Size (path/link is 0) uint64_t size; // Size (path/link is 0)
time_t timeModified; // Time file was last modified time_t timeModified; // Time file was last modified
// Set when info type >= storageInfoLevelDetail (undefined at lower levels)
mode_t mode; // Mode of path/file/link mode_t mode; // Mode of path/file/link
uid_t userId; // User that owns the file uid_t userId; // User that owns the file
uid_t groupId; // Group that owns the file uid_t groupId; // Group that owns the file

303
src/storage/posix/storage.c
View File

@ -42,57 +42,25 @@ struct StoragePosix
MemContext *memContext; // Object memory context MemContext *memContext; // Object memory context
}; };
/**********************************************************************************************************************************/
static bool
storagePosixExists(THIS_VOID, const String *file, StorageInterfaceExistsParam param)
{
THIS(StoragePosix);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_POSIX, this);
FUNCTION_LOG_PARAM(STRING, file);
(void)param; // No parameters are used
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(file != NULL);
bool result = false;
// Attempt to stat the file to determine if it exists
struct stat statFile;
// Any error other than entry not found should be reported
if (stat(strPtr(file), &statFile) == -1)
{
if (errno != ENOENT)
THROW_SYS_ERROR_FMT(FileOpenError, "unable to stat '%s'", strPtr(file));
}
// Else found
else
result = !S_ISDIR(statFile.st_mode);
FUNCTION_LOG_RETURN(BOOL, result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static StorageInfo static StorageInfo
storagePosixInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param) storagePosixInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param)
{ {
THIS(StoragePosix); THIS(StoragePosix);
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_POSIX, this); FUNCTION_LOG_PARAM(STORAGE_POSIX, this);
FUNCTION_LOG_PARAM(STRING, file); FUNCTION_LOG_PARAM(STRING, file);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(BOOL, param.followLink); FUNCTION_LOG_PARAM(BOOL, param.followLink);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(file != NULL); ASSERT(file != NULL);
StorageInfo result = {0}; StorageInfo result = {.level = level};
// Attempt to stat the file // Stat the file to check if it exists
struct stat statFile; struct stat statFile;
if ((param.followLink ? stat(strPtr(file), &statFile) : lstat(strPtr(file), &statFile)) == -1) if ((param.followLink ? stat(strPtr(file), &statFile) : lstat(strPtr(file), &statFile)) == -1)
@ -100,40 +68,50 @@ storagePosixInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param)
if (errno != ENOENT) if (errno != ENOENT)
THROW_SYS_ERROR_FMT(FileOpenError, STORAGE_ERROR_INFO, strPtr(file)); THROW_SYS_ERROR_FMT(FileOpenError, STORAGE_ERROR_INFO, strPtr(file));
} }
// On success load info into a structure // On success the file exists
else else
{ {
result.exists = true; result.exists = true;
result.groupId = statFile.st_gid;
result.group = groupNameFromId(result.groupId);
result.userId = statFile.st_uid;
result.user = userNameFromId(result.userId);
result.timeModified = statFile.st_mtime;
if (S_ISREG(statFile.st_mode)) // Add basic level info
if (result.level >= storageInfoLevelBasic)
{ {
result.type = storageTypeFile; result.timeModified = statFile.st_mtime;
result.size = (uint64_t)statFile.st_size;
if (S_ISREG(statFile.st_mode))
{
result.type = storageTypeFile;
result.size = (uint64_t)statFile.st_size;
}
else if (S_ISDIR(statFile.st_mode))
result.type = storageTypePath;
else if (S_ISLNK(statFile.st_mode))
result.type = storageTypeLink;
else
result.type = storageTypeSpecial;
} }
else if (S_ISDIR(statFile.st_mode))
result.type = storageTypePath; // Add detail level info
else if (S_ISLNK(statFile.st_mode)) if (result.level >= storageInfoLevelDetail)
{ {
result.type = storageTypeLink; result.groupId = statFile.st_gid;
result.group = groupNameFromId(result.groupId);
result.userId = statFile.st_uid;
result.user = userNameFromId(result.userId);
result.mode = statFile.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO);
char linkDestination[PATH_MAX]; if (result.type == storageTypeLink)
ssize_t linkDestinationSize = 0; {
char linkDestination[PATH_MAX];
ssize_t linkDestinationSize = 0;
THROW_ON_SYS_ERROR_FMT( THROW_ON_SYS_ERROR_FMT(
(linkDestinationSize = readlink(strPtr(file), linkDestination, sizeof(linkDestination) - 1)) == -1, (linkDestinationSize = readlink(strPtr(file), linkDestination, sizeof(linkDestination) - 1)) == -1,
FileReadError, "unable to get destination for link '%s'", strPtr(file)); FileReadError, "unable to get destination for link '%s'", strPtr(file));
result.linkDestination = strNewN(linkDestination, (size_t)linkDestinationSize); result.linkDestination = strNewN(linkDestination, (size_t)linkDestinationSize);
}
} }
else
result.type = storageTypeSpecial;
result.mode = statFile.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO);
} }
FUNCTION_LOG_RETURN(STORAGE_INFO, result); FUNCTION_LOG_RETURN(STORAGE_INFO, result);
@ -145,12 +123,14 @@ storagePosixInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param)
// get complete test coverage this function must be split out. // get complete test coverage this function must be split out.
static void static void
storagePosixInfoListEntry( storagePosixInfoListEntry(
StoragePosix *this, const String *path, const String *name, StorageInfoListCallback callback, void *callbackData) StoragePosix *this, const String *path, const String *name, StorageInfoLevel level, StorageInfoListCallback callback,
void *callbackData)
{ {
FUNCTION_TEST_BEGIN(); FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_POSIX, this); FUNCTION_TEST_PARAM(STORAGE_POSIX, this);
FUNCTION_TEST_PARAM(STRING, path); FUNCTION_TEST_PARAM(STRING, path);
FUNCTION_TEST_PARAM(STRING, name); FUNCTION_TEST_PARAM(STRING, name);
FUNCTION_TEST_PARAM(ENUM, level);
FUNCTION_TEST_PARAM(FUNCTIONP, callback); FUNCTION_TEST_PARAM(FUNCTIONP, callback);
FUNCTION_TEST_PARAM_P(VOID, callbackData); FUNCTION_TEST_PARAM_P(VOID, callbackData);
FUNCTION_TEST_END(); FUNCTION_TEST_END();
@ -160,17 +140,13 @@ storagePosixInfoListEntry(
ASSERT(name != NULL); ASSERT(name != NULL);
ASSERT(callback != NULL); ASSERT(callback != NULL);
if (!strEqZ(name, "..")) StorageInfo storageInfo = storageInterfaceInfoP(
this, strEq(name, DOT_STR) ? strDup(path) : strNewFmt("%s/%s", strPtr(path), strPtr(name)), level);
if (storageInfo.exists)
{ {
String *pathInfo = strEqZ(name, ".") ? strDup(path) : strNewFmt("%s/%s", strPtr(path), strPtr(name)); storageInfo.name = name;
callback(callbackData, &storageInfo);
StorageInfo storageInfo = storageInterfaceInfoP(this, pathInfo);
if (storageInfo.exists)
{
storageInfo.name = name;
callback(callbackData, &storageInfo);
}
} }
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
@ -178,13 +154,15 @@ storagePosixInfoListEntry(
static bool static bool
storagePosixInfoList( storagePosixInfoList(
THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
{ {
THIS(StoragePosix); THIS(StoragePosix);
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_POSIX, this); FUNCTION_LOG_PARAM(STORAGE_POSIX, this);
FUNCTION_LOG_PARAM(STRING, path); FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData); FUNCTION_LOG_PARAM_P(VOID, callbackData);
(void)param; // No parameters are used (void)param; // No parameters are used
@ -219,8 +197,21 @@ storagePosixInfoList(
while (dirEntry != NULL) while (dirEntry != NULL)
{ {
// Get info and perform callback const String *name = STR(dirEntry->d_name);
storagePosixInfoListEntry(this, path, STR(dirEntry->d_name), callback, callbackData);
// Always skip ..
if (!strEq(name, DOTDOT_STR))
{
// If only making a list of files that exist then no need to go get detailed info which requires calling
// stat() and is therefore relatively slow
if (level == storageInfoLevelExists)
{
callback(callbackData, &(StorageInfo){.name = name, .level = storageInfoLevelExists, .exists = true});
}
// Else more info is required which requires a call to stat()
else
storagePosixInfoListEntry(this, path, name, level, callback, callbackData);
}
// Get next entry // Get next entry
dirEntry = readdir(dir); dirEntry = readdir(dir);
@ -241,78 +232,9 @@ storagePosixInfoList(
FUNCTION_LOG_RETURN(BOOL, result); FUNCTION_LOG_RETURN(BOOL, result);
} }
/**********************************************************************************************************************************/
static StringList *
storagePosixList(THIS_VOID, const String *path, StorageInterfaceListParam param)
{
THIS(StoragePosix);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_POSIX, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(STRING, param.expression);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(path != NULL);
StringList *result = NULL;
DIR *dir = NULL;
TRY_BEGIN()
{
// Open the directory for read
dir = opendir(strPtr(path));
// If the directory could not be opened process errors but ignore missing directories when specified
if (!dir)
{
if (errno != ENOENT)
THROW_SYS_ERROR_FMT(PathOpenError, STORAGE_ERROR_LIST, strPtr(path));
}
else
{
MEM_CONTEXT_TEMP_BEGIN()
{
// Prepare regexp if an expression was passed
RegExp *regExp = param.expression == NULL ? NULL : regExpNew(param.expression);
// Create the string list now that we know the directory is valid
result = strLstNew();
// Read the directory entries
struct dirent *dirEntry = readdir(dir);
while (dirEntry != NULL)
{
const String *entry = STR(dirEntry->d_name);
// Exclude current/parent directory and apply the expression if specified
if (!strEqZ(entry, ".") && !strEqZ(entry, "..") && (regExp == NULL || regExpMatch(regExp, entry)))
strLstAdd(result, entry);
dirEntry = readdir(dir);
}
// Move finished list up to the old context
strLstMove(result, memContextPrior());
}
MEM_CONTEXT_TEMP_END();
}
}
FINALLY()
{
if (dir != NULL)
closedir(dir);
}
TRY_END();
FUNCTION_LOG_RETURN(STRING_LIST, result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static bool static bool
storagePosixMove(THIS_VOID, StorageRead *source, StorageWrite *destination, StorageInterfaceMoveParam param) storagePosixMove(THIS_VOID, StorageRead *source, StorageWrite *destination, StorageInterfaceMoveParam param)
{ {
THIS(StoragePosix); THIS(StoragePosix);
@ -341,8 +263,9 @@ storagePosixMove(THIS_VOID, StorageRead *source, StorageWrite *destination, Sto
// Determine which file/path is missing // Determine which file/path is missing
if (errno == ENOENT) if (errno == ENOENT)
{ {
if (!storageInterfaceExistsP(this, sourceFile)) // Check if the source is missing. Rename does not follow links so there is no need to set followLink.
THROW_SYS_ERROR_FMT(FileMissingError, "unable to move missing file '%s'", strPtr(sourceFile)); if (!storageInterfaceInfoP(this, sourceFile, storageInfoLevelExists).exists)
THROW_SYS_ERROR_FMT(FileMissingError, "unable to move missing source '%s'", strPtr(sourceFile));
if (!storageWriteCreatePath(destination)) if (!storageWriteCreatePath(destination))
{ {
@ -465,39 +388,45 @@ storagePosixPathCreate(
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static bool typedef struct StoragePosixPathRemoveData
storagePosixPathExists(THIS_VOID, const String *path, StorageInterfacePathExistsParam param)
{ {
THIS(StoragePosix); StoragePosix *driver; // Driver
const String *path; // Path
} StoragePosixPathRemoveData;
FUNCTION_LOG_BEGIN(logLevelTrace); static void
FUNCTION_LOG_PARAM(STORAGE_POSIX, this); storagePosixPathRemoveCallback(void *callbackData, const StorageInfo *info)
FUNCTION_LOG_PARAM(STRING, path); {
(void)param; // No parameters are used FUNCTION_TEST_BEGIN();
FUNCTION_LOG_END(); FUNCTION_TEST_PARAM_P(VOID, callbackData);
FUNCTION_TEST_PARAM_P(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(this != NULL); ASSERT(callbackData != NULL);
ASSERT(path != NULL); ASSERT(info != NULL);
bool result = false; if (!strEqZ(info->name, "."))
// Attempt to stat the file to determine if it exists
struct stat statPath;
// Any error other than entry not found should be reported
if (stat(strPtr(path), &statPath) == -1)
{ {
if (errno != ENOENT) StoragePosixPathRemoveData *data = callbackData;
THROW_SYS_ERROR_FMT(PathOpenError, "unable to stat '%s'", strPtr(path)); String *file = strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name));
}
// Else found
else
result = S_ISDIR(statPath.st_mode);
FUNCTION_LOG_RETURN(BOOL, result); // Rather than stat the file to discover what type it is, just try to unlink it and see what happens
if (unlink(strPtr(file)) == -1)
{
// These errors indicate that the entry is actually a path so we'll try to delete it that way
if (errno == EPERM || errno == EISDIR) // {uncovered_branch - no EPERM on tested systems}
{
storageInterfacePathRemoveP(data->driver, file, true);
}
// Else error
else
THROW_SYS_ERROR_FMT(PathRemoveError, STORAGE_ERROR_PATH_REMOVE_FILE, strPtr(file));
}
}
FUNCTION_TEST_RETURN_VOID();
} }
/**********************************************************************************************************************************/
static bool static bool
storagePosixPathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfacePathRemoveParam param) storagePosixPathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfacePathRemoveParam param)
{ {
@ -520,29 +449,14 @@ storagePosixPathRemove(THIS_VOID, const String *path, bool recurse, StorageInter
// Recurse if requested // Recurse if requested
if (recurse) if (recurse)
{ {
// Get a list of files in this path StoragePosixPathRemoveData data =
StringList *fileList = storageInterfaceListP(this, path);
// Only continue if the path exists
if (fileList != NULL)
{ {
// Delete all paths and files .driver = this,
for (unsigned int fileIdx = 0; fileIdx < strLstSize(fileList); fileIdx++) .path = path,
{ };
String *file = strNewFmt("%s/%s", strPtr(path), strPtr(strLstGet(fileList, fileIdx)));
// Rather than stat the file to discover what type it is, just try to unlink it and see what happens // Remove all sub paths/files
if (unlink(strPtr(file)) == -1) storageInterfaceInfoListP(this, path, storageInfoLevelExists, storagePosixPathRemoveCallback, &data);
{
// These errors indicate that the entry is actually a path so we'll try to delete it that way
if (errno == EPERM || errno == EISDIR) // {uncovered_branch - no EPERM on tested systems}
storageInterfacePathRemoveP(this, file, true);
// Else error
else
THROW_SYS_ERROR_FMT(PathRemoveError, STORAGE_ERROR_PATH_REMOVE_FILE, strPtr(file));
}
}
}
} }
// Delete the path // Delete the path
@ -635,15 +549,12 @@ static const StorageInterface storageInterfacePosix =
{ {
.feature = 1 << storageFeaturePath | 1 << storageFeatureCompress | 1 << storageFeatureLimitRead, .feature = 1 << storageFeaturePath | 1 << storageFeatureCompress | 1 << storageFeatureLimitRead,
.exists = storagePosixExists,
.info = storagePosixInfo, .info = storagePosixInfo,
.infoList = storagePosixInfoList, .infoList = storagePosixInfoList,
.list = storagePosixList,
.move = storagePosixMove, .move = storagePosixMove,
.newRead = storagePosixNewRead, .newRead = storagePosixNewRead,
.newWrite = storagePosixNewWrite, .newWrite = storagePosixNewWrite,
.pathCreate = storagePosixPathCreate, .pathCreate = storagePosixPathCreate,
.pathExists = storagePosixPathExists,
.pathRemove = storagePosixPathRemove, .pathRemove = storagePosixPathRemove,
.pathSync = storagePosixPathSync, .pathSync = storagePosixPathSync,
.remove = storagePosixRemove, .remove = storagePosixRemove,
@ -691,7 +602,9 @@ storagePosixNewInternal(
// If this is a posix driver then add link features // If this is a posix driver then add link features
if (strEq(type, STORAGE_POSIX_TYPE_STR)) if (strEq(type, STORAGE_POSIX_TYPE_STR))
driver->interface.feature |= (1 << storageFeatureHardLink | 1 << storageFeatureSymLink | 1 << storageFeaturePathSync); driver->interface.feature |=
1 << storageFeatureHardLink | 1 << storageFeatureSymLink | 1 << storageFeaturePathSync |
1 << storageFeatureInfoDetail;
this = storageNew(type, path, modeFile, modePath, write, pathExpressionFunction, driver, driver->interface); this = storageNew(type, path, modeFile, modePath, write, pathExpressionFunction, driver, driver->interface);
} }

51
src/storage/remote/protocol.c
View File

@ -24,15 +24,12 @@ Remote Storage Protocol Handler
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Constants Constants
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_EXISTS_STR, PROTOCOL_COMMAND_STORAGE_EXISTS);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_FEATURE_STR, PROTOCOL_COMMAND_STORAGE_FEATURE); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_FEATURE_STR, PROTOCOL_COMMAND_STORAGE_FEATURE);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_INFO_STR, PROTOCOL_COMMAND_STORAGE_INFO); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_INFO_STR, PROTOCOL_COMMAND_STORAGE_INFO);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR, PROTOCOL_COMMAND_STORAGE_INFO_LIST); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR, PROTOCOL_COMMAND_STORAGE_INFO_LIST);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_LIST_STR, PROTOCOL_COMMAND_STORAGE_LIST);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR, PROTOCOL_COMMAND_STORAGE_OPEN_READ); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR, PROTOCOL_COMMAND_STORAGE_OPEN_READ);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE_STR, PROTOCOL_COMMAND_STORAGE_OPEN_WRITE); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE_STR, PROTOCOL_COMMAND_STORAGE_OPEN_WRITE);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_CREATE_STR, PROTOCOL_COMMAND_STORAGE_PATH_CREATE); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_CREATE_STR, PROTOCOL_COMMAND_STORAGE_PATH_CREATE);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR, PROTOCOL_COMMAND_STORAGE_PATH_EXISTS);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE_STR, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_REMOVE_STR, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_SYNC_STR, PROTOCOL_COMMAND_STORAGE_PATH_SYNC); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_PATH_SYNC_STR, PROTOCOL_COMMAND_STORAGE_PATH_SYNC);
STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_REMOVE_STR, PROTOCOL_COMMAND_STORAGE_REMOVE); STRING_EXTERN(PROTOCOL_COMMAND_STORAGE_REMOVE_STR, PROTOCOL_COMMAND_STORAGE_REMOVE);
@ -134,6 +131,8 @@ storageRemoteInfoWriteType(ProtocolServer *server, StorageType type)
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
} }
// Helper to write storage info into the protocol. This function is not called unless the info exists so no need to write exists
// or check for level == storageInfoLevelExists.
static void static void
storageRemoteInfoWrite(ProtocolServer *server, const StorageInfo *info) storageRemoteInfoWrite(ProtocolServer *server, const StorageInfo *info)
{ {
@ -143,18 +142,22 @@ storageRemoteInfoWrite(ProtocolServer *server, const StorageInfo *info)
FUNCTION_TEST_END(); FUNCTION_TEST_END();
storageRemoteInfoWriteType(server, info->type); storageRemoteInfoWriteType(server, info->type);
protocolServerWriteLine(server, jsonFromUInt(info->userId));
protocolServerWriteLine(server, jsonFromStr(info->user));
protocolServerWriteLine(server, jsonFromUInt(info->groupId));
protocolServerWriteLine(server, jsonFromStr(info->group));
protocolServerWriteLine(server, jsonFromUInt(info->mode));
protocolServerWriteLine(server, jsonFromInt64(info->timeModified)); protocolServerWriteLine(server, jsonFromInt64(info->timeModified));
if (info->type == storageTypeFile) if (info->type == storageTypeFile)
protocolServerWriteLine(server, jsonFromUInt64(info->size)); protocolServerWriteLine(server, jsonFromUInt64(info->size));
if (info->type == storageTypeLink) if (info->level >= storageInfoLevelDetail)
protocolServerWriteLine(server, jsonFromStr(info->linkDestination)); {
protocolServerWriteLine(server, jsonFromUInt(info->userId));
protocolServerWriteLine(server, jsonFromStr(info->user));
protocolServerWriteLine(server, jsonFromUInt(info->groupId));
protocolServerWriteLine(server, jsonFromStr(info->group));
protocolServerWriteLine(server, jsonFromUInt(info->mode));
if (info->type == storageTypeLink)
protocolServerWriteLine(server, jsonFromStr(info->linkDestination));
}
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
} }
@ -199,11 +202,7 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
if (strEq(command, PROTOCOL_COMMAND_STORAGE_EXISTS_STR)) if (strEq(command, PROTOCOL_COMMAND_STORAGE_FEATURE_STR))
{
protocolServerResponse(server, VARBOOL(storageInterfaceExistsP(driver, varStr(varLstGet(paramList, 0)))));
}
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_FEATURE_STR))
{ {
protocolServerWriteLine(server, jsonFromStr(storagePathP(storage, NULL))); protocolServerWriteLine(server, jsonFromStr(storagePathP(storage, NULL)));
protocolServerWriteLine(server, jsonFromUInt64(interface.feature)); protocolServerWriteLine(server, jsonFromUInt64(interface.feature));
@ -213,7 +212,8 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_INFO_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_INFO_STR))
{ {
StorageInfo info = storageInterfaceInfoP( StorageInfo info = storageInterfaceInfoP(
driver, varStr(varLstGet(paramList, 0)), .followLink = varBool(varLstGet(paramList, 1))); driver, varStr(varLstGet(paramList, 0)), (StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)),
.followLink = varBool(varLstGet(paramList, 2)));
protocolServerResponse(server, VARBOOL(info.exists)); protocolServerResponse(server, VARBOOL(info.exists));
@ -226,20 +226,12 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR))
{ {
bool result = storageInterfaceInfoListP( bool result = storageInterfaceInfoListP(
driver, varStr(varLstGet(paramList, 0)), storageRemoteProtocolInfoListCallback, server); driver, varStr(varLstGet(paramList, 0)), (StorageInfoLevel)varUIntForce(varLstGet(paramList, 1)),
storageRemoteProtocolInfoListCallback, server);
protocolServerWriteLine(server, NULL); protocolServerWriteLine(server, NULL);
protocolServerResponse(server, VARBOOL(result)); protocolServerResponse(server, VARBOOL(result));
} }
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_LIST_STR))
{
protocolServerResponse(
server,
varNewVarLst(
varLstNewStrLst(
storageInterfaceListP(
driver, varStr(varLstGet(paramList, 0)), .expression = varStr(varLstGet(paramList, 1))))));
}
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR))
{ {
// Create the read object // Create the read object
@ -356,13 +348,6 @@ storageRemoteProtocol(const String *command, const VariantList *paramList, Proto
protocolServerResponse(server, NULL); protocolServerResponse(server, NULL);
} }
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR))
{
// Not all drivers implement pathExists()
CHECK(interface.pathExists != NULL);
protocolServerResponse(server, VARBOOL(storageInterfacePathExistsP(driver, varStr(varLstGet(paramList, 0)))));
}
else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE_STR)) else if (strEq(command, PROTOCOL_COMMAND_STORAGE_PATH_REMOVE_STR))
{ {
protocolServerResponse(server, protocolServerResponse(server,

6
src/storage/remote/protocol.h
View File

@ -13,24 +13,18 @@ Constants
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
#define PROTOCOL_BLOCK_HEADER "BRBLOCK" #define PROTOCOL_BLOCK_HEADER "BRBLOCK"
#define PROTOCOL_COMMAND_STORAGE_EXISTS "storageExists"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_EXISTS_STR);
#define PROTOCOL_COMMAND_STORAGE_FEATURE "storageFeature" #define PROTOCOL_COMMAND_STORAGE_FEATURE "storageFeature"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_FEATURE_STR); STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_FEATURE_STR);
#define PROTOCOL_COMMAND_STORAGE_INFO "storageInfo" #define PROTOCOL_COMMAND_STORAGE_INFO "storageInfo"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_INFO_STR); STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_INFO_STR);
#define PROTOCOL_COMMAND_STORAGE_INFO_LIST "storageInfoList" #define PROTOCOL_COMMAND_STORAGE_INFO_LIST "storageInfoList"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR); STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR);
#define PROTOCOL_COMMAND_STORAGE_LIST "storageList"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_LIST_STR);
#define PROTOCOL_COMMAND_STORAGE_OPEN_READ "storageOpenRead" #define PROTOCOL_COMMAND_STORAGE_OPEN_READ "storageOpenRead"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR); STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR);
#define PROTOCOL_COMMAND_STORAGE_OPEN_WRITE "storageOpenWrite" #define PROTOCOL_COMMAND_STORAGE_OPEN_WRITE "storageOpenWrite"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE_STR); STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE_STR);
#define PROTOCOL_COMMAND_STORAGE_PATH_CREATE "storagePathCreate" #define PROTOCOL_COMMAND_STORAGE_PATH_CREATE "storagePathCreate"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_PATH_CREATE_STR); STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_PATH_CREATE_STR);
#define PROTOCOL_COMMAND_STORAGE_PATH_EXISTS "storagePathExists"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR);
#define PROTOCOL_COMMAND_STORAGE_REMOVE "storageRemove" #define PROTOCOL_COMMAND_STORAGE_REMOVE "storageRemove"
STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_REMOVE_STR); STRING_DECLARE(PROTOCOL_COMMAND_STORAGE_REMOVE_STR);
#define PROTOCOL_COMMAND_STORAGE_PATH_REMOVE "storagePathRemove" #define PROTOCOL_COMMAND_STORAGE_PATH_REMOVE "storagePathRemove"

122
src/storage/remote/storage.c
View File

@ -29,35 +29,6 @@ struct StorageRemote
unsigned int compressLevel; // Protocol compression level unsigned int compressLevel; // Protocol compression level
}; };
/**********************************************************************************************************************************/
static bool
storageRemoteExists(THIS_VOID, const String *file, StorageInterfaceExistsParam param)
{
THIS(StorageRemote);
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_REMOTE, this);
FUNCTION_LOG_PARAM(STRING, file);
(void)param; // No parameters are used
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(file != NULL);
bool result = false;
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_EXISTS_STR);
protocolCommandParamAdd(command, VARSTR(file));
result = varBool(protocolClientExecute(this->client, command, true));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
// Helper to convert protocol storage type to an enum // Helper to convert protocol storage type to an enum
static StorageType static StorageType
@ -95,41 +66,47 @@ storageRemoteInfoParse(ProtocolClient *client, StorageInfo *info)
FUNCTION_TEST_END(); FUNCTION_TEST_END();
info->type = storageRemoteInfoParseType(strPtr(protocolClientReadLine(client))[0]); info->type = storageRemoteInfoParseType(strPtr(protocolClientReadLine(client))[0]);
info->userId = jsonToUInt(protocolClientReadLine(client));
info->user = jsonToStr(protocolClientReadLine(client));
info->groupId = jsonToUInt(protocolClientReadLine(client));
info->group = jsonToStr(protocolClientReadLine(client));
info->mode = jsonToUInt(protocolClientReadLine(client));
info->timeModified = (time_t)jsonToUInt64(protocolClientReadLine(client)); info->timeModified = (time_t)jsonToUInt64(protocolClientReadLine(client));
if (info->type == storageTypeFile) if (info->type == storageTypeFile)
info->size = jsonToUInt64(protocolClientReadLine(client)); info->size = jsonToUInt64(protocolClientReadLine(client));
if (info->type == storageTypeLink) if (info->level >= storageInfoLevelDetail)
info->linkDestination = jsonToStr(protocolClientReadLine(client)); {
info->userId = jsonToUInt(protocolClientReadLine(client));
info->user = jsonToStr(protocolClientReadLine(client));
info->groupId = jsonToUInt(protocolClientReadLine(client));
info->group = jsonToStr(protocolClientReadLine(client));
info->mode = jsonToUInt(protocolClientReadLine(client));
if (info->type == storageTypeLink)
info->linkDestination = jsonToStr(protocolClientReadLine(client));
}
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
} }
static StorageInfo static StorageInfo
storageRemoteInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param) storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param)
{ {
THIS(StorageRemote); THIS(StorageRemote);
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_REMOTE, this); FUNCTION_LOG_PARAM(STORAGE_REMOTE, this);
FUNCTION_LOG_PARAM(STRING, file); FUNCTION_LOG_PARAM(STRING, file);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(BOOL, param.followLink); FUNCTION_LOG_PARAM(BOOL, param.followLink);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
StorageInfo result = {.exists = false}; StorageInfo result = {.level = level};
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_STR); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_STR);
protocolCommandParamAdd(command, VARSTR(file)); protocolCommandParamAdd(command, VARSTR(file));
protocolCommandParamAdd(command, VARUINT(level));
protocolCommandParamAdd(command, VARBOOL(param.followLink)); protocolCommandParamAdd(command, VARBOOL(param.followLink));
result.exists = varBool(protocolClientExecute(this->client, command, true)); result.exists = varBool(protocolClientExecute(this->client, command, true));
@ -161,13 +138,15 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static bool static bool
storageRemoteInfoList( storageRemoteInfoList(
THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
{ {
THIS(StorageRemote); THIS(StorageRemote);
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_REMOTE, this); FUNCTION_LOG_PARAM(STORAGE_REMOTE, this);
FUNCTION_LOG_PARAM(STRING, path); FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData); FUNCTION_LOG_PARAM_P(VOID, callbackData);
(void)param; // No parameters are used (void)param; // No parameters are used
@ -183,6 +162,7 @@ storageRemoteInfoList(
{ {
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR); ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR);
protocolCommandParamAdd(command, VARSTR(path)); protocolCommandParamAdd(command, VARSTR(path));
protocolCommandParamAdd(command, VARUINT(level));
// Send command // Send command
protocolClientWriteCommand(this->client, command); protocolClientWriteCommand(this->client, command);
@ -195,7 +175,7 @@ storageRemoteInfoList(
while (strSize(name) != 0) while (strSize(name) != 0)
{ {
StorageInfo info = {.exists = true, .name = jsonToStr(name)}; StorageInfo info = {.exists = true, .level = level, .name = jsonToStr(name)};
storageRemoteInfoParse(this->client, &info); storageRemoteInfoParse(this->client, &info);
callback(callbackData, &info); callback(callbackData, &info);
@ -217,36 +197,6 @@ storageRemoteInfoList(
FUNCTION_LOG_RETURN(BOOL, result); FUNCTION_LOG_RETURN(BOOL, result);
} }
/**********************************************************************************************************************************/
static StringList *
storageRemoteList(THIS_VOID, const String *path, StorageInterfaceListParam param)
{
THIS(StorageRemote);
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_REMOTE, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(STRING, param.expression);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(path != NULL);
StringList *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_LIST_STR);
protocolCommandParamAdd(command, VARSTR(path));
protocolCommandParamAdd(command, VARSTR(param.expression));
result = strLstMove(strLstNewVarLst(varVarLst(protocolClientExecute(this->client, command, true))), memContextPrior());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING_LIST, result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static StorageRead * static StorageRead *
storageRemoteNewRead(THIS_VOID, const String *file, bool ignoreMissing, StorageInterfaceNewReadParam param) storageRemoteNewRead(THIS_VOID, const String *file, bool ignoreMissing, StorageInterfaceNewReadParam param)
@ -338,35 +288,6 @@ storageRemotePathCreate(
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }
/**********************************************************************************************************************************/
static bool
storageRemotePathExists(THIS_VOID, const String *path, StorageInterfacePathExistsParam param)
{
THIS(StorageRemote);
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_REMOTE, this);
FUNCTION_LOG_PARAM(STRING, path);
(void)param; // No parameters are used
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(path != NULL);
bool result = false;
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR);
protocolCommandParamAdd(command, VARSTR(path));
result = varBool(protocolClientExecute(this->client, command, true));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static bool static bool
storageRemotePathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfacePathRemoveParam param) storageRemotePathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfacePathRemoveParam param)
@ -456,14 +377,11 @@ storageRemoteRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam p
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static const StorageInterface storageInterfaceRemote = static const StorageInterface storageInterfaceRemote =
{ {
.exists = storageRemoteExists,
.info = storageRemoteInfo, .info = storageRemoteInfo,
.infoList = storageRemoteInfoList, .infoList = storageRemoteInfoList,
.list = storageRemoteList,
.newRead = storageRemoteNewRead, .newRead = storageRemoteNewRead,
.newWrite = storageRemoteNewWrite, .newWrite = storageRemoteNewWrite,
.pathCreate = storageRemotePathCreate, .pathCreate = storageRemotePathCreate,
.pathExists = storageRemotePathExists,
.pathRemove = storageRemotePathRemove, .pathRemove = storageRemotePathRemove,
.pathSync = storageRemotePathSync, .pathSync = storageRemotePathSync,
.remove = storageRemoteRemove, .remove = storageRemoteRemove,

131
src/storage/s3/storage.c
View File

@ -422,9 +422,6 @@ storageS3ListInternal(
{ {
const String *continuationToken = NULL; const String *continuationToken = NULL;
// Prepare regexp if an expression was passed
RegExp *regExp = expression == NULL ? NULL : regExpNew(expression);
// Build the base prefix by stripping off the initial / // Build the base prefix by stripping off the initial /
const String *basePrefix; const String *basePrefix;
@ -490,9 +487,8 @@ storageS3ListInternal(
// Strip off base prefix and final / // Strip off base prefix and final /
subPath = strSubN(subPath, strSize(basePrefix), strSize(subPath) - strSize(basePrefix) - 1); subPath = strSubN(subPath, strSize(basePrefix), strSize(subPath) - strSize(basePrefix) - 1);
// Add to list after checking expression if present // Add to list
if (regExp == NULL || regExpMatch(regExp, subPath)) callback(this, callbackData, subPath, storageTypePath, subPathNode);
callback(this, callbackData, subPath, storageTypePath, subPathNode);
} }
// Get file list // Get file list
@ -508,9 +504,8 @@ storageS3ListInternal(
// Strip off the base prefix when present // Strip off the base prefix when present
file = strEmpty(basePrefix) ? file : strSub(file, strSize(basePrefix)); file = strEmpty(basePrefix) ? file : strSub(file, strSize(basePrefix));
// Add to list after checking expression if present // Add to list
if (regExp == NULL || regExpMatch(regExp, file)) callback(this, callbackData, file, storageTypeFile, fileNode);
callback(this, callbackData, file, storageTypeFile, fileNode);
} }
// Get the continuation token and store it in the outer temp context // Get the continuation token and store it in the outer temp context
@ -529,56 +524,31 @@ storageS3ListInternal(
FUNCTION_LOG_RETURN_VOID(); FUNCTION_LOG_RETURN_VOID();
} }
/**********************************************************************************************************************************/
static bool
storageS3Exists(THIS_VOID, const String *file, StorageInterfaceExistsParam param)
{
THIS(StorageS3);
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_S3, this);
FUNCTION_LOG_PARAM(STRING, file);
(void)param; // No parameters are used
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(file != NULL);
bool result = false;
MEM_CONTEXT_TEMP_BEGIN()
{
result = httpClientResponseCodeOk(storageS3Request(this, HTTP_VERB_HEAD_STR, file, NULL, NULL, true, true).httpClient);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static StorageInfo static StorageInfo
storageS3Info(THIS_VOID, const String *file, StorageInterfaceInfoParam param) storageS3Info(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param)
{ {
THIS(StorageS3); THIS(StorageS3);
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_S3, this); FUNCTION_LOG_PARAM(STORAGE_S3, this);
FUNCTION_LOG_PARAM(STRING, file); FUNCTION_LOG_PARAM(STRING, file);
FUNCTION_LOG_PARAM(ENUM, level);
(void)param; // No parameters are used (void)param; // No parameters are used
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(file != NULL); ASSERT(file != NULL);
StorageInfo result = {0};
// Attempt to get file info // Attempt to get file info
StorageS3RequestResult httpResult = storageS3Request(this, HTTP_VERB_HEAD_STR, file, NULL, NULL, true, true); StorageS3RequestResult httpResult = storageS3Request(this, HTTP_VERB_HEAD_STR, file, NULL, NULL, true, true);
// On success load info into a structure // Does the file exist?
if (httpClientResponseCodeOk(httpResult.httpClient)) StorageInfo result = {.level = level, .exists = httpClientResponseCodeOk(httpResult.httpClient)};
// Add basic level info if requested and the file exists
if (result.level >= storageInfoLevelBasic && result.exists)
{ {
result.exists = true;
result.type = storageTypeFile; result.type = storageTypeFile;
result.size = cvtZToUInt64(strPtr(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_CONTENT_LENGTH_STR))); result.size = cvtZToUInt64(strPtr(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_CONTENT_LENGTH_STR)));
result.timeModified = httpLastModifiedToTime(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_LAST_MODIFIED_STR)); result.timeModified = httpLastModifiedToTime(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_LAST_MODIFIED_STR));
@ -590,6 +560,7 @@ storageS3Info(THIS_VOID, const String *file, StorageInterfaceInfoParam param)
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
typedef struct StorageS3InfoListData typedef struct StorageS3InfoListData
{ {
StorageInfoLevel level; // Level of info to set
StorageInfoListCallback callback; // User-supplied callback function StorageInfoListCallback callback; // User-supplied callback function
void *callbackData; // User-supplied callback data void *callbackData; // User-supplied callback data
} StorageS3InfoListData; } StorageS3InfoListData;
@ -630,13 +601,20 @@ storageS3InfoListCallback(StorageS3 *this, void *callbackData, const String *nam
StorageInfo info = StorageInfo info =
{ {
.type = type,
.name = name, .name = name,
.size = type == storageTypeFile ? cvtZToUInt64(strPtr(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_SIZE_STR, true)))) : 0, .level = data->level,
.timeModified = type == storageTypeFile ? .exists = true,
storageS3CvtTime(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_LAST_MODIFIED_STR, true))) : 0,
}; };
if (data->level >= storageInfoLevelBasic)
{
info.type = type;
info.size = type == storageTypeFile ?
cvtZToUInt64(strPtr(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_SIZE_STR, true)))) : 0;
info.timeModified = type == storageTypeFile ?
storageS3CvtTime(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_LAST_MODIFIED_STR, true))) : 0;
}
data->callback(data->callbackData, &info); data->callback(data->callbackData, &info);
FUNCTION_TEST_RETURN_VOID(); FUNCTION_TEST_RETURN_VOID();
@ -644,16 +622,18 @@ storageS3InfoListCallback(StorageS3 *this, void *callbackData, const String *nam
static bool static bool
storageS3InfoList( storageS3InfoList(
THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
{ {
THIS(StorageS3); THIS(StorageS3);
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_S3, this); FUNCTION_LOG_PARAM(STORAGE_S3, this);
FUNCTION_LOG_PARAM(STRING, path); FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData); FUNCTION_LOG_PARAM_P(VOID, callbackData);
(void)param; // No parameters are used FUNCTION_LOG_PARAM(STRING, param.expression);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
@ -662,65 +642,14 @@ storageS3InfoList(
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
StorageS3InfoListData data = {.callback = callback, .callbackData = callbackData}; StorageS3InfoListData data = {.level = level, .callback = callback, .callbackData = callbackData};
storageS3ListInternal(this, path, false, false, storageS3InfoListCallback, &data); storageS3ListInternal(this, path, param.expression, false, storageS3InfoListCallback, &data);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, true); FUNCTION_LOG_RETURN(BOOL, true);
} }
/**********************************************************************************************************************************/
static void
storageS3ListCallback(StorageS3 *this, void *callbackData, const String *name, StorageType type, const XmlNode *xml)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_S3, this);
FUNCTION_TEST_PARAM_P(VOID, callbackData);
FUNCTION_TEST_PARAM(STRING, name);
FUNCTION_TEST_PARAM(ENUM, type);
FUNCTION_TEST_PARAM(XML_NODE, xml);
FUNCTION_TEST_END();
(void)this;
ASSERT(callbackData != NULL);
ASSERT(name != NULL);
(void)type;
(void)xml;
strLstAdd((StringList *)callbackData, name);
FUNCTION_TEST_RETURN_VOID();
}
static StringList *
storageS3List(THIS_VOID, const String *path, StorageInterfaceListParam param)
{
THIS(StorageS3);
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_S3, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(STRING, param.expression);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(path != NULL);
StringList *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
result = strLstNew();
storageS3ListInternal(this, path, param.expression, false, storageS3ListCallback, result);
strLstMove(result, memContextPrior());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING_LIST, result);
}
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static StorageRead * static StorageRead *
storageS3NewRead(THIS_VOID, const String *file, bool ignoreMissing, StorageInterfaceNewReadParam param) storageS3NewRead(THIS_VOID, const String *file, bool ignoreMissing, StorageInterfaceNewReadParam param)
@ -909,10 +838,8 @@ storageS3Remove(THIS_VOID, const String *file, StorageInterfaceRemoveParam param
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static const StorageInterface storageInterfaceS3 = static const StorageInterface storageInterfaceS3 =
{ {
.exists = storageS3Exists,
.info = storageS3Info, .info = storageS3Info,
.infoList = storageS3InfoList, .infoList = storageS3InfoList,
.list = storageS3List,
.newRead = storageS3NewRead, .newRead = storageS3NewRead,
.newWrite = storageS3NewWrite, .newWrite = storageS3NewWrite,
.pathRemove = storageS3PathRemove, .pathRemove = storageS3PathRemove,

112
src/storage/storage.c
View File

@ -55,9 +55,7 @@ storageNew(
ASSERT(type != NULL); ASSERT(type != NULL);
ASSERT(strSize(path) >= 1 && strPtr(path)[0] == '/'); ASSERT(strSize(path) >= 1 && strPtr(path)[0] == '/');
ASSERT(driver != NULL); ASSERT(driver != NULL);
ASSERT(interface.exists != NULL);
ASSERT(interface.info != NULL); ASSERT(interface.info != NULL);
ASSERT(interface.list != NULL);
ASSERT(interface.infoList != NULL); ASSERT(interface.infoList != NULL);
ASSERT(interface.newRead != NULL); ASSERT(interface.newRead != NULL);
ASSERT(interface.newWrite != NULL); ASSERT(interface.newWrite != NULL);
@ -153,17 +151,19 @@ storageExists(const Storage *this, const String *pathExp, StorageExistsParam par
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Build the path to the file
String *path = storagePathP(this, pathExp);
// Create Wait object of timeout > 0 // Create Wait object of timeout > 0
Wait *wait = param.timeout != 0 ? waitNew(param.timeout) : NULL; Wait *wait = param.timeout != 0 ? waitNew(param.timeout) : NULL;
// Loop until file exists or timeout // Loop until file exists or timeout
do do
{ {
// Call driver function // storageInfoLevelBasic is required here because storageInfoLevelExists will not return the type and this function
result = storageInterfaceExistsP(this->driver, path); // specifically wants to test existence of a *file*, not just the existence of anything with the specified name.
StorageInfo info = storageInfoP(
this, pathExp, .level = storageInfoLevelBasic, .ignoreMissing = true, .followLink = true);
// Only exists if it is a file
result = info.exists && info.type == storageTypeFile;
} }
while (!result && wait != NULL && waitMore(wait)); while (!result && wait != NULL && waitMore(wait));
} }
@ -239,6 +239,7 @@ storageInfo(const Storage *this, const String *fileExp, StorageInfoParam param)
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE, this); FUNCTION_LOG_PARAM(STORAGE, this);
FUNCTION_LOG_PARAM(STRING, fileExp); FUNCTION_LOG_PARAM(STRING, fileExp);
FUNCTION_LOG_PARAM(ENUM, param.level);
FUNCTION_LOG_PARAM(BOOL, param.ignoreMissing); FUNCTION_LOG_PARAM(BOOL, param.ignoreMissing);
FUNCTION_LOG_PARAM(BOOL, param.followLink); FUNCTION_LOG_PARAM(BOOL, param.followLink);
FUNCTION_LOG_PARAM(BOOL, param.noPathEnforce); FUNCTION_LOG_PARAM(BOOL, param.noPathEnforce);
@ -255,7 +256,11 @@ storageInfo(const Storage *this, const String *fileExp, StorageInfoParam param)
String *file = storagePathP(this, fileExp, .noEnforce = param.noPathEnforce); String *file = storagePathP(this, fileExp, .noEnforce = param.noPathEnforce);
// Call driver function // Call driver function
result = storageInterfaceInfoP(this->driver, file, .followLink = param.followLink); if (param.level == storageInfoLevelDefault)
param.level = storageFeature(this, storageFeatureInfoDetail) ? storageInfoLevelDetail : storageInfoLevelBasic;
result = storageInterfaceInfoP(
this->driver, file, param.level, .followLink = param.followLink);
// Error if the file missing and not ignoring // Error if the file missing and not ignoring
if (!result.exists && !param.ignoreMissing) if (!result.exists && !param.ignoreMissing)
@ -311,11 +316,14 @@ storageInfoListSortCallback(void *data, const StorageInfo *info)
static bool static bool
storageInfoListSort( storageInfoListSort(
const Storage *this, const String *path, SortOrder sortOrder, StorageInfoListCallback callback, void *callbackData) const Storage *this, const String *path, StorageInfoLevel level, const String *expression, SortOrder sortOrder,
StorageInfoListCallback callback, void *callbackData)
{ {
FUNCTION_LOG_BEGIN(logLevelTrace); FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE, this); FUNCTION_LOG_PARAM(STORAGE, this);
FUNCTION_LOG_PARAM(STRING, path); FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(STRING, expression);
FUNCTION_LOG_PARAM(ENUM, sortOrder); FUNCTION_LOG_PARAM(ENUM, sortOrder);
FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData); FUNCTION_LOG_PARAM_P(VOID, callbackData);
@ -331,7 +339,7 @@ storageInfoListSort(
// If no sorting then use the callback directly // If no sorting then use the callback directly
if (sortOrder == sortOrderNone) if (sortOrder == sortOrderNone)
{ {
result = storageInterfaceInfoListP(this->driver, path, callback, callbackData); result = storageInterfaceInfoListP(this->driver, path, level, callback, callbackData, .expression = expression);
} }
// Else sort the info before sending it to the callback // Else sort the info before sending it to the callback
else else
@ -343,7 +351,8 @@ storageInfoListSort(
.infoList = lstNewP(sizeof(StorageInfo), .comparator = lstComparatorStr), .infoList = lstNewP(sizeof(StorageInfo), .comparator = lstComparatorStr),
}; };
result = storageInterfaceInfoListP(this->driver, path, storageInfoListSortCallback, &data); result = storageInterfaceInfoListP(
this->driver, path, level, storageInfoListSortCallback, &data, .expression = expression);
lstSort(data.infoList, sortOrder); lstSort(data.infoList, sortOrder);
MEM_CONTEXT_TEMP_RESET_BEGIN() MEM_CONTEXT_TEMP_RESET_BEGIN()
@ -370,7 +379,8 @@ typedef struct StorageInfoListData
const Storage *storage; // Storage object; const Storage *storage; // Storage object;
StorageInfoListCallback callbackFunction; // Original callback function StorageInfoListCallback callbackFunction; // Original callback function
void *callbackData; // Original callback data void *callbackData; // Original callback data
RegExp *expression; // Filter for names const String *expression; // Filter for names
RegExp *regExp; // Compiled filter for names
bool recurse; // Should we recurse? bool recurse; // Should we recurse?
SortOrder sortOrder; // Sort order SortOrder sortOrder; // Sort order
const String *path; // Top-level path for info const String *path; // Top-level path for info
@ -404,7 +414,7 @@ storageInfoListCallback(void *data, const StorageInfo *info)
infoUpdate.name = strNewFmt("%s/%s", strPtr(listData->subPath), strPtr(infoUpdate.name)); infoUpdate.name = strNewFmt("%s/%s", strPtr(listData->subPath), strPtr(infoUpdate.name));
// Only continue if there is no expression or the expression matches // Only continue if there is no expression or the expression matches
if (listData->expression == NULL || regExpMatch(listData->expression, infoUpdate.name)) if (listData->expression == NULL || regExpMatch(listData->regExp, infoUpdate.name))
{ {
if (listData->sortOrder != sortOrderDesc) if (listData->sortOrder != sortOrderDesc)
listData->callbackFunction(listData->callbackData, &infoUpdate); listData->callbackFunction(listData->callbackData, &infoUpdate);
@ -416,8 +426,8 @@ storageInfoListCallback(void *data, const StorageInfo *info)
data.subPath = infoUpdate.name; data.subPath = infoUpdate.name;
storageInfoListSort( storageInfoListSort(
data.storage, strNewFmt("%s/%s", strPtr(data.path), strPtr(data.subPath)), data.sortOrder, storageInfoListCallback, data.storage, strNewFmt("%s/%s", strPtr(data.path), strPtr(data.subPath)), infoUpdate.level, data.expression,
&data); data.sortOrder, storageInfoListCallback, &data);
} }
if (listData->sortOrder == sortOrderDesc) if (listData->sortOrder == sortOrderDesc)
@ -436,6 +446,7 @@ storageInfoList(
FUNCTION_LOG_PARAM(STRING, pathExp); FUNCTION_LOG_PARAM(STRING, pathExp);
FUNCTION_LOG_PARAM(FUNCTIONP, callback); FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData); FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_PARAM(ENUM, param.level);
FUNCTION_LOG_PARAM(BOOL, param.errorOnMissing); FUNCTION_LOG_PARAM(BOOL, param.errorOnMissing);
FUNCTION_LOG_PARAM(ENUM, param.sortOrder); FUNCTION_LOG_PARAM(ENUM, param.sortOrder);
FUNCTION_LOG_PARAM(STRING, param.expression); FUNCTION_LOG_PARAM(STRING, param.expression);
@ -451,6 +462,10 @@ storageInfoList(
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Info level
if (param.level == storageInfoLevelDefault)
param.level = storageFeature(this, storageFeatureInfoDetail) ? storageInfoLevelDetail : storageInfoLevelBasic;
// Build the path // Build the path
String *path = storagePathP(this, pathExp); String *path = storagePathP(this, pathExp);
@ -462,18 +477,20 @@ storageInfoList(
.storage = this, .storage = this,
.callbackFunction = callback, .callbackFunction = callback,
.callbackData = callbackData, .callbackData = callbackData,
.expression = param.expression,
.sortOrder = param.sortOrder, .sortOrder = param.sortOrder,
.recurse = param.recurse, .recurse = param.recurse,
.path = path, .path = path,
}; };
if (param.expression != NULL) if (data.expression != NULL)
data.expression = regExpNew(param.expression); data.regExp = regExpNew(param.expression);
result = storageInfoListSort(this, path, param.sortOrder, storageInfoListCallback, &data); result = storageInfoListSort(
this, path, param.level, param.expression, param.sortOrder, storageInfoListCallback, &data);
} }
else else
result = storageInfoListSort(this, path, param.sortOrder, callback, callbackData); result = storageInfoListSort(this, path, param.level, NULL, param.sortOrder, callback, callbackData);
if (!result && param.errorOnMissing) if (!result && param.errorOnMissing)
THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, strPtr(path)); THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, strPtr(path));
@ -484,6 +501,26 @@ storageInfoList(
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
static void
storageListCallback(void *data, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM_P(VOID, data);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
// Skip . path
if (strEq(info->name, DOT_STR))
{
FUNCTION_TEST_RETURN_VOID();
return;
}
strLstAdd((StringList *)data, info->name);
FUNCTION_TEST_RETURN_VOID();
}
StringList * StringList *
storageList(const Storage *this, const String *pathExp, StorageListParam param) storageList(const Storage *this, const String *pathExp, StorageListParam param)
{ {
@ -503,23 +540,16 @@ storageList(const Storage *this, const String *pathExp, StorageListParam param)
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Build the path result = strLstNew();
String *path = storagePathP(this, pathExp);
// Get the list // Build an empty list if the directory does not exist by default. This makes the logic in calling functions simpler when
result = storageInterfaceListP(this->driver, path, .expression = param.expression); // the caller doesn't care if the path is missing.
if (!storageInfoListP(
// If the path does not exist this, pathExp, storageListCallback, result, .level = storageInfoLevelExists, .errorOnMissing = param.errorOnMissing,
if (result == NULL) .expression = param.expression))
{ {
// Error if requested if (param.nullOnMissing)
if (param.errorOnMissing) result = NULL;
THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_MISSING, strPtr(path));
// Build an empty list if the directory does not exist by default. This makes the logic in calling functions simpler
// when they don't care if the path is missing.
if (!param.nullOnMissing)
result = strLstNew();
} }
// Move list up to the old context // Move list up to the old context
@ -781,17 +811,13 @@ storagePathExists(const Storage *this, const String *pathExp)
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(this != NULL); ASSERT(this != NULL);
ASSERT(this->interface.pathExists != NULL); ASSERT(storageFeature(this, storageFeaturePath));
bool result = false; // storageInfoLevelBasic is required here because storageInfoLevelExists will not return the type and this function specifically
// wants to test existence of a *path*, not just the existence of anything with the specified name.
StorageInfo info = storageInfoP(this, pathExp, .level = storageInfoLevelBasic, .ignoreMissing = true, .followLink = true);
MEM_CONTEXT_TEMP_BEGIN() FUNCTION_LOG_RETURN(BOOL, info.exists && info.type == storageTypePath);
{
result = storageInterfacePathExistsP(this->driver, storagePathP(this, pathExp));
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result);
} }
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/

5
src/storage/storage.h
View File

@ -50,6 +50,9 @@ typedef enum
// Does the storage support symlinks? Symlinks allow paths/files/links to be accessed from another path. // Does the storage support symlinks? Symlinks allow paths/files/links to be accessed from another path.
storageFeatureSymLink, storageFeatureSymLink,
// Does the storage support detailed info, i.e. user, group, mode, link destination, etc.
storageFeatureInfoDetail,
} StorageFeature; } StorageFeature;
/*********************************************************************************************************************************** /***********************************************************************************************************************************
@ -89,6 +92,7 @@ Buffer *storageGet(StorageRead *file, StorageGetParam param);
typedef struct StorageInfoParam typedef struct StorageInfoParam
{ {
VAR_PARAM_HEADER; VAR_PARAM_HEADER;
StorageInfoLevel level;
bool ignoreMissing; bool ignoreMissing;
bool followLink; bool followLink;
bool noPathEnforce; bool noPathEnforce;
@ -105,6 +109,7 @@ typedef void (*StorageInfoListCallback)(void *callbackData, const StorageInfo *i
typedef struct StorageInfoListParam typedef struct StorageInfoListParam
{ {
VAR_PARAM_HEADER; VAR_PARAM_HEADER;
StorageInfoLevel level;
bool errorOnMissing; bool errorOnMissing;
bool recurse; bool recurse;
SortOrder sortOrder; SortOrder sortOrder;

77
src/storage/storage.intern.h
View File

@ -25,9 +25,6 @@ Error messages
#define STORAGE_ERROR_INFO "unable to get info for path/file '%s'" #define STORAGE_ERROR_INFO "unable to get info for path/file '%s'"
#define STORAGE_ERROR_INFO_MISSING "unable to get info for missing path/file '%s'" #define STORAGE_ERROR_INFO_MISSING "unable to get info for missing path/file '%s'"
#define STORAGE_ERROR_LIST "unable to list files for path '%s'"
#define STORAGE_ERROR_LIST_MISSING "unable to list files for missing path '%s'"
#define STORAGE_ERROR_LIST_INFO "unable to list file info for path '%s'" #define STORAGE_ERROR_LIST_INFO "unable to list file info for path '%s'"
#define STORAGE_ERROR_LIST_INFO_MISSING "unable to list file info for missing path '%s'" #define STORAGE_ERROR_LIST_INFO_MISSING "unable to list file info for missing path '%s'"
@ -53,19 +50,12 @@ typedef String *StoragePathExpressionCallback(const String *expression, const St
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Required interface functions Required interface functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Does a file exist? This function is only for files, not paths. // Get information about a file/link/path
typedef struct StorageInterfaceExistsParam //
{ // The level parameter controls the amount of information that will be returned. See the StorageInfo type and StorageInfoLevel enum
VAR_PARAM_HEADER; // for details about information that must be provided at each level. The driver should only return the amount of information
} StorageInterfaceExistsParam; // requested even if more is available. All drivers must implement the storageInfoLevelExists and storageInfoLevelBasic levels. Only
// drivers with the storageFeatureInfoDetail feature must implement the storageInfoLevelDetail level.
typedef bool StorageInterfaceExists(void *thisVoid, const String *file, StorageInterfaceExistsParam param);
#define storageInterfaceExistsP(thisVoid, file, ...) \
STORAGE_COMMON_INTERFACE(thisVoid).exists(thisVoid, file, (StorageInterfaceExistsParam){VAR_PARAM_INIT, __VA_ARGS__})
// ---------------------------------------------------------------------------------------------------------------------------------
// Get information about a file
typedef struct StorageInterfaceInfoParam typedef struct StorageInterfaceInfoParam
{ {
VAR_PARAM_HEADER; VAR_PARAM_HEADER;
@ -74,25 +64,11 @@ typedef struct StorageInterfaceInfoParam
bool followLink; bool followLink;
} StorageInterfaceInfoParam; } StorageInterfaceInfoParam;
typedef StorageInfo StorageInterfaceInfo(void *thisVoid, const String *file, StorageInterfaceInfoParam param); typedef StorageInfo StorageInterfaceInfo(
void *thisVoid, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param);
#define storageInterfaceInfoP(thisVoid, file, ...) \ #define storageInterfaceInfoP(thisVoid, file, level, ...) \
STORAGE_COMMON_INTERFACE(thisVoid).info(thisVoid, file, (StorageInterfaceInfoParam){VAR_PARAM_INIT, __VA_ARGS__}) STORAGE_COMMON_INTERFACE(thisVoid).info(thisVoid, file, level, (StorageInterfaceInfoParam){VAR_PARAM_INIT, __VA_ARGS__})
// ---------------------------------------------------------------------------------------------------------------------------------
// Get a list of files
typedef struct StorageInterfaceListParam
{
VAR_PARAM_HEADER;
// Regular expression used to filter the results
const String *expression;
} StorageInterfaceListParam;
typedef StringList *StorageInterfaceList(void *thisVoid, const String *path, StorageInterfaceListParam param);
#define storageInterfaceListP(thisVoid, path, ...) \
STORAGE_COMMON_INTERFACE(thisVoid).list(thisVoid, path, (StorageInterfaceListParam){VAR_PARAM_INIT, __VA_ARGS__})
// --------------------------------------------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------------------------------------------
// Create a file read object. The file should not be opened immediately -- open() will be called on the IoRead interface when the // Create a file read object. The file should not be opened immediately -- open() will be called on the IoRead interface when the
@ -157,17 +133,29 @@ typedef StorageWrite *StorageInterfaceNewWrite(void *thisVoid, const String *fil
// --------------------------------------------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------------------------------------------
// Get info for a path and all paths/files in the path (does not recurse) // Get info for a path and all paths/files in the path (does not recurse)
//
// See storageInterfaceInfoP() for usage of the level parameter.
typedef struct StorageInterfaceInfoListParam typedef struct StorageInterfaceInfoListParam
{ {
VAR_PARAM_HEADER; VAR_PARAM_HEADER;
// Regular expression used to filter the results. The expression is always checked in the callback passed to
// storageInterfaceInfoListP() so checking the expression in the driver is entirely optional. The driver should only use the
// expression if it can improve performance or limit network transfer.
//
// Partial matching of the expression is fine as long as nothing that should match is excluded, e.g. it is OK to prefix match
// using the prefix returned from regExpPrefix(). This may cause extra results to be sent to the callback but won't exclude
// anything that matches the expression exactly.
const String *expression;
} StorageInterfaceInfoListParam; } StorageInterfaceInfoListParam;
typedef bool StorageInterfaceInfoList( typedef bool StorageInterfaceInfoList(
void *thisVoid, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param); void *thisVoid, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param);
#define storageInterfaceInfoListP(thisVoid, path, callback, callbackData, ...) \ #define storageInterfaceInfoListP(thisVoid, path, level, callback, callbackData, ...) \
STORAGE_COMMON_INTERFACE(thisVoid).infoList( \ STORAGE_COMMON_INTERFACE(thisVoid).infoList( \
thisVoid, path, callback, callbackData, (StorageInterfaceInfoListParam){VAR_PARAM_INIT, __VA_ARGS__}) thisVoid, path, level, callback, callbackData, (StorageInterfaceInfoListParam){VAR_PARAM_INIT, __VA_ARGS__})
// --------------------------------------------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------------------------------------------
// Remove a path (and optionally recurse) // Remove a path (and optionally recurse)
@ -227,18 +215,6 @@ typedef void StorageInterfacePathCreate(
STORAGE_COMMON_INTERFACE(thisVoid).pathCreate( \ STORAGE_COMMON_INTERFACE(thisVoid).pathCreate( \
thisVoid, path, errorOnExists, noParentCreate, mode, (StorageInterfacePathCreateParam){VAR_PARAM_INIT, __VA_ARGS__}) thisVoid, path, errorOnExists, noParentCreate, mode, (StorageInterfacePathCreateParam){VAR_PARAM_INIT, __VA_ARGS__})
// ---------------------------------------------------------------------------------------------------------------------------------
// Does a path exist?
typedef struct StorageInterfacePathExistsParam
{
VAR_PARAM_HEADER;
} StorageInterfacePathExistsParam;
typedef bool StorageInterfacePathExists(void *thisVoid, const String *path, StorageInterfacePathExistsParam param);
#define storageInterfacePathExistsP(thisVoid, path, ...) \
STORAGE_COMMON_INTERFACE(thisVoid).pathExists(thisVoid, path, (StorageInterfacePathExistsParam){VAR_PARAM_INIT, __VA_ARGS__})
// --------------------------------------------------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------------------------------------------------
// Sync a path // Sync a path
typedef struct StorageInterfacePathSyncParam typedef struct StorageInterfacePathSyncParam
@ -260,10 +236,8 @@ typedef struct StorageInterface
uint64_t feature; uint64_t feature;
// Required functions // Required functions
StorageInterfaceExists *exists;
StorageInterfaceInfo *info; StorageInterfaceInfo *info;
StorageInterfaceInfoList *infoList; StorageInterfaceInfoList *infoList;
StorageInterfaceList *list;
StorageInterfaceNewRead *newRead; StorageInterfaceNewRead *newRead;
StorageInterfaceNewWrite *newWrite; StorageInterfaceNewWrite *newWrite;
StorageInterfacePathRemove *pathRemove; StorageInterfacePathRemove *pathRemove;
@ -272,7 +246,6 @@ typedef struct StorageInterface
// Optional functions // Optional functions
StorageInterfaceMove *move; StorageInterfaceMove *move;
StorageInterfacePathCreate *pathCreate; StorageInterfacePathCreate *pathCreate;
StorageInterfacePathExists *pathExists;
StorageInterfacePathSync *pathSync; StorageInterfacePathSync *pathSync;
} StorageInterface; } StorageInterface;

4
test/define.yaml
View File

@ -409,6 +409,8 @@ unit:
storage/posix/read: full storage/posix/read: full
storage/posix/storage: full storage/posix/storage: full
storage/posix/write: full storage/posix/write: full
# Provide as much coverage as possible for these modules but some coverage needs to be provided by other driver tests
storage/helper: full storage/helper: full
storage/read: full storage/read: full
storage/storage: full storage/storage: full
@ -416,7 +418,7 @@ unit:
# ---------------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------------
- name: remote - name: remote
total: 12 total: 9
containerReq: true containerReq: true
binReq: true binReq: true

132
test/src/common/harnessStorage.c
View File

@ -21,92 +21,96 @@ hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info)
strCatFmt(data->content, "%s {", strPtr(info->name)); strCatFmt(data->content, "%s {", strPtr(info->name));
switch (info->type) if (info->level > storageInfoLevelExists)
{ {
case storageTypeFile: switch (info->type)
{ {
strCat(data->content, "file"); case storageTypeFile:
if (!data->sizeOmit)
{ {
uint64_t size = info->size; strCat(data->content, "file");
// If the file is compressed then decompress to get the real size. Note that only gz is used in unit tests since if (info->level >= storageInfoLevelBasic && !data->sizeOmit)
// it is the only compression type guaranteed to be present.
if (data->fileCompressed)
{ {
ASSERT(data->storage != NULL); uint64_t size = info->size;
StorageRead *read = storageNewReadP( // If the file is compressed then decompress to get the real size. Note that only gz is used in unit tests since
data->storage, // it is the only compression type guaranteed to be present.
data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name); if (data->fileCompressed)
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), decompressFilter(compressTypeGz)); {
size = bufUsed(storageGetP(read)); ASSERT(data->storage != NULL);
StorageRead *read = storageNewReadP(
data->storage,
data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name);
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), decompressFilter(compressTypeGz));
size = bufUsed(storageGetP(read));
}
strCatFmt(data->content, ", s=%" PRIu64, size);
} }
strCatFmt(data->content, ", s=%" PRIu64, size); break;
} }
break; case storageTypeLink:
}
case storageTypeLink:
{
strCatFmt(data->content, "link, d=%s", strPtr(info->linkDestination));
break;
}
case storageTypePath:
{
strCat(data->content, "path");
break;
}
case storageTypeSpecial:
{
strCat(data->content, "special");
break;
}
}
if (info->type != storageTypeSpecial)
{
if (info->type != storageTypeLink)
{
if (!data->modeOmit || (info->type == storageTypePath && data->modePath != info->mode) ||
(info->type == storageTypeFile && data->modeFile != info->mode))
{ {
strCatFmt(data->content, ", m=%04o", info->mode); strCatFmt(data->content, "link, d=%s", strPtr(info->linkDestination));
break;
}
case storageTypePath:
{
strCat(data->content, "path");
break;
}
case storageTypeSpecial:
{
strCat(data->content, "special");
break;
} }
} }
if (info->type == storageTypeFile) if (info->type != storageTypeSpecial)
{ {
if (!data->timestampOmit) if (info->type != storageTypeLink)
strCatFmt(data->content, ", t=%" PRIu64, (uint64_t)info->timeModified); {
} if (info->level >= storageInfoLevelDetail &&
(!data->modeOmit || (info->type == storageTypePath && data->modePath != info->mode) ||
(info->type == storageTypeFile && data->modeFile != info->mode)))
{
strCatFmt(data->content, ", m=%04o", info->mode);
}
}
if (!data->userOmit || userId() != info->userId) if (info->type == storageTypeFile && info->level >= storageInfoLevelBasic)
{
if (info->user != NULL)
{ {
strCatFmt(data->content, ", u=%s", strPtr(info->user)); if (!data->timestampOmit)
strCatFmt(data->content, ", t=%" PRIu64, (uint64_t)info->timeModified);
} }
else
{
strCatFmt(data->content, ", u=%d", (int)info->userId);
}
}
if (!data->groupOmit || groupId() != info->groupId) if (info->level >= storageInfoLevelDetail && (!data->userOmit || userId() != info->userId))
{
if (info->group != NULL)
{ {
strCatFmt(data->content, ", g=%s", strPtr(info->group)); if (info->user != NULL)
{
strCatFmt(data->content, ", u=%s", strPtr(info->user));
}
else
{
strCatFmt(data->content, ", u=%d", (int)info->userId);
}
} }
else
if (info->level >= storageInfoLevelDetail && (!data->groupOmit || groupId() != info->groupId))
{ {
strCatFmt(data->content, ", g=%d", (int)info->groupId); if (info->group != NULL)
{
strCatFmt(data->content, ", g=%s", strPtr(info->group));
}
else
{
strCatFmt(data->content, ", g=%d", (int)info->groupId);
}
} }
} }
} }

2
test/src/module/command/archiveGetTest.c
View File

@ -258,7 +258,7 @@ testRun(void)
TEST_ERROR_FMT( TEST_ERROR_FMT(
queueNeed(strNew("000000010000000100000001"), false, queueSize, walSegmentSize, PG_VERSION_92), queueNeed(strNew("000000010000000100000001"), false, queueSize, walSegmentSize, PG_VERSION_92),
PathMissingError, "unable to list files for missing path '%s/spool/archive/test1/in'", testPath()); PathMissingError, "unable to list file info for missing path '%s/spool/archive/test1/in'", testPath());
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
storagePathCreateP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN)); storagePathCreateP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN));

2
test/src/module/command/controlTest.c
View File

@ -92,7 +92,7 @@ testRun(void)
TEST_RESULT_VOID(storageRemoveP(storageData, strNew("lockpath/all" STOP_FILE_EXT)), "remove stop file"); TEST_RESULT_VOID(storageRemoveP(storageData, strNew("lockpath/all" STOP_FILE_EXT)), "remove stop file");
TEST_RESULT_INT(system(strPtr(strNewFmt("chmod 444 %s", strPtr(lockPath)))), 0, "change perms"); TEST_RESULT_INT(system(strPtr(strNewFmt("chmod 444 %s", strPtr(lockPath)))), 0, "change perms");
TEST_ERROR_FMT( TEST_ERROR_FMT(
cmdStop(), FileOpenError, "unable to stat '%s/all.stop': [13] Permission denied", strPtr(lockPath)); cmdStop(), FileOpenError, "unable to get info for path/file '%s/all.stop': [13] Permission denied", strPtr(lockPath));
TEST_RESULT_INT(system(strPtr(strNewFmt("chmod 700 %s", strPtr(lockPath)))), 0, "change perms"); TEST_RESULT_INT(system(strPtr(strNewFmt("chmod 700 %s", strPtr(lockPath)))), 0, "change perms");
TEST_RESULT_VOID( TEST_RESULT_VOID(
storagePathRemoveP(storageData, lockPath, .recurse = true, .errorOnMissing = true), " remove the lock path"); storagePathRemoveP(storageData, lockPath, .recurse = true, .errorOnMissing = true), " remove the lock path");

2
test/src/module/config/parseTest.c
View File

@ -179,7 +179,7 @@ testRun(void)
TEST_ERROR( TEST_ERROR(
cfgFileLoad(parseOptionList, backupCmdDefConfigValue, cfgFileLoad(parseOptionList, backupCmdDefConfigValue,
backupCmdDefConfigInclPathValue, oldConfigDefault), PathMissingError, backupCmdDefConfigInclPathValue, oldConfigDefault), PathMissingError,
"unable to list files for missing path '/BOGUS'"); "unable to list file info for missing path '/BOGUS'");
// --config-include-path valid, --config invalid (does not exist) // --config-include-path valid, --config invalid (does not exist)
value = strLstNew(); value = strLstNew();

28
test/src/module/performance/storageTest.c
View File

@ -29,29 +29,18 @@ stress testing as needed.
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Dummy functions and interface for constructing test drivers Dummy functions and interface for constructing test drivers
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
static bool
storageTestDummyExists(THIS_VOID, const String *file, StorageInterfaceExistsParam param)
{
(void)thisVoid; (void)file; (void)param; return false;
}
static StorageInfo static StorageInfo
storageTestDummyInfo(THIS_VOID, const String *file, StorageInterfaceInfoParam param) storageTestDummyInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param)
{ {
(void)thisVoid; (void)file; (void)param; return (StorageInfo){.exists = false}; (void)thisVoid; (void)file; (void)level; (void)param; return (StorageInfo){.exists = false};
} }
static bool static bool
storageTestDummyInfoList( storageTestDummyInfoList(
THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
{ {
(void)thisVoid; (void)path; (void)callback; (void)callbackData; (void)param; return false; (void)thisVoid; (void)path; (void)level; (void)callback; (void)callbackData; (void)param; return false;
}
static StringList *
storageTestDummyList(THIS_VOID, const String *path, StorageInterfaceListParam param)
{
(void)thisVoid; (void)path; (void)param; return NULL;
} }
static StorageRead * static StorageRead *
@ -80,10 +69,8 @@ storageTestDummyRemove(THIS_VOID, const String *file, StorageInterfaceRemovePara
static const StorageInterface storageInterfaceTestDummy = static const StorageInterface storageInterfaceTestDummy =
{ {
.exists = storageTestDummyExists,
.info = storageTestDummyInfo, .info = storageTestDummyInfo,
.infoList = storageTestDummyInfoList, .infoList = storageTestDummyInfoList,
.list = storageTestDummyList,
.newRead = storageTestDummyNewRead, .newRead = storageTestDummyNewRead,
.newWrite = storageTestDummyNewWrite, .newWrite = storageTestDummyNewWrite,
.pathRemove = storageTestDummyPathRemove, .pathRemove = storageTestDummyPathRemove,
@ -114,10 +101,11 @@ typedef struct
static bool static bool
storageTestPerfInfoList( storageTestPerfInfoList(
THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData, StorageInterfaceInfoListParam param) THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
{ {
THIS(StorageTestPerfInfoList); THIS(StorageTestPerfInfoList);
(void)path; (void)param; (void)path; (void)level; (void)param;
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {

25
test/src/module/storage/posixTest.c
View File

@ -102,7 +102,7 @@ testRun(void)
TEST_RESULT_BOOL(storageTest->write, true, " check write"); TEST_RESULT_BOOL(storageTest->write, true, " check write");
TEST_RESULT_BOOL(storageTest->pathExpressionFunction != NULL, true, " check expression function is set"); TEST_RESULT_BOOL(storageTest->pathExpressionFunction != NULL, true, " check expression function is set");
TEST_RESULT_PTR(storageInterface(storageTest).exists, storageTest->interface.exists, " check interface"); TEST_RESULT_PTR(storageInterface(storageTest).info, storageTest->interface.info, " check interface");
TEST_RESULT_PTR(storageDriver(storageTest), storageTest->driver, " check driver"); TEST_RESULT_PTR(storageDriver(storageTest), storageTest->driver, " check driver");
TEST_RESULT_PTR(storageType(storageTest), storageTest->type, " check type"); TEST_RESULT_PTR(storageType(storageTest), storageTest->type, " check type");
TEST_RESULT_BOOL(storageFeature(storageTest, storageFeaturePath), true, " check path feature"); TEST_RESULT_BOOL(storageFeature(storageTest, storageFeaturePath), true, " check path feature");
@ -127,16 +127,19 @@ testRun(void)
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR_FMT( TEST_ERROR_FMT(
storageExistsP(storageTest, fileNoPerm), FileOpenError, storageExistsP(storageTest, fileNoPerm), FileOpenError,
"unable to stat '%s': [13] Permission denied", strPtr(fileNoPerm)); "unable to get info for path/file '%s': [13] Permission denied", strPtr(fileNoPerm));
TEST_ERROR_FMT( TEST_ERROR_FMT(
storagePathExistsP(storageTest, fileNoPerm), PathOpenError, storagePathExistsP(storageTest, fileNoPerm), FileOpenError,
"unable to stat '%s': [13] Permission denied", strPtr(fileNoPerm)); "unable to get info for path/file '%s': [13] Permission denied", strPtr(fileNoPerm));
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
String *fileExists = strNewFmt("%s/exists", testPath()); String *fileExists = strNewFmt("%s/exists", testPath());
String *pathExists = strNewFmt("%s/pathExists", testPath());
TEST_RESULT_INT(system(strPtr(strNewFmt("touch %s", strPtr(fileExists)))), 0, "create exists file"); TEST_RESULT_INT(system(strPtr(strNewFmt("touch %s", strPtr(fileExists)))), 0, "create exists file");
TEST_SYSTEM_FMT("mkdir %s", strPtr(pathExists));
TEST_RESULT_BOOL(storageExistsP(storageTest, fileExists), true, "file exists"); TEST_RESULT_BOOL(storageExistsP(storageTest, fileExists), true, "file exists");
TEST_RESULT_BOOL(storageExistsP(storageTest, pathExists), false, "not a file");
TEST_RESULT_BOOL(storagePathExistsP(storageTest, fileExists), false, "not a path"); TEST_RESULT_BOOL(storagePathExistsP(storageTest, fileExists), false, "not a path");
TEST_RESULT_INT(system(strPtr(strNewFmt("sudo rm %s", strPtr(fileExists)))), 0, "remove exists file"); TEST_RESULT_INT(system(strPtr(strNewFmt("sudo rm %s", strPtr(fileExists)))), 0, "remove exists file");
@ -304,7 +307,7 @@ testRun(void)
TEST_RESULT_VOID( TEST_RESULT_VOID(
storagePosixInfoListEntry( storagePosixInfoListEntry(
(StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"), (StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"), storageInfoLevelBasic,
hrnStorageInfoListCallback, &callbackData), hrnStorageInfoListCallback, &callbackData),
"missing path"); "missing path");
TEST_RESULT_STR_Z(callbackData.content, "", " check content"); TEST_RESULT_STR_Z(callbackData.content, "", " check content");
@ -395,7 +398,7 @@ testRun(void)
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR_FMT( TEST_ERROR_FMT(
storageListP(storageTest, strNew(BOGUS_STR), .errorOnMissing = true), PathMissingError, STORAGE_ERROR_LIST_MISSING, storageListP(storageTest, strNew(BOGUS_STR), .errorOnMissing = true), PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING,
strPtr(strNewFmt("%s/BOGUS", testPath()))); strPtr(strNewFmt("%s/BOGUS", testPath())));
TEST_RESULT_PTR(storageListP(storageTest, strNew(BOGUS_STR), .nullOnMissing = true), NULL, "null for missing dir"); TEST_RESULT_PTR(storageListP(storageTest, strNew(BOGUS_STR), .nullOnMissing = true), NULL, "null for missing dir");
@ -404,12 +407,12 @@ testRun(void)
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR_FMT( TEST_ERROR_FMT(
storageListP(storageTest, pathNoPerm), PathOpenError, storageListP(storageTest, pathNoPerm), PathOpenError,
STORAGE_ERROR_LIST ": [13] Permission denied", strPtr(pathNoPerm)); STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathNoPerm));
// Should still error even when ignore missing // Should still error even when ignore missing
TEST_ERROR_FMT( TEST_ERROR_FMT(
storageListP(storageTest, pathNoPerm), PathOpenError, storageListP(storageTest, pathNoPerm), PathOpenError,
STORAGE_ERROR_LIST ": [13] Permission denied", strPtr(pathNoPerm)); STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathNoPerm));
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID( TEST_RESULT_VOID(
@ -464,7 +467,7 @@ testRun(void)
TEST_ERROR_FMT( TEST_ERROR_FMT(
storageMoveP(storageTest, source, destination), FileMissingError, storageMoveP(storageTest, source, destination), FileMissingError,
"unable to move missing file '%s': [2] No such file or directory", strPtr(sourceFile)); "unable to move missing source '%s': [2] No such file or directory", strPtr(sourceFile));
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
source = storageNewReadP(storageTest, fileNoPerm); source = storageNewReadP(storageTest, fileNoPerm);
@ -623,14 +626,14 @@ testRun(void)
strPtr(pathRemove2)); strPtr(pathRemove2));
TEST_ERROR_FMT( TEST_ERROR_FMT(
storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError, storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError,
STORAGE_ERROR_LIST ": [13] Permission denied", strPtr(pathRemove2)); STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathRemove2));
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_INT(system(strPtr(strNewFmt("sudo chmod 777 %s", strPtr(pathRemove1)))), 0, "top path can be removed"); TEST_RESULT_INT(system(strPtr(strNewFmt("sudo chmod 777 %s", strPtr(pathRemove1)))), 0, "top path can be removed");
TEST_ERROR_FMT( TEST_ERROR_FMT(
storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError, storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError,
STORAGE_ERROR_LIST ": [13] Permission denied", strPtr(pathRemove2)); STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathRemove2));
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
String *fileRemove = strNewFmt("%s/remove.txt", strPtr(pathRemove2)); String *fileRemove = strNewFmt("%s/remove.txt", strPtr(pathRemove2));

122
test/src/module/storage/remoteTest.c
View File

@ -83,34 +83,6 @@ testRun(void)
TEST_RESULT_BOOL(storageRemoteProtocol(strNew(BOGUS_STR), varLstNew(), server), false, "invalid function"); TEST_RESULT_BOOL(storageRemoteProtocol(strNew(BOGUS_STR), varLstNew(), server), false, "invalid function");
} }
// Do these tests against a pg remote for coverage
// *****************************************************************************************************************************
if (testBegin("storageExists()"))
{
Storage *storageRemote = NULL;
TEST_ASSIGN(storageRemote, storagePgGet(1, false), "get remote pg storage");
storagePathCreateP(storageTest, strNew("pg"));
TEST_RESULT_BOOL(storageExistsP(storageRemote, strNew("test.txt")), false, "file does not exist");
storagePutP(storageNewWriteP(storageTest, strNew("repo/test.txt")), BUFSTRDEF("TEST"));
TEST_RESULT_BOOL(storageExistsP(storageRemote, strNew("test.txt")), true, "file exists");
// Check protocol function directly
// -------------------------------------------------------------------------------------------------------------------------
cfgOptionSet(cfgOptRemoteType, cfgSourceParam, VARSTRDEF("pg"));
cfgOptionValidSet(cfgOptRemoteType, true);
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test.txt", testPath())));
TEST_RESULT_BOOL(
storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_EXISTS_STR, paramList, server), true, "protocol exists");
TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":true}\n", "check result");
bufUsedSet(serverWrite, 0);
}
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("storageInfo()")) if (testBegin("storageInfo()"))
{ {
@ -232,13 +204,13 @@ testRun(void)
bufUsedSet(serverWrite, 0); bufUsedSet(serverWrite, 0);
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("protocol output that is not tested elsewhere"); TEST_TITLE("protocol output that is not tested elsewhere (basic)");
info = (StorageInfo){.type = storageTypeLink, .linkDestination = STRDEF("../")}; info = (StorageInfo){.level = storageInfoLevelDetail, .type = storageTypeLink, .linkDestination = STRDEF("../")};
TEST_RESULT_VOID(storageRemoteInfoWrite(server, &info), "write link info"); TEST_RESULT_VOID(storageRemoteInfoWrite(server, &info), "write link info");
ioWriteFlush(serverWriteIo); ioWriteFlush(serverWriteIo);
TEST_RESULT_STR_Z(strNewBuf(serverWrite), ".l\n.0\n.null\n.0\n.null\n.0\n.0\n.\"../\"\n", "check result"); TEST_RESULT_STR_Z(strNewBuf(serverWrite), ".l\n.0\n.0\n.null\n.0\n.null\n.0\n.\"../\"\n", "check result");
bufUsedSet(serverWrite, 0); bufUsedSet(serverWrite, 0);
@ -247,6 +219,7 @@ testRun(void)
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStrZ(BOGUS_STR)); varLstAdd(paramList, varNewStrZ(BOGUS_STR));
varLstAdd(paramList, varNewUInt(storageInfoLevelBasic));
varLstAdd(paramList, varNewBool(false)); varLstAdd(paramList, varNewBool(false));
TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_STR, paramList, server), true, "protocol list"); TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_STR, paramList, server), true, "protocol list");
@ -255,10 +228,16 @@ testRun(void)
bufUsedSet(serverWrite, 0); bufUsedSet(serverWrite, 0);
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("check protocol function directly with a file"); TEST_TITLE("check protocol function directly with a file (basic level)");
// Do these tests against pg for coverage. We're not really going to get a pg remote here because the remote for this host
// id has already been created. This will just test that the pg storage is selected to provide coverage.
cfgOptionSet(cfgOptRemoteType, cfgSourceParam, VARSTRDEF("pg"));
cfgOptionValidSet(cfgOptRemoteType, true);
paramList = varLstNew(); paramList = varLstNew();
varLstAdd(paramList, varNewStrZ(hrnReplaceKey("{[path]}/repo/test"))); varLstAdd(paramList, varNewStrZ(hrnReplaceKey("{[path]}/repo/test")));
varLstAdd(paramList, varNewUInt(storageInfoLevelBasic));
varLstAdd(paramList, varNewBool(false)); varLstAdd(paramList, varNewBool(false));
TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_STR, paramList, server), true, "protocol list"); TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_STR, paramList, server), true, "protocol list");
@ -266,7 +245,26 @@ testRun(void)
strNewBuf(serverWrite), strNewBuf(serverWrite),
hrnReplaceKey( hrnReplaceKey(
"{\"out\":true}\n" "{\"out\":true}\n"
".f\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.416\n.1555160001\n.6\n" ".f\n.1555160001\n.6\n"
"{}\n"),
"check result");
bufUsedSet(serverWrite, 0);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("check protocol function directly with a file (detail level)");
paramList = varLstNew();
varLstAdd(paramList, varNewStrZ(hrnReplaceKey("{[path]}/repo/test")));
varLstAdd(paramList, varNewUInt(storageInfoLevelDetail));
varLstAdd(paramList, varNewBool(false));
TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_STR, paramList, server), true, "protocol list");
TEST_RESULT_STR_Z(
strNewBuf(serverWrite),
hrnReplaceKey(
"{\"out\":true}\n"
".f\n.1555160001\n.6\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.416\n"
"{}\n"), "{}\n"),
"check result"); "check result");
@ -319,13 +317,14 @@ testRun(void)
VariantList *paramList = varLstNew(); VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStrZ(hrnReplaceKey("{[path]}/repo"))); varLstAdd(paramList, varNewStrZ(hrnReplaceKey("{[path]}/repo")));
varLstAdd(paramList, varNewUInt(storageInfoLevelDetail));
TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR, paramList, server), true, "call protocol"); TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_INFO_LIST_STR, paramList, server), true, "call protocol");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
strNewBuf(serverWrite), strNewBuf(serverWrite),
hrnReplaceKey( hrnReplaceKey(
".\".\"\n.p\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.488\n.1555160000\n" ".\".\"\n.p\n.1555160000\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.488\n"
".\"test\"\n.f\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.416\n.1555160001\n.6\n" ".\"test\"\n.f\n.1555160001\n.6\n.{[user-id]}\n.\"{[user]}\"\n.{[group-id]}\n.\"{[group]}\"\n.416\n"
".\n" ".\n"
"{\"out\":true}\n"), "{\"out\":true}\n"),
"check result"); "check result");
@ -333,37 +332,6 @@ testRun(void)
bufUsedSet(serverWrite, 0); bufUsedSet(serverWrite, 0);
} }
// *****************************************************************************************************************************
if (testBegin("storageList()"))
{
Storage *storageRemote = NULL;
TEST_ASSIGN(storageRemote, storageRepoGet(strNew(STORAGE_TYPE_POSIX), false), "get remote repo storage");
storagePathCreateP(storageTest, strNew("repo"));
TEST_RESULT_PTR(storageListP(storageRemote, strNew(BOGUS_STR), .nullOnMissing = true), NULL, "null for missing dir");
TEST_RESULT_UINT(strLstSize(storageListP(storageRemote, NULL)), 0, "empty list for missing dir");
// -------------------------------------------------------------------------------------------------------------------------
storagePathCreateP(storageTest, strNew("repo/testy"));
TEST_RESULT_STR_Z(strLstJoin(storageListP(storageRemote, NULL), ","), "testy" , "list path");
storagePathCreateP(storageTest, strNew("repo/testy2\""));
TEST_RESULT_STR_Z(
strLstJoin(strLstSort(storageListP(storageRemote, strNewFmt("%s/repo", testPath())), sortOrderAsc), ","),
"testy,testy2\"" , "list 2 paths");
// Check protocol function directly
// -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNewFmt("%s/repo", testPath())));
varLstAdd(paramList, varNewStr(strNew("^testy$")));
TEST_RESULT_BOOL(storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_LIST_STR, paramList, server), true, "protocol list");
TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":[\"testy\"]}\n", "check result");
bufUsedSet(serverWrite, 0);
}
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("storageNewRead()")) if (testBegin("storageNewRead()"))
{ {
@ -650,28 +618,6 @@ testRun(void)
strNewBuf(storageGetP(storageNewReadP(storageTest, strNew("repo/test4.txt.pgbackrest.tmp")))), "", "check file"); strNewBuf(storageGetP(storageNewReadP(storageTest, strNew("repo/test4.txt.pgbackrest.tmp")))), "", "check file");
} }
// *****************************************************************************************************************************
if (testBegin("storagePathExists()"))
{
Storage *storageRemote = NULL;
TEST_ASSIGN(storageRemote, storageRepoGet(strNew(STORAGE_TYPE_POSIX), false), "get remote repo storage");
storagePathCreateP(storageTest, strNew("repo"));
TEST_RESULT_BOOL(storagePathExistsP(storageRemote, strNew("missing")), false, "path does not exist");
TEST_RESULT_BOOL(storagePathExistsP(storageRemote, NULL), true, "path exists");
// Check protocol function directly
// -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNewFmt("%s/repo/test.txt", testPath())));
TEST_RESULT_BOOL(
storageRemoteProtocol(PROTOCOL_COMMAND_STORAGE_PATH_EXISTS_STR, paramList, server), true, "protocol path exists");
TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":false}\n", "check result");
bufUsedSet(serverWrite, 0);
}
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("storagePathCreate()")) if (testBegin("storagePathCreate()"))
{ {

121
test/src/module/storage/s3Test.c
View File

@ -4,6 +4,7 @@ Test S3 Storage
#include <unistd.h> #include <unistd.h>
#include "common/harnessConfig.h" #include "common/harnessConfig.h"
#include "common/harnessStorage.h"
#include "common/harnessTls.h" #include "common/harnessTls.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
@ -200,7 +201,11 @@ testS3Server(void)
// File exists // File exists
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL, storageS3UriStyleHost)); harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:999", NULL)); harnessTlsServerReply(testS3ServerResponse(
200, "OK",
"content-length:999\r\n"
"Last-Modified: Wed, 21 Oct 2015 07:28:00 GMT",
NULL));
// Info() // Info()
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
@ -216,6 +221,14 @@ testS3Server(void)
"Last-Modified: Wed, 21 Oct 2015 07:28:00 GMT", "Last-Modified: Wed, 21 Oct 2015 07:28:00 GMT",
NULL)); NULL));
// File exists and only checking existence
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file2.txt", NULL, storageS3UriStyleHost));
harnessTlsServerReply(testS3ServerResponse(
200, "OK",
"content-length:777\r\n"
"Last-Modified: Wed, 22 Oct 2015 07:28:00 GMT",
NULL));
// InfoList() // InfoList()
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
harnessTlsServerExpect( harnessTlsServerExpect(
@ -506,24 +519,6 @@ testS3Server(void)
} }
} }
/***********************************************************************************************************************************
Callback and data for storageInfoList() tests
***********************************************************************************************************************************/
unsigned int testStorageInfoListSize = 0;
StorageInfo testStorageInfoList[256];
void
testStorageInfoListCallback(void *callbackData, const StorageInfo *info)
{
MEM_CONTEXT_BEGIN((MemContext *)callbackData)
{
testStorageInfoList[testStorageInfoListSize] = *info;
testStorageInfoList[testStorageInfoListSize].name = strDup(info->name);
testStorageInfoListSize++;
}
MEM_CONTEXT_END();
}
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Test Run Test Run
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
@ -847,26 +842,37 @@ testRun(void)
TEST_RESULT_UINT(info.size, 9999, " check exists"); TEST_RESULT_UINT(info.size, 9999, " check exists");
TEST_RESULT_INT(info.timeModified, 1445412480, " check time"); TEST_RESULT_INT(info.timeModified, 1445412480, " check time");
// InfoList() TEST_TITLE("file exists and only checking existence");
TEST_ASSIGN(info, storageInfoP(s3, strNew("subdir/file2.txt"), .level = storageInfoLevelExists), "file exists");
TEST_RESULT_BOOL(info.exists, true, " check exists");
TEST_RESULT_UINT(info.type, storageTypeFile, " check type");
TEST_RESULT_UINT(info.size, 0, " check exists");
TEST_RESULT_INT(info.timeModified, 0, " check time");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("list basic level");
HarnessStorageInfoListCallbackData callbackData =
{
.content = strNew(""),
};
TEST_ERROR( TEST_ERROR(
storageInfoListP(s3, strNew("/"), testStorageInfoListCallback, NULL, .errorOnMissing = true), storageInfoListP(s3, strNew("/"), hrnStorageInfoListCallback, NULL, .errorOnMissing = true),
AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed"); AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
TEST_RESULT_VOID( TEST_RESULT_VOID(
storageInfoListP(s3, strNew("/path/to"), testStorageInfoListCallback, (void *)memContextCurrent()), "info list files"); storageInfoListP(s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData), "info list files");
TEST_RESULT_STR_Z(
callbackData.content,
"test_path {path}\n"
"test_file {file, s=787, t=1255369830}\n",
" check content");
TEST_RESULT_UINT(testStorageInfoListSize, 2, " file and path returned");
TEST_RESULT_STR_Z(testStorageInfoList[0].name, "test_path", " check name");
TEST_RESULT_UINT(testStorageInfoList[0].size, 0, " check size");
TEST_RESULT_UINT(testStorageInfoList[0].type, storageTypePath, " check type");
TEST_RESULT_STR_Z(testStorageInfoList[1].name, "test_file", " check name");
TEST_RESULT_UINT(testStorageInfoList[1].size, 787, " check size");
TEST_RESULT_INT(testStorageInfoList[1].timeModified, 1255369830, " check time");
TEST_RESULT_UINT(testStorageInfoList[1].type, storageTypeFile, " check type");
// storageDriverList()
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("various errors");
TEST_ERROR( TEST_ERROR(
storageListP(s3, strNew("/"), .errorOnMissing = true), AssertError, storageListP(s3, strNew("/"), .errorOnMissing = true), AssertError,
"assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed"); "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
@ -910,16 +916,55 @@ testRun(void)
"<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>RequestTimeTooSkewed</Code>" "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Error><Code>RequestTimeTooSkewed</Code>"
"<Message>The difference between the request time and the current time is too large.</Message></Error>"); "<Message>The difference between the request time and the current time is too large.</Message></Error>");
TEST_RESULT_STR_Z(strLstJoin(storageListP(s3, strNew("/")), ","), "path1,test1.txt", "list a file/path in root"); // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("list exists level");
callbackData.content = strNew("");
TEST_RESULT_VOID(
storageInfoListP(s3, strNew("/"), hrnStorageInfoListCallback, &callbackData, .level = storageInfoLevelExists),
"list a file/path in root");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
strLstJoin(storageListP(s3, strNew("/"), .expression = strNew("^test.*$")), ","), "test1.txt", callbackData.content,
"path1 {}\n"
"test1.txt {}\n",
" check content");
callbackData.content = strNew("");
TEST_RESULT_VOID(
storageInfoListP(
s3, strNew("/"), hrnStorageInfoListCallback, &callbackData, .expression = strNew("^test.*$"),
.level = storageInfoLevelExists),
"list a file in root with expression"); "list a file in root with expression");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
strLstJoin(storageListP(s3, strNew("/path/to")), ","), callbackData.content,
"path1,test1.txt,test2.txt,path2,test3.txt", "list files with continuation"); "test1.txt {}\n",
" check content");
callbackData.content = strNew("");
TEST_RESULT_VOID(
storageInfoListP(s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData, .level = storageInfoLevelExists),
"list files with continuation");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
strLstJoin(storageListP(s3, strNew("/path/to"), .expression = strNew("^test(1|3)")), ","), callbackData.content,
"test1.path,test1.txt,test3.txt", "list files with expression"); "path1 {}\n"
"test1.txt {}\n"
"test2.txt {}\n"
"path2 {}\n"
"test3.txt {}\n",
" check content");
callbackData.content = strNew("");
TEST_RESULT_VOID(
storageInfoListP(
s3, strNew("/path/to"), hrnStorageInfoListCallback, &callbackData, .expression = strNew("^test(1|3)"),
.level = storageInfoLevelExists),
"list files with expression");
TEST_RESULT_STR_Z(
callbackData.content,
"test1.path {}\n"
"test1.txt {}\n"
"test3.txt {}\n",
" check content");
// storageDriverPathRemove() // storageDriverPathRemove()
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------