db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-14 10:13:05 +02:00
Code Issues Releases Activity

Add sorting, filters, and recursion to storageInfoList().

Browse Source 
These are needed for the ls command and are also useful for testing.
This commit is contained in:
David Steele 2019-09-12 16:03:05 -04:00
parent 92365fb801
commit e45baa1830
6 changed files with 390 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Makefile.in
View File

@ -597,7 +597,7 @@ storage/s3/storage.o: storage/s3/storage.c build.auto.h common/assert.h common/c
storage/s3/write.o: storage/s3/write.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/type/xml.h storage/info.h storage/read.h storage/read.intern.h storage/s3/storage.h storage/s3/storage.intern.h storage/s3/write.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/s3/write.c -o storage/s3/write.o
storage/storage.o: storage/storage.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h storage/info.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
storage/storage.o: storage/storage.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/read.intern.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/regExp.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h storage/info.h storage/read.h storage/read.intern.h storage/storage.h storage/storage.intern.h storage/write.h storage/write.intern.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c storage/storage.c -o storage/storage.o
storage/write.o: storage/write.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/write.h common/io/write.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h storage/write.h storage/write.intern.h version.h

178
src/storage/storage.c
View File

@ -8,9 +8,11 @@ Storage Interface
#include "common/debug.h"
#include "common/io/io.h"
#include "common/type/list.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/object.h"
#include "common/regExp.h"
#include "common/wait.h"
#include "storage/storage.intern.h"
@ -272,6 +274,157 @@ storageInfo(const Storage *this, const String *fileExp, StorageInfoParam param)
/***********************************************************************************************************************************
Info for all files/paths in a path
***********************************************************************************************************************************/
typedef struct StorageInfoListSortData
{
MemContext *memContext; // Mem context to use for allocating data in this struct
StringList *ownerList; // List of users and groups to reduce memory usage
List *infoList; // List of info
} StorageInfoListSortData;
static void
storageInfoListSortCallback(void *data, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM_P(VOID, data);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
StorageInfoListSortData *infoData = data;
MEM_CONTEXT_BEGIN(infoData->memContext)
{
// Copy info and dup strings
StorageInfo infoCopy = *info;
infoCopy.name = strDup(info->name);
infoCopy.linkDestination = strDup(info->linkDestination);
infoCopy.user = strLstAddIfMissing(infoData->ownerList, info->user);
infoCopy.group = strLstAddIfMissing(infoData->ownerList, info->group);
lstAdd(infoData->infoList, &infoCopy);
}
MEM_CONTEXT_END();
FUNCTION_TEST_RETURN_VOID();
}
static bool
storageInfoListSort(
const Storage *this, const String *path, SortOrder sortOrder, StorageInfoListCallback callback, void *callbackData)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, sortOrder);
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(callback != NULL);
bool result = false;
MEM_CONTEXT_TEMP_BEGIN()
{
// If no sorting then use the callback directly
if (sortOrder == sortOrderNone)
{
result = this->interface.infoList(this->driver, path, callback, callbackData);
}
// Else sort the info before sending it to the callback
else
{
StorageInfoListSortData data =
{
.memContext = MEM_CONTEXT_TEMP(),
.ownerList = strLstNew(),
.infoList = lstNewP(sizeof(StorageInfo), .comparator = lstComparatorStr),
};
result = this->interface.infoList(this->driver, path, storageInfoListSortCallback, &data);
lstSort(data.infoList, sortOrder);
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
for (unsigned int infoIdx = 0; infoIdx < lstSize(data.infoList); infoIdx++)
{
// Pass info to the caller
callback(callbackData, lstGet(data.infoList, infoIdx));
// Reset the memory context occasionally
MEM_CONTEXT_TEMP_RESET(1000);
}
}
MEM_CONTEXT_TEMP_END();
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result);
}
typedef struct StorageInfoListData
{
const Storage *storage; // Storage object;
StorageInfoListCallback callbackFunction; // Original callback function
void *callbackData; // Original callback data
RegExp *expression; // Filter for names
bool recurse; // Should we recurse?
SortOrder sortOrder; // Sort order
const String *path; // Top-level path for info
const String *subPath; // Path below the top-level path (starts as NULL)
} StorageInfoListData;
static void
storageInfoListCallback(void *data, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM_P(VOID, data);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
StorageInfoListData *listData = data;
// Is this the . path?
bool dotPath = info->type == storageTypePath && strEq(info->name, DOT_STR);
// Skip . paths when getting info for subpaths (since info was already reported in the parent path)
if (dotPath && listData->subPath != NULL)
{
FUNCTION_TEST_RETURN_VOID();
return;
}
// Update the name in info with the subpath
StorageInfo infoUpdate = *info;
if (listData->subPath != NULL)
infoUpdate.name = strNewFmt("%s/%s", strPtr(listData->subPath), strPtr(infoUpdate.name));
// Only continue if there is no expression or the expression matches
if (listData->expression == NULL || regExpMatch(listData->expression, infoUpdate.name))
{
if (listData->sortOrder != sortOrderDesc)
listData->callbackFunction(listData->callbackData, &infoUpdate);
// Recurse into paths
if (infoUpdate.type == storageTypePath && listData->recurse && !dotPath)
{
StorageInfoListData data = *listData;
data.subPath = infoUpdate.name;
storageInfoListSort(
data.storage, strNewFmt("%s/%s", strPtr(data.path), strPtr(data.subPath)), data.sortOrder, storageInfoListCallback,
&data);
}
if (listData->sortOrder == sortOrderDesc)
listData->callbackFunction(listData->callbackData, &infoUpdate);
}
FUNCTION_TEST_RETURN_VOID();
}
bool
storageInfoList(
const Storage *this, const String *pathExp, StorageInfoListCallback callback, void *callbackData, StorageInfoListParam param)
@ -282,6 +435,9 @@ storageInfoList(
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_PARAM(BOOL, param.errorOnMissing);
FUNCTION_LOG_PARAM(ENUM, param.sortOrder);
FUNCTION_LOG_PARAM(STRING, param.expression);
FUNCTION_LOG_PARAM(BOOL, param.recurse);
FUNCTION_LOG_END();
ASSERT(this != NULL);
@ -296,8 +452,26 @@ storageInfoList(
// Build the path
String *path = storagePathNP(this, pathExp);
// Call driver function
result = this->interface.infoList(this->driver, path, callback, callbackData);
// If there is an expression or recursion then the info will need to be filtered through a local callback
if (param.expression != NULL || param.recurse)
{
StorageInfoListData data =
{
.storage = this,
.callbackFunction = callback,
.callbackData = callbackData,
.sortOrder = param.sortOrder,
.recurse = param.recurse,
.path = path,
};
if (param.expression != NULL)
data.expression = regExpNew(param.expression);
result = storageInfoListSort(this, path, param.sortOrder, storageInfoListCallback, &data);
}
else
result = storageInfoListSort(this, path, param.sortOrder, callback, callbackData);
if (!result && param.errorOnMissing)
THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, strPtr(path));

3
src/storage/storage.h
View File

@ -109,6 +109,9 @@ typedef void (*StorageInfoListCallback)(void *callbackData, const StorageInfo *i
typedef struct StorageInfoListParam
{
bool errorOnMissing;
SortOrder sortOrder;
const String *expression;
bool recurse;
} StorageInfoListParam;
#define storageInfoListP(this, fileExp, callback, callbackData, ...) \

94
test/src/common/harnessStorage.c Normal file
View File

@ -0,0 +1,94 @@
/***********************************************************************************************************************************
Storage Test Harness
***********************************************************************************************************************************/
#include <inttypes.h>
#include "common/user.h"
#include "storage/storage.h"
#include "common/harnessStorage.h"
/**********************************************************************************************************************************/
void
hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info)
{
HarnessStorageInfoListCallbackData *data = callbackData;
// Add LF for after the first item
if (strSize(data->content) != 0)
strCat(data->content, "\n");
strCatFmt(data->content, "%s {", strPtr(info->name));
switch (info->type)
{
case storageTypeFile:
{
strCat(data->content, "file");
if (!data->sizeOmit)
strCatFmt(data->content, ", s=%" PRIu64, info->size);
break;
}
case storageTypeLink:
{
strCatFmt(data->content, "link, d=%s", strPtr(info->linkDestination));
break;
}
case storageTypePath:
{
strCat(data->content, "path");
break;
}
case storageTypeSpecial:
{
strCat(data->content, "special");
break;
}
}
if (info->type != storageTypeSpecial)
{
if (info->type != storageTypeLink)
{
if (!data->modeOmit || (info->type == storageTypePath && data->modePath != info->mode) ||
(info->type == storageTypeFile && data->modeFile != info->mode))
{
strCatFmt(data->content, ", m=%04o", info->mode);
}
if (!data->timestampOmit)
strCatFmt(data->content, ", t=%" PRIu64, (uint64_t)info->timeModified);
}
if (!data->userOmit || userId() != info->userId)
{
if (info->user != NULL)
{
strCatFmt(data->content, ", u=%s", strPtr(info->user));
}
else
{
strCatFmt(data->content, ", u=%d", (int)info->userId);
}
}
if (!data->groupOmit || groupId() != info->groupId)
{
if (info->group != NULL)
{
strCatFmt(data->content, ", g=%s", strPtr(info->group));
}
else
{
strCatFmt(data->content, ", g=%d", (int)info->groupId);
}
}
}
strCat(data->content, "}");
}

26
test/src/common/harnessStorage.h Normal file
View File

@ -0,0 +1,26 @@
/***********************************************************************************************************************************
Storage Test Harness
Helper functions for testing storage and related functions.
***********************************************************************************************************************************/
#ifndef TEST_COMMON_HARNESS_STORAGE_H
#define TEST_COMMON_HARNESS_STORAGE_H
/***********************************************************************************************************************************
Callback for formatting info list results
***********************************************************************************************************************************/
typedef struct HarnessStorageInfoListCallbackData
{
String *content; // String where content should be added
bool modeOmit; // Should the specified mode be ommitted?
mode_t modeFile; // File to omit if modeOmit is true
mode_t modePath; // Path mode to omit if modeOmit is true
bool timestampOmit; // Should the timestamp be ommitted?
bool userOmit; // Should the current user be ommitted?
bool groupOmit; // Should the current group be ommitted?
bool sizeOmit; // Should the size be ommitted
} HarnessStorageInfoListCallbackData;
void hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info);
#endif

121
test/src/module/storage/posixTest.c
View File

@ -11,6 +11,7 @@ Test Posix Storage
#include "common/harnessConfig.h"
#include "common/harnessFork.h"
#include "common/harnessStorage.h"
/***********************************************************************************************************************************
Test function for path expression
@ -38,24 +39,6 @@ Macro to create a path and file that cannot be accessed
strPtr(fileNoPerm)))), \
0, "create no perm path/file");
/***********************************************************************************************************************************
Callback and data for storageInfoList() tests
***********************************************************************************************************************************/
unsigned int testStorageInfoListSize = 0;
StorageInfo testStorageInfoList[256];
void
testStorageInfoListCallback(void *callbackData, const StorageInfo *info)
{
MEM_CONTEXT_BEGIN((MemContext *)callbackData)
{
testStorageInfoList[testStorageInfoListSize] = *info;
testStorageInfoList[testStorageInfoListSize].name = strDup(info->name);
testStorageInfoListSize++;
}
MEM_CONTEXT_END();
}
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
@ -283,24 +266,100 @@ testRun(void)
STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strPtr(pathNoPerm));
// -------------------------------------------------------------------------------------------------------------------------
testStorageInfoListSize = 0;
HarnessStorageInfoListCallbackData callbackData =
{
.content = strNew(""),
};
TEST_RESULT_VOID(
storagePosixInfoListEntry((StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"),
testStorageInfoListCallback, (void *)memContextCurrent()),
"missing file");
TEST_RESULT_UINT(testStorageInfoListSize, 0, " no file found");
storagePosixInfoListEntry(
(StoragePosix *)storageDriver(storageTest), strNew("pg"), strNew("missing"),
hrnStorageInfoListCallback, &callbackData),
"missing path");
TEST_RESULT_STR_Z(callbackData.content, "", " check content");
// -------------------------------------------------------------------------------------------------------------------------
storagePathCreateP(storageTest, strNew("pg/.include"), .mode = 0766);
storagePathCreateP(storageTest, strNew("pg"), .mode = 0766);
struct utimbuf utimeTest = {.actime = 1000000000, .modtime = 1555160777};
THROW_ON_SYS_ERROR_FMT(
utime(strPtr(strNewFmt("%s/pg", testPath())), &utimeTest) != 0, FileWriteError, "unable to set time for '%s'/pg",
testPath());
callbackData.content = strNew("");
TEST_RESULT_VOID(
storageInfoListNP(storageTest, strNew("pg"), testStorageInfoListCallback, (void *)memContextCurrent()),
"empty directory");
TEST_RESULT_UINT(testStorageInfoListSize, 2, " two paths returned");
TEST_RESULT_STR(
strPtr(testStorageInfoList[0].name), strEqZ(testStorageInfoList[1].name, ".") ? ".include" : ".", " check name");
TEST_RESULT_STR(
strPtr(testStorageInfoList[1].name), strEqZ(testStorageInfoList[0].name, ".") ? ".include" : ".", " check name");
storageInfoListP(storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData),
"directory with one dot file sorted");
TEST_RESULT_STR_Z(
callbackData.content, strPtr(strNewFmt(". {path, m=0766, t=1555160777, u=%s, g=%s}", testUser(), testGroup())),
" check content");
// -------------------------------------------------------------------------------------------------------------------------
storagePathCreateP(storageTest, strNew("pg/.include"), .mode = 0755);
ASSERT(system(strPtr(strNewFmt("sudo chown 77777:77777 %s/pg/.include", testPath()))) == 0);
storagePutNP(storageNewWriteP(storageTest, strNew("pg/file"), .modeFile = 0660), BUFSTRDEF("TESTDATA"));
ASSERT(system(strPtr(strNewFmt("ln -s ../file %s/pg/link", testPath()))) == 0);
ASSERT(system(strPtr(strNewFmt("mkfifo -m 777 %s/pg/pipe", testPath()))) == 0);
callbackData = (HarnessStorageInfoListCallbackData)
{
.content = strNew(""),
.timestampOmit = true,
.modeOmit = true,
.modePath = 0766,
.modeFile = 0600,
.userOmit = true,
.groupOmit = true,
};
TEST_RESULT_VOID(
storageInfoListP(storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderAsc),
"directory with one dot file sorted");
TEST_RESULT_STR_Z(
callbackData.content,
". {path}\n"
".include {path, m=0755, u=77777, g=77777}\n"
"file {file, s=8, m=0660}\n"
"link {link, d=../file}\n"
"pipe {special}",
" check content");
// -------------------------------------------------------------------------------------------------------------------------
ASSERT(system(strPtr(strNewFmt("sudo rmdir %s/pg/.include", testPath()))) == 0);
storagePathCreateP(storageTest, strNew("pg/path"), .mode = 0700);
storagePutNP(storageNewWriteP(storageTest, strNew("pg/path/file"), .modeFile = 0600), BUFSTRDEF("TESTDATA"));
callbackData.content = strNew("");
TEST_RESULT_VOID(
storageInfoListP(
storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderDesc, .recurse = true),
"recurse descending");
TEST_RESULT_STR_Z(
callbackData.content,
"pipe {special}\n"
"path/file {file, s=8}\n"
"path {path, m=0700}\n"
"link {link, d=../file}\n"
"file {file, s=8, m=0660}\n"
". {path}",
" check content");
// -------------------------------------------------------------------------------------------------------------------------
callbackData.content = strNew("");
TEST_RESULT_VOID(
storageInfoListP(
storageTest, strNew("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderAsc,
.expression = STRDEF("^path")),
"filter");
TEST_RESULT_STR_Z(
callbackData.content,
"path {path, m=0700}",
" check content");
}
// *****************************************************************************************************************************