db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-05 00:28:52 +02:00
Code Issues Releases Activity

Create snapshot when listing contents of a path.

Browse Source 
Previously a callback was used to list path contents and if no sort was specified then a snapshot was not required. When deleting files from the path some filesystems could omit files that still existed, which meant the path could not be removed.

Filter . out of lists in the Posix driver since this special entry was only used by test code (and filtered everywhere in the core code).

Also remove callbacks from the storage interface and replace with an iterator that should be easier to use and guarantees efficient use of the snapshots.
This commit is contained in:
David Steele
2022-07-08 17:21:39 -04:00
committed by GitHub
parent f9ac53db92
commit 75623d4583
34 changed files with 1798 additions and 1093 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
doc/xml/release.xml
View File

@ -32,6 +32,19 @@
<p><proper>OpenSSL 3</proper> support.</p>
</release-item>
<release-item>
<github-issue id="1754"/>
<github-pull-request id="1805"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="john.morris"/>
<release-item-reviewer id="stephen.frost"/>
</release-item-contributor-list>
<p>Create snapshot when listing contents of a path.</p>
</release-item>
<release-item>
<github-pull-request id="1758"/>

3
src/Makefile.in
View File

@ -25,6 +25,7 @@ SRCS_BUILD = \
common/regExp.c \
common/stackTrace.c \
common/time.c \
common/type/blob.c \
common/type/buffer.c \
common/type/convert.c \
common/type/keyValue.c \
@ -44,6 +45,8 @@ SRCS_BUILD = \
storage/posix/read.c \
storage/posix/storage.c \
storage/posix/write.c \
storage/iterator.c \
storage/list.c \
storage/read.c \
storage/storage.c \
storage/write.c

128
src/command/backup/backup.c
View File

@ -489,77 +489,75 @@ backupBuildIncr(const InfoBackup *infoBackup, Manifest *manifest, Manifest *mani
/***********************************************************************************************************************************
Check for a backup that can be resumed and merge into the manifest if found
***********************************************************************************************************************************/
typedef struct BackupResumeData
// Helper to clean invalid paths/files/links out of the resumable backup path
static void
backupResumeClean(
StorageIterator *const storageItr, Manifest *const manifest, const Manifest *const manifestResume,
const CompressType compressType, const bool delta, const String *const backupParentPath, const String *const manifestParentName)
{
Manifest *manifest; // New manifest
const Manifest *manifestResume; // Resumed manifest
const CompressType compressType; // Backup compression type
const bool delta; // Is this a delta backup?
const String *backupPath; // Path to the current level of the backup being cleaned
const String *manifestParentName; // Parent manifest name used to construct manifest name
} BackupResumeData;
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_ITERATOR, storageItr); // Storage info
FUNCTION_LOG_PARAM(MANIFEST, manifest); // New manifest
FUNCTION_LOG_PARAM(MANIFEST, manifestResume); // Resumed manifest
FUNCTION_LOG_PARAM(ENUM, compressType); // Backup compression type
FUNCTION_LOG_PARAM(BOOL, delta); // Is this a delta backup?
FUNCTION_LOG_PARAM(STRING, backupParentPath); // Path to the current level of the backup being cleaned
FUNCTION_LOG_PARAM(STRING, manifestParentName); // Parent manifest name used to construct manifest name
FUNCTION_LOG_END();
// Callback to clean invalid paths/files/links out of the resumable backup path
void backupResumeCallback(void *data, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(STORAGE_INFO, *storageInfo);
FUNCTION_TEST_END();
ASSERT(storageItr != NULL);
ASSERT(manifest != NULL);
ASSERT(manifestResume != NULL);
ASSERT(backupParentPath != NULL);
ASSERT(data != NULL);
ASSERT(info != NULL);
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
while (storageItrMore(storageItr))
{
const StorageInfo info = storageItrNext(storageItr);
BackupResumeData *resumeData = data;
// Skip all . paths because they have already been handled on the previous level of recursion
if (strEq(info->name, DOT_STR))
FUNCTION_TEST_RETURN_VOID();
// Skip backup.manifest.copy -- it must be preserved to allow resume again if this process throws an error before writing the
// manifest for the first time
if (resumeData->manifestParentName == NULL && strEqZ(info->name, BACKUP_MANIFEST_FILE INFO_COPY_EXT))
FUNCTION_TEST_RETURN_VOID();
// Skip backup.manifest.copy -- it must be preserved to allow resume again if this process throws an error before
// writing the manifest for the first time
if (manifestParentName == NULL && strEqZ(info.name, BACKUP_MANIFEST_FILE INFO_COPY_EXT))
continue;
// Build the name used to lookup files in the manifest
const String *manifestName = resumeData->manifestParentName != NULL ?
strNewFmt("%s/%s", strZ(resumeData->manifestParentName), strZ(info->name)) : info->name;
const String *manifestName = manifestParentName != NULL ?
strNewFmt("%s/%s", strZ(manifestParentName), strZ(info.name)) : info.name;
// Build the backup path used to remove files/links/paths that are invalid
const String *backupPath = strNewFmt("%s/%s", strZ(resumeData->backupPath), strZ(info->name));
const String *const backupPath = strNewFmt("%s/%s", strZ(backupParentPath), strZ(info.name));
// Process file types
switch (info->type)
switch (info.type)
{
// Check paths
// -------------------------------------------------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------------------------------------
case storageTypePath:
{
// If the path was not found in the new manifest then remove it
if (manifestPathFindDefault(resumeData->manifest, manifestName, NULL) == NULL)
if (manifestPathFindDefault(manifest, manifestName, NULL) == NULL)
{
LOG_DETAIL_FMT("remove path '%s' from resumed backup", strZ(storagePathP(storageRepo(), backupPath)));
storagePathRemoveP(storageRepoWrite(), backupPath, .recurse = true);
}
// Else recurse into the path
else
{
BackupResumeData resumeDataSub = *resumeData;
resumeDataSub.manifestParentName = manifestName;
resumeDataSub.backupPath = backupPath;
storageInfoListP(
storageRepo(), resumeDataSub.backupPath, backupResumeCallback, &resumeDataSub, .sortOrder = sortOrderAsc);
backupResumeClean(
storageNewItrP(storageRepo(), backupPath, .sortOrder = sortOrderAsc), manifest, manifestResume,
compressType, delta, backupPath, manifestName);
}
break;
}
// Check files
// -------------------------------------------------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------------------------------------
case storageTypeFile:
{
// If the file is compressed then strip off the extension before doing the lookup
CompressType fileCompressType = compressTypeFromName(manifestName);
const CompressType fileCompressType = compressTypeFromName(manifestName);
if (fileCompressType != compressTypeNone)
manifestName = compressExtStrip(manifestName, fileCompressType);
@ -567,21 +565,21 @@ void backupResumeCallback(void *data, const StorageInfo *info)
// Check if the file can be resumed or must be removed
const char *removeReason = NULL;
if (fileCompressType != resumeData->compressType)
if (fileCompressType != compressType)
removeReason = "mismatched compression type";
else if (!manifestFileExists(resumeData->manifest, manifestName))
else if (!manifestFileExists(manifest, manifestName))
removeReason = "missing in manifest";
else
{
const ManifestFile file = manifestFileFind(resumeData->manifest, manifestName);
const ManifestFile file = manifestFileFind(manifest, manifestName);
if (file.reference != NULL)
removeReason = "reference in manifest";
else if (!manifestFileExists(resumeData->manifestResume, manifestName))
else if (!manifestFileExists(manifestResume, manifestName))
removeReason = "missing in resumed manifest";
else
{
const ManifestFile fileResume = manifestFileFind(resumeData->manifestResume, manifestName);
const ManifestFile fileResume = manifestFileFind(manifestResume, manifestName);
if (fileResume.reference != NULL)
removeReason = "reference in resumed manifest";
@ -589,15 +587,15 @@ void backupResumeCallback(void *data, const StorageInfo *info)
removeReason = "no checksum in resumed manifest";
else if (file.size != fileResume.size)
removeReason = "mismatched size";
else if (!resumeData->delta && file.timestamp != fileResume.timestamp)
else if (!delta && file.timestamp != fileResume.timestamp)
removeReason = "mismatched timestamp";
else if (file.size == 0)
// ??? don't resume zero size files because Perl wouldn't -- this can be removed after the migration)
// ??? don't resume zero size files because Perl wouldn't -- can be removed after the migration)
removeReason = "zero size";
else
{
manifestFileUpdate(
resumeData->manifest, manifestName, file.size, fileResume.sizeRepo, fileResume.checksumSha1, NULL,
manifest, manifestName, file.size, fileResume.sizeRepo, fileResume.checksumSha1, NULL,
fileResume.checksumPage, fileResume.checksumPageError, fileResume.checksumPageErrorList, 0, 0);
}
}
@ -607,29 +605,36 @@ void backupResumeCallback(void *data, const StorageInfo *info)
if (removeReason != NULL)
{
LOG_DETAIL_FMT(
"remove file '%s' from resumed backup (%s)", strZ(storagePathP(storageRepo(), backupPath)), removeReason);
"remove file '%s' from resumed backup (%s)", strZ(storagePathP(storageRepo(), backupPath)),
removeReason);
storageRemoveP(storageRepoWrite(), backupPath);
}
break;
}
// Remove links. We could check that the link has not changed and preserve it but it doesn't seem worth the extra testing.
// The link will be recreated during the backup if needed.
// -------------------------------------------------------------------------------------------------------------------------
// Remove links. We could check that the link has not changed and preserve it but it doesn't seem worth the extra
// testing. The link will be recreated during the backup if needed.
// -----------------------------------------------------------------------------------------------------------------
case storageTypeLink:
storageRemoveP(storageRepoWrite(), backupPath);
break;
// Remove special files
// -------------------------------------------------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------------------------------------
case storageTypeSpecial:
LOG_WARN_FMT("remove special file '%s' from resumed backup", strZ(storagePathP(storageRepo(), backupPath)));
storageRemoveP(storageRepoWrite(), backupPath);
break;
}
}
FUNCTION_TEST_RETURN_VOID();
// Reset the memory context occasionally so we don't use too much memory or slow down processing
MEM_CONTEXT_TEMP_RESET(1000);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}
// Helper to find a resumable backup
@ -789,16 +794,11 @@ backupResume(Manifest *manifest, const String *cipherPassBackup)
manifestCipherSubPassSet(manifest, manifestCipherSubPass(manifestResume));
// Clean resumed backup
BackupResumeData resumeData =
{
.manifest = manifest,
.manifestResume = manifestResume,
.compressType = compressTypeEnum(cfgOptionStrId(cfgOptCompressType)),
.delta = cfgOptionBool(cfgOptDelta),
.backupPath = strNewFmt(STORAGE_REPO_BACKUP "/%s", strZ(manifestData(manifest)->backupLabel)),
};
const String *const backupPath = strNewFmt(STORAGE_REPO_BACKUP "/%s", strZ(manifestData(manifest)->backupLabel));
storageInfoListP(storageRepo(), resumeData.backupPath, backupResumeCallback, &resumeData, .sortOrder = sortOrderAsc);
backupResumeClean(
storageNewItrP(storageRepo(), backupPath, .sortOrder = sortOrderAsc), manifest, manifestResume,
compressTypeEnum(cfgOptionStrId(cfgOptCompressType)), cfgOptionBool(cfgOptDelta), backupPath, NULL);
}
}
MEM_CONTEXT_TEMP_END();

107
src/command/repo/ls.c
View File

@ -19,77 +19,59 @@ Repository List Command
/***********************************************************************************************************************************
Render storage list
***********************************************************************************************************************************/
typedef struct StorageListRenderCallbackData
{
IoWrite *write; // Where to write output
bool json; // Is this json output?
bool first; // Is this the first item?
} StorageListRenderCallbackData;
void
storageListRenderCallback(void *data, const StorageInfo *info)
static void
storageListRenderInfo(const StorageInfo *const info, IoWrite *const write, const bool json)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_PARAM(IO_WRITE, write);
FUNCTION_TEST_PARAM(BOOL, json);
FUNCTION_TEST_END();
ASSERT(data != NULL);
ASSERT(info != NULL);
StorageListRenderCallbackData *listData = (StorageListRenderCallbackData *)data;
// Skip . path if it is not first when json output
if (info->type == storageTypePath && strEq(info->name, DOT_STR) && (!listData->first || !listData->json))
FUNCTION_TEST_RETURN_VOID();
// Add separator character
if (!listData->first && listData->json)
ioWrite(listData->write, COMMA_BUF);
else
listData->first = false;
ASSERT(write != NULL);
// Render in json
if (listData->json)
if (json)
{
ioWriteStr(listData->write, jsonFromVar(VARSTR(info->name)));
ioWrite(listData->write, BUFSTRDEF(":{\"type\":\""));
ioWriteStr(write, jsonFromVar(VARSTR(info->name)));
ioWrite(write, BUFSTRDEF(":{\"type\":\""));
switch (info->type)
{
case storageTypeFile:
ioWrite(listData->write, BUFSTRDEF("file\""));
ioWrite(write, BUFSTRDEF("file\""));
break;
case storageTypeLink:
ioWrite(listData->write, BUFSTRDEF("link\""));
ioWrite(write, BUFSTRDEF("link\""));
break;
case storageTypePath:
ioWrite(listData->write, BUFSTRDEF("path\""));
ioWrite(write, BUFSTRDEF("path\""));
break;
case storageTypeSpecial:
ioWrite(listData->write, BUFSTRDEF("special\""));
ioWrite(write, BUFSTRDEF("special\""));
break;
}
if (info->type == storageTypeFile)
{
ioWriteStr(listData->write, strNewFmt(",\"size\":%" PRIu64, info->size));
ioWriteStr(listData->write, strNewFmt(",\"time\":%" PRId64, (int64_t)info->timeModified));
ioWriteStr(write, strNewFmt(",\"size\":%" PRIu64, info->size));
ioWriteStr(write, strNewFmt(",\"time\":%" PRId64, (int64_t)info->timeModified));
}
if (info->type == storageTypeLink)
ioWriteStr(listData->write, strNewFmt(",\"destination\":%s", strZ(jsonFromVar(VARSTR(info->linkDestination)))));
ioWriteStr(write, strNewFmt(",\"destination\":%s", strZ(jsonFromVar(VARSTR(info->linkDestination)))));
ioWrite(listData->write, BRACER_BUF);
ioWrite(write, BRACER_BUF);
}
// Render in text
else
{
ioWrite(listData->write, BUFSTR(info->name));
ioWrite(listData->write, LF_BUF);
ioWrite(write, BUFSTR(info->name));
ioWrite(write, LF_BUF);
}
FUNCTION_TEST_RETURN_VOID();
@ -133,18 +115,10 @@ storageListRender(IoWrite *write)
const String *expression = cfgOptionStrNull(cfgOptFilter);
RegExp *regExp = expression == NULL ? NULL : regExpNew(expression);
// Render the info list
StorageListRenderCallbackData data =
{
.write = write,
.json = json,
.first = true,
};
ioWriteOpen(write);
ioWriteOpen(data.write);
if (data.json)
ioWrite(data.write, BRACEL_BUF);
if (json)
ioWrite(write, BRACEL_BUF);
// Check if this is a file
StorageInfo info = storageInfoP(storageRepo(), path, .ignoreMissing = true);
@ -154,29 +128,46 @@ storageListRender(IoWrite *write)
if (regExp == NULL || regExpMatch(regExp, storagePathP(storageRepo(), path)))
{
info.name = DOT_STR;
storageListRenderCallback(&data, &info);
storageListRenderInfo(&info, write, json);
}
}
// Else try to list the path
else
{
// The path will always be reported as existing so we don't get different results from storage that does not support paths
if (data.json && (regExp == NULL || regExpMatch(regExp, DOT_STR)))
storageListRenderCallback(&data, &(StorageInfo){.type = storageTypePath, .name = DOT_STR});
bool first = true;
if (json && (regExp == NULL || regExpMatch(regExp, DOT_STR)))
{
storageListRenderInfo(&(StorageInfo){.type = storageTypePath, .name = DOT_STR}, write, json);
first = false;
}
// List content of the path
storageInfoListP(
storageRepo(), path, storageListRenderCallback, &data, .sortOrder = sortOrder, .expression = expression,
.recurse = cfgOptionBool(cfgOptRecurse));
}
StorageIterator *const storageItr = storageNewItrP(
storageRepo(), path, .sortOrder = sortOrder, .expression = expression, .recurse = cfgOptionBool(cfgOptRecurse));
if (data.json)
while (storageItrMore(storageItr))
{
ioWrite(data.write, BRACER_BUF);
ioWrite(data.write, LF_BUF);
const StorageInfo info = storageItrNext(storageItr);
// Add separator character
if (!first && json)
ioWrite(write, COMMA_BUF);
else
first = false;
storageListRenderInfo(&info, write, json);
}
}
ioWriteClose(data.write);
if (json)
{
ioWrite(write, BRACER_BUF);
ioWrite(write, LF_BUF);
}
ioWriteClose(write);
FUNCTION_LOG_RETURN_VOID();
}

77
src/command/restore/restore.c
View File

@ -892,27 +892,27 @@ restoreCleanMode(const String *pgPath, mode_t manifestMode, const StorageInfo *i
FUNCTION_TEST_RETURN_VOID();
}
// storageInfoList() callback that cleans the paths
// Recurse paths
static void
restoreCleanInfoListCallback(void *data, const StorageInfo *info)
restoreCleanBuildRecurse(StorageIterator *const storageItr, const RestoreCleanCallbackData *const cleanData)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_PARAM(STORAGE_ITERATOR, storageItr);
FUNCTION_TEST_PARAM_P(VOID, cleanData);
FUNCTION_TEST_END();
ASSERT(data != NULL);
ASSERT(info != NULL);
ASSERT(storageItr != NULL);
ASSERT(cleanData != NULL);
RestoreCleanCallbackData *cleanData = (RestoreCleanCallbackData *)data;
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
while (storageItrMore(storageItr))
{
const StorageInfo info = storageItrNext(storageItr);
// Don't include backup.manifest or recovery.conf (when preserved) in the comparison or empty directory check
if (cleanData->basePath && info->type == storageTypeFile && strLstExists(cleanData->fileIgnore, info->name))
FUNCTION_TEST_RETURN_VOID();
// Skip all . paths because they have already been cleaned on the previous level of recursion
if (strEq(info->name, DOT_STR))
FUNCTION_TEST_RETURN_VOID();
if (cleanData->basePath && info.type == storageTypeFile && strLstExists(cleanData->fileIgnore, info.name))
continue;
// If this is not a delta then error because the directory is expected to be empty. Ignore the . path.
if (!cleanData->delta)
@ -925,12 +925,12 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info)
}
// Construct the name used to find this file/link/path in the manifest
const String *manifestName = strNewFmt("%s/%s", strZ(cleanData->targetName), strZ(info->name));
const String *manifestName = strNewFmt("%s/%s", strZ(cleanData->targetName), strZ(info.name));
// Construct the path of this file/link/path in the PostgreSQL data directory
const String *pgPath = strNewFmt("%s/%s", strZ(cleanData->targetPath), strZ(info->name));
const String *pgPath = strNewFmt("%s/%s", strZ(cleanData->targetPath), strZ(info.name));
switch (info->type)
switch (info.type)
{
case storageTypeFile:
{
@ -941,8 +941,8 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info)
restoreCleanOwnership(
pgPath, manifestFile.user, cleanData->rootReplaceUser, manifestFile.group, cleanData->rootReplaceGroup,
info->userId, info->groupId, false);
restoreCleanMode(pgPath, manifestFile.mode, info);
info.userId, info.groupId, false);
restoreCleanMode(pgPath, manifestFile.mode, &info);
}
else
{
@ -959,7 +959,7 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info)
if (manifestLink != NULL)
{
if (!strEq(manifestLink->destination, info->linkDestination))
if (!strEq(manifestLink->destination, info.linkDestination))
{
LOG_DETAIL_FMT("remove link '%s' because destination changed", strZ(pgPath));
storageRemoveP(storageLocalWrite(), pgPath, .errorOnMissing = true);
@ -967,8 +967,8 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info)
else
{
restoreCleanOwnership(
pgPath, manifestLink->user, cleanData->rootReplaceUser, manifestLink->group, cleanData->rootReplaceGroup,
info->userId, info->groupId, false);
pgPath, manifestLink->user, cleanData->rootReplaceUser, manifestLink->group,
cleanData->rootReplaceGroup, info.userId, info.groupId, false);
}
}
else
@ -988,19 +988,20 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info)
{
// Check ownership/permissions
restoreCleanOwnership(
pgPath, manifestPath->user, cleanData->rootReplaceUser, manifestPath->group, cleanData->rootReplaceGroup,
info->userId, info->groupId, false);
restoreCleanMode(pgPath, manifestPath->mode, info);
pgPath, manifestPath->user, cleanData->rootReplaceUser, manifestPath->group,
cleanData->rootReplaceGroup, info.userId, info.groupId, false);
restoreCleanMode(pgPath, manifestPath->mode, &info);
// Recurse into the path
RestoreCleanCallbackData cleanDataSub = *cleanData;
cleanDataSub.targetName = strNewFmt("%s/%s", strZ(cleanData->targetName), strZ(info->name));
cleanDataSub.targetPath = strNewFmt("%s/%s", strZ(cleanData->targetPath), strZ(info->name));
cleanDataSub.targetName = strNewFmt("%s/%s", strZ(cleanData->targetName), strZ(info.name));
cleanDataSub.targetPath = strNewFmt("%s/%s", strZ(cleanData->targetPath), strZ(info.name));
cleanDataSub.basePath = false;
storageInfoListP(
storageLocalWrite(), cleanDataSub.targetPath, restoreCleanInfoListCallback, &cleanDataSub,
.errorOnMissing = true, .sortOrder = sortOrderAsc);
restoreCleanBuildRecurse(
storageNewItrP(
storageLocalWrite(), cleanDataSub.targetPath, .errorOnMissing = true, .sortOrder = sortOrderAsc),
&cleanDataSub);
}
else
{
@ -1018,6 +1019,12 @@ restoreCleanInfoListCallback(void *data, const StorageInfo *info)
break;
}
// Reset the memory context occasionally so we don't use too much memory or slow down processing
MEM_CONTEXT_TEMP_RESET(1000);
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_TEST_RETURN_VOID();
}
@ -1121,9 +1128,8 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac
{
if (cleanData->target->file == NULL)
{
storageInfoListP(
storageLocal(), cleanData->targetPath, restoreCleanInfoListCallback, cleanData,
.errorOnMissing = true);
restoreCleanBuildRecurse(
storageNewItrP(storageLocal(), cleanData->targetPath, .errorOnMissing = true), cleanData);
}
else
{
@ -1202,9 +1208,10 @@ restoreCleanBuild(const Manifest *const manifest, const String *const rootReplac
restoreCleanMode(cleanData->targetPath, manifestPath->mode, &info);
// Clean the target
storageInfoListP(
storageLocalWrite(), cleanData->targetPath, restoreCleanInfoListCallback, cleanData, .errorOnMissing = true,
.sortOrder = sortOrderAsc);
restoreCleanBuildRecurse(
storageNewItrP(
storageLocalWrite(), cleanData->targetPath, .errorOnMissing = true, .sortOrder = sortOrderAsc),
cleanData);
}
}
// If the target does not exist we'll attempt to create it

88
src/common/type/blob.c Normal file
View File

@ -0,0 +1,88 @@
/***********************************************************************************************************************************
Blob Handler
***********************************************************************************************************************************/
#include "build.auto.h"
#include <string.h>
#include "common/debug.h"
#include "common/memContext.h"
#include "common/type/blob.h"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct Blob
{
char *block; // Current block for writing
size_t pos; // Position in current block
};
/**********************************************************************************************************************************/
Blob *
blbNew(void)
{
FUNCTION_TEST_VOID();
Blob *this = NULL;
OBJ_NEW_BEGIN(Blob, .allocQty = MEM_CONTEXT_QTY_MAX)
{
this = OBJ_NEW_ALLOC();
*this = (Blob)
{
.block = memNew(BLOB_BLOCK_SIZE),
};
}
OBJ_NEW_END();
FUNCTION_TEST_RETURN(BLOB, this);
}
/**********************************************************************************************************************************/
const void *
blbAdd(Blob *const this, const void *const data, const size_t size)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BLOB, this);
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(SIZE, size);
FUNCTION_TEST_END();
void *result = NULL;
MEM_CONTEXT_OBJ_BEGIN(this)
{
// If data is >= block size then allocate a special block for the data
if (size >= BLOB_BLOCK_SIZE)
{
result = memNew(size);
memcpy(result, data, size);
}
else
{
// If data will fit in the current block
if (BLOB_BLOCK_SIZE - this->pos >= size)
{
result = (unsigned char *)this->block + this->pos;
memcpy(result, data, size);
this->pos += size;
}
// Else allocate a new block
else
{
this->block = memNew(BLOB_BLOCK_SIZE);
result = this->block;
memcpy(result, data, size);
this->pos = size;
}
}
}
MEM_CONTEXT_OBJ_END();
FUNCTION_TEST_RETURN_P(VOID, result);
}

58
src/common/type/blob.h Normal file
View File

@ -0,0 +1,58 @@
/***********************************************************************************************************************************
Blob Handler
Efficiently store large numbers of small allocations by packing them onto larger blocks. Note that the allocations will not be
aligned unless all of them are aligned.
A usage example is to store zero-terminated strings, which do not need to be aligned, and where alignment might waste significant
space.
Note that this code is fairly primitive, and this should probably be a capability of memory contexts. However, for now it reduces
memory requirements for large numbers of zero-terminated strings.
***********************************************************************************************************************************/
#ifndef COMMON_TYPE_BLOB_H
#define COMMON_TYPE_BLOB_H
/***********************************************************************************************************************************
Size of blocks allocated for blob data
***********************************************************************************************************************************/
#ifndef BLOB_BLOCK_SIZE
#define BLOB_BLOCK_SIZE (64 * 1024)
#endif
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct Blob Blob;
#include "common/type/object.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
Blob *blbNew(void);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Add data to the blob
const void *blbAdd(Blob *this, const void *data, size_t size);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
__attribute__((always_inline)) static inline void
strBlbFree(Blob *const this)
{
objFree(this);
}
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
#define FUNCTION_LOG_BLOB_TYPE \
Blob *
#define FUNCTION_LOG_BLOB_FORMAT(value, buffer, bufferSize) \
objToLog(value, "Blob", buffer, bufferSize)
#endif

168
src/info/manifest.c
View File

@ -680,8 +680,8 @@ manifestLinkCheck(const Manifest *this)
/**********************************************************************************************************************************/
typedef struct ManifestBuildData
{
Manifest *manifest;
const Storage *storagePg;
Manifest *manifest; // Manifest being build
const Storage *storagePg; // PostgreSQL storage
const String *tablespaceId; // Tablespace id if PostgreSQL version has one
bool online; // Is this an online backup?
bool checksumPage; // Are page checksums being checked?
@ -689,49 +689,46 @@ typedef struct ManifestBuildData
RegExp *dbPathExp; // Identify paths containing relations
RegExp *tempRelationExp; // Identify temp relations
const Pack *tablespaceList; // List of tablespaces in the database
ManifestLinkCheck linkCheck; // List of links found during build (used for prefix check)
ManifestLinkCheck *linkCheck; // List of links found during build (used for prefix check)
StringList *excludeContent; // Exclude contents of directories
StringList *excludeSingle; // Exclude a single file/link/path
// These change with each level of recursion
const String *manifestParentName; // Manifest name of this file/link/path's parent
const String *pgPath; // Current path in the PostgreSQL data directory
bool dbPath; // Does this path contain relations?
} ManifestBuildData;
// Callback to process files/links/paths and add them to the manifest
// Process files/links/paths and add them to the manifest
static void
manifestBuildCallback(void *data, const StorageInfo *info)
manifestBuildInfo(
ManifestBuildData *const buildData, const String *manifestParentName, const String *pgPath, const bool dbPath,
const StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_PARAM(STORAGE_INFO, *storageInfo);
FUNCTION_TEST_PARAM_P(VOID, buildData);
FUNCTION_TEST_PARAM(STRING, manifestParentName);
FUNCTION_TEST_PARAM(STRING, pgPath);
FUNCTION_TEST_PARAM(BOOL, dbPath);
FUNCTION_TEST_PARAM(STORAGE_INFO, *info);
FUNCTION_TEST_END();
ASSERT(data != NULL);
ASSERT(buildData != NULL);
ASSERT(manifestParentName != NULL);
ASSERT(pgPath != NULL);
ASSERT(info != NULL);
// Skip all . paths because they have already been recorded on the previous level of recursion
if (strEq(info->name, DOT_STR))
FUNCTION_TEST_RETURN_VOID();
// Skip any path/file/link that begins with pgsql_tmp. The files are removed when the server is restarted and the directories
// are recreated.
if (strBeginsWithZ(info->name, PG_PREFIX_PGSQLTMP))
FUNCTION_TEST_RETURN_VOID();
// Get build data
ManifestBuildData buildData = *(ManifestBuildData *)data;
unsigned int pgVersion = buildData.manifest->pub.data.pgVersion;
unsigned int pgVersion = buildData->manifest->pub.data.pgVersion;
// Construct the name used to identify this file/link/path in the manifest
const String *manifestName = strNewFmt("%s/%s", strZ(buildData.manifestParentName), strZ(info->name));
const String *manifestName = strNewFmt("%s/%s", strZ(manifestParentName), strZ(info->name));
// Skip excluded files/links/paths
if (buildData.excludeSingle != NULL && strLstExists(buildData.excludeSingle, manifestName))
if (buildData->excludeSingle != NULL && strLstExists(buildData->excludeSingle, manifestName))
{
LOG_INFO_FMT(
"exclude '%s/%s' from backup using '%s' exclusion", strZ(buildData.pgPath), strZ(info->name),
"exclude '%s/%s' from backup using '%s' exclusion", strZ(pgPath), strZ(info->name),
strZ(strSub(manifestName, sizeof(MANIFEST_TARGET_PGDATA))));
FUNCTION_TEST_RETURN_VOID();
@ -745,7 +742,7 @@ manifestBuildCallback(void *data, const StorageInfo *info)
case storageTypePath:
{
// There should not be any paths in pg_tblspc
if (strEqZ(buildData.manifestParentName, MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC))
if (strEqZ(manifestParentName, MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC))
{
THROW_FMT(
LinkExpectedError, "'%s' is not a symlink - " MANIFEST_TARGET_PGTBLSPC " should contain only symlinks",
@ -761,20 +758,20 @@ manifestBuildCallback(void *data, const StorageInfo *info)
.group = info->group,
};
manifestPathAdd(buildData.manifest, &path);
manifestPathAdd(buildData->manifest, &path);
// Skip excluded path content
if (buildData.excludeContent != NULL && strLstExists(buildData.excludeContent, manifestName))
if (buildData->excludeContent != NULL && strLstExists(buildData->excludeContent, manifestName))
{
LOG_INFO_FMT(
"exclude contents of '%s/%s' from backup using '%s/' exclusion", strZ(buildData.pgPath), strZ(info->name),
"exclude contents of '%s/%s' from backup using '%s/' exclusion", strZ(pgPath), strZ(info->name),
strZ(strSub(manifestName, sizeof(MANIFEST_TARGET_PGDATA))));
FUNCTION_TEST_RETURN_VOID();
}
// Skip the contents of these paths if they exist in the base path since they won't be reused after recovery
if (strEq(buildData.manifestParentName, MANIFEST_TARGET_PGDATA_STR))
if (strEq(manifestParentName, MANIFEST_TARGET_PGDATA_STR))
{
// Skip pg_dynshmem/* since these files cannot be reused on recovery
if (strEqZ(info->name, PG_PATH_PGDYNSHMEM) && pgVersion >= PG_VERSION_94)
@ -808,20 +805,30 @@ manifestBuildCallback(void *data, const StorageInfo *info)
}
// Skip the contents of archive_status when online
if (buildData.online && strEq(buildData.manifestParentName, buildData.manifestWalName) &&
if (buildData->online && strEq(manifestParentName, buildData->manifestWalName) &&
strEqZ(info->name, PG_PATH_ARCHIVE_STATUS))
{
FUNCTION_TEST_RETURN_VOID();
}
// Recurse into the path
ManifestBuildData buildDataSub = buildData;
buildDataSub.manifestParentName = manifestName;
buildDataSub.pgPath = strNewFmt("%s/%s", strZ(buildData.pgPath), strZ(info->name));
buildDataSub.dbPath = regExpMatch(buildData.dbPathExp, manifestName);
const String *const pgPathSub = strNewFmt("%s/%s", strZ(pgPath), strZ(info->name));
const bool dbPathSub = regExpMatch(buildData->dbPathExp, manifestName);
StorageIterator *const storageItr = storageNewItrP(buildData->storagePg, pgPathSub, .sortOrder = sortOrderAsc);
storageInfoListP(
buildDataSub.storagePg, buildDataSub.pgPath, manifestBuildCallback, &buildDataSub, .sortOrder = sortOrderAsc);
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
while (storageItrMore(storageItr))
{
const StorageInfo info = storageItrNext(storageItr);
manifestBuildInfo(buildData, manifestName, pgPathSub, dbPathSub, &info);
// Reset the memory context occasionally so we don't use too much memory or slow down processing
MEM_CONTEXT_TEMP_RESET(1000);
}
}
MEM_CONTEXT_TEMP_END();
break;
}
@ -831,7 +838,7 @@ manifestBuildCallback(void *data, const StorageInfo *info)
case storageTypeFile:
{
// There should not be any files in pg_tblspc
if (strEqZ(buildData.manifestParentName, MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC))
if (strEqZ(manifestParentName, MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC))
{
THROW_FMT(
LinkExpectedError, "'%s' is not a symlink - " MANIFEST_TARGET_PGTBLSPC " should contain only symlinks",
@ -842,7 +849,7 @@ manifestBuildCallback(void *data, const StorageInfo *info)
// the creating process id as the extension can exist so skip that as well. This seems to be a bug in PostgreSQL since
// the temp file should be removed on startup. Use regExpMatchOne() here instead of preparing a regexp in advance since
// the likelihood of needing the regexp should be very small.
if (buildData.dbPath && strBeginsWithZ(info->name, PG_FILE_PGINTERNALINIT) &&
if (dbPath && strBeginsWithZ(info->name, PG_FILE_PGINTERNALINIT) &&
(strSize(info->name) == sizeof(PG_FILE_PGINTERNALINIT) - 1 ||
regExpMatchOne(STRDEF("\\.[0-9]+"), strSub(info->name, sizeof(PG_FILE_PGINTERNALINIT) - 1))))
{
@ -850,7 +857,7 @@ manifestBuildCallback(void *data, const StorageInfo *info)
}
// Skip files in the root data path
if (strEq(buildData.manifestParentName, MANIFEST_TARGET_PGDATA_STR))
if (strEq(manifestParentName, MANIFEST_TARGET_PGDATA_STR))
{
// Skip recovery files
if (((strEqZ(info->name, PG_FILE_RECOVERYSIGNAL) || strEqZ(info->name, PG_FILE_STANDBYSIGNAL)) &&
@ -877,11 +884,11 @@ manifestBuildCallback(void *data, const StorageInfo *info)
// Skip the contents of the wal path when online. WAL will be restored from the archive or stored in the wal directory
// at the end of the backup if the archive-copy option is set.
if (buildData.online && strEq(buildData.manifestParentName, buildData.manifestWalName))
if (buildData->online && strEq(manifestParentName, buildData->manifestWalName))
FUNCTION_TEST_RETURN_VOID();
// Skip temp relations in db paths
if (buildData.dbPath && regExpMatch(buildData.tempRelationExp, info->name))
if (dbPath && regExpMatch(buildData->tempRelationExp, info->name))
FUNCTION_TEST_RETURN_VOID();
// Add file to manifest
@ -897,14 +904,14 @@ manifestBuildCallback(void *data, const StorageInfo *info)
};
// Determine if this file should be page checksummed
if (buildData.dbPath && buildData.checksumPage)
if (dbPath && buildData->checksumPage)
{
file.checksumPage =
!strEndsWithZ(manifestName, "/" PG_FILE_PGFILENODEMAP) && !strEndsWithZ(manifestName, "/" PG_FILE_PGVERSION) &&
!strEqZ(manifestName, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL);
}
manifestFileAdd(buildData.manifest, &file);
manifestFileAdd(buildData->manifest, &file);
break;
}
@ -915,16 +922,16 @@ manifestBuildCallback(void *data, const StorageInfo *info)
// If the destination is another link then error. In the future we'll allow this by following the link chain to the
// eventual destination but for now we are trying to maintain compatibility during the migration. To do this check we
// need to read outside of the data directory but it is a read-only operation so is considered safe.
const String *linkDestinationAbsolute = strPathAbsolute(info->linkDestination, buildData.pgPath);
const String *linkDestinationAbsolute = strPathAbsolute(info->linkDestination, pgPath);
StorageInfo linkedCheck = storageInfoP(
buildData.storagePg, linkDestinationAbsolute, .ignoreMissing = true, .noPathEnforce = true);
buildData->storagePg, linkDestinationAbsolute, .ignoreMissing = true, .noPathEnforce = true);
if (linkedCheck.exists && linkedCheck.type == storageTypeLink)
{
THROW_FMT(
LinkDestinationError, "link '%s/%s' cannot reference another link '%s'", strZ(buildData.pgPath),
strZ(info->name), strZ(linkDestinationAbsolute));
LinkDestinationError, "link '%s/%s' cannot reference another link '%s'", strZ(pgPath), strZ(info->name),
strZ(linkDestinationAbsolute));
}
// Initialize link and target
@ -946,7 +953,7 @@ manifestBuildCallback(void *data, const StorageInfo *info)
const String *linkName = info->name;
// Is this a tablespace?
if (strEq(buildData.manifestParentName, STRDEF(MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC)))
if (strEq(manifestParentName, STRDEF(MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC)))
{
// Strip pg_data off the manifest name so it begins with pg_tblspc instead. This reflects how the files are stored
// in the backup directory.
@ -957,10 +964,10 @@ manifestBuildCallback(void *data, const StorageInfo *info)
target.tablespaceId = cvtZToUInt(strZ(info->name));
// Look for this tablespace in the provided list (list may be null for off-line backup)
if (buildData.tablespaceList != NULL)
if (buildData->tablespaceList != NULL)
{
// Search list
PackRead *const read = pckReadNew(buildData.tablespaceList);
PackRead *const read = pckReadNew(buildData->tablespaceList);
while (!pckReadNullP(read))
{
@ -989,10 +996,10 @@ manifestBuildCallback(void *data, const StorageInfo *info)
// Add a dummy pg_tblspc path entry if it does not already exist. This entry will be ignored by restore but it is
// part of the original manifest format so we need to have it.
lstSort(buildData.manifest->pub.pathList, sortOrderAsc);
const ManifestPath *pathBase = manifestPathFind(buildData.manifest, MANIFEST_TARGET_PGDATA_STR);
lstSort(buildData->manifest->pub.pathList, sortOrderAsc);
const ManifestPath *pathBase = manifestPathFind(buildData->manifest, MANIFEST_TARGET_PGDATA_STR);
if (manifestPathFindDefault(buildData.manifest, MANIFEST_TARGET_PGTBLSPC_STR, NULL) == NULL)
if (manifestPathFindDefault(buildData->manifest, MANIFEST_TARGET_PGTBLSPC_STR, NULL) == NULL)
{
ManifestPath path =
{
@ -1002,14 +1009,14 @@ manifestBuildCallback(void *data, const StorageInfo *info)
.group = pathBase->group,
};
manifestPathAdd(buildData.manifest, &path);
manifestPathAdd(buildData->manifest, &path);
}
// The tablespace link destination path is not the path where data will be stored so we can just store it as a dummy
// path. This is because PostgreSQL creates a subpath with the version/catalog number so that multiple versions of
// PostgreSQL can share a tablespace, which makes upgrades easier.
const ManifestPath *pathTblSpc = manifestPathFind(
buildData.manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC));
buildData->manifest, STRDEF(MANIFEST_TARGET_PGDATA "/" MANIFEST_TARGET_PGTBLSPC));
ManifestPath path =
{
@ -1019,19 +1026,19 @@ manifestBuildCallback(void *data, const StorageInfo *info)
.group = pathTblSpc->group,
};
manifestPathAdd(buildData.manifest, &path);
manifestPathAdd(buildData->manifest, &path);
// Update build structure to reflect the path added above and the tablespace id
buildData.manifestParentName = manifestName;
manifestName = strNewFmt("%s/%s", strZ(manifestName), strZ(buildData.tablespaceId));
buildData.pgPath = strNewFmt("%s/%s", strZ(buildData.pgPath), strZ(info->name));
linkName = buildData.tablespaceId;
manifestParentName = manifestName;
manifestName = strNewFmt("%s/%s", strZ(manifestName), strZ(buildData->tablespaceId));
pgPath = strNewFmt("%s/%s", strZ(pgPath), strZ(info->name));
linkName = buildData->tablespaceId;
}
// Add info about the linked file/path
const String *linkPgPath = strNewFmt("%s/%s", strZ(buildData.pgPath), strZ(linkName));
const String *linkPgPath = strNewFmt("%s/%s", strZ(pgPath), strZ(linkName));
StorageInfo linkedInfo = storageInfoP(
buildData.storagePg, linkPgPath, .followLink = true, .ignoreMissing = true);
buildData->storagePg, linkPgPath, .followLink = true, .ignoreMissing = true);
linkedInfo.name = linkName;
// If the link destination exists then build the target
@ -1059,18 +1066,18 @@ manifestBuildCallback(void *data, const StorageInfo *info)
target.path = info->linkDestination;
// Add target and link
manifestTargetAdd(buildData.manifest, &target);
manifestLinkAdd(buildData.manifest, &link);
manifestTargetAdd(buildData->manifest, &target);
manifestLinkAdd(buildData->manifest, &link);
// Make sure the link is valid
manifestLinkCheckOne(buildData.manifest, &buildData.linkCheck, manifestTargetTotal(buildData.manifest) - 1);
manifestLinkCheckOne(buildData->manifest, buildData->linkCheck, manifestTargetTotal(buildData->manifest) - 1);
// If the link check was successful but the destination does not exist then check it again to generate an error
if (!linkedInfo.exists)
storageInfoP(buildData.storagePg, linkPgPath, .followLink = true);
storageInfoP(buildData->storagePg, linkPgPath, .followLink = true);
// Recurse into the link destination
manifestBuildCallback(&buildData, &linkedInfo);
manifestBuildInfo(buildData, manifestParentName, pgPath, dbPath, &linkedInfo);
break;
}
@ -1078,7 +1085,7 @@ manifestBuildCallback(void *data, const StorageInfo *info)
// Skip special files
// -------------------------------------------------------------------------------------------------------------------------
case storageTypeSpecial:
LOG_WARN_FMT("exclude special file '%s/%s' from backup", strZ(buildData.pgPath), strZ(info->name));
LOG_WARN_FMT("exclude special file '%s/%s' from backup", strZ(pgPath), strZ(info->name));
break;
}
@ -1127,6 +1134,8 @@ manifestNewBuild(
MEM_CONTEXT_TEMP_BEGIN()
{
// Data needed to build the manifest
ManifestLinkCheck linkCheck = manifestLinkCheckInit();
ManifestBuildData buildData =
{
.manifest = this,
@ -1135,10 +1144,8 @@ manifestNewBuild(
.online = online,
.checksumPage = checksumPage,
.tablespaceList = tablespaceList,
.linkCheck = manifestLinkCheckInit(),
.manifestParentName = MANIFEST_TARGET_PGDATA_STR,
.linkCheck = &linkCheck,
.manifestWalName = strNewFmt(MANIFEST_TARGET_PGDATA "/%s", strZ(pgWalPath(pgVersion))),
.pgPath = storagePathP(storagePg, NULL),
};
// Build expressions to identify databases paths and temp relations
@ -1180,7 +1187,8 @@ manifestNewBuild(
// Build manifest
// ---------------------------------------------------------------------------------------------------------------------
StorageInfo info = storageInfoP(storagePg, buildData.pgPath, .followLink = true);
const String *const pgPath = storagePathP(storagePg, NULL);
StorageInfo info = storageInfoP(storagePg, pgPath, .followLink = true);
ManifestPath path =
{
@ -1204,15 +1212,29 @@ manifestNewBuild(
ManifestTarget target =
{
.name = MANIFEST_TARGET_PGDATA_STR,
.path = buildData.pgPath,
.path = pgPath,
.type = manifestTargetTypePath,
};
manifestTargetAdd(this, &target);
// Gather info for the rest of the files/links/paths
storageInfoListP(
storagePg, buildData.pgPath, manifestBuildCallback, &buildData, .errorOnMissing = true, .sortOrder = sortOrderAsc);
StorageIterator *const storageItr = storageNewItrP(
storagePg, pgPath, .errorOnMissing = true, .sortOrder = sortOrderAsc);
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
while (storageItrMore(storageItr))
{
const StorageInfo info = storageItrNext(storageItr);
manifestBuildInfo(&buildData, MANIFEST_TARGET_PGDATA_STR, pgPath, false, &info);
// Reset the memory context occasionally so we don't use too much memory or slow down processing
MEM_CONTEXT_TEMP_RESET(1000);
}
}
MEM_CONTEXT_TEMP_END();
// These may not be in order even if the incoming data was sorted
lstSort(this->pub.fileList, sortOrderAsc);

3
src/meson.build
View File

@ -24,6 +24,7 @@ src_common = files(
'common/regExp.c',
'common/stackTrace.c',
'common/time.c',
'common/type/blob.c',
'common/type/buffer.c',
'common/type/convert.c',
'common/type/keyValue.c',
@ -42,6 +43,8 @@ src_common = files(
'storage/posix/read.c',
'storage/posix/storage.c',
'storage/posix/write.c',
'storage/iterator.c',
'storage/list.c',
'storage/read.c',
'storage/storage.c',
'storage/write.c',

35
src/storage/azure/storage.c
View File

@ -303,7 +303,7 @@ General function for listing files to be used by other list routines
static void
storageAzureListInternal(
StorageAzure *this, const String *path, StorageInfoLevel level, const String *expression, bool recurse,
StorageInfoListCallback callback, void *callbackData)
StorageListCallback callback, void *callbackData)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_AZURE, this);
@ -506,10 +506,24 @@ storageAzureInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageI
}
/**********************************************************************************************************************************/
static bool
storageAzureInfoList(
THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
static void
storageAzureListCallback(void *const callbackData, const StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, callbackData);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(callbackData != NULL);
ASSERT(info != NULL);
storageLstAdd(callbackData, info);
FUNCTION_TEST_RETURN_VOID();
}
static StorageList *
storageAzureList(THIS_VOID, const String *const path, const StorageInfoLevel level, const StorageInterfaceListParam param)
{
THIS(StorageAzure);
@ -517,18 +531,17 @@ storageAzureInfoList(
FUNCTION_LOG_PARAM(STORAGE_AZURE, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_PARAM(STRING, param.expression);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(path != NULL);
ASSERT(callback != NULL);
storageAzureListInternal(this, path, level, param.expression, false, callback, callbackData);
StorageList *const result = storageLstNew(level);
FUNCTION_LOG_RETURN(BOOL, true);
storageAzureListInternal(this, path, level, param.expression, false, storageAzureListCallback, result);
FUNCTION_LOG_RETURN(STORAGE_LIST, result);
}
/**********************************************************************************************************************************/
@ -677,7 +690,7 @@ storageAzureRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam pa
static const StorageInterface storageInterfaceAzure =
{
.info = storageAzureInfo,
.infoList = storageAzureInfoList,
.list = storageAzureList,
.newRead = storageAzureNewRead,
.newWrite = storageAzureNewWrite,
.pathRemove = storageAzurePathRemove,

39
src/storage/gcs/storage.c
View File

@ -532,7 +532,7 @@ storageGcsInfoFile(StorageInfo *info, const KeyValue *file)
static void
storageGcsListInternal(
StorageGcs *this, const String *path, StorageInfoLevel level, const String *expression, bool recurse,
StorageInfoListCallback callback, void *callbackData)
StorageListCallback callback, void *callbackData)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_GCS, this);
@ -744,10 +744,24 @@ storageGcsInfo(THIS_VOID, const String *const file, const StorageInfoLevel level
}
/**********************************************************************************************************************************/
static bool
storageGcsInfoList(
THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
static void
storageGcsListCallback(void *const callbackData, const StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, callbackData);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(callbackData != NULL);
ASSERT(info != NULL);
storageLstAdd(callbackData, info);
FUNCTION_TEST_RETURN_VOID();
}
static StorageList *
storageGcsList(THIS_VOID, const String *const path, const StorageInfoLevel level, const StorageInterfaceListParam param)
{
THIS(StorageGcs);
@ -755,22 +769,17 @@ storageGcsInfoList(
FUNCTION_LOG_PARAM(STORAGE_GCS, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_PARAM(STRING, param.expression);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(path != NULL);
ASSERT(callback != NULL);
MEM_CONTEXT_TEMP_BEGIN()
{
storageGcsListInternal(this, path, level, param.expression, false, callback, callbackData);
}
MEM_CONTEXT_TEMP_END();
StorageList *const result = storageLstNew(level);
FUNCTION_LOG_RETURN(BOOL, true);
storageGcsListInternal(this, path, level, param.expression, false, storageGcsListCallback, result);
FUNCTION_LOG_RETURN(STORAGE_LIST, result);
}
/**********************************************************************************************************************************/
@ -919,7 +928,7 @@ storageGcsRemove(THIS_VOID, const String *const file, const StorageInterfaceRemo
static const StorageInterface storageInterfaceGcs =
{
.info = storageGcsInfo,
.infoList = storageGcsInfoList,
.list = storageGcsList,
.newRead = storageGcsNewRead,
.newWrite = storageGcsNewWrite,
.pathRemove = storageGcsPathRemove,

277
src/storage/iterator.c Normal file
View File

@ -0,0 +1,277 @@
/***********************************************************************************************************************************
Storage Iterator
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/debug.h"
#include "common/log.h"
#include "common/regExp.h"
#include "common/type/list.h"
#include "storage/iterator.h"
#include "storage/list.h"
#include "storage/storage.h"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct StorageIterator
{
void *driver; // Storage driver
const String *path; // Path to iterate
StorageInfoLevel level; // Info level
bool recurse; // Recurse into paths
SortOrder sortOrder; // Sort order
const String *expression; // Match expression
RegExp *regExp; // Parsed match expression
List *stack; // Stack of info lists
bool returnedNext; // Next info was returned
StorageInfo infoNext; // Info to be returned by next
String *nameNext; // Name for next info
};
// Path info list
typedef struct StorageIteratorInfo
{
String *pathSub; // Subpath
StorageList *list; // Storage info list
unsigned int listIdx; // Current index in info list
bool pathContentSkip; // Skip reading path content
} StorageIteratorInfo;
/***********************************************************************************************************************************
Check a path and add it to the stack if it exists and has content
***********************************************************************************************************************************/
typedef struct StorageItrPathAddResult
{
bool exists; // Does the path exist?
bool content; // Does the path have content?
} StorageItrPathAddResult;
static StorageItrPathAddResult
storageItrPathAdd(StorageIterator *const this, const String *const pathSub)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_ITERATOR, this);
FUNCTION_LOG_PARAM(STRING, pathSub);
FUNCTION_LOG_END();
ASSERT(this != NULL);
StorageItrPathAddResult result = {0};
MEM_CONTEXT_TEMP_BEGIN()
{
// Get path content
StorageList *const list = storageInterfaceListP(
this->driver, pathSub == NULL ? this->path : strNewFmt("%s/%s", strZ(this->path), strZ(pathSub)), this->level,
.expression = this->expression);
// If path exists
if (list != NULL)
{
result.exists = true;
// If the path has content
if (!storageLstEmpty(list))
{
result.content = true;
// Sort if requested
if (this->sortOrder != sortOrderNone)
storageLstSort(list, this->sortOrder);
// Add path to top of stack
MEM_CONTEXT_OBJ_BEGIN(this->stack)
{
OBJ_NEW_BEGIN(StorageIteratorInfo, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = 1)
{
StorageIteratorInfo *const listInfo = OBJ_NEW_ALLOC();
*listInfo = (StorageIteratorInfo){
.pathSub = strDup(pathSub), .list = storageLstMove(list, memContextCurrent())};
lstAdd(this->stack, &listInfo);
}
OBJ_NEW_END();
}
MEM_CONTEXT_OBJ_END();
}
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_STRUCT(result);
}
/**********************************************************************************************************************************/
StorageIterator *
storageItrNew(
void *const driver, const String *const path, const StorageInfoLevel level, const bool errorOnMissing, const bool nullOnMissing,
const bool recurse, const SortOrder sortOrder, const String *const expression)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM_P(VOID, driver);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(BOOL, errorOnMissing);
FUNCTION_LOG_PARAM(BOOL, nullOnMissing);
FUNCTION_LOG_PARAM(BOOL, recurse);
FUNCTION_LOG_PARAM(ENUM, sortOrder);
FUNCTION_LOG_PARAM(STRING, expression);
FUNCTION_LOG_END();
ASSERT(driver != NULL);
ASSERT(path != NULL);
ASSERT(!recurse || level >= storageInfoLevelType);
StorageIterator *this = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
OBJ_NEW_BEGIN(StorageIterator, .childQty = MEM_CONTEXT_QTY_MAX)
{
// Create object
this = OBJ_NEW_ALLOC();
*this = (StorageIterator)
{
.driver = driver,
.path = strDup(path),
.level = level,
.recurse = recurse,
.sortOrder = sortOrder,
.expression = strDup(expression),
.stack = lstNewP(sizeof(StorageIteratorInfo *)),
.nameNext = strNew(),
.returnedNext = true,
};
// Compile regular expression
if (this->expression != NULL)
this->regExp = regExpNew(this->expression);
// If root path does not exist
if (!storageItrPathAdd(this, NULL).exists)
{
// Throw an error when requested
if (errorOnMissing)
THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, strZ(this->path));
// Return NULL when requested
if (nullOnMissing)
this = NULL;
}
}
OBJ_NEW_END();
storageItrMove(this, memContextPrior());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STORAGE_ITERATOR, this);
}
/**********************************************************************************************************************************/
bool
storageItrMore(StorageIterator *const this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_ITERATOR, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
// If next has not been called then return true
if (!this->returnedNext)
FUNCTION_TEST_RETURN(BOOL, true);
// Search stack for info
while (lstSize(this->stack) != 0)
{
// Always pull from the top of the stack
StorageIteratorInfo *const listInfo = *(StorageIteratorInfo **)lstGetLast(this->stack);
// Search list for info
for (; listInfo->listIdx < storageLstSize(listInfo->list); listInfo->listIdx++)
{
// Next info
this->infoNext = storageLstGet(listInfo->list, listInfo->listIdx);
// Update info name when in subpath
if (listInfo->pathSub != NULL)
{
strCatFmt(strTrunc(this->nameNext), "%s/%s", strZ(listInfo->pathSub), strZ(this->infoNext.name));
this->infoNext.name = this->nameNext;
}
// Does the path have content?
const bool pathContent =
this->infoNext.type == storageTypePath && this->recurse && !listInfo->pathContentSkip &&
storageItrPathAdd(this, this->infoNext.name).content;
// Clear path content skip flag if it was set on a previous iteration
listInfo->pathContentSkip = false;
// Skip info if it does match the provided expression
if (this->regExp != NULL && !regExpMatch(this->regExp, this->infoNext.name))
{
// Path content may match the expression even if the path does not. Break so the content on the top of the stack
// will be checked and increment the index so that path will be skipped.
if (pathContent)
{
listInfo->listIdx++;
break;
}
// Skip info and continue
continue;
}
// When sort order is descending the path will need to be output after the content. Break so that content gets checked
// but set a flag so on the next iteration the path will be output but the content skipped.
if (pathContent && this->sortOrder == sortOrderDesc)
{
listInfo->pathContentSkip = true;
break;
}
// Return next info
this->returnedNext = false;
listInfo->listIdx++;
FUNCTION_TEST_RETURN(BOOL, true);
}
// If no more info then free the list. This check is required because we may break out of the above loop early.
if (listInfo->listIdx >= storageLstSize(listInfo->list))
{
objFree(listInfo);
lstRemoveLast(this->stack);
}
}
FUNCTION_TEST_RETURN(BOOL, false);
}
/**********************************************************************************************************************************/
StorageInfo storageItrNext(StorageIterator *const this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_ITERATOR, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(!this->returnedNext);
this->returnedNext = true;
FUNCTION_TEST_RETURN(STORAGE_INFO, this->infoNext);
}
/**********************************************************************************************************************************/
String *
storageItrToLog(const StorageIterator *const this)
{
return strNewFmt("{stack: %s}", strZ(lstToLog(this->stack)));
}

58
src/storage/iterator.h Normal file
View File

@ -0,0 +1,58 @@
/***********************************************************************************************************************************
Storage Iterator
***********************************************************************************************************************************/
#ifndef STORAGE_ITERATOR_H
#define STORAGE_ITERATOR_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct StorageIterator StorageIterator;
#include "common/type/string.h"
#include "storage/info.h"
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
StorageIterator *storageItrNew(
void *driver, const String *path, StorageInfoLevel level, bool errorOnMissing, bool nullOnMissing, bool recurse,
SortOrder sortOrder, const String *expression);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Is there more info to be retrieved from the iterator?
bool storageItrMore(StorageIterator *this);
// Move to a new parent mem context
__attribute__((always_inline)) static inline StorageIterator *
storageItrMove(StorageIterator *const this, MemContext *const parentNew)
{
return objMove(this, parentNew);
}
// Get next info. An error will be thrown if there is no more data so use storageItrMore() to check. Note that StorageInfo pointer
// members (e.g. name) will be undefined after the next call to storageItrMore().
StorageInfo storageItrNext(StorageIterator *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
__attribute__((always_inline)) static inline void
storageItrFree(StorageIterator *const this)
{
objFree(this);
}
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
String *storageItrToLog(const StorageIterator *this);
#define FUNCTION_LOG_STORAGE_ITERATOR_TYPE \
StorageIterator *
#define FUNCTION_LOG_STORAGE_ITERATOR_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, storageItrToLog, buffer, bufferSize)
#endif

215
src/storage/list.c Normal file
View File

@ -0,0 +1,215 @@
/***********************************************************************************************************************************
Storage List
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/debug.h"
#include "common/log.h"
#include "common/type/blob.h"
#include "common/type/list.h"
#include "storage/list.h"
#include "storage/storage.h"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct StorageList
{
StorageListPub pub; // Publicly accessible variables
StringList *ownerList; // List of users/groups
Blob *blob; // Blob of info data
String *name; // Current info name
String *linkDestination; // Current link destination
};
/***********************************************************************************************************************************
A more space efficient version of StorageInfo. Each level is contained in a struct to ensure alignment when only using part of the
struct to store info. This also keeps the size calculations accurate if members are added to a level.
***********************************************************************************************************************************/
typedef struct StorageListInfo
{
// Set when info type >= storageInfoLevelExists
struct
{
const char *name; // Name of path/file/link
} exists;
// Mode is only provided at detail level but is included here to save space on 64-bit architectures
struct
{
// Set when info type >= storageInfoLevelType (undefined at lower levels)
StorageType type; // Type file/path/link)
// Set when info type >= storageInfoLevelDetail (undefined at lower levels)
mode_t mode; // Mode of path/file/link
} type;
// Set when info type >= storageInfoLevelBasic (undefined at lower levels)
struct
{
uint64_t size; // Size (path/link is 0)
time_t timeModified; // Time file was last modified
} basic;
// Set when info type >= storageInfoLevelDetail (undefined at lower levels)
struct
{
const String *user; // Name of user that owns the file
const String *group; // Name of group that owns the file
uid_t userId; // User that owns the file
uid_t groupId; // Group that owns the file
const char *linkDestination; // Destination if this is a link
} detail;
} StorageListInfo;
// Determine size of struct to store in the list. We only need to store the members that are used by the current level.
static const uint8_t storageLstInfoSize[storageInfoLevelDetail + 1] =
{
0, // storageInfoLevelDefault (should not be used)
offsetof(StorageListInfo, type), // storageInfoLevelExists
offsetof(StorageListInfo, basic), // storageInfoLevelType
offsetof(StorageListInfo, detail), // storageInfoLevelBasic
sizeof(StorageListInfo), // storageInfoLevelDetail
};
/**********************************************************************************************************************************/
StorageList *
storageLstNew(const StorageInfoLevel level)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(ENUM, level);
FUNCTION_TEST_END();
ASSERT(level != storageInfoLevelDefault);
StorageList *this = NULL;
OBJ_NEW_BEGIN(StorageList, .childQty = MEM_CONTEXT_QTY_MAX)
{
// Create object
this = OBJ_NEW_ALLOC();
*this = (StorageList)
{
.pub =
{
.list = lstNewP(storageLstInfoSize[level], .comparator = lstComparatorZ),
.level = level,
},
.ownerList = strLstNew(),
.blob = blbNew(),
.name = strNew(),
.linkDestination = strNew(),
};
}
OBJ_NEW_END();
FUNCTION_TEST_RETURN(STORAGE_LIST, this);
}
/**********************************************************************************************************************************/
void
storageLstInsert(StorageList *const this, const unsigned int idx, const StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_LIST, this);
FUNCTION_TEST_PARAM(UINT, idx);
FUNCTION_TEST_PARAM_P(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(info != NULL);
MEM_CONTEXT_OBJ_BEGIN(this->pub.list)
{
StorageListInfo listInfo = {.exists = {.name = blbAdd(this->blob, strZ(info->name), strSize(info->name) + 1)}};
switch (storageLstLevel(this))
{
case storageInfoLevelDetail:
{
listInfo.type.mode = info->mode;
listInfo.detail.user = strLstAddIfMissing(this->ownerList, info->user);
listInfo.detail.group = strLstAddIfMissing(this->ownerList, info->group);
listInfo.detail.userId = info->userId;
listInfo.detail.groupId = info->groupId;
if (info->linkDestination != NULL)
{
listInfo.detail.linkDestination = blbAdd(
this->blob, strZ(info->linkDestination), strSize(info->linkDestination) + 1);
}
}
case storageInfoLevelBasic:
listInfo.basic.size = info->size;
listInfo.basic.timeModified = info->timeModified;
case storageInfoLevelType:
listInfo.type.type = info->type;
default:
break;
}
lstInsert(this->pub.list, idx, &listInfo);
}
MEM_CONTEXT_OBJ_END();
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
StorageInfo
storageLstGet(StorageList *const this, const unsigned int idx)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_LIST, this);
FUNCTION_TEST_PARAM(UINT, idx);
FUNCTION_TEST_END();
ASSERT(this != NULL);
const StorageListInfo *const listInfo = lstGet(this->pub.list, idx);
StorageInfo result =
{
.name = strCatZ(strTrunc(this->name), listInfo->exists.name),
.exists = true,
.level = storageLstLevel(this)
};
switch (result.level)
{
case storageInfoLevelDetail:
{
result.mode = listInfo->type.mode;
result.user = listInfo->detail.user;
result.group = listInfo->detail.group;
result.userId = listInfo->detail.userId;
result.groupId = listInfo->detail.groupId;
if (listInfo->detail.linkDestination != NULL)
result.linkDestination = strCatZ(strTrunc(this->linkDestination), listInfo->detail.linkDestination);
}
case storageInfoLevelBasic:
result.size = listInfo->basic.size;
result.timeModified = listInfo->basic.timeModified;
case storageInfoLevelType:
result.type = listInfo->type.type;
default:
break;
}
FUNCTION_TEST_RETURN(STORAGE_INFO, result);
}
/**********************************************************************************************************************************/
String *
storageLstToLog(const StorageList *const this)
{
return strNewFmt("{size: %u}", lstSize(this->pub.list));
}

99
src/storage/list.h Normal file
View File

@ -0,0 +1,99 @@
/***********************************************************************************************************************************
Storage List
***********************************************************************************************************************************/
#ifndef STORAGE_LIST_H
#define STORAGE_LIST_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct StorageList StorageList;
#include "common/type/string.h"
#include "storage/info.h"
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
StorageList *storageLstNew(StorageInfoLevel level);
/***********************************************************************************************************************************
Getters/Setters
***********************************************************************************************************************************/
typedef struct StorageListPub
{
List *list; // Storage list
StorageInfoLevel level; // Storage info level
} StorageListPub;
// Empty?
__attribute__((always_inline)) static inline bool
storageLstEmpty(const StorageList *const this)
{
return lstEmpty(THIS_PUB(StorageList)->list);
}
// Storage info level
__attribute__((always_inline)) static inline StorageInfoLevel
storageLstLevel(const StorageList *const this)
{
return THIS_PUB(StorageList)->level;
}
// List size
__attribute__((always_inline)) static inline unsigned int
storageLstSize(const StorageList *const this)
{
return lstSize(THIS_PUB(StorageList)->list);
}
// List size
__attribute__((always_inline)) static inline void
storageLstSort(StorageList *const this, const SortOrder sortOrder)
{
lstSort(THIS_PUB(StorageList)->list, sortOrder);
}
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Insert info
void storageLstInsert(StorageList *this, unsigned int idx, const StorageInfo *info);
// Add info
__attribute__((always_inline)) static inline void
storageLstAdd(StorageList *const this, const StorageInfo *const info)
{
storageLstInsert(this, storageLstSize(this), info);
}
// Get info. Note that StorageInfo pointer members (e.g. name) will be undefined after the next call to storageLstGet().
StorageInfo storageLstGet(StorageList *this, unsigned int idx);
// Move to a new parent mem context
__attribute__((always_inline)) static inline StorageList *
storageLstMove(StorageList *const this, MemContext *const parentNew)
{
return objMove(this, parentNew);
}
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
__attribute__((always_inline)) static inline void
storageLstFree(StorageList *const this)
{
objFree(this);
}
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
String *storageLstToLog(const StorageList *this);
#define FUNCTION_LOG_STORAGE_LIST_TYPE \
StorageList *
#define FUNCTION_LOG_STORAGE_LIST_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, storageLstToLog, buffer, bufferSize)
#endif

144
src/storage/posix/storage.c
View File

@ -113,44 +113,40 @@ storagePosixInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageI
}
/**********************************************************************************************************************************/
// Helper function to get info for a file if it exists. This logic can't live directly in storagePosixInfoList() because there is
// a race condition where a file might exist while listing the directory but it is gone before stat() can be called. In order to
// get complete test coverage this function must be split out.
// Helper function to get info for a file if it exists. This logic can't live directly in storagePosixList() because there is a race
// condition where a file might exist while listing the directory but it is gone before stat() can be called. In order to get
// complete test coverage this function must be split out.
static void
storagePosixInfoListEntry(
StoragePosix *this, const String *path, const String *name, StorageInfoLevel level, StorageInfoListCallback callback,
void *callbackData)
storagePosixListEntry(
StoragePosix *const this, StorageList *const list, const String *const path, const char *const name,
const StorageInfoLevel level)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_POSIX, this);
FUNCTION_TEST_PARAM(STORAGE_LIST, list);
FUNCTION_TEST_PARAM(STRING, path);
FUNCTION_TEST_PARAM(STRING, name);
FUNCTION_TEST_PARAM(STRINGZ, name);
FUNCTION_TEST_PARAM(ENUM, level);
FUNCTION_TEST_PARAM(FUNCTIONP, callback);
FUNCTION_TEST_PARAM_P(VOID, callbackData);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(list != NULL);
ASSERT(path != NULL);
ASSERT(name != NULL);
ASSERT(callback != NULL);
StorageInfo storageInfo = storageInterfaceInfoP(
this, strEq(name, DOT_STR) ? strDup(path) : strNewFmt("%s/%s", strZ(path), strZ(name)), level);
StorageInfo info = storageInterfaceInfoP(this, strNewFmt("%s/%s", strZ(path), name), level);
if (storageInfo.exists)
if (info.exists)
{
storageInfo.name = name;
callback(callbackData, &storageInfo);
info.name = STR(name);
storageLstAdd(list, &info);
}
FUNCTION_TEST_RETURN_VOID();
}
static bool
storagePosixInfoList(
THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
static StorageList *
storagePosixList(THIS_VOID, const String *const path, const StorageInfoLevel level, const StorageInterfaceListParam param)
{
THIS(StoragePosix);
@ -158,19 +154,16 @@ storagePosixInfoList(
FUNCTION_LOG_PARAM(STORAGE_POSIX, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
(void)param; // No parameters are used
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(path != NULL);
ASSERT(callback != NULL);
bool result = false;
StorageList *result = NULL;
// Open the directory for read
DIR *dir = opendir(strZ(path));
DIR *const dir = opendir(strZ(path));
// If the directory could not be opened process errors and report missing directories
if (dir == NULL)
@ -178,34 +171,39 @@ storagePosixInfoList(
if (errno != ENOENT) // {vm_covered}
THROW_SYS_ERROR_FMT(PathOpenError, STORAGE_ERROR_LIST_INFO, strZ(path)); // {vm_covered}
}
// Directory was found
else
{
// Directory was found
result = true;
result = storageLstNew(level);
TRY_BEGIN()
{
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
// Read the directory entries
struct dirent *dirEntry = readdir(dir);
const struct dirent *dirEntry = readdir(dir);
while (dirEntry != NULL)
{
const String *name = STR(dirEntry->d_name);
// Always skip ..
if (!strEq(name, DOTDOT_STR))
// Always skip . and ..
if (!strEqZ(DOT_STR, dirEntry->d_name) && !strEqZ(DOTDOT_STR, dirEntry->d_name))
{
// If only making a list of files that exist then no need to go get detailed info which requires calling
// stat() and is therefore relatively slow
if (level == storageInfoLevelExists)
{
callback(callbackData, &(StorageInfo){.name = name, .level = storageInfoLevelExists, .exists = true});
storageLstAdd(
result,
&(StorageInfo)
{
.name = STR(dirEntry->d_name),
.level = storageInfoLevelExists,
.exists = true,
});
}
// Else more info is required which requires a call to stat()
else
storagePosixInfoListEntry(this, path, name, level, callback, callbackData);
storagePosixListEntry(this, result, path, dirEntry->d_name, level);
}
// Get next entry
@ -224,7 +222,7 @@ storagePosixInfoList(
TRY_END();
}
FUNCTION_LOG_RETURN(BOOL, result);
FUNCTION_LOG_RETURN(STORAGE_LIST, result);
}
/**********************************************************************************************************************************/
@ -392,47 +390,6 @@ storagePosixPathCreate(
}
/**********************************************************************************************************************************/
typedef struct StoragePosixPathRemoveData
{
StoragePosix *driver; // Driver
const String *path; // Path
} StoragePosixPathRemoveData;
static void
storagePosixPathRemoveCallback(void *const callbackData, const StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, callbackData);
FUNCTION_TEST_PARAM_P(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(callbackData != NULL);
ASSERT(info != NULL);
if (!strEqZ(info->name, "."))
{
StoragePosixPathRemoveData *const data = callbackData;
String *const file = strNewFmt("%s/%s", strZ(data->path), strZ(info->name));
// Rather than stat the file to discover what type it is, just try to unlink it and see what happens
if (unlink(strZ(file)) == -1) // {vm_covered}
{
// These errors indicate that the entry is actually a path so we'll try to delete it that way
if (errno == EPERM || errno == EISDIR) // {uncovered_branch - no EPERM on tested systems}
{
storageInterfacePathRemoveP(data->driver, file, true);
}
// Else error
else
THROW_SYS_ERROR_FMT(PathRemoveError, STORAGE_ERROR_PATH_REMOVE_FILE, strZ(file)); // {vm_covered}
}
strFree(file);
}
FUNCTION_TEST_RETURN_VOID();
}
static bool
storagePosixPathRemove(THIS_VOID, const String *path, bool recurse, StorageInterfacePathRemoveParam param)
{
@ -455,14 +412,35 @@ storagePosixPathRemove(THIS_VOID, const String *path, bool recurse, StorageInter
// Recurse if requested
if (recurse)
{
StoragePosixPathRemoveData data =
{
.driver = this,
.path = path,
};
StorageList *const list = storageInterfaceListP(this, path, storageInfoLevelExists);
// Remove all sub paths/files
storageInterfaceInfoListP(this, path, storageInfoLevelExists, storagePosixPathRemoveCallback, &data);
if (list != NULL)
{
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
for (unsigned int listIdx = 0; listIdx < storageLstSize(list); listIdx++)
{
const String *const file = strNewFmt("%s/%s", strZ(path), strZ(storageLstGet(list, listIdx).name));
// Rather than stat the file to discover what type it is, just try to unlink it and see what happens
if (unlink(strZ(file)) == -1) // {vm_covered}
{
// These errors indicate that the entry is actually a path so we'll try to delete it that way
if (errno == EPERM || errno == EISDIR) // {uncovered_branch - no EPERM on tested systems}
{
storageInterfacePathRemoveP(this, file, true);
}
// Else error
else
THROW_SYS_ERROR_FMT(PathRemoveError, STORAGE_ERROR_PATH_REMOVE_FILE, strZ(file)); // {vm_covered}
}
// Reset the memory context occasionally so we don't use too much memory or slow down processing
MEM_CONTEXT_TEMP_RESET(1000);
}
}
MEM_CONTEXT_TEMP_END();
}
}
// Delete the path
@ -556,7 +534,7 @@ static const StorageInterface storageInterfacePosix =
.feature = 1 << storageFeaturePath,
.info = storagePosixInfo,
.infoList = storagePosixInfoList,
.list = storagePosixList,
.move = storagePosixMove,
.newRead = storagePosixNewRead,
.newWrite = storagePosixNewWrite,

50
src/storage/remote/protocol.c
View File

@ -280,36 +280,8 @@ storageRemoteInfoProtocol(PackRead *const param, ProtocolServer *const server)
}
/**********************************************************************************************************************************/
typedef struct StorageRemoteProtocolInfoListCallbackData
{
ProtocolServer *const server;
StorageRemoteInfoProtocolWriteData writeData;
} StorageRemoteProtocolInfoListCallbackData;
static void
storageRemoteProtocolInfoListCallback(void *const dataVoid, const StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM_P(VOID, dataVoid);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(dataVoid != NULL);
ASSERT(info != NULL);
StorageRemoteProtocolInfoListCallbackData *const data = dataVoid;
PackWrite *const write = protocolPackNew();
pckWriteStrP(write, info->name);
storageRemoteInfoProtocolPut(&data->writeData, write, info);
protocolServerDataPut(data->server, write);
pckWriteFree(write);
FUNCTION_TEST_RETURN_VOID();
}
void
storageRemoteInfoListProtocol(PackRead *const param, ProtocolServer *const server)
storageRemoteListProtocol(PackRead *const param, ProtocolServer *const server)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(PACK_READ, param);
@ -324,15 +296,27 @@ storageRemoteInfoListProtocol(PackRead *const param, ProtocolServer *const serve
{
const String *const path = pckReadStrP(param);
const StorageInfoLevel level = (StorageInfoLevel)pckReadU32P(param);
StorageRemoteInfoProtocolWriteData writeData = {.memContext = memContextCurrent()};
StorageList *const list = storageInterfaceListP(storageRemoteProtocolLocal.driver, path, level);
StorageRemoteProtocolInfoListCallbackData data = {.server = server, .writeData = {.memContext = memContextCurrent()}};
// Put list
if (list != NULL)
{
for (unsigned int listIdx = 0; listIdx < storageLstSize(list); listIdx++)
{
const StorageInfo info = storageLstGet(list, listIdx);
const bool result = storageInterfaceInfoListP(
storageRemoteProtocolLocal.driver, path, level, storageRemoteProtocolInfoListCallback, &data);
PackWrite *const write = protocolPackNew();
pckWriteStrP(write, info.name);
storageRemoteInfoProtocolPut(&writeData, write, &info);
protocolServerDataPut(server, write);
pckWriteFree(write);
}
}
// Indicate whether or not the path was found
PackWrite *write = protocolPackNew();
pckWriteBoolP(write, result, .defaultWrite = true);
pckWriteBoolP(write, list != NULL, .defaultWrite = true);
protocolServerDataPut(server, write);
protocolServerDataEndPut(server);

6
src/storage/remote/protocol.h
View File

@ -13,7 +13,7 @@ Functions
// Process storage protocol requests
void storageRemoteFeatureProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteInfoProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteInfoListProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteListProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteOpenReadProtocol(PackRead *param, ProtocolServer *server);
void storageRemoteOpenWriteProtocol(PackRead *param, ProtocolServer *server);
void storageRemotePathCreateProtocol(PackRead *param, ProtocolServer *server);
@ -26,7 +26,7 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
***********************************************************************************************************************************/
#define PROTOCOL_COMMAND_STORAGE_FEATURE STRID5("s-f", 0x1b730)
#define PROTOCOL_COMMAND_STORAGE_INFO STRID5("s-i", 0x27730)
#define PROTOCOL_COMMAND_STORAGE_INFO_LIST STRID5("s-l", 0x33730)
#define PROTOCOL_COMMAND_STORAGE_LIST STRID5("s-l", 0x33730)
#define PROTOCOL_COMMAND_STORAGE_OPEN_READ STRID5("s-or", 0x93f730)
#define PROTOCOL_COMMAND_STORAGE_OPEN_WRITE STRID5("s-ow", 0xbbf730)
#define PROTOCOL_COMMAND_STORAGE_PATH_CREATE STRID5("s-pc", 0x1c3730)
@ -37,7 +37,7 @@ Protocol commands for ProtocolServerHandler arrays passed to protocolServerProce
#define PROTOCOL_SERVER_HANDLER_STORAGE_REMOTE_LIST \
{.command = PROTOCOL_COMMAND_STORAGE_FEATURE, .handler = storageRemoteFeatureProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_INFO, .handler = storageRemoteInfoProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_INFO_LIST, .handler = storageRemoteInfoListProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_LIST, .handler = storageRemoteListProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_OPEN_READ, .handler = storageRemoteOpenReadProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_OPEN_WRITE, .handler = storageRemoteOpenWriteProtocol}, \
{.command = PROTOCOL_COMMAND_STORAGE_PATH_CREATE, .handler = storageRemotePathCreateProtocol}, \

24
src/storage/remote/storage.c
View File

@ -168,10 +168,8 @@ storageRemoteInfo(THIS_VOID, const String *file, StorageInfoLevel level, Storage
}
/**********************************************************************************************************************************/
static bool
storageRemoteInfoList(
THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
static StorageList *
storageRemoteList(THIS_VOID, const String *const path, const StorageInfoLevel level, const StorageInterfaceListParam param)
{
THIS(StorageRemote);
@ -179,20 +177,17 @@ storageRemoteInfoList(
FUNCTION_LOG_PARAM(STORAGE_REMOTE, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
(void)param; // No parameters are used
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(path != NULL);
ASSERT(callback != NULL);
bool result = false;
StorageList *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_INFO_LIST);
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_LIST);
PackWrite *const commandParam = protocolCommandParam(command);
pckWriteStrP(commandParam, path);
@ -203,6 +198,7 @@ storageRemoteInfoList(
// Read list
StorageRemoteInfoData parseData = {.memContext = memContextCurrent()};
result = storageLstNew(level);
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
@ -214,7 +210,7 @@ storageRemoteInfoList(
StorageInfo info = {.exists = true, .level = level, .name = pckReadStrP(read)};
storageRemoteInfoGet(&parseData, read, &info);
callback(callbackData, &info);
storageLstAdd(result, &info);
// Reset the memory context occasionally so we don't use too much memory or slow down processing
MEM_CONTEXT_TEMP_RESET(1000);
@ -223,15 +219,17 @@ storageRemoteInfoList(
pckReadNext(read);
}
result = pckReadBoolP(read);
if (!pckReadBoolP(read))
result = NULL;
}
MEM_CONTEXT_TEMP_END();
protocolClientDataEndGet(this->client);
storageLstMove(result, memContextPrior());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result);
FUNCTION_LOG_RETURN(STORAGE_LIST, result);
}
/**********************************************************************************************************************************/
@ -422,7 +420,7 @@ storageRemoteRemove(THIS_VOID, const String *file, StorageInterfaceRemoveParam p
static const StorageInterface storageInterfaceRemote =
{
.info = storageRemoteInfo,
.infoList = storageRemoteInfoList,
.list = storageRemoteList,
.newRead = storageRemoteNewRead,
.newWrite = storageRemoteNewWrite,
.pathCreate = storageRemotePathCreate,

35
src/storage/s3/storage.c
View File

@ -608,7 +608,7 @@ General function for listing files to be used by other list routines
static void
storageS3ListInternal(
StorageS3 *this, const String *path, StorageInfoLevel level, const String *expression, bool recurse,
StorageInfoListCallback callback, void *callbackData)
StorageListCallback callback, void *callbackData)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE_S3, this);
@ -813,10 +813,24 @@ storageS3Info(THIS_VOID, const String *const file, const StorageInfoLevel level,
}
/**********************************************************************************************************************************/
static bool
storageS3InfoList(
THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
static void
storageS3ListCallback(void *const callbackData, const StorageInfo *const info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, callbackData);
FUNCTION_TEST_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
ASSERT(callbackData != NULL);
ASSERT(info != NULL);
storageLstAdd(callbackData, info);
FUNCTION_TEST_RETURN_VOID();
}
static StorageList *
storageS3List(THIS_VOID, const String *const path, const StorageInfoLevel level, const StorageInterfaceListParam param)
{
THIS(StorageS3);
@ -824,18 +838,17 @@ storageS3InfoList(
FUNCTION_LOG_PARAM(STORAGE_S3, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_PARAM(STRING, param.expression);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(path != NULL);
ASSERT(callback != NULL);
storageS3ListInternal(this, path, level, param.expression, false, callback, callbackData);
StorageList *const result = storageLstNew(level);
FUNCTION_LOG_RETURN(BOOL, true);
storageS3ListInternal(this, path, level, param.expression, false, storageS3ListCallback, result);
FUNCTION_LOG_RETURN(STORAGE_LIST, result);
}
/**********************************************************************************************************************************/
@ -1066,7 +1079,7 @@ storageS3Remove(THIS_VOID, const String *const file, const StorageInterfaceRemov
static const StorageInterface storageInterfaceS3 =
{
.info = storageS3Info,
.infoList = storageS3InfoList,
.list = storageS3List,
.newRead = storageS3NewRead,
.newWrite = storageS3NewWrite,
.pathRemove = storageS3PathRemove,

244
src/storage/storage.c
View File

@ -50,7 +50,7 @@ storageNew(
ASSERT(strSize(path) >= 1 && strZ(path)[0] == '/');
ASSERT(driver != NULL);
ASSERT(interface.info != NULL);
ASSERT(interface.infoList != NULL);
ASSERT(interface.list != NULL);
ASSERT(interface.newRead != NULL);
ASSERT(interface.newWrite != NULL);
ASSERT(interface.pathRemove != NULL);
@ -279,182 +279,26 @@ storageInfo(const Storage *this, const String *fileExp, StorageInfoParam param)
}
/**********************************************************************************************************************************/
typedef struct StorageInfoListSortData
{
MemContext *memContext; // Mem context to use for allocating data in this struct
StringList *ownerList; // List of users and groups to reduce memory usage
List *infoList; // List of info
} StorageInfoListSortData;
static void
storageInfoListSortCallback(void *data, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM_P(VOID, data);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
StorageInfoListSortData *infoData = data;
MEM_CONTEXT_BEGIN(infoData->memContext)
{
// Copy info and dup strings
StorageInfo infoCopy = *info;
infoCopy.name = strDup(info->name);
infoCopy.linkDestination = strDup(info->linkDestination);
infoCopy.user = strLstAddIfMissing(infoData->ownerList, info->user);
infoCopy.group = strLstAddIfMissing(infoData->ownerList, info->group);
lstAdd(infoData->infoList, &infoCopy);
}
MEM_CONTEXT_END();
FUNCTION_TEST_RETURN_VOID();
}
static bool
storageInfoListSort(
const Storage *this, const String *path, StorageInfoLevel level, const String *expression, SortOrder sortOrder,
StorageInfoListCallback callback, void *callbackData)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE, this);
FUNCTION_LOG_PARAM(STRING, path);
FUNCTION_LOG_PARAM(ENUM, level);
FUNCTION_LOG_PARAM(STRING, expression);
FUNCTION_LOG_PARAM(ENUM, sortOrder);
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(callback != NULL);
bool result = false;
MEM_CONTEXT_TEMP_BEGIN()
{
// If no sorting then use the callback directly
if (sortOrder == sortOrderNone)
{
result = storageInterfaceInfoListP(storageDriver(this), path, level, callback, callbackData, .expression = expression);
}
// Else sort the info before sending it to the callback
else
{
StorageInfoListSortData data =
{
.memContext = MEM_CONTEXT_TEMP(),
.ownerList = strLstNew(),
.infoList = lstNewP(sizeof(StorageInfo), .comparator = lstComparatorStr),
};
result = storageInterfaceInfoListP(
storageDriver(this), path, level, storageInfoListSortCallback, &data, .expression = expression);
lstSort(data.infoList, sortOrder);
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
for (unsigned int infoIdx = 0; infoIdx < lstSize(data.infoList); infoIdx++)
{
// Pass info to the caller
callback(callbackData, lstGet(data.infoList, infoIdx));
// Reset the memory context occasionally
MEM_CONTEXT_TEMP_RESET(1000);
}
}
MEM_CONTEXT_TEMP_END();
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result);
}
typedef struct StorageInfoListData
{
const Storage *storage; // Storage object;
StorageInfoListCallback callbackFunction; // Original callback function
void *callbackData; // Original callback data
const String *expression; // Filter for names
RegExp *regExp; // Compiled filter for names
bool recurse; // Should we recurse?
SortOrder sortOrder; // Sort order
const String *path; // Top-level path for info
const String *subPath; // Path below the top-level path (starts as NULL)
} StorageInfoListData;
static void
storageInfoListCallback(void *data, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM_P(VOID, data);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
StorageInfoListData *listData = data;
// Is this the . path?
bool dotPath = info->type == storageTypePath && strEq(info->name, DOT_STR);
// Skip . paths when getting info for subpaths (since info was already reported in the parent path)
if (dotPath && listData->subPath != NULL)
FUNCTION_TEST_RETURN_VOID();
// Update the name in info with the subpath
StorageInfo infoUpdate = *info;
if (listData->subPath != NULL)
infoUpdate.name = strNewFmt("%s/%s", strZ(listData->subPath), strZ(infoUpdate.name));
// Is this file a match?
bool match = listData->expression == NULL || regExpMatch(listData->regExp, infoUpdate.name);
// Callback before checking path contents when not descending
if (match && listData->sortOrder != sortOrderDesc)
listData->callbackFunction(listData->callbackData, &infoUpdate);
// Recurse into paths
if (infoUpdate.type == storageTypePath && listData->recurse && !dotPath)
{
StorageInfoListData data = *listData;
data.subPath = infoUpdate.name;
storageInfoListSort(
data.storage, strNewFmt("%s/%s", strZ(data.path), strZ(data.subPath)), infoUpdate.level, data.expression,
data.sortOrder, storageInfoListCallback, &data);
}
// Callback after checking path contents when descending
if (match && listData->sortOrder == sortOrderDesc)
listData->callbackFunction(listData->callbackData, &infoUpdate);
FUNCTION_TEST_RETURN_VOID();
}
bool
storageInfoList(
const Storage *this, const String *pathExp, StorageInfoListCallback callback, void *callbackData, StorageInfoListParam param)
StorageIterator *
storageNewItr(const Storage *const this, const String *const pathExp, StorageNewItrParam param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE, this);
FUNCTION_LOG_PARAM(STRING, pathExp);
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
FUNCTION_LOG_PARAM_P(VOID, callbackData);
FUNCTION_LOG_PARAM(ENUM, param.level);
FUNCTION_LOG_PARAM(BOOL, param.errorOnMissing);
FUNCTION_LOG_PARAM(BOOL, param.recurse);
FUNCTION_LOG_PARAM(BOOL, param.nullOnMissing);
FUNCTION_LOG_PARAM(ENUM, param.sortOrder);
FUNCTION_LOG_PARAM(STRING, param.expression);
FUNCTION_LOG_PARAM(BOOL, param.recurse);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(callback != NULL);
ASSERT(this->pub.interface.infoList != NULL);
ASSERT(this->pub.interface.list != NULL);
ASSERT(!param.errorOnMissing || storageFeature(this, storageFeaturePath));
bool result = false;
StorageIterator *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
@ -462,58 +306,18 @@ storageInfoList(
if (param.level == storageInfoLevelDefault)
param.level = storageFeature(this, storageFeatureInfoDetail) ? storageInfoLevelDetail : storageInfoLevelBasic;
// Build the path
String *path = storagePathP(this, pathExp);
// If there is an expression or recursion then the info will need to be filtered through a local callback
if (param.expression != NULL || param.recurse)
{
StorageInfoListData data =
{
.storage = this,
.callbackFunction = callback,
.callbackData = callbackData,
.expression = param.expression,
.sortOrder = param.sortOrder,
.recurse = param.recurse,
.path = path,
};
if (data.expression != NULL)
data.regExp = regExpNew(param.expression);
result = storageInfoListSort(
this, path, param.level, param.expression, param.sortOrder, storageInfoListCallback, &data);
}
else
result = storageInfoListSort(this, path, param.level, NULL, param.sortOrder, callback, callbackData);
if (!result && param.errorOnMissing)
THROW_FMT(PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, strZ(path));
result = storageItrMove(
storageItrNew(
storageDriver(this), storagePathP(this, pathExp), param.level, param.errorOnMissing, param.nullOnMissing,
param.recurse, param.sortOrder, param.expression),
memContextPrior());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result);
FUNCTION_LOG_RETURN(STORAGE_ITERATOR, result);
}
/**********************************************************************************************************************************/
static void
storageListCallback(void *data, const StorageInfo *info)
{
FUNCTION_TEST_BEGIN();
FUNCTION_LOG_PARAM_P(VOID, data);
FUNCTION_LOG_PARAM(STORAGE_INFO, info);
FUNCTION_TEST_END();
// Skip . path
if (strEq(info->name, DOT_STR))
FUNCTION_TEST_RETURN_VOID();
strLstAdd((StringList *)data, info->name);
FUNCTION_TEST_RETURN_VOID();
}
StringList *
storageList(const Storage *this, const String *pathExp, StorageListParam param)
{
@ -527,26 +331,24 @@ storageList(const Storage *this, const String *pathExp, StorageListParam param)
ASSERT(this != NULL);
ASSERT(!param.errorOnMissing || !param.nullOnMissing);
ASSERT(!param.errorOnMissing || storageFeature(this, storageFeaturePath));
StringList *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
StorageIterator *const storageItr = storageNewItrP(
this, pathExp, .errorOnMissing = param.errorOnMissing, .nullOnMissing = param.nullOnMissing,
.expression = param.expression);
if (storageItr != NULL)
{
result = strLstNew();
// Build an empty list if the directory does not exist by default. This makes the logic in calling functions simpler when
// the caller doesn't care if the path is missing.
if (!storageInfoListP(
this, pathExp, storageListCallback, result, .level = storageInfoLevelExists, .errorOnMissing = param.errorOnMissing,
.expression = param.expression))
{
if (param.nullOnMissing)
result = NULL;
}
while (storageItrMore(storageItr))
strLstAdd(result, storageItrNext(storageItr).name);
// Move list up to the old context
result = strLstMove(result, memContextPrior());
strLstMove(result, memContextPrior());
}
}
MEM_CONTEXT_TEMP_END();

17
src/storage/storage.h
View File

@ -17,6 +17,7 @@ typedef struct Storage Storage;
#include "common/time.h"
#include "common/type/param.h"
#include "storage/info.h"
#include "storage/iterator.h"
#include "storage/read.h"
#include "storage/storage.intern.h"
#include "storage/write.h"
@ -94,24 +95,22 @@ typedef struct StorageInfoParam
StorageInfo storageInfo(const Storage *this, const String *fileExp, StorageInfoParam param);
// Info for all files/paths in a path
typedef void (*StorageInfoListCallback)(void *callbackData, const StorageInfo *info);
typedef struct StorageInfoListParam
// Iterator for all files/links/paths in a path which returns different info based on the value of the level parameter
typedef struct StorageNewItrParam
{
VAR_PARAM_HEADER;
StorageInfoLevel level;
bool errorOnMissing;
bool nullOnMissing;
bool recurse;
SortOrder sortOrder;
const String *expression;
} StorageInfoListParam;
} StorageNewItrParam;
#define storageInfoListP(this, fileExp, callback, callbackData, ...) \
storageInfoList(this, fileExp, callback, callbackData, (StorageInfoListParam){VAR_PARAM_INIT, __VA_ARGS__})
#define storageNewItrP(this, fileExp, ...) \
storageNewItr(this, fileExp, (StorageNewItrParam){VAR_PARAM_INIT, __VA_ARGS__})
bool storageInfoList(
const Storage *this, const String *pathExp, StorageInfoListCallback callback, void *callbackData, StorageInfoListParam param);
StorageIterator *storageNewItr(const Storage *this, const String *pathExp, StorageNewItrParam param);
// Get a list of files from a directory
typedef struct StorageListParam

27
src/storage/storage.intern.h
View File

@ -15,6 +15,7 @@ in the description of each function.
#include "common/type/param.h"
#include "storage/info.h"
#include "storage/list.h"
#include "storage/read.h"
#include "storage/write.h"
@ -57,6 +58,11 @@ Path expression callback function type - used to modify paths based on expressio
***********************************************************************************************************************************/
typedef String *StoragePathExpressionCallback(const String *expression, const String *path);
/***********************************************************************************************************************************
Storage info callback function type - used to return storage info
***********************************************************************************************************************************/
typedef void (*StorageListCallback)(void *callbackData, const StorageInfo *info);
/***********************************************************************************************************************************
Required interface functions
***********************************************************************************************************************************/
@ -143,30 +149,29 @@ typedef StorageWrite *StorageInterfaceNewWrite(void *thisVoid, const String *fil
STORAGE_COMMON_INTERFACE(thisVoid).newWrite(thisVoid, file, (StorageInterfaceNewWriteParam){VAR_PARAM_INIT, __VA_ARGS__})
// ---------------------------------------------------------------------------------------------------------------------------------
// Get info for a path and all paths/files in the path (does not recurse)
// Get info for all files/links/paths in a path (does not recurse or return info about the path passed to the function)
//
// See storageInterfaceInfoP() for usage of the level parameter.
typedef struct StorageInterfaceInfoListParam
typedef struct StorageInterfaceListParam
{
VAR_PARAM_HEADER;
// Regular expression used to filter the results. The expression is always checked in the callback passed to
// storageInterfaceInfoListP() so checking the expression in the driver is entirely optional. The driver should only use the
// storageInterfaceIterP() so checking the expression in the driver is entirely optional. The driver should only use the
// expression if it can improve performance or limit network transfer.
//
// Partial matching of the expression is fine as long as nothing that should match is excluded, e.g. it is OK to prefix match
// using the prefix returned from regExpPrefix(). This may cause extra results to be sent to the callback but won't exclude
// anything that matches the expression exactly.
const String *expression;
} StorageInterfaceInfoListParam;
} StorageInterfaceListParam;
typedef bool StorageInterfaceInfoList(
void *thisVoid, const String *path, StorageInfoLevel level, void (*callback)(void *callbackData, const StorageInfo *info),
void *callbackData, StorageInterfaceInfoListParam param);
typedef StorageList *StorageInterfaceList(
void *thisVoid, const String *path, StorageInfoLevel level, StorageInterfaceListParam param);
#define storageInterfaceInfoListP(thisVoid, path, level, callback, callbackData, ...) \
STORAGE_COMMON_INTERFACE(thisVoid).infoList( \
thisVoid, path, level, callback, callbackData, (StorageInterfaceInfoListParam){VAR_PARAM_INIT, __VA_ARGS__})
#define storageInterfaceListP(thisVoid, path, level, ...) \
STORAGE_COMMON_INTERFACE(thisVoid).list( \
thisVoid, path, level, (StorageInterfaceListParam){VAR_PARAM_INIT, __VA_ARGS__})
// ---------------------------------------------------------------------------------------------------------------------------------
// Remove a path (and optionally recurse)
@ -263,7 +268,7 @@ typedef struct StorageInterface
// Required functions
StorageInterfaceInfo *info;
StorageInterfaceInfoList *infoList;
StorageInterfaceList *list;
StorageInterfaceNewRead *newRead;
StorageInterfaceNewWrite *newWrite;
StorageInterfacePathRemove *pathRemove;

11
test/define.yaml
View File

@ -134,6 +134,13 @@ unit:
coverage:
- common/type/object
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-blob
total: 1
coverage:
- common/type/blob
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-string
total: 27
@ -305,6 +312,8 @@ unit:
- storage/posix/read
- storage/posix/storage
- storage/posix/write
- storage/iterator
- storage/list
- storage/read
- storage/storage
- storage/write
@ -483,6 +492,8 @@ unit:
- storage/posix/storage
- storage/posix/write
- storage/helper
- storage/iterator
- storage/list
- storage/read
- storage/storage
- storage/write

118
test/src/common/harnessStorage.c
View File

@ -29,12 +29,10 @@ storageTestDummyInfo(THIS_VOID, const String *file, StorageInfoLevel level, Stor
(void)thisVoid; (void)file; (void)level; (void)param; return (StorageInfo){.exists = false};
}
static bool
storageTestDummyInfoList(
THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
static StorageList *
storageTestDummyList(THIS_VOID, const String *path, StorageInfoLevel level, StorageInterfaceListParam param)
{
(void)thisVoid; (void)path; (void)level; (void)callback; (void)callbackData; (void)param; return false;
(void)thisVoid; (void)path; (void)level; (void)param; return false;
}
static StorageRead *
@ -64,7 +62,7 @@ storageTestDummyRemove(THIS_VOID, const String *file, StorageInterfaceRemovePara
const StorageInterface storageInterfaceTestDummy =
{
.info = storageTestDummyInfo,
.infoList = storageTestDummyInfoList,
.list = storageTestDummyList,
.newRead = storageTestDummyNewRead,
.newWrite = storageTestDummyNewWrite,
.pathRemove = storageTestDummyPathRemove,
@ -140,28 +138,18 @@ testStorageExists(const Storage *const storage, const char *const file, const Te
}
/**********************************************************************************************************************************/
static void
hrnStorageListCallback(void *list, const StorageInfo *info)
{
MEM_CONTEXT_BEGIN(lstMemContext(list))
{
StorageInfo infoCopy = *info;
infoCopy.name = strDup(infoCopy.name);
infoCopy.user = strDup(infoCopy.user);
infoCopy.group = strDup(infoCopy.group);
infoCopy.linkDestination = strDup(infoCopy.linkDestination);
lstAdd(list, &infoCopy);
}
MEM_CONTEXT_END();
}
void
hrnStorageList(const Storage *const storage, const char *const path, const char *const expected, const HrnStorageListParam param)
{
// Check if paths are supported
const bool featurePath = storageFeature(storage, storageFeaturePath);
// Determine sort order
const SortOrder sortOrder = param.sortOrder == sortOrderNone ? sortOrderAsc : param.sortOrder;
// Determine level
StorageInfoLevel level = param.level == storageInfoLevelDefault && !param.levelForce ? storageInfoLevelType : param.level;
// Log list test
hrnTestResultBegin(__func__, false);
@ -173,37 +161,41 @@ hrnStorageList(const Storage *const storage, const char *const path, const char
hrnTestResultComment(param.comment);
// Generate a list of files/paths/etc
List *list = lstNewP(sizeof(StorageInfo));
StorageList *const list = storageLstNew(level == storageInfoLevelDefault ? storageInfoLevelDetail : level);
storageInfoListP(
storage, pathFull, hrnStorageListCallback, list, .recurse = !param.noRecurse,
.sortOrder = param.sortOrder == sortOrderNone ? sortOrderAsc : param.sortOrder,
.level = param.level == storageInfoLevelDefault && !param.levelForce ? storageInfoLevelType : param.level,
StorageIterator *const storageItr = storageNewItrP(
storage, pathFull, .recurse = !param.noRecurse, .sortOrder = sortOrder, .level = level,
.expression = param.expression != NULL ? STR(param.expression) : NULL);
while (storageItrMore(storageItr))
{
StorageInfo info = storageItrNext(storageItr);
storageLstAdd(list, &info);
}
// Remove files if requested
if (param.remove)
{
for (unsigned int listIdx = 0; listIdx < lstSize(list); listIdx++)
for (unsigned int listIdx = 0; listIdx < storageLstSize(list); listIdx++)
{
const StorageInfo *const info = lstGet(list, listIdx);
const StorageInfo info = storageLstGet(list, listIdx);
if (strEq(info->name, DOT_STR))
if (strEq(info.name, DOT_STR))
continue;
// Only remove at the top level since path remove will recurse
if (strChr(info->name, '/') == -1)
if (strChr(info.name, '/') == -1)
{
// Remove a path recursively
if (info->type == storageTypePath)
if (info.type == storageTypePath)
{
storagePathRemoveP(
storage, strNewFmt("%s/%s", strZ(pathFull), strZ(info->name)), .errorOnMissing = featurePath,
storage, strNewFmt("%s/%s", strZ(pathFull), strZ(info.name)), .errorOnMissing = featurePath,
.recurse = true);
}
// Remove file, link, or special
else
storageRemoveP(storage, strNewFmt("%s/%s", strZ(pathFull), strZ(info->name)), .errorOnMissing = true);
storageRemoveP(storage, strNewFmt("%s/%s", strZ(pathFull), strZ(info.name)), .errorOnMissing = true);
}
}
}
@ -211,15 +203,29 @@ hrnStorageList(const Storage *const storage, const char *const path, const char
// Generate list for comparison
StringList *listStr = strLstNew();
for (unsigned int listIdx = 0; listIdx < lstSize(list); listIdx++)
{
const StorageInfo *const info = lstGet(list, listIdx);
String *const item = strCat(strNew(), info->name);
if (level == storageInfoLevelDefault)
level = storageFeature(storage, storageFeatureInfoDetail) ? storageInfoLevelDetail : storageInfoLevelBasic;
if (strEq(info->name, DOT_STR) && !param.includeDot)
if (param.includeDot)
{
StorageInfo info = storageInfoP(storage, pathFull);
info.name = DOT_STR;
if (sortOrder == sortOrderAsc)
storageLstInsert(list, 0, &info);
else
storageLstAdd(list, &info);
}
for (unsigned int listIdx = 0; listIdx < storageLstSize(list); listIdx++)
{
const StorageInfo info = storageLstGet(list, listIdx);
String *const item = strCat(strNew(), info.name);
if (strEq(info.name, DOT_STR) && !param.includeDot)
continue;
switch (info->type)
switch (info.type)
{
case storageTypeFile:
break;
@ -237,40 +243,40 @@ hrnStorageList(const Storage *const storage, const char *const path, const char
break;
}
if (((info->type == storageTypeFile || info->type == storageTypeLink) && info->level >= storageInfoLevelBasic) ||
(info->type == storageTypePath && info->level >= storageInfoLevelDetail))
if (((info.type == storageTypeFile || info.type == storageTypeLink) && level >= storageInfoLevelBasic) ||
(info.type == storageTypePath && level >= storageInfoLevelDetail))
{
strCatZ(item, " {");
if (info->type == storageTypeFile)
strCatFmt(item, "s=%" PRIu64 ", t=%" PRId64, info->size, (int64_t)info->timeModified);
else if (info->type == storageTypeLink)
if (info.type == storageTypeFile)
strCatFmt(item, "s=%" PRIu64 ", t=%" PRId64, info.size, (int64_t)info.timeModified);
else if (info.type == storageTypeLink)
{
const StorageInfo infoLink = storageInfoP(
storage,
strEq(info->name, DOT_STR) ? pathFull : strNewFmt("%s/%s", strZ(pathFull), strZ(info->name)),
strEq(info.name, DOT_STR) ? pathFull : strNewFmt("%s/%s", strZ(pathFull), strZ(info.name)),
.level = storageInfoLevelDetail);
strCatFmt(item, "d=%s", strZ(infoLink.linkDestination));
}
if (info->level >= storageInfoLevelDetail)
if (level >= storageInfoLevelDetail)
{
if (info->type != storageTypePath)
if (info.type != storageTypePath)
strCatZ(item, ", ");
if (info->user != NULL)
strCatFmt(item, "u=%s", strZ(info->user));
if (info.user != NULL)
strCatFmt(item, "u=%s", strZ(info.user));
else
strCatFmt(item, "u=%d", (int)info->userId);
strCatFmt(item, "u=%d", (int)info.userId);
if (info->group != NULL)
strCatFmt(item, ", g=%s", strZ(info->group));
if (info.group != NULL)
strCatFmt(item, ", g=%s", strZ(info.group));
else
strCatFmt(item, ", g=%d", (int)info->groupId);
strCatFmt(item, ", g=%d", (int)info.groupId);
if (info->type != storageTypeLink)
strCatFmt(item, ", m=%04o", info->mode);
if (info.type != storageTypeLink)
strCatFmt(item, ", m=%04o", info.mode);
}
strCatZ(item, "}");

156
test/src/module/command/backupTest.c
View File

@ -19,98 +19,100 @@ Test Backup Command
/***********************************************************************************************************************************
Get a list of all files in the backup and a redacted version of the manifest that can be tested against a static string
***********************************************************************************************************************************/
typedef struct TestBackupValidateCallbackData
static String *
testBackupValidateList(
const Storage *const storage, const String *const path, Manifest *const manifest, const ManifestData *const manifestData,
String *const result)
{
const Storage *storage; // Storage object when needed (e.g. fileCompressed = true)
const String *path; // Subpath when storage is specified
Manifest *manifest; // Manifest to check for files/links/paths
const ManifestData *manifestData; // Manifest data
String *content; // String where content should be added
} TestBackupValidateCallbackData;
// Output root path if it is a link so we can verify the destination
const StorageInfo dotInfo = storageInfoP(storage, path);
static void
testBackupValidateCallback(void *callbackData, const StorageInfo *info)
{
TestBackupValidateCallbackData *data = callbackData;
if (dotInfo.type == storageTypeLink)
strCatFmt(result, ". {link, d=%s}\n", strZ(dotInfo.linkDestination));
// Don't include . when it is a path (we'll still include it when it is a link so we can see the destination)
if (info->type == storageTypePath && strEq(info->name, DOT_STR))
return;
// Output path contents
StorageIterator *const storageItr = storageNewItrP(storage, path, .recurse = true, .sortOrder = sortOrderAsc);
while (storageItrMore(storageItr))
{
const StorageInfo info = storageItrNext(storageItr);
// Don't include backup.manifest or copy. We'll test that they are present elsewhere
if (info->type == storageTypeFile &&
(strEqZ(info->name, BACKUP_MANIFEST_FILE) || strEqZ(info->name, BACKUP_MANIFEST_FILE INFO_COPY_EXT)))
return;
if (info.type == storageTypeFile &&
(strEqZ(info.name, BACKUP_MANIFEST_FILE) || strEqZ(info.name, BACKUP_MANIFEST_FILE INFO_COPY_EXT)))
{
continue;
}
switch (info->type)
switch (info.type)
{
case storageTypeFile:
{
// Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
// mode and current user/group.
// ---------------------------------------------------------------------------------------------------------------------
if (info->mode != 0640)
THROW_FMT(AssertError, "'%s' mode is not 0640", strZ(info->name));
// -----------------------------------------------------------------------------------------------------------------
if (info.mode != 0640)
THROW_FMT(AssertError, "'%s' mode is not 0640", strZ(info.name));
if (!strEq(info->user, TEST_USER_STR))
THROW_FMT(AssertError, "'%s' user should be '" TEST_USER "'", strZ(info->name));
if (!strEq(info.user, TEST_USER_STR))
THROW_FMT(AssertError, "'%s' user should be '" TEST_USER "'", strZ(info.name));
if (!strEq(info->group, TEST_GROUP_STR))
THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(info->name));
if (!strEq(info.group, TEST_GROUP_STR))
THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(info.name));
// Build file list (needed because bundles can contain multiple files)
// ---------------------------------------------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------------------------------------
List *const fileList = lstNewP(sizeof(ManifestFilePack **));
bool bundle = strBeginsWithZ(info->name, "bundle/");
bool bundle = strBeginsWithZ(info.name, "bundle/");
if (bundle)
{
const uint64_t bundleId = cvtZToUInt64(strZ(info->name) + sizeof("bundle"));
const uint64_t bundleId = cvtZToUInt64(strZ(info.name) + sizeof("bundle"));
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(data->manifest); fileIdx++)
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
ManifestFilePack **const filePack = lstGet(data->manifest->pub.fileList, fileIdx);
ManifestFilePack **const filePack = lstGet(manifest->pub.fileList, fileIdx);
if (manifestFileUnpack(data->manifest, *filePack).bundleId == bundleId)
if (manifestFileUnpack(manifest, *filePack).bundleId == bundleId)
lstAdd(fileList, &filePack);
}
}
else
{
const String *manifestName = info->name;
const String *manifestName = info.name;
if (data->manifestData->backupOptionCompressType != compressTypeNone)
if (manifestData->backupOptionCompressType != compressTypeNone)
{
manifestName = strSubN(
info->name, 0, strSize(info->name) - strSize(compressExtStr(data->manifestData->backupOptionCompressType)));
info.name, 0, strSize(info.name) - strSize(compressExtStr(manifestData->backupOptionCompressType)));
}
ManifestFilePack **const filePack = manifestFilePackFindInternal(data->manifest, manifestName);
ManifestFilePack **const filePack = manifestFilePackFindInternal(manifest, manifestName);
lstAdd(fileList, &filePack);
}
// Check files
// ---------------------------------------------------------------------------------------------------------------------
// -----------------------------------------------------------------------------------------------------------------
for (unsigned int fileIdx = 0; fileIdx < lstSize(fileList); fileIdx++)
{
ManifestFilePack **const filePack = *(ManifestFilePack ***)lstGet(fileList, fileIdx);
ManifestFile file = manifestFileUnpack(data->manifest, *filePack);
ManifestFile file = manifestFileUnpack(manifest, *filePack);
if (bundle)
strCatFmt(data->content, "%s/%s {file", strZ(info->name), strZ(file.name));
strCatFmt(result, "%s/%s {file", strZ(info.name), strZ(file.name));
else
strCatFmt(data->content, "%s {file", strZ(info->name));
strCatFmt(result, "%s {file", strZ(info.name));
// Calculate checksum/size and decompress if needed
// -----------------------------------------------------------------------------------------------------------------
// -------------------------------------------------------------------------------------------------------------
StorageRead *read = storageNewReadP(
data->storage, strNewFmt("%s/%s", strZ(data->path), strZ(info->name)), .offset = file.bundleOffset,
storage, strNewFmt("%s/%s", strZ(path), strZ(info.name)), .offset = file.bundleOffset,
.limit = VARUINT64(file.sizeRepo));
if (data->manifestData->backupOptionCompressType != compressTypeNone)
if (manifestData->backupOptionCompressType != compressTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)), decompressFilter(data->manifestData->backupOptionCompressType));
ioReadFilterGroup(storageReadIo(read)), decompressFilter(manifestData->backupOptionCompressType));
}
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(hashTypeSha1));
@ -119,84 +121,86 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
const String *checksum = pckReadStrP(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE));
strCatFmt(data->content, ", s=%" PRIu64, size);
strCatFmt(result, ", s=%" PRIu64, size);
if (!strEqZ(checksum, file.checksumSha1))
THROW_FMT(AssertError, "'%s' checksum does match manifest", strZ(file.name));
// Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even the
// same compression algorithm can give slightly different results based on the version so repo-size is not
// Test size and repo-size. If compressed then set the repo-size to size so it will not be in test output. Even
// the same compression algorithm can give slightly different results based on the version so repo-size is not
// deterministic for compression.
// -----------------------------------------------------------------------------------------------------------------
// -------------------------------------------------------------------------------------------------------------
if (size != file.size)
THROW_FMT(AssertError, "'%s' size does match manifest", strZ(file.name));
// Repo size can only be compared to file size when not bundled
if (!bundle)
{
if (info->size != file.sizeRepo)
if (info.size != file.sizeRepo)
THROW_FMT(AssertError, "'%s' repo size does match manifest", strZ(file.name));
}
if (data->manifestData->backupOptionCompressType != compressTypeNone)
if (manifestData->backupOptionCompressType != compressTypeNone)
file.sizeRepo = file.size;
// Bundle id/offset are too noisy so remove them. They are checked size/checksum and listed with the files.
// -----------------------------------------------------------------------------------------------------------------
// -------------------------------------------------------------------------------------------------------------
file.bundleId = 0;
file.bundleOffset = 0;
// pg_control and WAL headers have different checksums depending on cpu architecture so remove the checksum from the
// test output.
// -----------------------------------------------------------------------------------------------------------------
// pg_control and WAL headers have different checksums depending on cpu architecture so remove the checksum from
// the test output.
// -------------------------------------------------------------------------------------------------------------
if (strEqZ(file.name, MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL) ||
strBeginsWith(
file.name, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(data->manifestData->pgVersion)))))
file.name, strNewFmt(MANIFEST_TARGET_PGDATA "/%s/", strZ(pgWalPath(manifestData->pgVersion)))))
{
file.checksumSha1[0] = '\0';
}
strCatZ(data->content, "}\n");
strCatZ(result, "}\n");
// Update changes to manifest file
manifestFilePackUpdate(data->manifest, filePack, &file);
manifestFilePackUpdate(manifest, filePack, &file);
}
break;
}
case storageTypeLink:
strCatFmt(data->content, "%s {link, d=%s}\n", strZ(info->name), strZ(info->linkDestination));
strCatFmt(result, "%s {link, d=%s}\n", strZ(info.name), strZ(info.linkDestination));
break;
case storageTypePath:
{
strCatFmt(data->content, "%s {path", strZ(info->name));
strCatFmt(result, "%s {path", strZ(info.name));
// Check against the manifest
// ---------------------------------------------------------------------------------------------------------------------
if (!strEq(info->name, STRDEF("bundle")))
manifestPathFind(data->manifest, info->name);
// -----------------------------------------------------------------------------------------------------------------
if (!strEq(info.name, STRDEF("bundle")))
manifestPathFind(manifest, info.name);
// Test mode, user, group. These values are not in the manifest but we know what they should be based on the default
// mode and current user/group.
if (info->mode != 0750)
THROW_FMT(AssertError, "'%s' mode is not 00750", strZ(info->name));
if (info.mode != 0750)
THROW_FMT(AssertError, "'%s' mode is not 00750", strZ(info.name));
if (!strEq(info->user, TEST_USER_STR))
THROW_FMT(AssertError, "'%s' user should be '" TEST_USER "'", strZ(info->name));
if (!strEq(info.user, TEST_USER_STR))
THROW_FMT(AssertError, "'%s' user should be '" TEST_USER "'", strZ(info.name));
if (!strEq(info->group, TEST_GROUP_STR))
THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(info->name));
if (!strEq(info.group, TEST_GROUP_STR))
THROW_FMT(AssertError, "'%s' group should be '" TEST_GROUP "'", strZ(info.name));
strCatZ(data->content, "}\n");
strCatZ(result, "}\n");
break;
}
case storageTypeSpecial:
THROW_FMT(AssertError, "unexpected special file '%s'", strZ(info->name));
THROW_FMT(AssertError, "unexpected special file '%s'", strZ(info.name));
}
}
return result;
}
static String *
@ -210,24 +214,14 @@ testBackupValidate(const Storage *storage, const String *path)
ASSERT(storage != NULL);
ASSERT(path != NULL);
String *result = strNew();
String *const result = strNew();
MEM_CONTEXT_TEMP_BEGIN()
{
// Build a list of files in the backup path and verify against the manifest
// -------------------------------------------------------------------------------------------------------------------------
Manifest *manifest = manifestLoadFile(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE, strZ(path)), cipherTypeNone, NULL);
TestBackupValidateCallbackData callbackData =
{
.storage = storage,
.path = path,
.content = result,
.manifest = manifest,
.manifestData = manifestData(manifest),
};
storageInfoListP(storage, path, testBackupValidateCallback, &callbackData, .recurse = true, .sortOrder = sortOrderAsc);
testBackupValidateList(storage, path, manifest, manifestData(manifest), result);
// Make sure both backup.manifest files exist since we skipped them in the callback above
if (!storageExistsP(storage, strNewFmt("%s/" BACKUP_MANIFEST_FILE, strZ(path))))

52
test/src/module/common/typeBlobTest.c Normal file
View File

@ -0,0 +1,52 @@
/***********************************************************************************************************************************
Test Blob
***********************************************************************************************************************************/
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
static void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("Blob"))
{
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("Add enough data to allocate a new page");
#define TEST_DATA_MAX 5
#define TEST_DATA_SIZE (BLOB_BLOCK_SIZE / (TEST_DATA_MAX - 1))
void *dataList[TEST_DATA_MAX];
const void *dataListPtr[TEST_DATA_MAX];
Blob *blob = NULL;
TEST_ASSIGN(blob, blbNew(), "new blob");
for (int dataIdx = 0; dataIdx < TEST_DATA_MAX; dataIdx++)
{
dataList[dataIdx] = memNew(TEST_DATA_SIZE);
memset(dataList[dataIdx], dataIdx, TEST_DATA_SIZE);
TEST_ASSIGN(dataListPtr[dataIdx], blbAdd(blob, dataList[dataIdx], TEST_DATA_SIZE), "data add");
}
for (int dataIdx = 0; dataIdx < TEST_DATA_MAX; dataIdx++)
TEST_RESULT_INT(memcmp(dataList[dataIdx], dataListPtr[dataIdx], TEST_DATA_SIZE), 0, "data check");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("Add data >= than block size");
void *data = memNew(BLOB_BLOCK_SIZE);
const void *dataPtr = NULL;
memset(data, 255, BLOB_BLOCK_SIZE);
TEST_ASSIGN(dataPtr, blbAdd(blob, data, BLOB_BLOCK_SIZE), "data add");
TEST_RESULT_INT(memcmp(data, dataPtr, BLOB_BLOCK_SIZE), 0, "data check");
}
FUNCTION_HARNESS_RETURN_VOID();
}

75
test/src/module/performance/storageTest.c
View File

@ -28,53 +28,31 @@ stress testing as needed.
#include "storage/remote/protocol.h"
/***********************************************************************************************************************************
Dummy callback functions
***********************************************************************************************************************************/
static void
storageTestDummyInfoListCallback(void *data, const StorageInfo *info)
{
(void)info;
// Do some work in the mem context to blow up the total time if this is not efficient, i.e. if the current mem context is not
// being freed regularly
memResize(memNew(16), 32);
// Increment callback total
(*(uint64_t *)data)++;
}
/***********************************************************************************************************************************
Driver to test storageInfoList
Driver to test storageNewItrP()
***********************************************************************************************************************************/
typedef struct
{
STORAGE_COMMON_MEMBER;
uint64_t fileTotal;
} StorageTestPerfInfoList;
} StorageTestPerfList;
static bool
storageTestPerfInfoList(
THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
static StorageList *
storageTestPerfList(THIS_VOID, const String *path, StorageInfoLevel level, StorageInterfaceListParam param)
{
THIS(StorageTestPerfInfoList);
THIS(StorageTestPerfList);
(void)path; (void)level; (void)param;
MEM_CONTEXT_TEMP_BEGIN()
{
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
for (uint64_t fileIdx = 0; fileIdx < this->fileTotal; fileIdx++)
{
callback(callbackData, &(StorageInfo){.exists = true, .name = STRDEF("name")});
MEM_CONTEXT_TEMP_RESET(1000);
}
}
MEM_CONTEXT_TEMP_END();
}
MEM_CONTEXT_TEMP_END();
StorageList *result = NULL;
return this->fileTotal != 0;
if (this->fileTotal != 0)
{
result = storageLstNew(storageInfoLevelExists);
for (uint64_t fileIdx = 0; fileIdx < this->fileTotal; fileIdx++)
storageLstAdd(result, &(StorageInfo){.exists = true, .name = STRDEF("name")});
}
return result;
}
/***********************************************************************************************************************************
@ -144,7 +122,7 @@ testRun(void)
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("storageInfoList()"))
if (testBegin("storageNewItrP()"))
{
TEST_TITLE_FMT("list %d million files", TEST_SCALE);
@ -163,14 +141,14 @@ testRun(void)
hrnCfgArgRawStrId(argList, cfgOptRemoteType, protocolStorageTypeRepo);
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleRemote);
// Create a driver to test remote performance of storageInfoList() and inject it into storageRepo()
StorageTestPerfInfoList driver =
// Create a driver to test remote performance of storageNewItrP() and inject it into storageRepo()
StorageTestPerfList driver =
{
.interface = storageInterfaceTestDummy,
.fileTotal = fileTotal,
};
driver.interface.infoList = storageTestPerfInfoList;
driver.interface.list = storageTestPerfList;
Storage *storageTest = storageNew(strIdFromZ("test"), STRDEF("/"), 0, 0, false, NULL, &driver, driver.interface);
storageHelper.storageRepoWrite = memNew(sizeof(Storage *));
@ -200,13 +178,18 @@ testRun(void)
TimeMSec timeBegin = timeMSec();
// Storage info list
uint64_t fileCallbackTotal = 0;
uint64_t fileTotal = 0;
StorageIterator *storageItr = NULL;
TEST_RESULT_VOID(
storageInfoListP(storageRemote, NULL, storageTestDummyInfoListCallback, &fileCallbackTotal),
"list remote files");
TEST_ASSIGN(storageItr, storageNewItrP(storageRemote, NULL), "list remote files");
TEST_RESULT_UINT(fileCallbackTotal, fileTotal, "check callback total");
while (storageItrMore(storageItr))
{
storageItrNext(storageItr);
fileTotal++;
}
TEST_RESULT_UINT(fileTotal, fileTotal, "check callback total");
TEST_LOG_FMT("list transferred in %ums", (unsigned int)(timeMSec() - timeBegin));

34
test/src/module/performance/typeTest.c
View File

@ -93,17 +93,17 @@ storageTestManifestNewBuildInfo(THIS_VOID, const String *file, StorageInfoLevel
return result;
}
static bool
storageTestManifestNewBuildInfoList(
THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
StorageInterfaceInfoListParam param)
static StorageList *
storageTestManifestNewBuildList(THIS_VOID, const String *path, StorageInfoLevel level, StorageInterfaceListParam param)
{
THIS(StorageTestManifestNewBuild);
(void)path; (void)level; (void)param;
StorageList *const result = storageLstNew(storageInfoLevelDetail);
MEM_CONTEXT_TEMP_RESET_BEGIN()
{
StorageInfo result =
StorageInfo info =
{
.level = storageInfoLevelDetail,
.exists = true,
@ -117,25 +117,25 @@ storageTestManifestNewBuildInfoList(
if (strEq(path, STRDEF("/pg")))
{
result.name = STRDEF("base");
callback(callbackData, &result);
info.name = STRDEF("base");
storageLstAdd(result, &info);
}
else if (strEq(path, STRDEF("/pg/base")))
{
result.name = STRDEF("1000000000");
callback(callbackData, &result);
info.name = STRDEF("1000000000");
storageLstAdd(result, &info);
}
else if (strEq(path, STRDEF("/pg/base/1000000000")))
{
result.type = storageTypeFile;
result.size = 8192;
result.mode = 0600;
result.timeModified = 1595627966;
info.type = storageTypeFile;
info.size = 8192;
info.mode = 0600;
info.timeModified = 1595627966;
for (unsigned int fileIdx = 0; fileIdx < this->fileTotal; fileIdx++)
{
result.name = strNewFmt("%u", 1000000000 + fileIdx);
callback(callbackData, &result);
info.name = strNewFmt("%u", 1000000000 + fileIdx);
storageLstAdd(result, &info);
MEM_CONTEXT_TEMP_RESET(10000);
}
}
@ -144,7 +144,7 @@ storageTestManifestNewBuildInfoList(
}
MEM_CONTEXT_TEMP_END();
return true;
return result;
}
/***********************************************************************************************************************************
@ -255,7 +255,7 @@ testRun(void)
};
driver.interface.info = storageTestManifestNewBuildInfo;
driver.interface.infoList = storageTestManifestNewBuildInfoList;
driver.interface.list = storageTestManifestNewBuildList;
const Storage *const storagePg = storageNew(
strIdFromZ("test"), STRDEF("/pg"), 0, 0, false, NULL, &driver, driver.interface);

4
test/src/module/storage/azureTest.c
View File

@ -718,8 +718,8 @@ testRun(void)
"</EnumerationResults>");
TEST_ERROR(
storageInfoListP(storage, STRDEF("/"), (void *)1, NULL, .errorOnMissing = true),
AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
storageNewItrP(storage, STRDEF("/"), .errorOnMissing = true), AssertError,
"assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
TEST_STORAGE_LIST(
storage, "/path/to",

4
test/src/module/storage/gcsTest.c
View File

@ -714,8 +714,8 @@ testRun(void)
"}");
TEST_ERROR(
storageInfoListP(storage, STRDEF("/"), (void *)1, NULL, .errorOnMissing = true),
AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
storageNewItrP(storage, STRDEF("/"), .errorOnMissing = true), AssertError,
"assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
TEST_STORAGE_LIST(
storage, "/path/to",

62
test/src/module/storage/posixTest.c
View File

@ -340,7 +340,7 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("storageInfoList()"))
if (testBegin("storageNewItrP()"))
{
#ifdef TEST_CONTAINER_REQUIRED
TEST_CREATE_NOPERM();
@ -350,30 +350,29 @@ testRun(void)
TEST_TITLE("path missing");
TEST_ERROR_FMT(
storageInfoListP(storageTest, STRDEF(BOGUS_STR), (StorageInfoListCallback)1, NULL, .errorOnMissing = true),
PathMissingError, STORAGE_ERROR_LIST_INFO_MISSING, TEST_PATH "/BOGUS");
storageNewItrP(storageTest, STRDEF(BOGUS_STR), .errorOnMissing = true), PathMissingError,
STORAGE_ERROR_LIST_INFO_MISSING, TEST_PATH "/BOGUS");
TEST_RESULT_BOOL(
storageInfoListP(storageTest, STRDEF(BOGUS_STR), (StorageInfoListCallback)1, NULL), false, "ignore missing dir");
TEST_RESULT_PTR(storageNewItrP(storageTest, STRDEF(BOGUS_STR), .nullOnMissing = true), NULL, "ignore missing dir");
#ifdef TEST_CONTAINER_REQUIRED
TEST_ERROR_FMT(
storageInfoListP(storageTest, pathNoPerm, (StorageInfoListCallback)1, NULL), PathOpenError,
STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strZ(pathNoPerm));
storageNewItrP(storageTest, pathNoPerm), PathOpenError, STORAGE_ERROR_LIST_INFO ": [13] Permission denied",
strZ(pathNoPerm));
// Should still error even when ignore missing
TEST_ERROR_FMT(
storageInfoListP(storageTest, pathNoPerm, (StorageInfoListCallback)1, NULL), PathOpenError,
STORAGE_ERROR_LIST_INFO ": [13] Permission denied", strZ(pathNoPerm));
storageNewItrP(storageTest, pathNoPerm), PathOpenError, STORAGE_ERROR_LIST_INFO ": [13] Permission denied",
strZ(pathNoPerm));
#endif // TEST_CONTAINER_REQUIRED
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("helper function - storagePosixInfoListEntry()");
TEST_TITLE("helper function - storagePosixListEntry()");
TEST_RESULT_VOID(
storagePosixInfoListEntry(
(StoragePosix *)storageDriver(storageTest), STRDEF("pg"), STRDEF("missing"), storageInfoLevelBasic, (void *)1,
NULL),
storagePosixListEntry(
(StoragePosix *)storageDriver(storageTest), storageLstNew(storageInfoLevelBasic), STRDEF("pg"), "missing",
storageInfoLevelBasic),
"missing path");
// -------------------------------------------------------------------------------------------------------------------------
@ -382,8 +381,8 @@ testRun(void)
storagePathCreateP(storageTest, STRDEF("pg"), .mode = 0766);
TEST_STORAGE_LIST(
storageTest,
"pg", "./ {u=" TEST_USER ", g=" TEST_GROUP ", m=0766}\n",
storageTest, "pg",
"./ {u=" TEST_USER ", g=" TEST_GROUP ", m=0766}\n",
.level = storageInfoLevelDetail, .includeDot = true);
// -------------------------------------------------------------------------------------------------------------------------
@ -416,7 +415,16 @@ testRun(void)
#endif // TEST_CONTAINER_REQUIRED
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path - recurse");
TEST_TITLE("storageItrMore() twice in a row");
StorageIterator *storageItr = NULL;
TEST_ASSIGN(storageItr, storageNewItrP(storageTest, STRDEF("pg")), "new iterator");
TEST_RESULT_BOOL(storageItrMore(storageItr), true, "check more");
TEST_RESULT_BOOL(storageItrMore(storageItr), true, "check more again");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path - recurse desc");
storagePathCreateP(storageTest, STRDEF("pg/path"), .mode = 0700);
storagePutP(
@ -433,6 +441,20 @@ testRun(void)
"./\n",
.level = storageInfoLevelBasic, .includeDot = true, .sortOrder = sortOrderDesc);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path - recurse asc");
TEST_STORAGE_LIST(
storageTest, "pg",
"./\n"
"file {s=8, t=1656433838}\n"
"link> {d=../file}\n"
"path/\n"
"path/file {s=8, t=1656434296}\n"
"pipe*\n",
.level = storageInfoLevelBasic, .includeDot = true);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path basic info - recurse");
@ -451,12 +473,14 @@ testRun(void)
storageTest->pub.interface.feature ^= 1 << storageFeatureInfoDetail;
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path - filter");
TEST_TITLE("empty path - filter");
storagePathCreateP(storageTest, STRDEF("pg/empty"), .mode = 0700);
TEST_STORAGE_LIST(
storageTest, "pg",
"path/\n",
.noRecurse = true, .level = storageInfoLevelType, .expression = "^path");
"empty/\n",
.level = storageInfoLevelType, .expression = "^empty");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("filter in subpath during recursion");

4
test/src/module/storage/remoteTest.c
View File

@ -193,11 +193,11 @@ testRun(void)
}
// *****************************************************************************************************************************
if (testBegin("storageInfoList()"))
if (testBegin("storageNewItrP()"))
{
TEST_TITLE("path not found");
TEST_RESULT_BOOL(storageInfoListP(storageRepo, STRDEF(BOGUS_STR), (void *)1, NULL), false, "path missing");
TEST_RESULT_PTR(storageNewItrP(storageRepo, STRDEF(BOGUS_STR), .nullOnMissing = true), NULL, "path missing");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("list path and file (no user/group");

4
test/src/module/storage/s3Test.c
View File

@ -1065,8 +1065,8 @@ testRun(void)
"</ListBucketResult>");
TEST_ERROR(
storageInfoListP(s3, STRDEF("/"), (void *)1, NULL, .errorOnMissing = true),
AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
storageNewItrP(s3, STRDEF("/"), .errorOnMissing = true), AssertError,
"assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
TEST_STORAGE_LIST(
s3, "/path/to",