From 7f2f535460e3499c1811b5beca9130f84b78c982 Mon Sep 17 00:00:00 2001 From: David Steele Date: Mon, 10 Jun 2019 16:09:38 -0400 Subject: [PATCH] Add info() and infoList() to S3 driver. These should be the last functions required to complete the implementation of the S3 driver. --- src/storage/s3/storage.c | 103 ++++++++++++++++++++++++++++++- test/src/module/storage/s3Test.c | 72 +++++++++++++++++++++ 2 files changed, 173 insertions(+), 2 deletions(-) diff --git a/src/storage/s3/storage.c b/src/storage/s3/storage.c index 7021b637e..7fc7636f9 100644 --- a/src/storage/s3/storage.c +++ b/src/storage/s3/storage.c @@ -58,6 +58,7 @@ STRING_STATIC(S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, "NextContinu STRING_STATIC(S3_XML_TAG_OBJECT_STR, "Object"); STRING_STATIC(S3_XML_TAG_PREFIX_STR, "Prefix"); STRING_STATIC(S3_XML_TAG_QUIET_STR, "Quiet"); +STRING_STATIC(S3_XML_TAG_SIZE_STR, "Size"); /*********************************************************************************************************************************** AWS authentication v4 constants @@ -497,6 +498,103 @@ storageS3Exists(THIS_VOID, const String *file) FUNCTION_LOG_RETURN(BOOL, result); } +/*********************************************************************************************************************************** +File info +***********************************************************************************************************************************/ +static StorageInfo +storageS3Info(THIS_VOID, const String *file, bool followLink) +{ + THIS(StorageS3); + + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(STORAGE_S3, this); + FUNCTION_LOG_PARAM(STRING, file); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(file != NULL); + (void)followLink; + + StorageInfo result = {0}; + + // Attempt to get file info + StorageS3RequestResult httpResult = storageS3Request(this, HTTP_VERB_HEAD_STR, file, NULL, NULL, false, true); + + // On success load info into a structure + if (httpClientResponseCodeOk(this->httpClient)) + { + result.exists = true; + result.size = cvtZToUInt64(strPtr(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_CONTENT_LENGTH_STR))); + } + + FUNCTION_LOG_RETURN(STORAGE_INFO, result); +} + +/*********************************************************************************************************************************** +Info for all files/paths in a path +***********************************************************************************************************************************/ +typedef struct StorageS3InfoListData +{ + StorageInfoListCallback callback; // User-supplied callback function + void *callbackData; // User-supplied callback data +} StorageS3InfoListData; + +static void +storageS3InfoListCallback(StorageS3 *this, void *callbackData, const String *name, StorageType type, const XmlNode *xml) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(STORAGE_S3, this); + FUNCTION_TEST_PARAM_P(VOID, callbackData); + FUNCTION_TEST_PARAM(STRING, name); + FUNCTION_TEST_PARAM(ENUM, type); + FUNCTION_TEST_PARAM(XML_NODE, xml); + FUNCTION_TEST_END(); + + (void)this; + ASSERT(callbackData != NULL); + ASSERT(name != NULL); + ASSERT(xml != NULL); + + StorageS3InfoListData *data = (StorageS3InfoListData *)callbackData; + + StorageInfo info = + { + .type = type, + .name = name, + .size = type == storageTypeFile ? cvtZToUInt64(strPtr(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_SIZE_STR, true)))) : 0, + }; + + data->callback(data->callbackData, &info); + + FUNCTION_TEST_RETURN_VOID(); +} + +static bool +storageS3InfoList(THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData) +{ + THIS(StorageS3); + + FUNCTION_LOG_BEGIN(logLevelTrace); + FUNCTION_LOG_PARAM(STORAGE_S3, this); + FUNCTION_LOG_PARAM(STRING, path); + FUNCTION_LOG_PARAM(FUNCTIONP, callback); + FUNCTION_LOG_PARAM_P(VOID, callbackData); + FUNCTION_LOG_END(); + + ASSERT(this != NULL); + ASSERT(path != NULL); + ASSERT(callback != NULL); + + MEM_CONTEXT_TEMP_BEGIN() + { + StorageS3InfoListData data = {.callback = callback, .callbackData = callbackData}; + storageS3ListInternal(this, path, false, false, storageS3InfoListCallback, &data); + } + MEM_CONTEXT_TEMP_END(); + + FUNCTION_LOG_RETURN(BOOL, true); +} + /*********************************************************************************************************************************** Get a list of files from a directory ***********************************************************************************************************************************/ @@ -828,8 +926,9 @@ storageS3New( this = storageNewP( STORAGE_S3_TYPE_STR, path, 0, 0, write, pathExpressionFunction, driver, - .exists = storageS3Exists, .list = storageS3List, .newRead = storageS3NewRead, .newWrite = storageS3NewWrite, - .pathRemove = storageS3PathRemove, .remove = storageS3Remove); + .exists = storageS3Exists, .info = storageS3Info, .infoList = storageS3InfoList, .list = storageS3List, + .newRead = storageS3NewRead, .newWrite = storageS3NewWrite, .pathRemove = storageS3PathRemove, + .remove = storageS3Remove); } MEM_CONTEXT_NEW_END(); diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c index 1b859b97c..43b3035f9 100644 --- a/test/src/module/storage/s3Test.c +++ b/test/src/module/storage/s3Test.c @@ -179,6 +179,34 @@ testS3Server(void) harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL)); harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:999", NULL)); + // Info() + // ------------------------------------------------------------------------------------------------------------------------- + // File missing + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL)); + harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL)); + + // File exists + harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL)); + harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:9999", NULL)); + + // InfoList() + // ------------------------------------------------------------------------------------------------------------------------- + harnessTlsServerExpect( + testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL)); + harnessTlsServerReply( + testS3ServerResponse( + 200, "OK", NULL, + "" + "" + " " + " path/to/test_file" + " 787" + " " + " " + " path/to/test_path/" + " " + "")); + // storageDriverList() // ------------------------------------------------------------------------------------------------------------------------- // Throw error @@ -404,6 +432,24 @@ testS3Server(void) } } +/*********************************************************************************************************************************** +Callback and data for storageInfoList() tests +***********************************************************************************************************************************/ +unsigned int testStorageInfoListSize = 0; +StorageInfo testStorageInfoList[256]; + +void +testStorageInfoListCallback(void *callbackData, const StorageInfo *info) +{ + MEM_CONTEXT_BEGIN((MemContext *)callbackData) + { + testStorageInfoList[testStorageInfoListSize] = *info; + testStorageInfoList[testStorageInfoListSize].name = strDup(info->name); + testStorageInfoListSize++; + } + MEM_CONTEXT_END(); +} + /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -712,6 +758,32 @@ testRun(void) TEST_RESULT_BOOL(storageExistsNP(s3, strNew("BOGUS")), false, "file does not exist"); TEST_RESULT_BOOL(storageExistsNP(s3, strNew("subdir/file1.txt")), true, "file exists"); + // Info() + // ------------------------------------------------------------------------------------------------------------------------- + TEST_RESULT_BOOL(storageInfoP(s3, strNew("BOGUS"), .ignoreMissing = true).exists, false, "file does not exist"); + + StorageInfo info; + TEST_ASSIGN(info, storageInfoNP(s3, strNew("subdir/file1.txt")), "file exists"); + TEST_RESULT_BOOL(info.exists, true, " check exists"); + TEST_RESULT_UINT(info.size, 9999, " check exists"); + + // InfoList() + // ------------------------------------------------------------------------------------------------------------------------- + TEST_ERROR( + storageInfoListP(s3, strNew("/"), testStorageInfoListCallback, NULL, .errorOnMissing = true), + AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed"); + + TEST_RESULT_VOID( + storageInfoListNP(s3, strNew("/path/to"), testStorageInfoListCallback, (void *)memContextCurrent()), "info list files"); + + TEST_RESULT_UINT(testStorageInfoListSize, 2, " file and path returned"); + TEST_RESULT_STR(strPtr(testStorageInfoList[0].name), "test_path", " check name"); + TEST_RESULT_UINT(testStorageInfoList[0].size, 0, " check size"); + TEST_RESULT_UINT(testStorageInfoList[0].type, storageTypePath, " check type"); + TEST_RESULT_STR(strPtr(testStorageInfoList[1].name), "test_file", " check name"); + TEST_RESULT_UINT(testStorageInfoList[1].size, 787, " check size"); + TEST_RESULT_UINT(testStorageInfoList[1].type, storageTypeFile, " check type"); + // storageDriverList() // ------------------------------------------------------------------------------------------------------------------------- TEST_ERROR(