mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-01-18 04:58:51 +02:00
Add info() and infoList() to S3 driver.
These should be the last functions required to complete the implementation of the S3 driver.
This commit is contained in:
David Steele
parent
9d1b03781f
commit
7f2f535460
@ -58,6 +58,7 @@ STRING_STATIC(S3_XML_TAG_NEXT_CONTINUATION_TOKEN_STR, "NextContinu
|
||||
STRING_STATIC(S3_XML_TAG_OBJECT_STR, "Object");
|
||||
STRING_STATIC(S3_XML_TAG_PREFIX_STR, "Prefix");
|
||||
STRING_STATIC(S3_XML_TAG_QUIET_STR, "Quiet");
|
||||
STRING_STATIC(S3_XML_TAG_SIZE_STR, "Size");
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
AWS authentication v4 constants
|
||||
@ -497,6 +498,103 @@ storageS3Exists(THIS_VOID, const String *file)
|
||||
FUNCTION_LOG_RETURN(BOOL, result);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
File info
|
||||
***********************************************************************************************************************************/
|
||||
static StorageInfo
|
||||
storageS3Info(THIS_VOID, const String *file, bool followLink)
|
||||
{
|
||||
THIS(StorageS3);
|
||||
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(STORAGE_S3, this);
|
||||
FUNCTION_LOG_PARAM(STRING, file);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
ASSERT(file != NULL);
|
||||
(void)followLink;
|
||||
|
||||
StorageInfo result = {0};
|
||||
|
||||
// Attempt to get file info
|
||||
StorageS3RequestResult httpResult = storageS3Request(this, HTTP_VERB_HEAD_STR, file, NULL, NULL, false, true);
|
||||
|
||||
// On success load info into a structure
|
||||
if (httpClientResponseCodeOk(this->httpClient))
|
||||
{
|
||||
result.exists = true;
|
||||
result.size = cvtZToUInt64(strPtr(httpHeaderGet(httpResult.responseHeader, HTTP_HEADER_CONTENT_LENGTH_STR)));
|
||||
}
|
||||
|
||||
FUNCTION_LOG_RETURN(STORAGE_INFO, result);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Info for all files/paths in a path
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct StorageS3InfoListData
|
||||
{
|
||||
StorageInfoListCallback callback; // User-supplied callback function
|
||||
void *callbackData; // User-supplied callback data
|
||||
} StorageS3InfoListData;
|
||||
|
||||
static void
|
||||
storageS3InfoListCallback(StorageS3 *this, void *callbackData, const String *name, StorageType type, const XmlNode *xml)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STORAGE_S3, this);
|
||||
FUNCTION_TEST_PARAM_P(VOID, callbackData);
|
||||
FUNCTION_TEST_PARAM(STRING, name);
|
||||
FUNCTION_TEST_PARAM(ENUM, type);
|
||||
FUNCTION_TEST_PARAM(XML_NODE, xml);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
(void)this;
|
||||
ASSERT(callbackData != NULL);
|
||||
ASSERT(name != NULL);
|
||||
ASSERT(xml != NULL);
|
||||
|
||||
StorageS3InfoListData *data = (StorageS3InfoListData *)callbackData;
|
||||
|
||||
StorageInfo info =
|
||||
{
|
||||
.type = type,
|
||||
.name = name,
|
||||
.size = type == storageTypeFile ? cvtZToUInt64(strPtr(xmlNodeContent(xmlNodeChild(xml, S3_XML_TAG_SIZE_STR, true)))) : 0,
|
||||
};
|
||||
|
||||
data->callback(data->callbackData, &info);
|
||||
|
||||
FUNCTION_TEST_RETURN_VOID();
|
||||
}
|
||||
|
||||
static bool
|
||||
storageS3InfoList(THIS_VOID, const String *path, StorageInfoListCallback callback, void *callbackData)
|
||||
{
|
||||
THIS(StorageS3);
|
||||
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(STORAGE_S3, this);
|
||||
FUNCTION_LOG_PARAM(STRING, path);
|
||||
FUNCTION_LOG_PARAM(FUNCTIONP, callback);
|
||||
FUNCTION_LOG_PARAM_P(VOID, callbackData);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
ASSERT(path != NULL);
|
||||
ASSERT(callback != NULL);
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
StorageS3InfoListData data = {.callback = callback, .callbackData = callbackData};
|
||||
storageS3ListInternal(this, path, false, false, storageS3InfoListCallback, &data);
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(BOOL, true);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get a list of files from a directory
|
||||
***********************************************************************************************************************************/
|
||||
@ -828,8 +926,9 @@ storageS3New(
|
||||
|
||||
this = storageNewP(
|
||||
STORAGE_S3_TYPE_STR, path, 0, 0, write, pathExpressionFunction, driver,
|
||||
.exists = storageS3Exists, .list = storageS3List, .newRead = storageS3NewRead, .newWrite = storageS3NewWrite,
|
||||
.pathRemove = storageS3PathRemove, .remove = storageS3Remove);
|
||||
.exists = storageS3Exists, .info = storageS3Info, .infoList = storageS3InfoList, .list = storageS3List,
|
||||
.newRead = storageS3NewRead, .newWrite = storageS3NewWrite, .pathRemove = storageS3PathRemove,
|
||||
.remove = storageS3Remove);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
|
||||
@ -179,6 +179,34 @@ testS3Server(void)
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL));
|
||||
harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:999", NULL));
|
||||
|
||||
// Info()
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
// File missing
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/BOGUS", NULL));
|
||||
harnessTlsServerReply(testS3ServerResponse(404, "Not Found", NULL, NULL));
|
||||
|
||||
// File exists
|
||||
harnessTlsServerExpect(testS3ServerRequest(HTTP_VERB_HEAD, "/subdir/file1.txt", NULL));
|
||||
harnessTlsServerReply(testS3ServerResponse(200, "OK", "content-length:9999", NULL));
|
||||
|
||||
// InfoList()
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
harnessTlsServerExpect(
|
||||
testS3ServerRequest(HTTP_VERB_GET, "/?delimiter=%2F&list-type=2&prefix=path%2Fto%2F", NULL));
|
||||
harnessTlsServerReply(
|
||||
testS3ServerResponse(
|
||||
200, "OK", NULL,
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
"<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">"
|
||||
" <Contents>"
|
||||
" <Key>path/to/test_file</Key>"
|
||||
" <Size>787</Size>"
|
||||
" </Contents>"
|
||||
" <CommonPrefixes>"
|
||||
" <Prefix>path/to/test_path/</Prefix>"
|
||||
" </CommonPrefixes>"
|
||||
"</ListBucketResult>"));
|
||||
|
||||
// storageDriverList()
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
// Throw error
|
||||
@ -404,6 +432,24 @@ testS3Server(void)
|
||||
}
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Callback and data for storageInfoList() tests
|
||||
***********************************************************************************************************************************/
|
||||
unsigned int testStorageInfoListSize = 0;
|
||||
StorageInfo testStorageInfoList[256];
|
||||
|
||||
void
|
||||
testStorageInfoListCallback(void *callbackData, const StorageInfo *info)
|
||||
{
|
||||
MEM_CONTEXT_BEGIN((MemContext *)callbackData)
|
||||
{
|
||||
testStorageInfoList[testStorageInfoListSize] = *info;
|
||||
testStorageInfoList[testStorageInfoListSize].name = strDup(info->name);
|
||||
testStorageInfoListSize++;
|
||||
}
|
||||
MEM_CONTEXT_END();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Test Run
|
||||
***********************************************************************************************************************************/
|
||||
@ -712,6 +758,32 @@ testRun(void)
|
||||
TEST_RESULT_BOOL(storageExistsNP(s3, strNew("BOGUS")), false, "file does not exist");
|
||||
TEST_RESULT_BOOL(storageExistsNP(s3, strNew("subdir/file1.txt")), true, "file exists");
|
||||
|
||||
// Info()
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_RESULT_BOOL(storageInfoP(s3, strNew("BOGUS"), .ignoreMissing = true).exists, false, "file does not exist");
|
||||
|
||||
StorageInfo info;
|
||||
TEST_ASSIGN(info, storageInfoNP(s3, strNew("subdir/file1.txt")), "file exists");
|
||||
TEST_RESULT_BOOL(info.exists, true, " check exists");
|
||||
TEST_RESULT_UINT(info.size, 9999, " check exists");
|
||||
|
||||
// InfoList()
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_ERROR(
|
||||
storageInfoListP(s3, strNew("/"), testStorageInfoListCallback, NULL, .errorOnMissing = true),
|
||||
AssertError, "assertion '!param.errorOnMissing || storageFeature(this, storageFeaturePath)' failed");
|
||||
|
||||
TEST_RESULT_VOID(
|
||||
storageInfoListNP(s3, strNew("/path/to"), testStorageInfoListCallback, (void *)memContextCurrent()), "info list files");
|
||||
|
||||
TEST_RESULT_UINT(testStorageInfoListSize, 2, " file and path returned");
|
||||
TEST_RESULT_STR(strPtr(testStorageInfoList[0].name), "test_path", " check name");
|
||||
TEST_RESULT_UINT(testStorageInfoList[0].size, 0, " check size");
|
||||
TEST_RESULT_UINT(testStorageInfoList[0].type, storageTypePath, " check type");
|
||||
TEST_RESULT_STR(strPtr(testStorageInfoList[1].name), "test_file", " check name");
|
||||
TEST_RESULT_UINT(testStorageInfoList[1].size, 787, " check size");
|
||||
TEST_RESULT_UINT(testStorageInfoList[1].type, storageTypeFile, " check type");
|
||||
|
||||
// storageDriverList()
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_ERROR(
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user