db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-06-20 01:17:49 +02:00
Code Issues Releases Activity

Add batch delete for Azure storage.

Browse Source 
Path remove (used by expire and friends) issued one HTTP DELETE per file. Use the Azure Blob Batch API to remove up to 256 objects per request, as the GCS driver already does, by sending the deletes as multipart sub-requests.

Azure's batch parser is stricter than the MIME/OData spec it is based on: it requires the body to begin with the boundary delimiter (no leading CRLF preamble) and splits header lines on ": ", so the multipart builder now emits the opening delimiter without a leading CRLF and writes the MIME part headers and embedded sub-request headers in the "Name: value" form. The multipart request code is shared with the GCS driver, which accepts either form.

Azure omits the blank line that terminates the headers of an empty-body sub-response, relying on the boundary's CRLF as the terminator, so multipart response header parsing now allows eof to end the header block.

Sub-requests that fail (not 2xx or 404) are retried individually. A failed part is mapped back to the original request by its position in the response rather than the echoed content-id header, since Azure omits content-id on some error responses.
This commit is contained in:
David Steele
2026-06-17 11:24:30 +07:00
committed by GitHub
parent 34bba828c6
commit ffa325b8a4
10 changed files with 466 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/xml/release/2020s/2026/2.59.0.xml
+12
View File
@@ -12,6 +12,18 @@
<p><proper>PostgreSQL 19beta1</proper> support.</p>
</release-item>
<release-item>
<github-pull-request id="2786"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="douglas.j.hunley"/>
<release-item-reviewer id="crispy"/>
</release-item-contributor-list>
<p>Add batch delete for <proper>Azure</proper> storage.</p>
</release-item>
<release-item>
<github-pull-request id="2762"/>
doc/xml/release/contributor.xml
+5
View File
@@ -284,6 +284,11 @@
<contributor-id type="github">cjames53</contributor-id>
</contributor>
<contributor id="crispy">
<contributor-name-display>Crispy</contributor-name-display>
<contributor-id type="github">Crispy1975</contributor-id>
</contributor>
<contributor id="ctenuun">
<contributor-name-display>Tenuun</contributor-name-display>
<contributor-id type="github">ctenuun</contributor-id>
src/common/io/http/request.c
+18 -12
View File
@@ -66,7 +66,7 @@ Format the request (excluding content)
static String *
httpRequestFmt(
const String *const verb, const String *const path, const HttpQuery *const query, const HttpHeader *const header,
const bool agent)
const bool agent, const bool headerSpace)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, verb);
@@ -74,6 +74,7 @@ httpRequestFmt(
FUNCTION_TEST_PARAM(HTTP_QUERY, query);
FUNCTION_TEST_PARAM(HTTP_HEADER, header);
FUNCTION_TEST_PARAM(BOOL, agent);
FUNCTION_TEST_PARAM(BOOL, headerSpace);
FUNCTION_TEST_END();
ASSERT(verb != NULL);
@@ -93,14 +94,16 @@ httpRequestFmt(
if (agent)
strCatZ(result, HTTP_HEADER_USER_AGENT ":" PROJECT_NAME "/" PROJECT_VERSION "\r\n");
// Add headers
// Add headers. A space after the colon is used for multipart sub-requests since Azure's batch parser requires the canonical
// "Name: value" form, but is omitted for normal requests to preserve existing behavior.
StringList *const headerList = httpHeaderList(header);
for (unsigned int headerIdx = 0; headerIdx < strLstSize(headerList); headerIdx++)
{
const String *const headerKey = strLstGet(headerList, headerIdx);
strCatFmt(result, "%s:%s\r\n", strZ(headerKey), strZ(httpHeaderGet(header, headerKey)));
strCatFmt(
result, headerSpace ? "%s: %s\r\n" : "%s:%s\r\n", strZ(headerKey), strZ(httpHeaderGet(header, headerKey)));
}
// Add blank line to end of headers
@@ -167,7 +170,7 @@ httpRequestProcess(HttpRequest *const this, const bool waitForResponse, const bo
BUFSTR(
httpRequestFmt(
httpRequestVerb(this), httpRequestPath(this), httpRequestQuery(this), httpRequestHeader(this),
true)));
true, false)));
// Write out content if any
if (this->content != NULL)
@@ -393,13 +396,14 @@ httpRequestMultiAdd(
MEM_CONTEXT_TEMP_BEGIN()
{
// Construct request header
// Construct request header. The MIME part headers use the canonical "Name: value" form (capitalized name, space after the
// colon) since Azure's batch parser is stricter than the MIME/OData spec and rejects other forms.
String *const request = strNew();
strCatZ(request, HTTP_HEADER_CONTENT_TYPE ":" HTTP_HEADER_CONTENT_TYPE_HTTP "\r\n");
strCatZ(request, HTTP_HEADER_CONTENT_TRANSFER_ENCODING ":" HTTP_HEADER_CONTENT_TRANSFER_ENCODING_BINARY "\r\n");
strCatFmt(request, HTTP_HEADER_CONTENT_ID ":%s\r\n\r\n", strZ(contentId));
strCat(request, httpRequestFmt(verb, path, param.query, param.header, false));
strCatZ(request, "Content-Type: " HTTP_HEADER_CONTENT_TYPE_HTTP "\r\n");
strCatZ(request, "Content-Transfer-Encoding: " HTTP_HEADER_CONTENT_TRANSFER_ENCODING_BINARY "\r\n");
strCatFmt(request, "Content-ID: %s\r\n\r\n", strZ(contentId));
strCat(request, httpRequestFmt(verb, path, param.query, param.header, false, true));
MEM_CONTEXT_OBJ_BEGIN(this->contentList)
{
@@ -453,13 +457,15 @@ httpRequestMultiContent(HttpRequestMulti *const this)
MEM_CONTEXT_TEMP_BEGIN()
{
// Generate boundary
// Generate boundary. The leading CRLF is conceptually part of the delimiter that precedes each subsequent part, but the
// opening delimiter is emitted without it below since Azure's batch parser requires the body to begin with the delimiter.
Buffer *const boundary = bufNew(boundarySize);
bufCat(boundary, BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_PRE));
bufCat(boundary, this->boundaryRaw);
// Add first boundary
bufCat(result, boundary);
// Add first boundary (without the leading CRLF so the body begins with the delimiter)
bufCat(result, BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_OPEN));
bufCat(result, this->boundaryRaw);
bufCat(result, BUFSTRDEF(HTTP_MULTIPART_BOUNDARY_POST));
// Add content and boundaries
src/common/io/http/request.h
+1
View File
@@ -77,6 +77,7 @@ STRING_DECLARE(HTTP_HEADER_RANGE_STR);
#define HTTP_MULTIPART_BOUNDARY_INIT "QKX4EYg4"
#define HTTP_MULTIPART_BOUNDARY_NEXT 4
#define HTTP_MULTIPART_BOUNDARY_EXTRA "LARJ52gF4F239iNVrrC5w5aYskcGrWCXIFlMp5IxswggIhcX2A0gF9nrgN8q"
#define HTTP_MULTIPART_BOUNDARY_OPEN "--"
#define HTTP_MULTIPART_BOUNDARY_PRE "\r\n--"
#define HTTP_MULTIPART_BOUNDARY_POST "\r\n"
#define HTTP_MULTIPART_BOUNDARY_POST_LAST "--\r\n"
src/common/io/http/response.c
+9 -7
View File
@@ -287,19 +287,21 @@ httpResponseStatusRead(HttpResponse *const this, IoRead *const read)
Read response headers
***********************************************************************************************************************************/
static void
httpResponseHeaderRead(HttpResponse *const this, IoRead *const read)
httpResponseHeaderRead(HttpResponse *const this, IoRead *const read, const bool allowEof)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(HTTP_RESPONSE, this);
FUNCTION_TEST_PARAM(IO_READ, read);
FUNCTION_TEST_PARAM(BOOL, allowEof);
FUNCTION_TEST_END();
MEM_CONTEXT_TEMP_BEGIN()
{
do
{
// Read the next header
String *const header = strTrim(ioReadLine(read));
// Read the next header. In a multipart part the headers may be terminated by the boundary rather than a blank line (the
// boundary's leading CRLF doubles as the terminator), so allow eof to end the header block in that case.
String *const header = strTrim(ioReadLineParam(read, allowEof));
// If the header is empty then we have reached the end of the headers
if (strSize(header) == 0)
@@ -400,7 +402,7 @@ httpResponseNew(HttpSession *const session, const String *const verb, const bool
httpResponseStatusRead(this, httpSessionIoReadP(this->session));
// Read headers
httpResponseHeaderRead(this, httpSessionIoReadP(this->session));
httpResponseHeaderRead(this, httpSessionIoReadP(this->session), false);
// Was content returned in the response? HEAD will report content but not actually return any.
this->contentExists =
@@ -553,7 +555,7 @@ httpResponseMultiNext(HttpResponseMulti *const this)
IoRead *const responseIo = ioBufferReadNewOpen(response);
// Read multipart headers
httpResponseHeaderRead(result, responseIo);
httpResponseHeaderRead(result, responseIo, true);
CHECK(
FormatError,
@@ -563,8 +565,8 @@ httpResponseMultiNext(HttpResponseMulti *const this)
// Read status
httpResponseStatusRead(result, responseIo);
// Read headers
httpResponseHeaderRead(result, responseIo);
// Read headers (eof is allowed since the boundary, not a blank line, may terminate the embedded response headers)
httpResponseHeaderRead(result, responseIo, true);
// Read content
CHECK(FormatError, !result->contentChunked, "chunked encoding not supported in multipart");
src/storage/azure/storage.c
+188 -34
View File
@@ -14,12 +14,18 @@ Azure Storage
#include "common/io/tls/client.h"
#include "common/log.h"
#include "common/regExp.h"
#include "common/stat.h"
#include "common/type/json.h"
#include "common/type/object.h"
#include "common/type/xml.h"
#include "storage/azure/read.h"
#include "storage/azure/write.h"
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
#define STORAGE_AZURE_DELETE_MAX 256
/***********************************************************************************************************************************
Azure http headers
***********************************************************************************************************************************/
@@ -31,6 +37,7 @@ STRING_STATIC(AZURE_HEADER_VERSION_VALUE_STR, "2024-08-04"
Azure query tokens
***********************************************************************************************************************************/
STRING_STATIC(AZURE_QUERY_MARKER_STR, "marker");
STRING_STATIC(AZURE_QUERY_BATCH_STR, "batch");
STRING_EXTERN(AZURE_QUERY_COMP_STR, AZURE_QUERY_COMP);
STRING_STATIC(AZURE_QUERY_DELIMITER_STR, "delimiter");
STRING_STATIC(AZURE_QUERY_INCLUDE_STR, "include");
@@ -71,6 +78,14 @@ STRING_STATIC(AZURE_CREDENTIAL_HOST_STR, "169.254.169
VARIANT_STRDEF_STATIC(AZURE_JSON_TAG_ACCESS_TOKEN_VAR, "access_token");
VARIANT_STRDEF_STATIC(AZURE_JSON_TAG_EXPIRES_IN_VAR, "expires_in");
/***********************************************************************************************************************************
Statistics constants
***********************************************************************************************************************************/
STRING_STATIC(AZURE_STAT_REMOVE_STR, "azure.rm");
STRING_STATIC(AZURE_STAT_REMOVE_BATCH_STR, "azure.rm.batch");
STRING_STATIC(AZURE_STAT_REMOVE_BATCH_PART_STR, "azure.rm.batch.part");
STRING_STATIC(AZURE_STAT_REMOVE_BATCH_RETRY_STR, "azure.rm.batch.retry");
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@@ -88,6 +103,7 @@ struct StorageAzure
const HttpQuery *sasKey; // SAS key
const String *host; // Host name
size_t blockSize; // Block size for multi-block upload
unsigned int deleteMax; // Maximum objects that can be deleted in one request
const String *tag; // Tags to be applied to objects
const String *pathPrefix; // Account/container prefix
@@ -108,7 +124,7 @@ Based on the documentation at https://docs.microsoft.com/en-us/rest/api/storages
static void
storageAzureAuth(
StorageAzure *const this, const String *const verb, const String *const path, HttpQuery *const query,
const String *const dateTime, HttpHeader *const httpHeader)
const String *const dateTime, HttpHeader *const httpHeader, const bool subRequest)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STORAGE_AZURE, this);
@@ -117,6 +133,7 @@ storageAzureAuth(
FUNCTION_TEST_PARAM(HTTP_QUERY, query);
FUNCTION_TEST_PARAM(STRING, dateTime);
FUNCTION_TEST_PARAM(KEY_VALUE, httpHeader);
FUNCTION_TEST_PARAM(BOOL, subRequest);
FUNCTION_TEST_END();
ASSERT(this != NULL);
@@ -128,15 +145,17 @@ storageAzureAuth(
MEM_CONTEXT_TEMP_BEGIN()
{
// Set required headers
httpHeaderPut(httpHeader, HTTP_HEADER_HOST_STR, this->host);
// Host header is required for all authentication types but must not be set on batch sub-requests
if (!subRequest)
httpHeaderPut(httpHeader, HTTP_HEADER_HOST_STR, this->host);
// Date header is required for shared key authentication (for signing)
if (this->keyType == storageAzureKeyTypeShared)
httpHeaderPut(httpHeader, HTTP_HEADER_DATE_STR, dateTime);
// Set version header (required for shared key and auto auth types, not for SAS)
if (this->keyType != storageAzureKeyTypeSas)
// Set version header (required for shared key and auto auth types, not for SAS) but never on batch sub-requests,
// which inherit the version from the top-level batch request
if (this->keyType != storageAzureKeyTypeSas && !subRequest)
httpHeaderPut(httpHeader, AZURE_HEADER_VERSION_STR, AZURE_HEADER_VERSION_VALUE_STR);
// Shared key authentication
@@ -177,6 +196,7 @@ storageAzureAuth(
// Generate string to sign
const String *const contentLength = httpHeaderGet(httpHeader, HTTP_HEADER_CONTENT_LENGTH_STR);
const String *const contentMd5 = httpHeaderGet(httpHeader, HTTP_HEADER_CONTENT_MD5_STR);
const String *const contentType = httpHeaderGet(httpHeader, HTTP_HEADER_CONTENT_TYPE_STR);
const String *const range = httpHeaderGet(httpHeader, HTTP_HEADER_RANGE_STR);
const String *const stringToSign = strNewFmt(
@@ -185,7 +205,7 @@ storageAzureAuth(
"\n" // content-language
"%s\n" // content-length
"%s\n" // content-md5
"\n" // content-type
"%s\n" // content-type
"%s\n" // date
"\n" // If-Modified-Since
"\n" // If-Match
@@ -196,8 +216,8 @@ storageAzureAuth(
"/%s%s" // Canonicalized account/path
"%s", // Canonicalized query
strZ(verb), strEq(contentLength, ZERO_STR) ? "" : strZ(contentLength), contentMd5 == NULL ? "" : strZ(contentMd5),
strZ(dateTime), range == NULL ? "" : strZ(range), strZ(headerCanonical), strZ(this->account), strZ(path),
strZ(queryCanonical));
contentType == NULL ? "" : strZ(contentType), strZ(dateTime), range == NULL ? "" : strZ(range),
strZ(headerCanonical), strZ(this->account), strZ(path), strZ(queryCanonical));
// Generate authorization header
httpHeaderPut(
@@ -280,6 +300,7 @@ storageAzureRequestAsync(StorageAzure *const this, const String *const verb, Sto
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
FUNCTION_LOG_PARAM(BUFFER, param.content);
FUNCTION_LOG_PARAM(LIST, param.contentList);
FUNCTION_LOG_PARAM(BOOL, param.tag);
FUNCTION_LOG_END();
@@ -293,21 +314,49 @@ storageAzureRequestAsync(StorageAzure *const this, const String *const verb, Sto
// Prepend path prefix
param.path = param.path == NULL ? this->pathPrefix : strNewFmt("%s%s", strZ(this->pathPrefix), strZ(param.path));
// Create header list and add content length
// Create header list
HttpHeader *requestHeader =
param.header == NULL ? httpHeaderNew(this->headerRedactList) : httpHeaderDup(param.header, this->headerRedactList);
// Set content or construct multipart content
const String *const date = httpDateFromTime(time(NULL));
const Buffer *content = param.content;
if (param.contentList != NULL)
{
ASSERT(param.content == NULL);
ASSERT(!lstEmpty(param.contentList));
HttpRequestMulti *const requestMulti = httpRequestMultiNew();
for (unsigned int contentIdx = 0; contentIdx < lstSize(param.contentList); contentIdx++)
{
const StorageAzureRequestPart *const requestPart = lstGet(param.contentList, contentIdx);
HttpHeader *const partHeader = httpHeaderNew(this->headerRedactList);
HttpQuery *const partQuery = this->sasKey != NULL ? httpQueryNewP(.redactList = this->queryRedactList) : NULL;
const String *const path = strNewFmt("%s%s", strZ(this->pathPrefix), strZ(requestPart->path));
httpHeaderAdd(partHeader, HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
storageAzureAuth(this, requestPart->verb, path, partQuery, date, partHeader, true);
httpRequestMultiAddP(
requestMulti, strNewFmt("%u", contentIdx), requestPart->verb, path, .header = partHeader, .query = partQuery);
}
httpRequestMultiHeaderAdd(requestMulti, requestHeader);
content = httpRequestMultiContent(requestMulti);
}
// Set content length
httpHeaderAdd(
requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR,
param.content == NULL || bufEmpty(param.content) ? ZERO_STR : strNewFmt("%zu", bufUsed(param.content)));
content == NULL || bufEmpty(content) ? ZERO_STR : strNewFmt("%zu", bufUsed(content)));
// Calculate content-md5 header if there is content
if (param.content != NULL)
if (content != NULL)
{
httpHeaderAdd(
requestHeader, HTTP_HEADER_CONTENT_MD5_STR,
strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, param.content)));
requestHeader, HTTP_HEADER_CONTENT_MD5_STR, strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, content)));
}
// Set tags when requested and available
@@ -324,13 +373,13 @@ storageAzureRequestAsync(StorageAzure *const this, const String *const verb, Sto
httpQueryDupP(param.query, .redactList = this->queryRedactList);
// Generate authorization header
storageAzureAuth(this, verb, path, query, httpDateFromTime(time(NULL)), requestHeader);
storageAzureAuth(this, verb, path, query, date, requestHeader, false);
// Send request
MEM_CONTEXT_PRIOR_BEGIN()
{
result = httpRequestNewP(
this->httpClient, verb, path, .query = query, .header = requestHeader, .content = param.content);
this->httpClient, verb, path, .query = query, .header = requestHeader, .content = content);
}
MEM_CONTEXT_END();
}
@@ -379,13 +428,15 @@ storageAzureRequest(StorageAzure *const this, const String *const verb, const St
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
FUNCTION_LOG_PARAM(BUFFER, param.content);
FUNCTION_LOG_PARAM(LIST, param.contentList);
FUNCTION_LOG_PARAM(BOOL, param.allowMissing);
FUNCTION_LOG_PARAM(BOOL, param.contentIo);
FUNCTION_LOG_PARAM(BOOL, param.tag);
FUNCTION_LOG_END();
HttpRequest *const request = storageAzureRequestAsyncP(
this, verb, .path = param.path, .header = param.header, .query = param.query, .content = param.content, .tag = param.tag);
this, verb, .path = param.path, .header = param.header, .query = param.query, .content = param.content,
.contentList = param.contentList, .tag = param.tag);
HttpResponse *const result = storageAzureResponseP(request, .allowMissing = param.allowMissing, .contentIo = param.contentIo);
httpRequestFree(request);
@@ -734,9 +785,94 @@ typedef struct StorageAzurePathRemoveData
StorageAzure *this; // Storage object
MemContext *memContext; // Mem context to create requests in
HttpRequest *request; // Async remove request
List *requestContentList; // Content list for async request
List *contentList; // Content list currently being built
const String *path; // Root path of remove
} StorageAzurePathRemoveData;
static void
storageAzurePathRemoveInternal(StorageAzurePathRemoveData *const data)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, data);
FUNCTION_TEST_END();
ASSERT(data != NULL);
ASSERT(data->this != NULL);
// Get response for async request
if (data->request != NULL)
{
MEM_CONTEXT_TEMP_BEGIN()
{
HttpResponse *const response = storageAzureResponseP(data->request);
HttpResponseMulti *const responseMulti = httpResponseMultiNew(
httpResponseContent(response), httpHeaderGet(httpResponseHeader(response), HTTP_HEADER_CONTENT_TYPE_STR));
// Loop through all response parts. Parts are mapped to the original request by ordinal position since the service
// returns one response part per sub-request in request order. The echoed content-id is intentionally not used because
// Azure omits it on some error responses, which would otherwise make a failed part impossible to retry.
unsigned int partIdx = 0;
HttpResponse *responsePart = httpResponseMultiNext(responseMulti);
CHECK(FormatError, responsePart != NULL, "at least one response part is required");
do
{
// If not OK and not missing then retry
if (!httpResponseCodeOk(responsePart) && httpResponseCode(responsePart) != HTTP_RESPONSE_CODE_NOT_FOUND)
{
// Get the original request for this part by position
CHECK_FMT(
FormatError, partIdx < lstSize(data->requestContentList), "response part %u is out of range", partIdx);
const StorageAzureRequestPart *const content = lstGet(data->requestContentList, partIdx);
// Retry remove
statInc(AZURE_STAT_REMOVE_BATCH_RETRY_STR);
httpResponseFree(storageAzureRequestP(data->this, content->verb, .path = content->path, .allowMissing = true));
}
else
statInc(AZURE_STAT_REMOVE_BATCH_PART_STR);
httpResponseFree(responsePart);
responsePart = httpResponseMultiNext(responseMulti);
partIdx++;
}
while (responsePart != NULL);
}
MEM_CONTEXT_TEMP_END();
// Free request
httpRequestFree(data->request);
data->request = NULL;
// Free content list
lstFree(data->requestContentList);
}
// Send new async request if there is more to remove
if (data->contentList != NULL)
{
statInc(AZURE_STAT_REMOVE_BATCH_STR);
MEM_CONTEXT_BEGIN(data->memContext)
{
data->request = storageAzureRequestAsyncP(
data->this, HTTP_VERB_POST_STR,
.query = httpQueryAdd(
httpQueryAdd(httpQueryNewP(), AZURE_QUERY_COMP_STR, AZURE_QUERY_BATCH_STR),
AZURE_QUERY_RESTYPE_STR, AZURE_QUERY_VALUE_CONTAINER_STR),
.contentList = data->contentList);
}
MEM_CONTEXT_END();
// Store the content list for use in error handling
data->requestContentList = data->contentList;
data->contentList = NULL;
}
FUNCTION_TEST_RETURN_VOID();
}
static void
storageAzurePathRemoveCallback(void *const callbackData, const StorageInfo *const info)
{
@@ -748,25 +884,34 @@ storageAzurePathRemoveCallback(void *const callbackData, const StorageInfo *cons
ASSERT(callbackData != NULL);
ASSERT(info != NULL);
StorageAzurePathRemoveData *const data = callbackData;
// Get response from prior async request
if (data->request != NULL)
{
httpResponseFree(storageAzureResponseP(data->request, .allowMissing = true));
httpRequestFree(data->request);
data->request = NULL;
}
// Only delete files since paths don't really exist
if (info->type == storageTypeFile)
{
MEM_CONTEXT_BEGIN(data->memContext)
StorageAzurePathRemoveData *const data = callbackData;
if (data->contentList == NULL)
{
data->request = storageAzureRequestAsyncP(
data->this, HTTP_VERB_DELETE_STR, strNewFmt("%s/%s", strZ(data->path), strZ(info->name)));
MEM_CONTEXT_BEGIN(data->memContext)
{
data->contentList = lstNewP(sizeof(StorageAzureRequestPart));
}
MEM_CONTEXT_END();
}
MEM_CONTEXT_END();
MEM_CONTEXT_OBJ_BEGIN(data->contentList)
{
const StorageAzureRequestPart content =
{
.verb = HTTP_VERB_DELETE_STR,
.path = strNewFmt("%s/%s", strZ(data->path), strZ(info->name)),
};
lstAdd(data->contentList, &content);
}
MEM_CONTEXT_OBJ_END();
if (lstSize(data->contentList) == data->this->deleteMax)
storageAzurePathRemoveInternal(data);
}
FUNCTION_TEST_RETURN_VOID();
@@ -796,11 +941,18 @@ storageAzurePathRemove(THIS_VOID, const String *const path, const bool recurse,
.path = strEq(path, FSLASH_STR) ? EMPTY_STR : path,
};
storageAzureListInternal(this, path, storageInfoLevelType, NULL, true, 0, storageAzurePathRemoveCallback, &data);
MEM_CONTEXT_TEMP_BEGIN()
{
storageAzureListInternal(this, path, storageInfoLevelType, NULL, true, 0, storageAzurePathRemoveCallback, &data);
// Check response on last async request
if (data.request != NULL)
storageAzureResponseP(data.request, .allowMissing = true);
// Call if there is more to be removed
if (data.contentList != NULL)
storageAzurePathRemoveInternal(&data);
// Check response on last async request
storageAzurePathRemoveInternal(&data);
}
MEM_CONTEXT_TEMP_END();
}
MEM_CONTEXT_TEMP_END();
@@ -823,6 +975,7 @@ storageAzureRemove(THIS_VOID, const String *const file, const StorageInterfaceRe
ASSERT(file != NULL);
ASSERT(!param.errorOnMissing);
statInc(AZURE_STAT_REMOVE_STR);
httpResponseFree(storageAzureRequestP(this, HTTP_VERB_DELETE_STR, file, .allowMissing = true));
FUNCTION_LOG_RETURN_VOID();
@@ -889,6 +1042,7 @@ storageAzureNew(
.pathPrefix =
uriStyle == storageAzureUriStyleHost ?
strNewFmt("/%s", strZ(container)) : strNewFmt("/%s/%s", strZ(account), strZ(container)),
.deleteMax = STORAGE_AZURE_DELETE_MAX,
.keyType = keyType,
};
src/storage/azure/storage.intern.h
+11
View File
@@ -22,6 +22,15 @@ STRING_DECLARE(AZURE_QUERY_RESTYPE_STR);
#define AZURE_QUERY_VALUE_CONTAINER "container"
STRING_DECLARE(AZURE_QUERY_VALUE_CONTAINER_STR);
/***********************************************************************************************************************************
Multi-Part request data
***********************************************************************************************************************************/
typedef struct StorageAzureRequestPart
{
const String *path; // Request path
const String *verb; // Verb (GET, PUT, etc)
} StorageAzureRequestPart;
/***********************************************************************************************************************************
Perform an Azure Request
***********************************************************************************************************************************/
@@ -33,6 +42,7 @@ typedef struct StorageAzureRequestAsyncParam
const HttpHeader *header; // Request headers
const HttpQuery *query; // Query parameters
const Buffer *content; // Request content
const List *contentList; // Request content part list
bool tag; // Add tags when available?
} StorageAzureRequestAsyncParam;
@@ -61,6 +71,7 @@ typedef struct StorageAzureRequestParam
const HttpHeader *header; // Request headers
const HttpQuery *query; // Query parameters
const Buffer *content; // Request content
const List *contentList; // Request content part list
bool allowMissing; // Allow missing files (caller can check response code)
bool contentIo; // Is IoRead interface required to read content?
bool tag; // Add tags when available?
test/src/module/common/ioHttpTest.c
+10 -9
View File
@@ -961,17 +961,17 @@ testRun(void)
"GET / HTTP/1.1\r\n" TEST_USER_AGENT
"content-type:multipart/mixed; boundary=QKX4EYg4LARJ\r\n"
"hdr1:1\r\n\r\n"
"\r\n--QKX4EYg4LARJ\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:0\r\n\r\n"
"--QKX4EYg4LARJ\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 0\r\n\r\n"
"GET / HTTP/1.1\r\n\r\n"
"\r\n--QKX4EYg4LARJ\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:1\r\n\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 1\r\n\r\n"
"POST /ack HTTP/1.1\r\n"
"content-length:3\r\n\r\n"
"content-length: 3\r\n\r\n"
HTTP_MULTIPART_BOUNDARY_INIT
"\r\n--QKX4EYg4LARJ--\r\n");
hrnServerScriptReplyZ(
@@ -981,7 +981,8 @@ testRun(void)
"--XXX\r\n"
"content-type:application/http\r\n"
"content-id:0\r\n\r\n"
"HTTP/1.1 200 OK\r\n\r\n"
// Headers terminated by the boundary rather than a blank line (as Azure does for empty-body parts)
"HTTP/1.1 200 OK\r\n"
"\r\n--XXX\r\n"
"content-type:application/http\r\n"
"content-id:1\r\n"
test/src/module/storage/azureTest.c
+189 -15
View File
@@ -37,6 +37,7 @@ typedef struct TestRequestParam
const char *blobType;
const char *range;
const char *tag;
bool multiPart;
} TestRequestParam;
#define testRequestP(write, verb, path, ...) \
@@ -45,7 +46,7 @@ typedef struct TestRequestParam
static void
testRequest(IoWrite *write, const char *verb, const char *path, TestRequestParam param)
{
String *request = strCatFmt(strNew(), "%s /" TEST_ACCOUNT "/" TEST_CONTAINER, verb);
String *const request = strCatFmt(strNew(), "%s /" TEST_ACCOUNT "/" TEST_CONTAINER, verb);
// When SAS spit out the query and merge in the SAS key
if (driver->sasKey != NULL)
@@ -78,13 +79,24 @@ testRequest(IoWrite *write, const char *verb, const char *path, TestRequestParam
// Add content-length
strCatFmt(request, "content-length:%zu\r\n", param.content == NULL ? 0 : strlen(param.content));
// Add md5
// Add md5. When the content contains wildcards (a shared-key batch with non-deterministic embedded signatures) the md5 is also
// wildcarded since it cannot be computed from the wildcarded content.
if (param.content != NULL)
{
strCatFmt(
request, "content-md5:%s\r\n", strZ(strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, BUFSTRZ(param.content)))));
if (strchr(param.content, '?') != NULL)
strCatZ(request, "content-md5:????????????????????????\r\n");
else
{
strCatFmt(
request, "content-md5:%s\r\n",
strZ(strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, BUFSTRZ(param.content)))));
}
}
// Add multipart content-type
if (param.multiPart)
strCatZ(request, "content-type:multipart/mixed; boundary=" HTTP_MULTIPART_BOUNDARY_INIT "\r\n");
// Add date
if (driver->sharedKey != NULL)
strCatZ(request, "date:???, ?? ??? ???? ??:??:?? GMT\r\n");
@@ -127,6 +139,7 @@ typedef struct TestResponseParam
unsigned int code;
const char *header;
const char *content;
bool multiPart;
const Variant *contentSize;
} TestResponseParam;
@@ -161,6 +174,10 @@ testResponse(IoWrite *write, TestResponseParam param)
if (param.header != NULL)
strCatFmt(response, "%s\r\n", param.header);
// Add multipart content-type
if (param.multiPart)
strCatZ(response, "content-type:multipart/mixed; boundary=" HTTP_MULTIPART_BOUNDARY_INIT "\r\n");
// Content
if (param.content != NULL)
{
@@ -413,7 +430,7 @@ testRun(void)
header = httpHeaderAdd(httpHeaderNew(NULL), HTTP_HEADER_CONTENT_LENGTH_STR, ZERO_STR);
TEST_RESULT_VOID(storageAzureAuth(storage, HTTP_VERB_GET_STR, STRDEF("/path"), NULL, dateTime, header), "auth");
TEST_RESULT_VOID(storageAzureAuth(storage, HTTP_VERB_GET_STR, STRDEF("/path"), NULL, dateTime, header, false), "auth");
TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(header, httpHeaderToLog, logBuf, sizeof(logBuf)), "httpHeaderToLog");
TEST_RESULT_Z(
logBuf,
@@ -429,7 +446,8 @@ testRun(void)
HttpQuery *query = httpQueryAdd(httpQueryNewP(), STRDEF("a"), STRDEF("b"));
TEST_RESULT_VOID(storageAzureAuth(storage, HTTP_VERB_GET_STR, STRDEF("/path/file"), query, dateTime, header), "auth");
TEST_RESULT_VOID(
storageAzureAuth(storage, HTTP_VERB_GET_STR, STRDEF("/path/file"), query, dateTime, header, false), "auth");
TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(header, httpHeaderToLog, logBuf, sizeof(logBuf)), "httpHeaderToLog");
TEST_RESULT_Z(
logBuf,
@@ -453,7 +471,8 @@ testRun(void)
query = httpQueryAdd(httpQueryNewP(), STRDEF("a"), STRDEF("b"));
header = httpHeaderAdd(httpHeaderNew(NULL), HTTP_HEADER_CONTENT_LENGTH_STR, STRDEF("66"));
TEST_RESULT_VOID(storageAzureAuth(storage, HTTP_VERB_GET_STR, STRDEF("/path/file"), query, dateTime, header), "auth");
TEST_RESULT_VOID(
storageAzureAuth(storage, HTTP_VERB_GET_STR, STRDEF("/path/file"), query, dateTime, header, false), "auth");
TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(header, httpHeaderToLog, logBuf, sizeof(logBuf)), "httpHeaderToLog");
TEST_RESULT_Z(logBuf, "{content-length: '66', host: 'account.blob.core.usgovcloudapi.net'}", "check headers");
TEST_RESULT_STR_Z(httpQueryRenderP(query), "a=b&sig=key", "check query");
@@ -703,6 +722,83 @@ testRun(void)
TEST_RESULT_BOOL(storageInfoP(storage, NULL, .ignoreMissing = true).exists, false, "info for /");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("remove files with shared key batched one per request");
// Force one delete per batch to exercise the deleteMax flush with shared-key signed sub-requests
driver->deleteMax = 1;
testRequestP(service, HTTP_VERB_GET, "?comp=list&prefix=path%2F&restype=container");
testResponseP(
service,
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<EnumerationResults>"
" <Blobs>"
" <Blob>"
" <Name>path/test1.txt</Name>"
" <Properties/>"
" </Blob>"
" <Blob>"
" <Name>path/test2.txt</Name>"
" <Properties/>"
" </Blob>"
" </Blobs>"
" <NextMarker/>"
"</EnumerationResults>");
testRequestP(
service, HTTP_VERB_POST, "?comp=batch&restype=container", .multiPart = true,
.content =
"--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 0\r\n"
"\r\n"
"DELETE /account/container/path/test1.txt HTTP/1.1\r\n"
"authorization: SharedKey account:????????????????????????????????????????????\r\n"
"content-length: 0\r\n"
"date: ???, ?? ??? ???? ??:??:?? GMT\r\n"
"\r\n\r\n"
"--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(
service, .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:0\r\n"
"\r\n"
"HTTP/1.1 202 Accepted\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testRequestP(
service, HTTP_VERB_POST, "?comp=batch&restype=container", .multiPart = true,
.content =
"--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 0\r\n"
"\r\n"
"DELETE /account/container/path/test2.txt HTTP/1.1\r\n"
"authorization: SharedKey account:????????????????????????????????????????????\r\n"
"content-length: 0\r\n"
"date: ???, ?? ??? ???? ??:??:?? GMT\r\n"
"\r\n\r\n"
"--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(
service, .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:0\r\n"
"\r\n"
"HTTP/1.1 202 Accepted\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
TEST_RESULT_VOID(storagePathRemoveP(storage, STRDEF("/path"), .recurse = true), "remove");
driver->deleteMax = STORAGE_AZURE_DELETE_MAX;
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("switch to managed identity");
@@ -1058,6 +1154,10 @@ testRun(void)
" <Name>path1/xxx.zzz</Name>"
" <Properties/>"
" </Blob>"
" <Blob>"
" <Name>path2/file2</Name>"
" <Properties/>"
" </Blob>"
" <BlobPrefix>"
" <Name>not-deleted/</Name>"
" </BlobPrefix>"
@@ -1065,10 +1165,55 @@ testRun(void)
" <NextMarker/>"
"</EnumerationResults>");
testRequestP(service, HTTP_VERB_DELETE, "/test1.txt");
testResponseP(service);
testRequestP(
service, HTTP_VERB_POST, "?comp=batch&restype=container", .multiPart = true,
.content =
"--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 0\r\n"
"\r\n"
"DELETE /account/container/test1.txt?sig=key HTTP/1.1\r\n"
"content-length: 0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 1\r\n"
"\r\n"
"DELETE /account/container/path1/xxx.zzz?sig=key HTTP/1.1\r\n"
"content-length: 0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 2\r\n"
"\r\n"
"DELETE /account/container/path2/file2?sig=key HTTP/1.1\r\n"
"content-length: 0\r\n"
"\r\n\r\n"
"--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(
service, .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:0\r\n"
"\r\n"
"HTTP/1.1 404 Missing\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:1\r\n"
"\r\n"
"HTTP/1.1 200 OK\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
// No content-id on the error part (as Azure omits it on some errors) to verify the retry maps by position
"\r\n"
"HTTP/1.1 300 Error\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testRequestP(service, HTTP_VERB_DELETE, "/path1/xxx.zzz");
testRequestP(service, HTTP_VERB_DELETE, "/path2/file2");
testResponseP(service);
TEST_RESULT_VOID(storagePathRemoveP(storage, STRDEF("/"), .recurse = true), "remove");
@@ -1098,11 +1243,40 @@ testRun(void)
" <NextMarker/>"
"</EnumerationResults>");
testRequestP(service, HTTP_VERB_DELETE, "/path/test1.txt");
testResponseP(service);
testRequestP(service, HTTP_VERB_DELETE, "/path/path1/xxx.zzz");
testResponseP(service);
testRequestP(
service, HTTP_VERB_POST, "?comp=batch&restype=container", .multiPart = true,
.content =
"--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 0\r\n"
"\r\n"
"DELETE /account/container/path/test1.txt?sig=key HTTP/1.1\r\n"
"content-length: 0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 1\r\n"
"\r\n"
"DELETE /account/container/path/path1/xxx.zzz?sig=key HTTP/1.1\r\n"
"content-length: 0\r\n"
"\r\n\r\n"
"--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(
service, .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:0\r\n"
"\r\n"
"HTTP/1.1 200 OK\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-id:1\r\n"
"\r\n"
"HTTP/1.1 200 OK\r\n\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
TEST_RESULT_VOID(storagePathRemoveP(storage, STRDEF("/path"), .recurse = true), "remove");
test/src/module/storage/gcsTest.c
+23 -23
View File
@@ -977,29 +977,29 @@ testRun(void)
testRequestP(
service, HTTP_VERB_POST, .path = "/batch/storage/v1", .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:0\r\n"
"--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 0\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path%2Fto%2Ftest1.txt HTTP/1.1\r\n"
"content-length:0\r\n"
"content-length: 0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:1\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 1\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path1%2Fxxx.zzz HTTP/1.1\r\n"
"content-length:0\r\n"
"content-length: 0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:2\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 2\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path2%2Ffile2 HTTP/1.1\r\n"
"content-length:0\r\n"
"content-length: 0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(
@@ -1053,13 +1053,13 @@ testRun(void)
testRequestP(
service, HTTP_VERB_POST, .path = "/batch/storage/v1", .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:0\r\n"
"--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 0\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path%2Ftest1.txt HTTP/1.1\r\n"
"content-length:0\r\n"
"content-length: 0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(
@@ -1075,13 +1075,13 @@ testRun(void)
testRequestP(
service, HTTP_VERB_POST, .path = "/batch/storage/v1", .multiPart = true,
.content =
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"content-type:application/http\r\n"
"content-transfer-encoding:binary\r\n"
"content-id:0\r\n"
"--" HTTP_MULTIPART_BOUNDARY_INIT "\r\n"
"Content-Type: application/http\r\n"
"Content-Transfer-Encoding: binary\r\n"
"Content-ID: 0\r\n"
"\r\n"
"DELETE /storage/v1/b/bucket/o/path%2Fpath1%2Fxxx.zzz HTTP/1.1\r\n"
"content-length:0\r\n"
"content-length: 0\r\n"
"\r\n"
"\r\n--" HTTP_MULTIPART_BOUNDARY_INIT "--\r\n");
testResponseP(