You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-07 00:35:37 +02:00
Calculate content-md5 on S3 only when required.
The content-md5 header was generated for all requests with content but it is only required for batch delete requests. It is not clear why this header is required when x-amz-content-sha256 is also provided or why it is required only for this request but the documentation is clear on the matter. However, the content for these requests is relatively small compared to uploading files so omitting content-md5 where possible will save some CPU cycles. Current AWS S3 and recent Minio don't complain if this header is missing but since it is still required by older versions of Minio and it is specified in the documentation for batch delete it is makes sense to keep it.
This commit is contained in:
@ -131,6 +131,17 @@
|
||||
<p>Use <proper>lz4</proper> for protocol compression.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<github-pull-request id="2592"/>
|
||||
|
||||
<release-item-contributor-list>
|
||||
<release-item-contributor id="david.steele"/>
|
||||
<release-item-reviewer id="david.christensen"/>
|
||||
</release-item-contributor-list>
|
||||
|
||||
<p>Calculate <id>content-md5</id> on <proper>S3</proper> only when required.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<commit subject="Add detail logging to expire test."/>
|
||||
<commit subject="Add detail logging for expired archive path.">
|
||||
|
@ -452,6 +452,7 @@ storageS3RequestAsync(StorageS3 *const this, const String *const verb, const Str
|
||||
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
|
||||
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
|
||||
FUNCTION_LOG_PARAM(BUFFER, param.content);
|
||||
FUNCTION_LOG_PARAM(BOOL, param.contentMd5);
|
||||
FUNCTION_LOG_PARAM(BOOL, param.sseKms);
|
||||
FUNCTION_LOG_PARAM(BOOL, param.sseC);
|
||||
FUNCTION_LOG_PARAM(BOOL, param.tag);
|
||||
@ -473,9 +474,11 @@ storageS3RequestAsync(StorageS3 *const this, const String *const verb, const Str
|
||||
requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR,
|
||||
param.content == NULL || bufEmpty(param.content) ? ZERO_STR : strNewFmt("%zu", bufUsed(param.content)));
|
||||
|
||||
// Calculate content-md5 header if there is content
|
||||
if (param.content != NULL)
|
||||
// Calculate content-md5 header when required
|
||||
if (param.contentMd5)
|
||||
{
|
||||
ASSERT(param.content != NULL && !bufEmpty(param.content));
|
||||
|
||||
httpHeaderAdd(
|
||||
requestHeader, HTTP_HEADER_CONTENT_MD5_STR,
|
||||
strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, param.content)));
|
||||
@ -607,6 +610,7 @@ storageS3Request(StorageS3 *const this, const String *const verb, const String *
|
||||
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
|
||||
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
|
||||
FUNCTION_LOG_PARAM(BUFFER, param.content);
|
||||
FUNCTION_LOG_PARAM(BOOL, param.contentMd5);
|
||||
FUNCTION_LOG_PARAM(BOOL, param.allowMissing);
|
||||
FUNCTION_LOG_PARAM(BOOL, param.contentIo);
|
||||
FUNCTION_LOG_PARAM(BOOL, param.sseKms);
|
||||
@ -615,8 +619,8 @@ storageS3Request(StorageS3 *const this, const String *const verb, const String *
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
HttpRequest *const request = storageS3RequestAsyncP(
|
||||
this, verb, path, .header = param.header, .query = param.query, .content = param.content, .sseKms = param.sseKms,
|
||||
.sseC = param.sseC, .tag = param.tag);
|
||||
this, verb, path, .header = param.header, .query = param.query, .content = param.content, .contentMd5 = param.contentMd5,
|
||||
.sseKms = param.sseKms, .sseC = param.sseC, .tag = param.tag);
|
||||
HttpResponse *const result = storageS3ResponseP(
|
||||
request, .allowMissing = param.allowMissing, .contentIo = param.contentIo);
|
||||
|
||||
@ -1041,7 +1045,8 @@ storageS3PathRemoveInternal(StorageS3 *const this, HttpRequest *const request, X
|
||||
HttpQuery *const query = httpQueryAdd(httpQueryNewP(), S3_QUERY_DELETE_STR, EMPTY_STR);
|
||||
Buffer *const content = xmlDocumentBuf(xml);
|
||||
|
||||
result = storageS3RequestAsyncP(this, HTTP_VERB_POST_STR, FSLASH_STR, .query = query, .content = content);
|
||||
result = storageS3RequestAsyncP(
|
||||
this, HTTP_VERB_POST_STR, FSLASH_STR, .query = query, .content = content, .contentMd5 = true);
|
||||
|
||||
httpQueryFree(query);
|
||||
bufFree(content);
|
||||
|
@ -22,6 +22,7 @@ typedef struct StorageS3RequestAsyncParam
|
||||
const HttpHeader *header; // Headers
|
||||
const HttpQuery *query; // Query parameters
|
||||
const Buffer *content; // Request content
|
||||
bool contentMd5; // MD5 content checksum required?
|
||||
bool sseKms; // Enable server-side encryption?
|
||||
bool sseC; // Enable server-side encryption with customer-provided keys?
|
||||
bool tag; // Add tags when available?
|
||||
@ -52,6 +53,7 @@ typedef struct StorageS3RequestParam
|
||||
const HttpHeader *header; // Headers
|
||||
const HttpQuery *query; // Query parameters
|
||||
const Buffer *content; // Request content
|
||||
bool contentMd5; // MD5 content checksum required?
|
||||
bool allowMissing; // Allow missing files (caller can check response code)
|
||||
bool contentIo; // Is IoRead interface required to read content?
|
||||
bool sseKms; // Enable server-side encryption?
|
||||
|
@ -34,6 +34,7 @@ typedef struct TestRequestParam
|
||||
const char *token;
|
||||
const char *tag;
|
||||
bool requesterPays;
|
||||
bool contentMd5;
|
||||
} TestRequestParam;
|
||||
|
||||
#define testRequestP(write, s3, verb, path, ...) \
|
||||
@ -68,7 +69,7 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
|
||||
"authorization:AWS4-HMAC-SHA256 Credential=%s/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request,SignedHeaders=",
|
||||
param.accessKey == NULL ? strZ(driver->accessKey) : param.accessKey);
|
||||
|
||||
if (param.content != NULL)
|
||||
if (param.contentMd5)
|
||||
strCatZ(request, "content-md5;");
|
||||
|
||||
strCatZ(request, "host;");
|
||||
@ -105,7 +106,7 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
|
||||
strCatFmt(request, "content-length:%zu\r\n", param.content != NULL ? strlen(param.content) : 0);
|
||||
|
||||
// Add md5
|
||||
if (param.content != NULL)
|
||||
if (param.contentMd5)
|
||||
{
|
||||
strCatFmt(
|
||||
request, "content-md5:%s\r\n", strZ(strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, BUFSTRZ(param.content)))));
|
||||
@ -869,7 +870,6 @@ testRun(void)
|
||||
"*** Request Headers ***:\n"
|
||||
"authorization: <redacted>\n"
|
||||
"content-length: 205\n"
|
||||
"content-md5: 37smUM6Ah2/EjZbp420dPw==\n"
|
||||
"host: bucket.s3.amazonaws.com\n"
|
||||
"x-amz-content-sha256: 0838a79dfbddc2128d28fb4fa8d605e0a8e6d1355094000f39b6eb3feff4641f\n"
|
||||
"x-amz-date: <redacted>\n"
|
||||
@ -1317,7 +1317,7 @@ testRun(void)
|
||||
"</ListBucketResult>");
|
||||
|
||||
testRequestP(
|
||||
service, s3, HTTP_VERB_POST, "/bucket/?delete=",
|
||||
service, s3, HTTP_VERB_POST, "/bucket/?delete=", .contentMd5 = true,
|
||||
.content =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
|
||||
"<Delete><Quiet>true</Quiet>"
|
||||
@ -1378,7 +1378,7 @@ testRun(void)
|
||||
"</ListBucketResult>");
|
||||
|
||||
testRequestP(
|
||||
service, s3, HTTP_VERB_POST, "/bucket/?delete=",
|
||||
service, s3, HTTP_VERB_POST, "/bucket/?delete=", .contentMd5 = true,
|
||||
.content =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
|
||||
"<Delete><Quiet>true</Quiet>"
|
||||
@ -1388,7 +1388,7 @@ testRun(void)
|
||||
testResponseP(service);
|
||||
|
||||
testRequestP(
|
||||
service, s3, HTTP_VERB_POST, "/bucket/?delete=",
|
||||
service, s3, HTTP_VERB_POST, "/bucket/?delete=", .contentMd5 = true,
|
||||
.content =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
|
||||
"<Delete><Quiet>true</Quiet>"
|
||||
@ -1417,7 +1417,7 @@ testRun(void)
|
||||
"</ListBucketResult>");
|
||||
|
||||
testRequestP(
|
||||
service, s3, HTTP_VERB_POST, "/bucket/?delete=",
|
||||
service, s3, HTTP_VERB_POST, "/bucket/?delete=", .contentMd5 = true,
|
||||
.content =
|
||||
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
|
||||
"<Delete><Quiet>true</Quiet>"
|
||||
|
Reference in New Issue
Block a user