db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-07 00:35:37 +02:00
Code Issues Releases Activity

Calculate content-md5 on S3 only when required.

Browse Source 
The content-md5 header was generated for all requests with content but it is only required for batch delete requests. It is not clear why this header is required when x-amz-content-sha256 is also provided or why it
is required only for this request but the documentation is clear on the matter. However, the content for these requests is relatively small compared to uploading files so omitting content-md5 where possible will
save some CPU cycles.

Current AWS S3 and recent Minio don't complain if this header is missing but since it is still required by older versions of Minio and it is specified in the documentation for batch delete it is makes sense to
keep it.
This commit is contained in:
David Steele
2025-04-09 12:27:27 -05:00
committed by GitHub
parent c925832e17
commit 20bfd14b73
4 changed files with 30 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
doc/xml/release/2025/2.55.0.xml
View File

@ -131,6 +131,17 @@
<p>Use <proper>lz4</proper> for protocol compression.</p>
</release-item>
<release-item>
<github-pull-request id="2592"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="david.christensen"/>
</release-item-contributor-list>
<p>Calculate <id>content-md5</id> on <proper>S3</proper> only when required.</p>
</release-item>
<release-item>
<commit subject="Add detail logging to expire test."/>
<commit subject="Add detail logging for expired archive path.">

15
src/storage/s3/storage.c
View File

@ -452,6 +452,7 @@ storageS3RequestAsync(StorageS3 *const this, const String *const verb, const Str
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
FUNCTION_LOG_PARAM(BUFFER, param.content);
FUNCTION_LOG_PARAM(BOOL, param.contentMd5);
FUNCTION_LOG_PARAM(BOOL, param.sseKms);
FUNCTION_LOG_PARAM(BOOL, param.sseC);
FUNCTION_LOG_PARAM(BOOL, param.tag);
@ -473,9 +474,11 @@ storageS3RequestAsync(StorageS3 *const this, const String *const verb, const Str
requestHeader, HTTP_HEADER_CONTENT_LENGTH_STR,
param.content == NULL || bufEmpty(param.content) ? ZERO_STR : strNewFmt("%zu", bufUsed(param.content)));
// Calculate content-md5 header if there is content
if (param.content != NULL)
// Calculate content-md5 header when required
if (param.contentMd5)
{
ASSERT(param.content != NULL && !bufEmpty(param.content));
httpHeaderAdd(
requestHeader, HTTP_HEADER_CONTENT_MD5_STR,
strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, param.content)));
@ -607,6 +610,7 @@ storageS3Request(StorageS3 *const this, const String *const verb, const String *
FUNCTION_LOG_PARAM(HTTP_HEADER, param.header);
FUNCTION_LOG_PARAM(HTTP_QUERY, param.query);
FUNCTION_LOG_PARAM(BUFFER, param.content);
FUNCTION_LOG_PARAM(BOOL, param.contentMd5);
FUNCTION_LOG_PARAM(BOOL, param.allowMissing);
FUNCTION_LOG_PARAM(BOOL, param.contentIo);
FUNCTION_LOG_PARAM(BOOL, param.sseKms);
@ -615,8 +619,8 @@ storageS3Request(StorageS3 *const this, const String *const verb, const String *
FUNCTION_LOG_END();
HttpRequest *const request = storageS3RequestAsyncP(
this, verb, path, .header = param.header, .query = param.query, .content = param.content, .sseKms = param.sseKms,
.sseC = param.sseC, .tag = param.tag);
this, verb, path, .header = param.header, .query = param.query, .content = param.content, .contentMd5 = param.contentMd5,
.sseKms = param.sseKms, .sseC = param.sseC, .tag = param.tag);
HttpResponse *const result = storageS3ResponseP(
request, .allowMissing = param.allowMissing, .contentIo = param.contentIo);
@ -1041,7 +1045,8 @@ storageS3PathRemoveInternal(StorageS3 *const this, HttpRequest *const request, X
HttpQuery *const query = httpQueryAdd(httpQueryNewP(), S3_QUERY_DELETE_STR, EMPTY_STR);
Buffer *const content = xmlDocumentBuf(xml);
result = storageS3RequestAsyncP(this, HTTP_VERB_POST_STR, FSLASH_STR, .query = query, .content = content);
result = storageS3RequestAsyncP(
this, HTTP_VERB_POST_STR, FSLASH_STR, .query = query, .content = content, .contentMd5 = true);
httpQueryFree(query);
bufFree(content);

2
src/storage/s3/storage.intern.h
View File

@ -22,6 +22,7 @@ typedef struct StorageS3RequestAsyncParam
const HttpHeader *header; // Headers
const HttpQuery *query; // Query parameters
const Buffer *content; // Request content
bool contentMd5; // MD5 content checksum required?
bool sseKms; // Enable server-side encryption?
bool sseC; // Enable server-side encryption with customer-provided keys?
bool tag; // Add tags when available?
@ -52,6 +53,7 @@ typedef struct StorageS3RequestParam
const HttpHeader *header; // Headers
const HttpQuery *query; // Query parameters
const Buffer *content; // Request content
bool contentMd5; // MD5 content checksum required?
bool allowMissing; // Allow missing files (caller can check response code)
bool contentIo; // Is IoRead interface required to read content?
bool sseKms; // Enable server-side encryption?

14
test/src/module/storage/s3Test.c
View File

@ -34,6 +34,7 @@ typedef struct TestRequestParam
const char *token;
const char *tag;
bool requesterPays;
bool contentMd5;
} TestRequestParam;
#define testRequestP(write, s3, verb, path, ...) \
@ -68,7 +69,7 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
"authorization:AWS4-HMAC-SHA256 Credential=%s/\?\?\?\?\?\?\?\?/us-east-1/s3/aws4_request,SignedHeaders=",
param.accessKey == NULL ? strZ(driver->accessKey) : param.accessKey);
if (param.content != NULL)
if (param.contentMd5)
strCatZ(request, "content-md5;");
strCatZ(request, "host;");
@ -105,7 +106,7 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
strCatFmt(request, "content-length:%zu\r\n", param.content != NULL ? strlen(param.content) : 0);
// Add md5
if (param.content != NULL)
if (param.contentMd5)
{
strCatFmt(
request, "content-md5:%s\r\n", strZ(strNewEncode(encodingBase64, cryptoHashOne(hashTypeMd5, BUFSTRZ(param.content)))));
@ -869,7 +870,6 @@ testRun(void)
"*** Request Headers ***:\n"
"authorization: <redacted>\n"
"content-length: 205\n"
"content-md5: 37smUM6Ah2/EjZbp420dPw==\n"
"host: bucket.s3.amazonaws.com\n"
"x-amz-content-sha256: 0838a79dfbddc2128d28fb4fa8d605e0a8e6d1355094000f39b6eb3feff4641f\n"
"x-amz-date: <redacted>\n"
@ -1317,7 +1317,7 @@ testRun(void)
"</ListBucketResult>");
testRequestP(
service, s3, HTTP_VERB_POST, "/bucket/?delete=",
service, s3, HTTP_VERB_POST, "/bucket/?delete=", .contentMd5 = true,
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
@ -1378,7 +1378,7 @@ testRun(void)
"</ListBucketResult>");
testRequestP(
service, s3, HTTP_VERB_POST, "/bucket/?delete=",
service, s3, HTTP_VERB_POST, "/bucket/?delete=", .contentMd5 = true,
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
@ -1388,7 +1388,7 @@ testRun(void)
testResponseP(service);
testRequestP(
service, s3, HTTP_VERB_POST, "/bucket/?delete=",
service, s3, HTTP_VERB_POST, "/bucket/?delete=", .contentMd5 = true,
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"
@ -1417,7 +1417,7 @@ testRun(void)
"</ListBucketResult>");
testRequestP(
service, s3, HTTP_VERB_POST, "/bucket/?delete=",
service, s3, HTTP_VERB_POST, "/bucket/?delete=", .contentMd5 = true,
.content =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
"<Delete><Quiet>true</Quiet>"