db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Reduce buffer-size default to 1MiB.

Browse Source 
The prior default was determined by benchmarking the Perl code prior to the 1.0 release. In general buffer allocation was more expensive in Perl so large buffers gave the best performance. This was due to multiple buffer allocations for each filter in an IO operation.

The C code allocates fixed buffers for each IO operation so the cost for buffer allocation is lower than Perl. That being the case it made sense to benchmark the C code to determine the optimal buffer default.

The performance/storage tests were used to measure the performance of a variety of filters. 1GiB of data was processed by each filter 10 times and the results of the tests were averaged.

While most buffer sizes gave similar performance, 1MiB appeared to perform the best overall. Of course, different architectures are likely to yield different results but this seems like a sensible default. The buffer-size option may still need to be manually configured to give optimal results.

Raw test data for reference:

4MB buffer (prior default)

copy time 1807ms, avg time 180ms, avg throughput: 5942MB/s
md5 time 14200ms, avg time 1420ms, avg throughput: 756MB/s
sha1 time 11431ms, avg time 1143ms, avg throughput: 939MB/s
sha256 time 23463ms, avg time 2346ms, avg throughput: 457MB/s
gzip -6 time 381199ms, avg time 38119ms, avg throughput: 28MB/s
lz4 -1 time 15484ms, avg time 1548ms, avg throughput: 693MB/s

1MB buffer (new default)

copy time 1760ms, avg time 176ms, avg throughput: 6100MB/s
md5 time 13739ms, avg time 1373ms, avg throughput: 781MB/s
sha1 time 11025ms, avg time 1102ms, avg throughput: 973MB/s
sha256 time 22539ms, avg time 2253ms, avg throughput: 476MB/s
gzip -6 time 372995ms, avg time 37299ms, avg throughput: 28MB/s
lz4 -1 time 15118ms, avg time 1511ms, avg throughput: 710MB/s

512K buffer

copy time 1782ms, avg time 178ms, avg throughput: 6025MB/s
md5 time 13724ms, avg time 1372ms, avg throughput: 782MB/s
sha1 time 10959ms, avg time 1095ms, avg throughput: 979MB/s
sha256 time 22982ms, avg time 2298ms, avg throughput: 467MB/s
gzip -6 time 378120ms, avg time 37812ms, avg throughput: 28MB/s
lz4 -1 time 15484ms, avg time 1548ms, avg throughput: 693MB/s

256K buffer

copy time 1805ms, avg time 180ms, avg throughput: 5948MB/s
md5 time 13706ms, avg time 1370ms, avg throughput: 783MB/s
sha1 time 11074ms, avg time 1107ms, avg throughput: 969MB/s
sha256 time 22588ms, avg time 2258ms, avg throughput: 475MB/s
gzip -6 time 372645ms, avg time 37264ms, avg throughput: 28MB/s
lz4 -1 time 16346ms, avg time 1634ms, avg throughput: 656MB/s
This commit is contained in:
David Steele 2020-05-19 16:58:49 -04:00
parent f773d909be
commit ea9147e2e0
8 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build/lib/pgBackRestBuild/Config/Data.pm
View File

@ -1093,7 +1093,7 @@ my %hConfigDefine =
{
&CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL,
&CFGDEF_TYPE => CFGDEF_TYPE_SIZE,
&CFGDEF_DEFAULT => 4194304,
&CFGDEF_DEFAULT => 1048576,
&CFGDEF_ALLOW_LIST =>
[
&CFGDEF_DEFAULT_BUFFER_SIZE_MIN,

8
doc/xml/release.xml
View File

@ -122,6 +122,14 @@
<p>WAL prior to the first full backup was previously expired after the first full backup. Now it is preserved according to retention settings.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="stephen.frost"/>
</release-item-contributor-list>
<p>Reduce <br-option>buffer-size</br-option> default to <id>1MiB</id>.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-contributor id="cynthia.shang"/>

2
src/config/define.auto.c
View File

@ -607,7 +607,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
"16777216"
)
CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("4194304")
CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("1048576")
)
)

6
test/src/module/command/helpTest.c
View File

@ -147,7 +147,7 @@ testRun(void)
"General Options:\n"
"\n"
" --buffer-size buffer size for file operations\n"
" [current=32768, default=4194304]\n"
" [current=32768, default=1048576]\n"
" --cmd-ssh path to ssh client executable [default=ssh]\n"
" --compress-level-network network compression level [default=3]\n"
" --config pgBackRest configuration file\n"
@ -281,12 +281,12 @@ testRun(void)
strLstAddZ(argList, "buffer-size");
TEST_RESULT_VOID(
configParse(strLstSize(argList), strLstPtr(argList), false), "help for archive-push command, buffer-size option");
TEST_RESULT_STR(helpRender(), strNewFmt("%s\ndefault: 4194304\n", optionHelp), " check text");
TEST_RESULT_STR(helpRender(), strNewFmt("%s\ndefault: 1048576\n", optionHelp), " check text");
strLstAddZ(argList, "--buffer-size=32768");
TEST_RESULT_VOID(
configParse(strLstSize(argList), strLstPtr(argList), false), "help for archive-push command, buffer-size option");
TEST_RESULT_STR(helpRender(), strNewFmt("%s\ncurrent: 32768\ndefault: 4194304\n", optionHelp), " check text");
TEST_RESULT_STR(helpRender(), strNewFmt("%s\ncurrent: 32768\ndefault: 1048576\n", optionHelp), " check text");
// -------------------------------------------------------------------------------------------------------------------------
optionHelp = strPtr(strNewFmt(

2
test/src/module/config/loadTest.c
View File

@ -445,7 +445,7 @@ testRun(void)
strLstAdd(argList, strNew("--repo1-retention-full=2"));
TEST_RESULT_VOID(cfgLoad(strLstSize(argList), strLstPtr(argList)), "help command for backup");
TEST_RESULT_UINT(ioBufferSize(), 4 * 1024 * 1024, "buffer size set to option default");
TEST_RESULT_UINT(ioBufferSize(), 1048576, "buffer size set to option default");
// Command takes lock and opens log file and uses custom tcp settings
// -------------------------------------------------------------------------------------------------------------------------

2
test/src/module/config/parseTest.c
View File

@ -1171,7 +1171,7 @@ testRun(void)
TEST_RESULT_INT(cfgOptionSource(cfgOptRepoS3KeySecret), cfgSourceConfig, " repo1-s3-secret is source env");
TEST_RESULT_BOOL(cfgOptionBool(cfgOptOnline), false, " online is not set");
TEST_RESULT_INT(cfgOptionSource(cfgOptOnline), cfgSourceParam, " online is source default");
TEST_RESULT_INT(cfgOptionInt(cfgOptBufferSize), 4194304, " buffer-size is set");
TEST_RESULT_INT(cfgOptionInt(cfgOptBufferSize), 1048576, " buffer-size is set");
TEST_RESULT_INT(cfgOptionSource(cfgOptBufferSize), cfgSourceDefault, " buffer-size is source default");
unsetenv("PGBACKREST_REPO1_S3_KEY");

2
test/src/module/info/infoBackupTest.c
View File

@ -465,7 +465,7 @@ testRun(void)
"option-archive-check=true\n"
"option-archive-copy=false\n"
"option-backup-standby=false\n"
"option-buffer-size=4194304\n"
"option-buffer-size=1048576\n"
"option-checksum-page=false\n"
"option-compress=true\n"
"option-compress-level=6\n"

2
test/src/module/performance/typeTest.c
View File

@ -145,7 +145,7 @@ testRun(void)
"option-archive-check=true\n"
"option-archive-copy=false\n"
"option-backup-standby=false\n"
"option-buffer-size=4194304\n"
"option-buffer-size=1048576\n"
"option-checksum-page=true\n"
"option-compress=true\n"
"option-compress-level=9\n"