db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-17 01:12:23 +02:00
Code Issues Releases Activity

Improve storage filter performance tests.

Browse Source 
Improve the accuracy of the calculations in several areas with better integer expressions.

Make the input buffer size configurable. Previously it was always 1mb, i.e. block size.

Use a macro for output results to reduce code duplication.
This commit is contained in:
David Steele
2020-05-19 14:35:20 -04:00
parent a3d9d9a387
commit f773d909be
1 changed files with 44 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
test/src/module/performance/storageTest.c
View File

@ -15,6 +15,7 @@ stress testing as needed.
#include "common/compress/lz4/compress.h"
#include "common/io/filter/filter.intern.h"
#include "common/io/filter/sink.h"
#include "common/io/bufferRead.h"
#include "common/io/bufferWrite.h"
#include "common/io/handleRead.h"
#include "common/io/handleWrite.h"
@ -154,7 +155,7 @@ testIoRateProcess(THIS_VOID, const Buffer *input)
this->byteTotal += bufUsed(input);
// Determine how many ms these bytes should take to go through the filter and sleep if greater than elapsed time
uint64_t timeRate = this->byteTotal / this->bytesPerSec * MSEC_PER_SEC;
uint64_t timeRate = this->byteTotal * MSEC_PER_SEC / this->bytesPerSec;
if (timeElapsed < timeRate)
sleepMSec(timeRate - timeElapsed);
@ -175,7 +176,7 @@ testIoRateNew(uint64_t bytesPerSec)
.bytesPerSec = bytesPerSec,
};
this = ioFilterNewP(STRDEF("TestIoRate"), driver, NULL, .in = testIoRateProcess);
this = ioFilterNewP(strNew("TestIoRate"), driver, NULL, .in = testIoRateProcess);
}
MEM_CONTEXT_NEW_END();
@ -263,7 +264,7 @@ testRun(void)
if (testBegin("benchmark filters"))
{
// 4MB buffers are the current default
ioBufferSizeSet(1024 * 1024);
ioBufferSizeSet(4 * 1024 * 1024);
// 1MB is a fairly normal table size
CHECK(testScale() <= 1024 * 1024 * 1024);
@ -273,35 +274,56 @@ testRun(void)
unsigned int iteration = 1;
// Set rate
uint64_t rateIn = 100000;
uint64_t rateOut = 100000;
uint64_t rateIn = 0; // MB/s (0 disables)
uint64_t rateOut = 0; // MB/s (0 disables)
// Get the sample pages from disk
Buffer *block = storageGetP(storageNewReadP(storagePosixNewP(STR(testRepoPath())), STRDEF("test/data/filecopy.table.bin")));
ASSERT(bufUsed(block) == 1024 * 1024);
// Build the input buffer
Buffer *input = bufNew(blockTotal * bufSize(block));
for (unsigned int blockIdx = 0; blockIdx < blockTotal; blockIdx++)
memcpy(bufPtr(input) + (blockIdx * bufSize(block)), bufPtr(block), bufSize(block));
bufUsedSet(input, bufSize(input));
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE_FMT(
"%u iteration(s) of %" PRIu64 "MiB with %" PRIu64 "MB/s input, %" PRIu64 "MB/s output", iteration, blockTotal, rateIn,
rateOut);
"%u iteration(s) of %" PRIu64 "MiB with %" PRIu64 "MB/s input, %" PRIu64 "MB/s output", iteration,
bufUsed(input) / bufUsed(block), rateIn, rateOut);
#define BENCHMARK_BEGIN() \
IoWrite *write = ioBufferWriteNew(bufNew(0)); \
ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateIn * 1000 * 1000));
#define BENCHMARK_FILTER_ADD(filter) \
ioFilterGroupAdd(ioWriteFilterGroup(write), filter);
#define BENCHMARK_END(addTo) \
ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateOut * 1000 * 1000)); \
if (rateOut != 0) \
ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateOut * 1000 * 1000)); \
ioFilterGroupAdd(ioWriteFilterGroup(write), ioSinkNew()); \
ioWriteOpen(write); \
\
IoRead *read = ioBufferReadNew(input); \
if (rateIn != 0) \
ioFilterGroupAdd(ioReadFilterGroup(read), testIoRateNew(rateIn * 1000 * 1000)); \
ioReadOpen(read); \
\
uint64_t benchMarkBegin = timeMSec(); \
\
for (uint64_t blockIdx = 0; blockIdx < blockTotal; blockIdx++) \
ioWrite(write, block); \
Buffer *buffer = bufNew(ioBufferSize()); \
\
do \
{ \
ioRead(read, buffer); \
ioWrite(write, buffer); \
bufUsedZero(buffer); \
} \
while (!ioReadEof(read)); \
\
ioReadClose(read); \
ioWriteClose(write); \
\
addTo += timeMSec() - benchMarkBegin;
@ -385,12 +407,17 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("results");
TEST_LOG_FMT("copy average: %" PRIu64 "MiB/s", blockTotal * 1000 / copyTotal / iteration);
TEST_LOG_FMT("md5 average: %" PRIu64 "MiB/s", blockTotal * 1000 / md5Total / iteration);
TEST_LOG_FMT("sha1 average: %" PRIu64 "MiB/s", blockTotal * 1000 / sha1Total / iteration);
TEST_LOG_FMT("sha256 average: %" PRIu64 "MiB/s", blockTotal * 1000 / sha256Total / iteration);
TEST_LOG_FMT("gzip -6 average: %" PRIu64 "MiB/s", blockTotal * 1000 / gzip6Total / iteration);
TEST_LOG_FMT("lz4 -1 average: %" PRIu64 "MiB/s", blockTotal * 1000 / lz41Total / iteration);
#define TEST_RESULT(name, total) \
TEST_LOG_FMT( \
"%s time %" PRIu64"ms, avg time %" PRIu64"ms, avg throughput: %" PRIu64 "MB/s", name, total, total / iteration, \
iteration * blockTotal * 1024 * 1024 * 1000 / total / 1000000);
TEST_RESULT("copy", copyTotal);
TEST_RESULT("md5", md5Total);
TEST_RESULT("sha1", sha1Total);
TEST_RESULT("sha256", sha256Total);
TEST_RESULT("gzip -6", gzip6Total);
TEST_RESULT("lz4 -1", lz41Total);
}
FUNCTION_HARNESS_RESULT_VOID();