1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-09 00:45:49 +02:00

Improve storage filter performance tests.

Improve the accuracy of the calculations in several areas with better integer expressions.

Make the input buffer size configurable. Previously it was always 1mb, i.e. block size.

Use a macro for output results to reduce code duplication.
This commit is contained in:
David Steele
2020-05-19 14:35:20 -04:00
parent a3d9d9a387
commit f773d909be

View File

@ -15,6 +15,7 @@ stress testing as needed.
#include "common/compress/lz4/compress.h"
#include "common/io/filter/filter.intern.h"
#include "common/io/filter/sink.h"
#include "common/io/bufferRead.h"
#include "common/io/bufferWrite.h"
#include "common/io/handleRead.h"
#include "common/io/handleWrite.h"
@ -154,7 +155,7 @@ testIoRateProcess(THIS_VOID, const Buffer *input)
this->byteTotal += bufUsed(input);
// Determine how many ms these bytes should take to go through the filter and sleep if greater than elapsed time
uint64_t timeRate = this->byteTotal / this->bytesPerSec * MSEC_PER_SEC;
uint64_t timeRate = this->byteTotal * MSEC_PER_SEC / this->bytesPerSec;
if (timeElapsed < timeRate)
sleepMSec(timeRate - timeElapsed);
@ -175,7 +176,7 @@ testIoRateNew(uint64_t bytesPerSec)
.bytesPerSec = bytesPerSec,
};
this = ioFilterNewP(STRDEF("TestIoRate"), driver, NULL, .in = testIoRateProcess);
this = ioFilterNewP(strNew("TestIoRate"), driver, NULL, .in = testIoRateProcess);
}
MEM_CONTEXT_NEW_END();
@ -263,7 +264,7 @@ testRun(void)
if (testBegin("benchmark filters"))
{
// 4MB buffers are the current default
ioBufferSizeSet(1024 * 1024);
ioBufferSizeSet(4 * 1024 * 1024);
// 1MB is a fairly normal table size
CHECK(testScale() <= 1024 * 1024 * 1024);
@ -273,35 +274,56 @@ testRun(void)
unsigned int iteration = 1;
// Set rate
uint64_t rateIn = 100000;
uint64_t rateOut = 100000;
uint64_t rateIn = 0; // MB/s (0 disables)
uint64_t rateOut = 0; // MB/s (0 disables)
// Get the sample pages from disk
Buffer *block = storageGetP(storageNewReadP(storagePosixNewP(STR(testRepoPath())), STRDEF("test/data/filecopy.table.bin")));
ASSERT(bufUsed(block) == 1024 * 1024);
// Build the input buffer
Buffer *input = bufNew(blockTotal * bufSize(block));
for (unsigned int blockIdx = 0; blockIdx < blockTotal; blockIdx++)
memcpy(bufPtr(input) + (blockIdx * bufSize(block)), bufPtr(block), bufSize(block));
bufUsedSet(input, bufSize(input));
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE_FMT(
"%u iteration(s) of %" PRIu64 "MiB with %" PRIu64 "MB/s input, %" PRIu64 "MB/s output", iteration, blockTotal, rateIn,
rateOut);
"%u iteration(s) of %" PRIu64 "MiB with %" PRIu64 "MB/s input, %" PRIu64 "MB/s output", iteration,
bufUsed(input) / bufUsed(block), rateIn, rateOut);
#define BENCHMARK_BEGIN() \
IoWrite *write = ioBufferWriteNew(bufNew(0)); \
ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateIn * 1000 * 1000));
#define BENCHMARK_FILTER_ADD(filter) \
ioFilterGroupAdd(ioWriteFilterGroup(write), filter);
#define BENCHMARK_END(addTo) \
ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateOut * 1000 * 1000)); \
if (rateOut != 0) \
ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateOut * 1000 * 1000)); \
ioFilterGroupAdd(ioWriteFilterGroup(write), ioSinkNew()); \
ioWriteOpen(write); \
\
IoRead *read = ioBufferReadNew(input); \
if (rateIn != 0) \
ioFilterGroupAdd(ioReadFilterGroup(read), testIoRateNew(rateIn * 1000 * 1000)); \
ioReadOpen(read); \
\
uint64_t benchMarkBegin = timeMSec(); \
\
for (uint64_t blockIdx = 0; blockIdx < blockTotal; blockIdx++) \
ioWrite(write, block); \
Buffer *buffer = bufNew(ioBufferSize()); \
\
do \
{ \
ioRead(read, buffer); \
ioWrite(write, buffer); \
bufUsedZero(buffer); \
} \
while (!ioReadEof(read)); \
\
ioReadClose(read); \
ioWriteClose(write); \
\
addTo += timeMSec() - benchMarkBegin;
@ -385,12 +407,17 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("results");
TEST_LOG_FMT("copy average: %" PRIu64 "MiB/s", blockTotal * 1000 / copyTotal / iteration);
TEST_LOG_FMT("md5 average: %" PRIu64 "MiB/s", blockTotal * 1000 / md5Total / iteration);
TEST_LOG_FMT("sha1 average: %" PRIu64 "MiB/s", blockTotal * 1000 / sha1Total / iteration);
TEST_LOG_FMT("sha256 average: %" PRIu64 "MiB/s", blockTotal * 1000 / sha256Total / iteration);
TEST_LOG_FMT("gzip -6 average: %" PRIu64 "MiB/s", blockTotal * 1000 / gzip6Total / iteration);
TEST_LOG_FMT("lz4 -1 average: %" PRIu64 "MiB/s", blockTotal * 1000 / lz41Total / iteration);
#define TEST_RESULT(name, total) \
TEST_LOG_FMT( \
"%s time %" PRIu64"ms, avg time %" PRIu64"ms, avg throughput: %" PRIu64 "MB/s", name, total, total / iteration, \
iteration * blockTotal * 1024 * 1024 * 1000 / total / 1000000);
TEST_RESULT("copy", copyTotal);
TEST_RESULT("md5", md5Total);
TEST_RESULT("sha1", sha1Total);
TEST_RESULT("sha256", sha256Total);
TEST_RESULT("gzip -6", gzip6Total);
TEST_RESULT("lz4 -1", lz41Total);
}
FUNCTION_HARNESS_RESULT_VOID();