You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-09 00:45:49 +02:00
Improve storage filter performance tests.
Improve the accuracy of the calculations in several areas with better integer expressions. Make the input buffer size configurable. Previously it was always 1mb, i.e. block size. Use a macro for output results to reduce code duplication.
This commit is contained in:
@ -15,6 +15,7 @@ stress testing as needed.
|
||||
#include "common/compress/lz4/compress.h"
|
||||
#include "common/io/filter/filter.intern.h"
|
||||
#include "common/io/filter/sink.h"
|
||||
#include "common/io/bufferRead.h"
|
||||
#include "common/io/bufferWrite.h"
|
||||
#include "common/io/handleRead.h"
|
||||
#include "common/io/handleWrite.h"
|
||||
@ -154,7 +155,7 @@ testIoRateProcess(THIS_VOID, const Buffer *input)
|
||||
this->byteTotal += bufUsed(input);
|
||||
|
||||
// Determine how many ms these bytes should take to go through the filter and sleep if greater than elapsed time
|
||||
uint64_t timeRate = this->byteTotal / this->bytesPerSec * MSEC_PER_SEC;
|
||||
uint64_t timeRate = this->byteTotal * MSEC_PER_SEC / this->bytesPerSec;
|
||||
|
||||
if (timeElapsed < timeRate)
|
||||
sleepMSec(timeRate - timeElapsed);
|
||||
@ -175,7 +176,7 @@ testIoRateNew(uint64_t bytesPerSec)
|
||||
.bytesPerSec = bytesPerSec,
|
||||
};
|
||||
|
||||
this = ioFilterNewP(STRDEF("TestIoRate"), driver, NULL, .in = testIoRateProcess);
|
||||
this = ioFilterNewP(strNew("TestIoRate"), driver, NULL, .in = testIoRateProcess);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
@ -263,7 +264,7 @@ testRun(void)
|
||||
if (testBegin("benchmark filters"))
|
||||
{
|
||||
// 4MB buffers are the current default
|
||||
ioBufferSizeSet(1024 * 1024);
|
||||
ioBufferSizeSet(4 * 1024 * 1024);
|
||||
|
||||
// 1MB is a fairly normal table size
|
||||
CHECK(testScale() <= 1024 * 1024 * 1024);
|
||||
@ -273,35 +274,56 @@ testRun(void)
|
||||
unsigned int iteration = 1;
|
||||
|
||||
// Set rate
|
||||
uint64_t rateIn = 100000;
|
||||
uint64_t rateOut = 100000;
|
||||
uint64_t rateIn = 0; // MB/s (0 disables)
|
||||
uint64_t rateOut = 0; // MB/s (0 disables)
|
||||
|
||||
// Get the sample pages from disk
|
||||
Buffer *block = storageGetP(storageNewReadP(storagePosixNewP(STR(testRepoPath())), STRDEF("test/data/filecopy.table.bin")));
|
||||
ASSERT(bufUsed(block) == 1024 * 1024);
|
||||
|
||||
// Build the input buffer
|
||||
Buffer *input = bufNew(blockTotal * bufSize(block));
|
||||
|
||||
for (unsigned int blockIdx = 0; blockIdx < blockTotal; blockIdx++)
|
||||
memcpy(bufPtr(input) + (blockIdx * bufSize(block)), bufPtr(block), bufSize(block));
|
||||
|
||||
bufUsedSet(input, bufSize(input));
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE_FMT(
|
||||
"%u iteration(s) of %" PRIu64 "MiB with %" PRIu64 "MB/s input, %" PRIu64 "MB/s output", iteration, blockTotal, rateIn,
|
||||
rateOut);
|
||||
"%u iteration(s) of %" PRIu64 "MiB with %" PRIu64 "MB/s input, %" PRIu64 "MB/s output", iteration,
|
||||
bufUsed(input) / bufUsed(block), rateIn, rateOut);
|
||||
|
||||
#define BENCHMARK_BEGIN() \
|
||||
IoWrite *write = ioBufferWriteNew(bufNew(0)); \
|
||||
ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateIn * 1000 * 1000));
|
||||
|
||||
#define BENCHMARK_FILTER_ADD(filter) \
|
||||
ioFilterGroupAdd(ioWriteFilterGroup(write), filter);
|
||||
|
||||
#define BENCHMARK_END(addTo) \
|
||||
ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateOut * 1000 * 1000)); \
|
||||
if (rateOut != 0) \
|
||||
ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateOut * 1000 * 1000)); \
|
||||
ioFilterGroupAdd(ioWriteFilterGroup(write), ioSinkNew()); \
|
||||
ioWriteOpen(write); \
|
||||
\
|
||||
IoRead *read = ioBufferReadNew(input); \
|
||||
if (rateIn != 0) \
|
||||
ioFilterGroupAdd(ioReadFilterGroup(read), testIoRateNew(rateIn * 1000 * 1000)); \
|
||||
ioReadOpen(read); \
|
||||
\
|
||||
uint64_t benchMarkBegin = timeMSec(); \
|
||||
\
|
||||
for (uint64_t blockIdx = 0; blockIdx < blockTotal; blockIdx++) \
|
||||
ioWrite(write, block); \
|
||||
Buffer *buffer = bufNew(ioBufferSize()); \
|
||||
\
|
||||
do \
|
||||
{ \
|
||||
ioRead(read, buffer); \
|
||||
ioWrite(write, buffer); \
|
||||
bufUsedZero(buffer); \
|
||||
} \
|
||||
while (!ioReadEof(read)); \
|
||||
\
|
||||
ioReadClose(read); \
|
||||
ioWriteClose(write); \
|
||||
\
|
||||
addTo += timeMSec() - benchMarkBegin;
|
||||
@ -385,12 +407,17 @@ testRun(void)
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("results");
|
||||
|
||||
TEST_LOG_FMT("copy average: %" PRIu64 "MiB/s", blockTotal * 1000 / copyTotal / iteration);
|
||||
TEST_LOG_FMT("md5 average: %" PRIu64 "MiB/s", blockTotal * 1000 / md5Total / iteration);
|
||||
TEST_LOG_FMT("sha1 average: %" PRIu64 "MiB/s", blockTotal * 1000 / sha1Total / iteration);
|
||||
TEST_LOG_FMT("sha256 average: %" PRIu64 "MiB/s", blockTotal * 1000 / sha256Total / iteration);
|
||||
TEST_LOG_FMT("gzip -6 average: %" PRIu64 "MiB/s", blockTotal * 1000 / gzip6Total / iteration);
|
||||
TEST_LOG_FMT("lz4 -1 average: %" PRIu64 "MiB/s", blockTotal * 1000 / lz41Total / iteration);
|
||||
#define TEST_RESULT(name, total) \
|
||||
TEST_LOG_FMT( \
|
||||
"%s time %" PRIu64"ms, avg time %" PRIu64"ms, avg throughput: %" PRIu64 "MB/s", name, total, total / iteration, \
|
||||
iteration * blockTotal * 1024 * 1024 * 1000 / total / 1000000);
|
||||
|
||||
TEST_RESULT("copy", copyTotal);
|
||||
TEST_RESULT("md5", md5Total);
|
||||
TEST_RESULT("sha1", sha1Total);
|
||||
TEST_RESULT("sha256", sha256Total);
|
||||
TEST_RESULT("gzip -6", gzip6Total);
|
||||
TEST_RESULT("lz4 -1", lz41Total);
|
||||
}
|
||||
|
||||
FUNCTION_HARNESS_RESULT_VOID();
|
||||
|
Reference in New Issue
Block a user