From a29e25a845bed5d3b635766dd3ea7ca6a793ac47 Mon Sep 17 00:00:00 2001 From: David Steele Date: Sun, 29 Mar 2020 21:25:48 -0400 Subject: [PATCH] Add storage filter performance test. This test allows the important storage filters to be benchmarked by MiB/s. --- test/define.yaml | 2 +- test/src/module/performance/storageTest.c | 194 +++++++++++++++++++++- 2 files changed, 193 insertions(+), 3 deletions(-) diff --git a/test/define.yaml b/test/define.yaml index b0cbffeae..a4e112c7a 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -736,7 +736,7 @@ performance: # ---------------------------------------------------------------------------------------------------------------------------- - name: storage - total: 1 + total: 2 include: - storage/helper diff --git a/test/src/module/performance/storageTest.c b/test/src/module/performance/storageTest.c index c64313f1c..13022e027 100644 --- a/test/src/module/performance/storageTest.c +++ b/test/src/module/performance/storageTest.c @@ -5,17 +5,24 @@ Test the performance of various storage functions, in particular when implemente Generally speaking, the starting values should be high enough to "blow up" in terms of execution time if there are performance problems without taking very long if everything is running smoothly. These starting values can then be scaled up for profiling and -stress testing as needed. In general we hope to scale to 1000 without running out of memory on the test systems or taking an undue -amount of time. It should be noted that in this context scaling to 1000 is nowhere near to turning it up to 11. +stress testing as needed. ***********************************************************************************************************************************/ #include "common/harnessConfig.h" #include "common/harnessFork.h" +#include "common/crypto/hash.h" +#include "common/compress/gz/compress.h" +#include "common/compress/lz4/compress.h" +#include "common/io/filter/filter.intern.h" +#include "common/io/filter/sink.h" +#include "common/io/bufferWrite.h" #include "common/io/handleRead.h" #include "common/io/handleWrite.h" +#include "common/io/io.h" #include "common/object.h" #include "protocol/client.h" #include "protocol/server.h" +#include "storage/posix/storage.h" #include "storage/remote/protocol.h" #include "storage/storage.intern.h" @@ -129,6 +136,64 @@ storageTestPerfInfoList( return this->fileTotal != 0; } +/*********************************************************************************************************************************** +Test filter to simulate throughput via rate limiting +***********************************************************************************************************************************/ +typedef struct TestIoRate +{ + MemContext *memContext; // Mem context of filter + + uint64_t timeBegin; // Time when filter started processing data in ms + uint64_t byteTotal; // Total bytes processed + uint64_t bytesPerSec; // Rate in bytes per second to enforce +} TestIoRate; + +static void +testIoRateProcess(THIS_VOID, const Buffer *input) +{ + THIS(TestIoRate); + + // Determine the elapsed time since the filter began processing data. The begin time is not set in the constructor because an + // unknown amount of time can elapse between the filter being created and acually used. + uint64_t timeElapsed = 0; + + if (this->timeBegin == 0) + this->timeBegin = timeMSec(); + else + timeElapsed = timeMSec() - this->timeBegin; + + // Add buffer used to the byte total + this->byteTotal += bufUsed(input); + + // Determine how many ms these bytes should take to go through the filter and sleep if greater than elapsed time + uint64_t timeRate = this->byteTotal / this->bytesPerSec * MSEC_PER_SEC; + + if (timeElapsed < timeRate) + sleepMSec(timeRate - timeElapsed); +} + +static IoFilter * +testIoRateNew(uint64_t bytesPerSec) +{ + IoFilter *this = NULL; + + MEM_CONTEXT_NEW_BEGIN("TestIoRate") + { + TestIoRate *driver = memNew(sizeof(TestIoRate)); + + *driver = (TestIoRate) + { + .memContext = memContextCurrent(), + .bytesPerSec = bytesPerSec, + }; + + this = ioFilterNewP(STRDEF("TestIoRate"), driver, NULL, .in = testIoRateProcess); + } + MEM_CONTEXT_NEW_END(); + + return this; +} + /*********************************************************************************************************************************** Test Run ***********************************************************************************************************************************/ @@ -206,5 +271,130 @@ testRun(void) HARNESS_FORK_END(); } + // ***************************************************************************************************************************** + if (testBegin("benchmark filters")) + { + // 4MB buffers are the current default + ioBufferSizeSet(1024 * 1024); + + // 1MB is a fairly normal table size + CHECK(testScale() <= 1024 * 1024 * 1024); + uint64_t blockTotal = (uint64_t)1 * testScale(); + + // Set iteration + unsigned int iteration = 1; + + // Set rate + uint64_t rateIn = 100000; + uint64_t rateOut = 100000; + + // Get the sample pages from disk + Buffer *block = storageGetP( + storageNewReadP( + storagePosixNew(STR(testRepoPath()), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, false, NULL), + STRDEF("test/data/filecopy.table.bin"))); + + ASSERT(bufUsed(block) == 1024 * 1024); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE_FMT( + "%u iteration(s) of %" PRIu64 "MiB with %" PRIu64 "MB/s input, %" PRIu64 "MB/s output", iteration, blockTotal, rateIn, + rateOut); + + #define BENCHMARK_BEGIN() \ + IoWrite *write = ioBufferWriteNew(bufNew(0)); \ + ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateIn * 1000 * 1000)); + + #define BENCHMARK_FILTER_ADD(filter) \ + ioFilterGroupAdd(ioWriteFilterGroup(write), filter); + + #define BENCHMARK_END(addTo) \ + ioFilterGroupAdd(ioWriteFilterGroup(write), testIoRateNew(rateOut * 1000 * 1000)); \ + ioFilterGroupAdd(ioWriteFilterGroup(write), ioSinkNew()); \ + ioWriteOpen(write); \ + \ + uint64_t benchMarkBegin = timeMSec(); \ + \ + for (uint64_t blockIdx = 0; blockIdx < blockTotal; blockIdx++) \ + ioWrite(write, block); \ + \ + ioWriteClose(write); \ + \ + addTo += timeMSec() - benchMarkBegin; + + // Start totals to 1ms just in case something takes 0ms to run + uint64_t copyTotal = 1; + uint64_t sha1Total = 1; + uint64_t sha256Total = 1; + uint64_t gzip6Total = 1; + uint64_t lz41Total = 1; + + for (unsigned int idx = 0; idx < iteration; idx++) + { + // ------------------------------------------------------------------------------------------------------------------------- + TEST_LOG_FMT("copy iteration %u", idx + 1); + + MEM_CONTEXT_TEMP_BEGIN() + { + BENCHMARK_BEGIN(); + BENCHMARK_END(copyTotal); + } + MEM_CONTEXT_TEMP_END(); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_LOG_FMT("sha1 iteration %u", idx + 1); + + MEM_CONTEXT_TEMP_BEGIN() + { + BENCHMARK_BEGIN(); + BENCHMARK_FILTER_ADD(cryptoHashNew(HASH_TYPE_SHA1_STR)); + BENCHMARK_END(sha1Total); + } + MEM_CONTEXT_TEMP_END(); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_LOG_FMT("sha256 iteration %u", idx + 1); + + MEM_CONTEXT_TEMP_BEGIN() + { + BENCHMARK_BEGIN(); + BENCHMARK_FILTER_ADD(cryptoHashNew(HASH_TYPE_SHA256_STR)); + BENCHMARK_END(sha256Total); + } + MEM_CONTEXT_TEMP_END(); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_LOG_FMT("gzip -6 iteration %u", idx + 1); + + MEM_CONTEXT_TEMP_BEGIN() + { + BENCHMARK_BEGIN(); + BENCHMARK_FILTER_ADD(gzCompressNew(6)); + BENCHMARK_END(gzip6Total); + } + MEM_CONTEXT_TEMP_END(); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_LOG_FMT("lz4 -1 iteration %u", idx + 1); + + MEM_CONTEXT_TEMP_BEGIN() + { + BENCHMARK_BEGIN(); + BENCHMARK_FILTER_ADD(lz4CompressNew(1)); + BENCHMARK_END(lz41Total); + } + MEM_CONTEXT_TEMP_END(); + } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("results"); + + TEST_LOG_FMT("copy average: %" PRIu64 "MiB/s", blockTotal * 1000 / copyTotal / iteration); + TEST_LOG_FMT("sha1 average: %" PRIu64 "MiB/s", blockTotal * 1000 / sha1Total / iteration); + TEST_LOG_FMT("sha256 average: %" PRIu64 "MiB/s", blockTotal * 1000 / sha256Total / iteration); + TEST_LOG_FMT("gzip -6 average: %" PRIu64 "MiB/s", blockTotal * 1000 / gzip6Total / iteration); + TEST_LOG_FMT("lz4 -1 average: %" PRIu64 "MiB/s", blockTotal * 1000 / lz41Total / iteration); + } + FUNCTION_HARNESS_RESULT_VOID(); }