From ed699e692d59f3f4857837a7ee0a1f40e0695b0e Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 16 Dec 2015 02:37:24 +0100 Subject: [PATCH] benchmark multiple files --- programs/bench.c | 288 ++++++++++++++++++++++++----------------------- 1 file changed, 148 insertions(+), 140 deletions(-) diff --git a/programs/bench.c b/programs/bench.c index c48112f1d..f55219b4b 100644 --- a/programs/bench.c +++ b/programs/bench.c @@ -72,6 +72,10 @@ # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) #endif +#ifdef _MSC_VER +#define snprintf sprintf_s +#endif + /* ************************************* * Constants @@ -90,9 +94,28 @@ static U32 g_compressibilityDefault = 50; /* ************************************* -* Macros +* console display ***************************************/ -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ + + +/* ************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__); +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} /* ************************************* @@ -156,13 +179,27 @@ static int BMK_GetMilliSpan( int nTimeStart ) return nSpan; } +static U64 BMK_getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + return (U64)statbuf.st_size; +} + /* ******************************************************** * Bench functions **********************************************************/ typedef struct { - char* srcPtr; + const char* srcPtr; size_t srcSize; char* cPtr; size_t cRoom; @@ -175,53 +212,52 @@ typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, s #define MIN(a,b) ((a)<(b) ? (a) : (b)) -static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel) +static int BMK_benchMem(const void* srcBuffer, size_t srcSize, + const char* displayName, int cLevel, + const char** fileNames, U32 nbFiles) { const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ - const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize); - blockParam_t* const blockTable = (blockParam_t*) malloc(nbBlocks * sizeof(blockParam_t)); - const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize); + const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); + const size_t maxCompressedSize = (size_t)maxNbBlocks * ZSTD_compressBound(blockSize); void* const compressedBuffer = malloc(maxCompressedSize); void* const resultBuffer = malloc(srcSize); const compressor_t compressor = ZSTD_compress; - U64 crcOrig; + U64 crcOrig = XXH64(srcBuffer, srcSize, 0); + U32 nbBlocks = 0; /* init */ - if (strlen(fileName)>16) - fileName += strlen(fileName)-16; + if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ /* Memory allocation & restrictions */ if (!compressedBuffer || !resultBuffer || !blockTable) - { - DISPLAY("\nError: not enough memory!\n"); - free(compressedBuffer); - free(resultBuffer); - free(blockTable); - return 12; - } - - /* Calculating input Checksum */ - crcOrig = XXH64(srcBuffer, srcSize, 0); + EXM_THROW(31, "not enough memory"); /* Init blockTable data */ { - U32 i; - size_t remaining = srcSize; - char* srcPtr = (char*)srcBuffer; + U32 fileNb; + const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; char* resPtr = (char*)resultBuffer; - for (i=0; i=2) ? BMK_getFileSize(fileNames[fileNb]) : srcSize; + size_t remaining = (size_t)fileSize; + U32 nbBlocksforThisFile = (U32)((fileSize + (blockSize-1)) / blockSize); + U32 blockEnd = nbBlocks + nbBlocksforThisFile; + for ( ; nbBlocks\r", loopNb, fileName, (U32)srcSize); + DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize); memset(compressedBuffer, 0xE5, maxCompressedSize); nbLoops = 0; @@ -264,7 +300,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i cSize += blockTable[blockNb].cSize; if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops; ratio = (double)srcSize / (double)cSize; - DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.); + DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.); #if 1 /* Decompression */ @@ -283,7 +319,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i milliTime = BMK_GetMilliSpan(milliTime); if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops; - DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); + DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); /* CRC Checking */ crcCheck = XXH64(resultBuffer, srcSize, 0); @@ -291,10 +327,10 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i { unsigned u; unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize)); - DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck); + DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); for (u=0; u%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); + DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.); } - /* End cleaning */ + /* clean up */ free(compressedBuffer); free(resultBuffer); return 0; } -static U64 BMK_GetFileSize(const char* infilename) -{ - int r; -#if defined(_MSC_VER) - struct _stat64 statbuf; - r = _stat64(infilename, &statbuf); -#else - struct stat statbuf; - r = stat(infilename, &statbuf); -#endif - if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ - return (U64)statbuf.st_size; -} - static size_t BMK_findMaxMem(U64 requiredMem) { size_t step = 64 MB; @@ -349,103 +371,99 @@ static size_t BMK_findMaxMem(U64 requiredMem) return (size_t)(requiredMem - step); } -static int BMK_benchOneFile(const char* inFileName, int cLevel) +static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, + const char* displayName, int cLevel, + const char** fileNamesTable, unsigned nbFiles) { - FILE* inFile; - U64 inFileSize; - size_t benchedSize, readSize; - void* srcBuffer; - int result=0; - - /* Check file existence */ - inFile = fopen(inFileName, "rb"); - if (inFile == NULL) - { - DISPLAY("Pb opening %s\n", inFileName); - return 11; - } - - /* Memory allocation & restrictions */ - inFileSize = BMK_GetFileSize(inFileName); - benchedSize = BMK_findMaxMem(inFileSize * 3) / 3; - if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; - if (benchedSize < inFileSize) - DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize >> 20)); - srcBuffer = malloc(benchedSize); - if (!srcBuffer) - { - DISPLAY("\nError: not enough memory!\n"); - fclose(inFile); - return 12; - } - - /* Fill input buffer */ - DISPLAY("Loading %s... \r", inFileName); - readSize = fread(srcBuffer, 1, benchedSize, inFile); - fclose(inFile); - - if (readSize != benchedSize) - { - DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); - free(srcBuffer); - return 13; - } - - /* Bench */ - if (cLevel<0) + if (cLevel < 0) { int l; for (l=1; l <= -cLevel; l++) - result = BMK_benchMem(srcBuffer, benchedSize, inFileName, l); + BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileNamesTable, nbFiles); + return; } - else - result = BMK_benchMem(srcBuffer, benchedSize, inFileName, cLevel); + BMK_benchMem(srcBuffer, benchedSize, displayName, cLevel, fileNamesTable, nbFiles); +} + +static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles) +{ + U64 total = 0; + unsigned n; + for (n=0; n bufferSize-pos) fileSize = bufferSize-pos; + readSize = fread(buff+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + pos += readSize; + fclose(f); + } +} + +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, int cLevel) +{ + void* srcBuffer; + size_t benchedSize; + U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles); + char mfName[20] = {0}; + const char* displayName = NULL; + + /* Memory allocation & restrictions */ + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; + if (benchedSize < totalSizeToLoad) + DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20)); + srcBuffer = malloc(benchedSize); + if (!srcBuffer) EXM_THROW(12, "not enough memory"); + + /* Load input buffer */ + BMK_loadFiles(srcBuffer, benchedSize, fileNamesTable, nbFiles); + + /* Bench */ + snprintf (mfName, sizeof(mfName), " %u files", nbFiles); + if (nbFiles > 1) displayName = mfName; + else displayName = fileNamesTable[0]; + + BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, fileNamesTable, nbFiles); /* clean up */ free(srcBuffer); - DISPLAY("\n"); - return result; } -static int BMK_syntheticTest(int cLevel, double compressibility) +static void BMK_syntheticTest(int cLevel, double compressibility) { + char name[20] = {0}; size_t benchedSize = 10000000; void* srcBuffer = malloc(benchedSize); - int result=0; - char name[20] = {0}; /* Memory allocation */ - if (!srcBuffer) - { - DISPLAY("\nError: not enough memory!\n"); - free(srcBuffer); - return 12; - } + if (!srcBuffer) EXM_THROW(21, "not enough memory"); /* Fill input buffer */ RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); /* Bench */ -#ifdef _MSC_VER - sprintf_s(name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100)); -#else - snprintf (name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100)); -#endif - /* Bench */ - if (cLevel<0) - { - int l; - for (l=1; l <= -cLevel; l++) - result = BMK_benchMem(srcBuffer, benchedSize, name, l); - } - else - result = BMK_benchMem(srcBuffer, benchedSize, name, cLevel); + snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, NULL, 1); - /* End */ + /* clean up */ free(srcBuffer); - DISPLAY("\n"); - return result; } @@ -454,19 +472,9 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLeve double compressibility = (double)g_compressibilityDefault / 100; if (nbFiles == 0) - { BMK_syntheticTest(cLevel, compressibility); - } else - { - /* Loop for each file */ - unsigned fileIdx = 0; - while (fileIdx