krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-06 16:56:49 +02:00
Code Issues Releases 1 Activity

benchmark multiple files

Browse Source
This commit is contained in:
Yann Collet 2015-12-16 02:37:24 +01:00
parent 7a3ab588c7
commit ed699e692d
1 changed files with 148 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

288
programs/bench.c
View File

@ -72,6 +72,10 @@
# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
#endif
#ifdef _MSC_VER
#define snprintf sprintf_s
#endif
/* *************************************
* Constants
@ -90,9 +94,28 @@ static U32 g_compressibilityDefault = 50;
/* *************************************
* Macros
* console display
***************************************/
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
static U32 g_displayLevel = 2; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */
/* *************************************
* Exceptions
***************************************/
#ifndef DEBUG
# define DEBUG 0
#endif
#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
#define EXM_THROW(error, ...) \
{ \
DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
DISPLAYLEVEL(1, "Error %i : ", error); \
DISPLAYLEVEL(1, __VA_ARGS__); \
DISPLAYLEVEL(1, "\n"); \
exit(error); \
}
/* *************************************
@ -156,13 +179,27 @@ static int BMK_GetMilliSpan( int nTimeStart )
return nSpan;
}
static U64 BMK_getFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
return (U64)statbuf.st_size;
}
/* ********************************************************
* Bench functions
**********************************************************/
typedef struct
{
char* srcPtr;
const char* srcPtr;
size_t srcSize;
char* cPtr;
size_t cRoom;
@ -175,53 +212,52 @@ typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, s
#define MIN(a,b) ((a)<(b) ? (a) : (b))
static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
const char* displayName, int cLevel,
const char** fileNames, U32 nbFiles)
{
const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize); /* avoid div by 0 */
const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
blockParam_t* const blockTable = (blockParam_t*) malloc(nbBlocks * sizeof(blockParam_t));
const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
const size_t maxCompressedSize = (size_t)maxNbBlocks * ZSTD_compressBound(blockSize);
void* const compressedBuffer = malloc(maxCompressedSize);
void* const resultBuffer = malloc(srcSize);
const compressor_t compressor = ZSTD_compress;
U64 crcOrig;
U64 crcOrig = XXH64(srcBuffer, srcSize, 0);
U32 nbBlocks = 0;
/* init */
if (strlen(fileName)>16)
fileName += strlen(fileName)-16;
if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */
/* Memory allocation & restrictions */
if (!compressedBuffer || !resultBuffer || !blockTable)
{
DISPLAY("\nError: not enough memory!\n");
free(compressedBuffer);
free(resultBuffer);
free(blockTable);
return 12;
}
/* Calculating input Checksum */
crcOrig = XXH64(srcBuffer, srcSize, 0);
EXM_THROW(31, "not enough memory");
/* Init blockTable data */
{
U32 i;
size_t remaining = srcSize;
char* srcPtr = (char*)srcBuffer;
U32 fileNb;
const char* srcPtr = (const char*)srcBuffer;
char* cPtr = (char*)compressedBuffer;
char* resPtr = (char*)resultBuffer;
for (i=0; i<nbBlocks; i++)
for (fileNb=0; fileNb<nbFiles; fileNb++)
{
size_t thisBlockSize = MIN(remaining, blockSize);
blockTable[i].srcPtr = srcPtr;
blockTable[i].cPtr = cPtr;
blockTable[i].resPtr = resPtr;
blockTable[i].srcSize = thisBlockSize;
blockTable[i].cRoom = ZSTD_compressBound(thisBlockSize);
srcPtr += thisBlockSize;
cPtr += blockTable[i].cRoom;
resPtr += thisBlockSize;
remaining -= thisBlockSize;
U64 fileSize = (nbFiles>=2) ? BMK_getFileSize(fileNames[fileNb]) : srcSize;
size_t remaining = (size_t)fileSize;
U32 nbBlocksforThisFile = (U32)((fileSize + (blockSize-1)) / blockSize);
U32 blockEnd = nbBlocks + nbBlocksforThisFile;
for ( ; nbBlocks<blockEnd; nbBlocks++)
{
size_t thisBlockSize = MIN(remaining, blockSize);
blockTable[nbBlocks].srcPtr = srcPtr;
blockTable[nbBlocks].cPtr = cPtr;
blockTable[nbBlocks].resPtr = resPtr;
blockTable[nbBlocks].srcSize = thisBlockSize;
blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize);
srcPtr += thisBlockSize;
cPtr += blockTable[nbBlocks].cRoom;
resPtr += thisBlockSize;
remaining -= thisBlockSize;
}
}
}
@ -244,7 +280,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
U32 blockNb;
/* Compression */
DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, fileName, (U32)srcSize);
DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize);
memset(compressedBuffer, 0xE5, maxCompressedSize);
nbLoops = 0;
@ -264,7 +300,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
cSize += blockTable[blockNb].cSize;
if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
ratio = (double)srcSize / (double)cSize;
DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
#if 1
/* Decompression */
@ -283,7 +319,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
milliTime = BMK_GetMilliSpan(milliTime);
if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
/* CRC Checking */
crcCheck = XXH64(resultBuffer, srcSize, 0);
@ -291,10 +327,10 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
{
unsigned u;
unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
for (u=0; u<srcSize; u++)
{
if (((BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u])
if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u])
{
printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
break;
@ -306,30 +342,16 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
}
if (crcOrig == crcCheck)
DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
}
/* End cleaning */
/* clean up */
free(compressedBuffer);
free(resultBuffer);
return 0;
}
static U64 BMK_GetFileSize(const char* infilename)
{
int r;
#if defined(_MSC_VER)
struct _stat64 statbuf;
r = _stat64(infilename, &statbuf);
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
#endif
if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */
return (U64)statbuf.st_size;
}
static size_t BMK_findMaxMem(U64 requiredMem)
{
size_t step = 64 MB;
@ -349,103 +371,99 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem - step);
}
static int BMK_benchOneFile(const char* inFileName, int cLevel)
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel,
const char** fileNamesTable, unsigned nbFiles)
{
FILE* inFile;
U64 inFileSize;
size_t benchedSize, readSize;
void* srcBuffer;
int result=0;
/* Check file existence */
inFile = fopen(inFileName, "rb");
if (inFile == NULL)
{
DISPLAY("Pb opening %s\n", inFileName);
return 11;
}
/* Memory allocation & restrictions */
inFileSize = BMK_GetFileSize(inFileName);
benchedSize = BMK_findMaxMem(inFileSize * 3) / 3;
if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
if (benchedSize < inFileSize)
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize >> 20));
srcBuffer = malloc(benchedSize);
if (!srcBuffer)
{
DISPLAY("\nError: not enough memory!\n");
fclose(inFile);
return 12;
}
/* Fill input buffer */
DISPLAY("Loading %s... \r", inFileName);
readSize = fread(srcBuffer, 1, benchedSize, inFile);
fclose(inFile);
if (readSize != benchedSize)
{
DISPLAY("\nError: problem reading file '%s' !! \n", inFileName);
free(srcBuffer);
return 13;
}
/* Bench */
if (cLevel<0)
if (cLevel < 0)
{
int l;
for (l=1; l <= -cLevel; l++)
result = BMK_benchMem(srcBuffer, benchedSize, inFileName, l);
BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileNamesTable, nbFiles);
return;
}
else
result = BMK_benchMem(srcBuffer, benchedSize, inFileName, cLevel);
BMK_benchMem(srcBuffer, benchedSize, displayName, cLevel, fileNamesTable, nbFiles);
}
static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
{
U64 total = 0;
unsigned n;
for (n=0; n<nbFiles; n++)
total += BMK_getFileSize(fileNamesTable[n]);
return total;
}
static void BMK_loadFiles(void* buffer, size_t bufferSize, const char** fileNamesTable, unsigned nbFiles)
{
BYTE* buff = (BYTE*)buffer;
size_t pos = 0;
unsigned n;
for (n=0; n<nbFiles; n++)
{
size_t readSize;
U64 fileSize = BMK_getFileSize(fileNamesTable[n]);
FILE* f = fopen(fileNamesTable[n], "rb");
if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]);
if (fileSize > bufferSize-pos) fileSize = bufferSize-pos;
readSize = fread(buff+pos, 1, (size_t)fileSize, f);
if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
pos += readSize;
fclose(f);
}
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, int cLevel)
{
void* srcBuffer;
size_t benchedSize;
U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles);
char mfName[20] = {0};
const char* displayName = NULL;
/* Memory allocation & restrictions */
benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
if (benchedSize < totalSizeToLoad)
DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
srcBuffer = malloc(benchedSize);
if (!srcBuffer) EXM_THROW(12, "not enough memory");
/* Load input buffer */
BMK_loadFiles(srcBuffer, benchedSize, fileNamesTable, nbFiles);
/* Bench */
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
if (nbFiles > 1) displayName = mfName;
else displayName = fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, fileNamesTable, nbFiles);
/* clean up */
free(srcBuffer);
DISPLAY("\n");
return result;
}
static int BMK_syntheticTest(int cLevel, double compressibility)
static void BMK_syntheticTest(int cLevel, double compressibility)
{
char name[20] = {0};
size_t benchedSize = 10000000;
void* srcBuffer = malloc(benchedSize);
int result=0;
char name[20] = {0};
/* Memory allocation */
if (!srcBuffer)
{
DISPLAY("\nError: not enough memory!\n");
free(srcBuffer);
return 12;
}
if (!srcBuffer) EXM_THROW(21, "not enough memory");
/* Fill input buffer */
RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
/* Bench */
#ifdef _MSC_VER
sprintf_s(name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100));
#else
snprintf (name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100));
#endif
/* Bench */
if (cLevel<0)
{
int l;
for (l=1; l <= -cLevel; l++)
result = BMK_benchMem(srcBuffer, benchedSize, name, l);
}
else
result = BMK_benchMem(srcBuffer, benchedSize, name, cLevel);
snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, NULL, 1);
/* End */
/* clean up */
free(srcBuffer);
DISPLAY("\n");
return result;
}
@ -454,19 +472,9 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLeve
double compressibility = (double)g_compressibilityDefault / 100;
if (nbFiles == 0)
{
BMK_syntheticTest(cLevel, compressibility);
}
else
{
/* Loop for each file */
unsigned fileIdx = 0;
while (fileIdx<nbFiles)
{
BMK_benchOneFile(fileNamesTable[fileIdx], cLevel);
fileIdx++;
}
}
BMK_benchFileTable(fileNamesTable, nbFiles, cLevel);
return 0;
}