benchmark multiple files

programs/bench.c

@@ -72,6 +72,10 @@
 #  define S_ISREG(x) (((x) & S_IFMT) == S_IFREG)
 #endif
 
+#ifdef _MSC_VER
+#define snprintf sprintf_s
+#endif
+
 
 /* *************************************
 *  Constants
@@ -90,9 +94,28 @@ static U32 g_compressibilityDefault = 50;
 
 
 /* *************************************
-*  Macros
+*  console display
 ***************************************/
-#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
+#define DISPLAY(...)         fprintf(stderr, __VA_ARGS__)
+#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
+static U32 g_displayLevel = 2;   /* 0 : no display;   1: errors;   2 : + result + interaction + warnings;   3 : + progression;   4 : + information */
+
+
+/* *************************************
+*  Exceptions
+***************************************/
+#ifndef DEBUG
+#  define DEBUG 0
+#endif
+#define DEBUGOUTPUT(...) if (DEBUG) DISPLAY(__VA_ARGS__);
+#define EXM_THROW(error, ...)                                             \
+{                                                                         \
+    DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \
+    DISPLAYLEVEL(1, "Error %i : ", error);                                \
+    DISPLAYLEVEL(1, __VA_ARGS__);                                         \
+    DISPLAYLEVEL(1, "\n");                                                \
+    exit(error);                                                          \
+}
 
 
 /* *************************************
@@ -156,13 +179,27 @@ static int BMK_GetMilliSpan( int nTimeStart )
   return nSpan;
 }
 
+static U64 BMK_getFileSize(const char* infilename)
+{
+    int r;
+#if defined(_MSC_VER)
+    struct _stat64 statbuf;
+    r = _stat64(infilename, &statbuf);
+#else
+    struct stat statbuf;
+    r = stat(infilename, &statbuf);
+#endif
+    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
+    return (U64)statbuf.st_size;
+}
+
 
 /* ********************************************************
 *  Bench functions
 **********************************************************/
 typedef struct
 {
-    char*  srcPtr;
+    const char* srcPtr;
     size_t srcSize;
     char*  cPtr;
     size_t cRoom;
@@ -175,53 +212,52 @@ typedef size_t (*compressor_t) (void* dst, size_t maxDstSize, const void* src, s
 
 #define MIN(a,b) ((a)<(b) ? (a) : (b))
 
-static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, int cLevel)
+static int BMK_benchMem(const void* srcBuffer, size_t srcSize,
+                        const char* displayName, int cLevel,
+                        const char** fileNames, U32 nbFiles)
 {
     const size_t blockSize = (g_blockSize ? g_blockSize : srcSize) + (!srcSize);   /* avoid div by 0 */
-    const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize);
+    const U32 maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles;
-    blockParam_t* const blockTable = (blockParam_t*) malloc(nbBlocks * sizeof(blockParam_t));
+    blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t));
-    const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize);
+    const size_t maxCompressedSize = (size_t)maxNbBlocks * ZSTD_compressBound(blockSize);
     void* const compressedBuffer = malloc(maxCompressedSize);
     void* const resultBuffer = malloc(srcSize);
     const compressor_t compressor = ZSTD_compress;
-    U64 crcOrig;
+    U64 crcOrig = XXH64(srcBuffer, srcSize, 0);
+    U32 nbBlocks = 0;
 
     /* init */
-    if (strlen(fileName)>16)
+    if (strlen(displayName)>17) displayName += strlen(displayName)-17;   /* can only display 17 characters */
-        fileName += strlen(fileName)-16;
 
     /* Memory allocation & restrictions */
     if (!compressedBuffer || !resultBuffer || !blockTable)
-    {
+        EXM_THROW(31, "not enough memory");
-        DISPLAY("\nError: not enough memory!\n");
-        free(compressedBuffer);
-        free(resultBuffer);
-        free(blockTable);
-        return 12;
-    }
-
-    /* Calculating input Checksum */
-    crcOrig = XXH64(srcBuffer, srcSize, 0);
 
     /* Init blockTable data */
     {
-        U32 i;
+        U32 fileNb;
-        size_t remaining = srcSize;
+        const char* srcPtr = (const char*)srcBuffer;
-        char* srcPtr = (char*)srcBuffer;
         char* cPtr = (char*)compressedBuffer;
         char* resPtr = (char*)resultBuffer;
-        for (i=0; i<nbBlocks; i++)
+        for (fileNb=0; fileNb<nbFiles; fileNb++)
         {
-            size_t thisBlockSize = MIN(remaining, blockSize);
+            U64 fileSize = (nbFiles>=2) ? BMK_getFileSize(fileNames[fileNb]) : srcSize;
-            blockTable[i].srcPtr = srcPtr;
+            size_t remaining = (size_t)fileSize;
-            blockTable[i].cPtr = cPtr;
+            U32 nbBlocksforThisFile = (U32)((fileSize + (blockSize-1)) / blockSize);
-            blockTable[i].resPtr = resPtr;
+            U32 blockEnd = nbBlocks + nbBlocksforThisFile;
-            blockTable[i].srcSize = thisBlockSize;
+            for ( ; nbBlocks<blockEnd; nbBlocks++)
-            blockTable[i].cRoom = ZSTD_compressBound(thisBlockSize);
+            {
-            srcPtr += thisBlockSize;
+                size_t thisBlockSize = MIN(remaining, blockSize);
-            cPtr += blockTable[i].cRoom;
+                blockTable[nbBlocks].srcPtr = srcPtr;
-            resPtr += thisBlockSize;
+                blockTable[nbBlocks].cPtr = cPtr;
-            remaining -= thisBlockSize;
+                blockTable[nbBlocks].resPtr = resPtr;
+                blockTable[nbBlocks].srcSize = thisBlockSize;
+                blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize);
+                srcPtr += thisBlockSize;
+                cPtr += blockTable[nbBlocks].cRoom;
+                resPtr += thisBlockSize;
+                remaining -= thisBlockSize;
+            }
         }
     }
 
@@ -244,7 +280,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
             U32 blockNb;
 
             /* Compression */
-            DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, fileName, (U32)srcSize);
+            DISPLAY("%2i-%-17.17s :%10u ->\r", loopNb, displayName, (U32)srcSize);
             memset(compressedBuffer, 0xE5, maxCompressedSize);
 
             nbLoops = 0;
@@ -264,7 +300,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
                 cSize += blockTable[blockNb].cSize;
             if ((double)milliTime < fastestC*nbLoops) fastestC = (double)milliTime / nbLoops;
             ratio = (double)srcSize / (double)cSize;
-            DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
+            DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000.);
 
 #if 1
             /* Decompression */
@@ -283,7 +319,7 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
             milliTime = BMK_GetMilliSpan(milliTime);
 
             if ((double)milliTime < fastestD*nbLoops) fastestD = (double)milliTime / nbLoops;
-            DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
+            DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", loopNb, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
 
             /* CRC Checking */
             crcCheck = XXH64(resultBuffer, srcSize, 0);
@@ -291,10 +327,10 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
             {
                 unsigned u;
                 unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize));
-                DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", fileName, (unsigned)crcOrig, (unsigned)crcCheck);
+                DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", displayName, (unsigned)crcOrig, (unsigned)crcCheck);
                 for (u=0; u<srcSize; u++)
                 {
-                    if (((BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u])
+                    if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u])
                     {
                         printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize);
                         break;
@@ -306,30 +342,16 @@ static int BMK_benchMem(void* srcBuffer, size_t srcSize, const char* fileName, i
         }
 
         if (crcOrig == crcCheck)
-            DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, fileName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
+            DISPLAY("%2i-%-17.17s :%10i ->%10i (%5.3f),%6.1f MB/s ,%6.1f MB/s \n", cLevel, displayName, (int)srcSize, (int)cSize, ratio, (double)srcSize / fastestC / 1000., (double)srcSize / fastestD / 1000.);
     }
 
-    /* End cleaning */
+    /* clean up */
     free(compressedBuffer);
     free(resultBuffer);
     return 0;
 }
 
 
-static U64 BMK_GetFileSize(const char* infilename)
-{
-    int r;
-#if defined(_MSC_VER)
-    struct _stat64 statbuf;
-    r = _stat64(infilename, &statbuf);
-#else
-    struct stat statbuf;
-    r = stat(infilename, &statbuf);
-#endif
-    if (r || !S_ISREG(statbuf.st_mode)) return 0;   /* No good... */
-    return (U64)statbuf.st_size;
-}
-
 static size_t BMK_findMaxMem(U64 requiredMem)
 {
     size_t step = 64 MB;
@@ -349,103 +371,99 @@ static size_t BMK_findMaxMem(U64 requiredMem)
     return (size_t)(requiredMem - step);
 }
 
-static int BMK_benchOneFile(const char* inFileName, int cLevel)
+static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
+                            const char* displayName, int cLevel,
+                            const char** fileNamesTable, unsigned nbFiles)
 {
-    FILE*  inFile;
+    if (cLevel < 0)
-    U64    inFileSize;
-    size_t benchedSize, readSize;
-    void* srcBuffer;
-    int result=0;
-
-    /* Check file existence */
-    inFile = fopen(inFileName, "rb");
-    if (inFile == NULL)
-    {
-        DISPLAY("Pb opening %s\n", inFileName);
-        return 11;
-    }
-
-    /* Memory allocation & restrictions */
-    inFileSize = BMK_GetFileSize(inFileName);
-    benchedSize = BMK_findMaxMem(inFileSize * 3) / 3;
-    if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize;
-    if (benchedSize < inFileSize)
-        DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize >> 20));
-    srcBuffer = malloc(benchedSize);
-    if (!srcBuffer)
-    {
-        DISPLAY("\nError: not enough memory!\n");
-        fclose(inFile);
-        return 12;
-    }
-
-    /* Fill input buffer */
-    DISPLAY("Loading %s...       \r", inFileName);
-    readSize = fread(srcBuffer, 1, benchedSize, inFile);
-    fclose(inFile);
-
-    if (readSize != benchedSize)
-    {
-        DISPLAY("\nError: problem reading file '%s' !!    \n", inFileName);
-        free(srcBuffer);
-        return 13;
-    }
-
-    /* Bench */
-    if (cLevel<0)
     {
         int l;
         for (l=1; l <= -cLevel; l++)
-            result = BMK_benchMem(srcBuffer, benchedSize, inFileName, l);
+            BMK_benchMem(srcBuffer, benchedSize, displayName, l, fileNamesTable, nbFiles);
+        return;
     }
-    else
+    BMK_benchMem(srcBuffer, benchedSize, displayName, cLevel, fileNamesTable, nbFiles);
-        result = BMK_benchMem(srcBuffer, benchedSize, inFileName, cLevel);
+}
+
+static U64 BMK_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
+{
+    U64 total = 0;
+    unsigned n;
+    for (n=0; n<nbFiles; n++)
+        total += BMK_getFileSize(fileNamesTable[n]);
+    return total;
+}
+
+static void BMK_loadFiles(void* buffer, size_t bufferSize, const char** fileNamesTable, unsigned nbFiles)
+{
+    BYTE* buff = (BYTE*)buffer;
+    size_t pos = 0;
+    unsigned n;
+
+    for (n=0; n<nbFiles; n++)
+    {
+        size_t readSize;
+        U64 fileSize = BMK_getFileSize(fileNamesTable[n]);
+        FILE* f = fopen(fileNamesTable[n], "rb");
+        if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]);
+        DISPLAYLEVEL(2, "Loading %s...       \r", fileNamesTable[n]);
+        if (fileSize > bufferSize-pos) fileSize = bufferSize-pos;
+        readSize = fread(buff+pos, 1, (size_t)fileSize, f);
+        if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]);
+        pos += readSize;
+        fclose(f);
+    }
+}
+
+static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, int cLevel)
+{
+    void* srcBuffer;
+    size_t benchedSize;
+    U64 totalSizeToLoad = BMK_getTotalFileSize(fileNamesTable, nbFiles);
+    char mfName[20] = {0};
+    const char* displayName = NULL;
+
+    /* Memory allocation & restrictions */
+    benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3;
+    if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad;
+    if (benchedSize < totalSizeToLoad)
+        DISPLAY("Not enough memory; testing %u MB only...\n", (U32)(benchedSize >> 20));
+    srcBuffer = malloc(benchedSize);
+    if (!srcBuffer) EXM_THROW(12, "not enough memory");
+
+    /* Load input buffer */
+    BMK_loadFiles(srcBuffer, benchedSize, fileNamesTable, nbFiles);
+
+    /* Bench */
+    snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
+    if (nbFiles > 1) displayName = mfName;
+    else displayName = fileNamesTable[0];
+
+    BMK_benchCLevel(srcBuffer, benchedSize, displayName, cLevel, fileNamesTable, nbFiles);
 
     /* clean up */
     free(srcBuffer);
-    DISPLAY("\n");
-    return result;
 }
 
 
-static int BMK_syntheticTest(int cLevel, double compressibility)
+static void BMK_syntheticTest(int cLevel, double compressibility)
 {
+    char name[20] = {0};
     size_t benchedSize = 10000000;
     void* srcBuffer = malloc(benchedSize);
-    int result=0;
-    char name[20] = {0};
 
     /* Memory allocation */
-    if (!srcBuffer)
+    if (!srcBuffer) EXM_THROW(21, "not enough memory");
-    {
-        DISPLAY("\nError: not enough memory!\n");
-        free(srcBuffer);
-        return 12;
-    }
 
     /* Fill input buffer */
     RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0);
 
     /* Bench */
-#ifdef _MSC_VER
+    snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100));
-    sprintf_s(name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100));
+    BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, NULL, 1);
-#else
-    snprintf (name, 20, "Synthetic %2u%%", (unsigned)(compressibility*100));
-#endif
-    /* Bench */
-    if (cLevel<0)
-    {
-        int l;
-        for (l=1; l <= -cLevel; l++)
-            result = BMK_benchMem(srcBuffer, benchedSize, name, l);
-    }
-    else
-        result = BMK_benchMem(srcBuffer, benchedSize, name, cLevel);
 
-    /* End */
+    /* clean up */
     free(srcBuffer);
-    DISPLAY("\n");
-    return result;
 }
 
 
@@ -454,19 +472,9 @@ int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, unsigned cLeve
     double compressibility = (double)g_compressibilityDefault / 100;
 
     if (nbFiles == 0)
-    {
         BMK_syntheticTest(cLevel, compressibility);
-    }
     else
-    {
+        BMK_benchFileTable(fileNamesTable, nbFiles, cLevel);
-        /* Loop for each file */
-        unsigned fileIdx = 0;
-        while (fileIdx<nbFiles)
-        {
-            BMK_benchOneFile(fileNamesTable[fileIdx], cLevel);
-            fileIdx++;
-        }
-    }
     return 0;
 }