krak/zstd
krak/zstd
1
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2025-03-07 09:26:03 +02:00
Code Issues Releases 1 Activity

Remove global parameters, pass into public functions instead

Browse Source
This commit is contained in:
Karl Ostmo 2019-01-22 17:31:13 -08:00
parent 31ba3f7e64
commit 5e220bf4b5
3 changed files with 330 additions and 224 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

400
programs/fileio.c
View File

@ -79,23 +79,29 @@
/*-************************************* /*-*************************************
* Macros * Macros
***************************************/ ***************************************/
struct FIO_display_prefs_s {
int displayLevel; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */
U32 noProgress;
};
static FIO_display_prefs_t g_display_prefs = {2, 0};
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) #define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } #define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
static int g_displayLevel = 2; /* 0 : no display; 1: errors; 2: + result + interaction + warnings; 3: + progression; 4: + information */
void FIO_setNotificationLevel(unsigned level) { g_displayLevel=level; }
static const U64 g_refreshRate = SEC_TO_MICRO / 6; static const U64 g_refreshRate = SEC_TO_MICRO / 6;
static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
#define READY_FOR_UPDATE() (!g_noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) #define READY_FOR_UPDATE() (!g_display_prefs.noProgress && UTIL_clockSpanMicro(g_displayClock) > g_refreshRate)
#define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); } #define DELAY_NEXT_UPDATE() { g_displayClock = UTIL_getTime(); }
#define DISPLAYUPDATE(l, ...) { \ #define DISPLAYUPDATE(l, ...) { \
if (g_displayLevel>=l && !g_noProgress) { \ if (g_display_prefs.displayLevel>=l && !g_display_prefs.noProgress) { \
if (READY_FOR_UPDATE() || (g_displayLevel>=4)) { \ if (READY_FOR_UPDATE() || (g_display_prefs.displayLevel>=4)) { \
DELAY_NEXT_UPDATE(); \ DELAY_NEXT_UPDATE(); \
DISPLAY(__VA_ARGS__); \ DISPLAY(__VA_ARGS__); \
if (g_displayLevel>=4) fflush(stderr); \ if (g_display_prefs.displayLevel>=4) fflush(stderr); \
} } } } } }
#undef MIN /* in case it would be already defined */ #undef MIN /* in case it would be already defined */
@ -269,96 +275,170 @@ void FIO_addAbortHandler()
/*-************************************* /*-*************************************
* Local Parameters - Not thread safe * Parameters: Typedefs
***************************************/ ***************************************/
static FIO_compressionType_t g_compressionType = FIO_zstdCompression;
void FIO_setCompressionType(FIO_compressionType_t compressionType) { g_compressionType = compressionType; } struct FIO_prefs_s {
static U32 g_overwrite = 0;
void FIO_overwriteMode(void) { g_overwrite = 1; } /* Algorithm preferences */
static U32 g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */ FIO_compressionType_t compressionType;
void FIO_setSparseWrite(unsigned sparse) { g_sparseFileSupport = sparse; } U32 sparseFileSupport; /* 0: no sparse allowed; 1: auto (file yes, stdout no); 2: force sparse */
static U32 g_dictIDFlag = 1; U32 dictIDFlag;
void FIO_setDictIDFlag(unsigned dictIDFlag) { g_dictIDFlag = dictIDFlag; } U32 checksumFlag;
static U32 g_checksumFlag = 1; U32 blockSize;
void FIO_setChecksumFlag(unsigned checksumFlag) { g_checksumFlag = checksumFlag; } unsigned overlapLog;
static U32 g_removeSrcFile = 0; U32 adaptiveMode;
void FIO_setRemoveSrcFile(unsigned flag) { g_removeSrcFile = (flag>0); } U32 rsyncable;
static unsigned g_memLimit = 0; int minAdaptLevel;
void FIO_setMemLimit(unsigned memLimit) { g_memLimit = memLimit; } int maxAdaptLevel;
static unsigned g_nbWorkers = 1; U32 ldmFlag;
void FIO_setNbWorkers(unsigned nbWorkers) { U32 ldmHashLog;
U32 ldmMinMatch;
U32 ldmBucketSizeLog;
U32 ldmHashRateLog;
/* IO preferences */
U32 removeSrcFile;
U32 overwrite;
/* Computation resources preferences */
unsigned memLimit;
unsigned nbWorkers;
};
/*-*************************************
* Parameters: Initialization
***************************************/
#define FIO_OVERLAP_LOG_NOTSET 9999
#define FIO_LDM_PARAM_NOTSET 9999
FIO_prefs_t* FIO_createPreferences(void)
{
FIO_prefs_t* const ret = (FIO_prefs_t*)malloc(sizeof(FIO_prefs_t));
if (!ret) EXM_THROW(21, "Allocation error : not enough memory");
ret->compressionType = FIO_zstdCompression;
ret->overwrite = 0;
ret->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
ret->dictIDFlag = 1;
ret->checksumFlag = 1;
ret->removeSrcFile = 0;
ret->memLimit = 0;
ret->nbWorkers = 1;
ret->blockSize = 0;
ret->overlapLog = FIO_OVERLAP_LOG_NOTSET;
ret->adaptiveMode = 0;
ret->rsyncable = 0;
ret->minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
ret->maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */
ret->ldmFlag = 0;
ret->ldmHashLog = 0;
ret->ldmMinMatch = 0;
ret->ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET;
ret->ldmHashRateLog = FIO_LDM_PARAM_NOTSET;
return ret;
}
void FIO_freePreferences(FIO_prefs_t* const prefs)
{
free(prefs);
}
/*-*************************************
* Parameters: Display Options
***************************************/
void FIO_setNotificationLevel(unsigned level) { g_display_prefs.displayLevel=level; }
void FIO_setNoProgress(unsigned noProgress) { g_display_prefs.noProgress = noProgress; }
/*-*************************************
* Parameters: Setters
***************************************/
void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType) { prefs->compressionType = compressionType; }
void FIO_overwriteMode(FIO_prefs_t* const prefs) { prefs->overwrite = 1; }
void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse) { prefs->sparseFileSupport = sparse; }
void FIO_setDictIDFlag(FIO_prefs_t* const prefs, unsigned dictIDFlag) { prefs->dictIDFlag = dictIDFlag; }
void FIO_setChecksumFlag(FIO_prefs_t* const prefs, unsigned checksumFlag) { prefs->checksumFlag = checksumFlag; }
void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag) { prefs->removeSrcFile = (flag>0); }
void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit) { prefs->memLimit = memLimit; }
void FIO_setNbWorkers(FIO_prefs_t* const prefs, unsigned nbWorkers) {
#ifndef ZSTD_MULTITHREAD #ifndef ZSTD_MULTITHREAD
if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n"); if (nbWorkers > 0) DISPLAYLEVEL(2, "Note : multi-threading is disabled \n");
#endif #endif
g_nbWorkers = nbWorkers; prefs->nbWorkers = nbWorkers;
} }
static U32 g_blockSize = 0;
void FIO_setBlockSize(unsigned blockSize) { void FIO_setBlockSize(FIO_prefs_t* const prefs, unsigned blockSize) {
if (blockSize && g_nbWorkers==0) if (blockSize && prefs->nbWorkers==0)
DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n");
g_blockSize = blockSize; prefs->blockSize = blockSize;
} }
#define FIO_OVERLAP_LOG_NOTSET 9999
static unsigned g_overlapLog = FIO_OVERLAP_LOG_NOTSET; void FIO_setOverlapLog(FIO_prefs_t* const prefs, unsigned overlapLog){
void FIO_setOverlapLog(unsigned overlapLog){ if (overlapLog && prefs->nbWorkers==0)
if (overlapLog && g_nbWorkers==0)
DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n"); DISPLAYLEVEL(2, "Setting overlapLog is useless in single-thread mode \n");
g_overlapLog = overlapLog; prefs->overlapLog = overlapLog;
} }
static U32 g_adaptiveMode = 0;
void FIO_setAdaptiveMode(unsigned adapt) { void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt) {
if ((adapt>0) && (g_nbWorkers==0)) if ((adapt>0) && (prefs->nbWorkers==0))
EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n"); EXM_THROW(1, "Adaptive mode is not compatible with single thread mode \n");
g_adaptiveMode = adapt; prefs->adaptiveMode = adapt;
} }
static U32 g_rsyncable = 0;
void FIO_setRsyncable(unsigned rsyncable) { void FIO_setRsyncable(FIO_prefs_t* const prefs, unsigned rsyncable) {
if ((rsyncable>0) && (g_nbWorkers==0)) if ((rsyncable>0) && (prefs->nbWorkers==0))
EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n"); EXM_THROW(1, "Rsyncable mode is not compatible with single thread mode \n");
g_rsyncable = rsyncable; prefs->rsyncable = rsyncable;
} }
static int g_minAdaptLevel = -50; /* initializing this value requires a constant, so ZSTD_minCLevel() doesn't work */
void FIO_setAdaptMin(int minCLevel) void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel)
{ {
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
assert(minCLevel >= ZSTD_minCLevel()); assert(minCLevel >= ZSTD_minCLevel());
#endif #endif
g_minAdaptLevel = minCLevel; prefs->minAdaptLevel = minCLevel;
} }
static int g_maxAdaptLevel = 22; /* initializing this value requires a constant, so ZSTD_maxCLevel() doesn't work */
void FIO_setAdaptMax(int maxCLevel) void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel)
{ {
g_maxAdaptLevel = maxCLevel; prefs->maxAdaptLevel = maxCLevel;
} }
static U32 g_ldmFlag = 0; void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag) {
void FIO_setLdmFlag(unsigned ldmFlag) { prefs->ldmFlag = (ldmFlag>0);
g_ldmFlag = (ldmFlag>0);
}
static U32 g_ldmHashLog = 0;
void FIO_setLdmHashLog(unsigned ldmHashLog) {
g_ldmHashLog = ldmHashLog;
}
static U32 g_ldmMinMatch = 0;
void FIO_setLdmMinMatch(unsigned ldmMinMatch) {
g_ldmMinMatch = ldmMinMatch;
} }
#define FIO_LDM_PARAM_NOTSET 9999 void FIO_setLdmHashLog(FIO_prefs_t* const prefs, unsigned ldmHashLog) {
static U32 g_ldmBucketSizeLog = FIO_LDM_PARAM_NOTSET; prefs->ldmHashLog = ldmHashLog;
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog) {
g_ldmBucketSizeLog = ldmBucketSizeLog;
} }
static U32 g_ldmHashRateLog = FIO_LDM_PARAM_NOTSET; void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, unsigned ldmMinMatch) {
void FIO_setLdmHashRateLog(unsigned ldmHashRateLog) { prefs->ldmMinMatch = ldmMinMatch;
g_ldmHashRateLog = ldmHashRateLog;
}
static U32 g_noProgress = 0;
void FIO_setNoProgress(unsigned noProgress) {
g_noProgress = noProgress;
} }
void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, unsigned ldmBucketSizeLog) {
prefs->ldmBucketSizeLog = ldmBucketSizeLog;
}
void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, unsigned ldmHashRateLog) {
prefs->ldmHashRateLog = ldmHashRateLog;
}
/*-************************************* /*-*************************************
@ -414,14 +494,14 @@ static FILE* FIO_openSrcFile(const char* srcFileName)
/** FIO_openDstFile() : /** FIO_openDstFile() :
* condition : `dstFileName` must be non-NULL. * condition : `dstFileName` must be non-NULL.
* @result : FILE* to `dstFileName`, or NULL if it fails */ * @result : FILE* to `dstFileName`, or NULL if it fails */
static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName) static FILE* FIO_openDstFile(FIO_prefs_t* const prefs, const char* srcFileName, const char* dstFileName)
{ {
assert(dstFileName != NULL); assert(dstFileName != NULL);
if (!strcmp (dstFileName, stdoutmark)) { if (!strcmp (dstFileName, stdoutmark)) {
DISPLAYLEVEL(4,"Using stdout for output \n"); DISPLAYLEVEL(4,"Using stdout for output \n");
SET_BINARY_MODE(stdout); SET_BINARY_MODE(stdout);
if (g_sparseFileSupport == 1) { if (prefs->sparseFileSupport == 1) {
g_sparseFileSupport = 0; prefs->sparseFileSupport = 0;
DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n"); DISPLAYLEVEL(4, "Sparse File Support is automatically disabled on stdout ; try --sparse \n");
} }
return stdout; return stdout;
@ -451,8 +531,8 @@ static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName)
#endif #endif
} }
if (g_sparseFileSupport == 1) { if (prefs->sparseFileSupport == 1) {
g_sparseFileSupport = ZSTD_SPARSE_DEFAULT; prefs->sparseFileSupport = ZSTD_SPARSE_DEFAULT;
} }
if (UTIL_isRegularFile(dstFileName)) { if (UTIL_isRegularFile(dstFileName)) {
@ -464,8 +544,8 @@ static FILE* FIO_openDstFile(const char* srcFileName, const char* dstFileName)
} }
if (fCheck != NULL) { /* dst file exists, authorization prompt */ if (fCheck != NULL) { /* dst file exists, authorization prompt */
fclose(fCheck); fclose(fCheck);
if (!g_overwrite) { if (!prefs->overwrite) {
if (g_displayLevel <= 1) { if (g_display_prefs.displayLevel <= 1) {
/* No interaction possible */ /* No interaction possible */
DISPLAY("zstd: %s already exists; not overwritten \n", DISPLAY("zstd: %s already exists; not overwritten \n",
dstFileName); dstFileName);
@ -543,7 +623,8 @@ typedef struct {
ZSTD_CStream* cctx; ZSTD_CStream* cctx;
} cRess_t; } cRess_t;
static cRess_t FIO_createCResources(const char* dictFileName, int cLevel, static cRess_t FIO_createCResources(FIO_prefs_t* const prefs,
const char* dictFileName, int cLevel,
U64 srcSize, U64 srcSize,
ZSTD_compressionParameters comprParams) { ZSTD_compressionParameters comprParams) {
cRess_t ress; cRess_t ress;
@ -567,23 +648,23 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
if (dictFileName && (dictBuffer==NULL)) if (dictFileName && (dictBuffer==NULL))
EXM_THROW(32, "allocation error : can't create dictBuffer"); EXM_THROW(32, "allocation error : can't create dictBuffer");
if (g_adaptiveMode && !g_ldmFlag && !comprParams.windowLog) if (prefs->adaptiveMode && !prefs->ldmFlag && !comprParams.windowLog)
comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT; comprParams.windowLog = ADAPT_WINDOWLOG_DEFAULT;
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */ CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_contentSizeFlag, 1) ); /* always enable content size when available (note: supposed to be default) */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, g_dictIDFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_dictIDFlag, prefs->dictIDFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, g_checksumFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_checksumFlag, prefs->checksumFlag) );
/* compression level */ /* compression level */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, cLevel) );
/* long distance matching */ /* long distance matching */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, g_ldmFlag) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_enableLongDistanceMatching, prefs->ldmFlag) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, g_ldmHashLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashLog, prefs->ldmHashLog) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, g_ldmMinMatch) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmMinMatch, prefs->ldmMinMatch) );
if (g_ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) { if (prefs->ldmBucketSizeLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, g_ldmBucketSizeLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmBucketSizeLog, prefs->ldmBucketSizeLog) );
} }
if (g_ldmHashRateLog != FIO_LDM_PARAM_NOTSET) { if (prefs->ldmHashRateLog != FIO_LDM_PARAM_NOTSET) {
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, g_ldmHashRateLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_ldmHashRateLog, prefs->ldmHashRateLog) );
} }
/* compression parameters */ /* compression parameters */
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_windowLog, comprParams.windowLog) );
@ -595,14 +676,14 @@ static cRess_t FIO_createCResources(const char* dictFileName, int cLevel,
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_strategy, comprParams.strategy) );
/* multi-threading */ /* multi-threading */
#ifdef ZSTD_MULTITHREAD #ifdef ZSTD_MULTITHREAD
DISPLAYLEVEL(5,"set nb workers = %u \n", g_nbWorkers); DISPLAYLEVEL(5,"set nb workers = %u \n", prefs->nbWorkers);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, g_nbWorkers) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_nbWorkers, prefs->nbWorkers) );
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, g_blockSize) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_jobSize, prefs->blockSize) );
if (g_overlapLog != FIO_OVERLAP_LOG_NOTSET) { if (prefs->overlapLog != FIO_OVERLAP_LOG_NOTSET) {
DISPLAYLEVEL(3,"set overlapLog = %u \n", g_overlapLog); DISPLAYLEVEL(3,"set overlapLog = %u \n", prefs->overlapLog);
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, g_overlapLog) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_overlapLog, prefs->overlapLog) );
} }
CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, g_rsyncable) ); CHECK( ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_rsyncable, prefs->rsyncable) );
#endif #endif
/* dictionary */ /* dictionary */
CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */ CHECK( ZSTD_CCtx_setPledgedSrcSize(ress.cctx, srcSize) ); /* set the value temporarily for dictionary loading, to adapt compression parameters */
@ -880,7 +961,8 @@ FIO_compressLz4Frame(cRess_t* ress,
static unsigned long long static unsigned long long
FIO_compressZstdFrame(const cRess_t* ressPtr, FIO_compressZstdFrame(FIO_prefs_t* const prefs,
const cRess_t* ressPtr,
const char* srcFileName, U64 fileSize, const char* srcFileName, U64 fileSize,
int compressionLevel, U64* readsize) int compressionLevel, U64* readsize)
{ {
@ -951,7 +1033,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100; double const cShare = (double)zfp.produced / (zfp.consumed + !zfp.consumed/*avoid div0*/) * 100;
/* display progress notifications */ /* display progress notifications */
if (g_displayLevel >= 3) { if (g_display_prefs.displayLevel >= 3) {
DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ", DISPLAYUPDATE(3, "\r(L%i) Buffered :%4u MB - Consumed :%4u MB - Compressed :%4u MB => %.2f%% ",
compressionLevel, compressionLevel,
(unsigned)((zfp.ingested - zfp.consumed) >> 20), (unsigned)((zfp.ingested - zfp.consumed) >> 20),
@ -967,7 +1049,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
} }
/* adaptive mode : statistics measurement and speed correction */ /* adaptive mode : statistics measurement and speed correction */
if (g_adaptiveMode) { if (prefs->adaptiveMode) {
/* check output speed */ /* check output speed */
if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */ if (zfp.currentJobID > 1) { /* only possible if nbWorkers >= 1 */
@ -975,7 +1057,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced; unsigned long long newlyProduced = zfp.produced - previous_zfp_update.produced;
unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed; unsigned long long newlyFlushed = zfp.flushed - previous_zfp_update.flushed;
assert(zfp.produced >= previous_zfp_update.produced); assert(zfp.produced >= previous_zfp_update.produced);
assert(g_nbWorkers >= 1); assert(prefs->nbWorkers >= 1);
/* test if compression is blocked /* test if compression is blocked
* either because output is slow and all buffers are full * either because output is slow and all buffers are full
@ -1004,7 +1086,7 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
DISPLAYLEVEL(6, "compression level adaptation check \n") DISPLAYLEVEL(6, "compression level adaptation check \n")
/* check input speed */ /* check input speed */
if (zfp.currentJobID > g_nbWorkers+1) { /* warm up period, to fill all workers */ if (zfp.currentJobID > prefs->nbWorkers+1) { /* warm up period, to fill all workers */
if (inputBlocked <= 0) { if (inputBlocked <= 0) {
DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n"); DISPLAYLEVEL(6, "input is never blocked => input is slower than ingestion \n");
speedChange = slower; speedChange = slower;
@ -1036,14 +1118,14 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
DISPLAYLEVEL(6, "slower speed , higher compression \n") DISPLAYLEVEL(6, "slower speed , higher compression \n")
compressionLevel ++; compressionLevel ++;
if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel(); if (compressionLevel > ZSTD_maxCLevel()) compressionLevel = ZSTD_maxCLevel();
if (compressionLevel > g_maxAdaptLevel) compressionLevel = g_maxAdaptLevel; if (compressionLevel > prefs->maxAdaptLevel) compressionLevel = prefs->maxAdaptLevel;
compressionLevel += (compressionLevel == 0); /* skip 0 */ compressionLevel += (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
} }
if (speedChange == faster) { if (speedChange == faster) {
DISPLAYLEVEL(6, "faster speed , lighter compression \n") DISPLAYLEVEL(6, "faster speed , lighter compression \n")
compressionLevel --; compressionLevel --;
if (compressionLevel < g_minAdaptLevel) compressionLevel = g_minAdaptLevel; if (compressionLevel < prefs->minAdaptLevel) compressionLevel = prefs->minAdaptLevel;
compressionLevel -= (compressionLevel == 0); /* skip 0 */ compressionLevel -= (compressionLevel == 0); /* skip 0 */
ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel); ZSTD_CCtx_setParameter(ress.cctx, ZSTD_c_compressionLevel, compressionLevel);
} }
@ -1073,7 +1155,8 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
* 1 : missing or pb opening srcFileName * 1 : missing or pb opening srcFileName
*/ */
static int static int
FIO_compressFilename_internal(cRess_t ress, FIO_compressFilename_internal(FIO_prefs_t* const prefs,
cRess_t ress,
const char* dstFileName, const char* srcFileName, const char* dstFileName, const char* srcFileName,
int compressionLevel) int compressionLevel)
{ {
@ -1083,10 +1166,10 @@ FIO_compressFilename_internal(cRess_t ress,
DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize); DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (unsigned)fileSize);
/* compression format selection */ /* compression format selection */
switch (g_compressionType) { switch (prefs->compressionType) {
default: default:
case FIO_zstdCompression: case FIO_zstdCompression:
compressedfilesize = FIO_compressZstdFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize); compressedfilesize = FIO_compressZstdFrame(prefs, &ress, srcFileName, fileSize, compressionLevel, &readsize);
break; break;
case FIO_gzipCompression: case FIO_gzipCompression:
@ -1102,7 +1185,7 @@ FIO_compressFilename_internal(cRess_t ress,
case FIO_xzCompression: case FIO_xzCompression:
case FIO_lzmaCompression: case FIO_lzmaCompression:
#ifdef ZSTD_LZMACOMPRESS #ifdef ZSTD_LZMACOMPRESS
compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, g_compressionType==FIO_lzmaCompression); compressedfilesize = FIO_compressLzmaFrame(&ress, srcFileName, fileSize, compressionLevel, &readsize, prefs->compressionType==FIO_lzmaCompression);
#else #else
(void)compressionLevel; (void)compressionLevel;
EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n", EXM_THROW(20, "zstd: %s: file cannot be compressed as xz/lzma (zstd compiled without ZSTD_LZMACOMPRESS) -- ignored \n",
@ -1142,7 +1225,8 @@ FIO_compressFilename_internal(cRess_t ress,
* @return : 0 : compression completed correctly, * @return : 0 : compression completed correctly,
* 1 : pb * 1 : pb
*/ */
static int FIO_compressFilename_dstFile(cRess_t ress, static int FIO_compressFilename_dstFile(FIO_prefs_t* const prefs,
cRess_t ress,
const char* dstFileName, const char* dstFileName,
const char* srcFileName, const char* srcFileName,
int compressionLevel) int compressionLevel)
@ -1157,7 +1241,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
if (ress.dstFile == NULL) { if (ress.dstFile == NULL) {
closeDstFile = 1; closeDstFile = 1;
DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName); DISPLAYLEVEL(6, "FIO_compressFilename_dstFile: opening dst: %s", dstFileName);
ress.dstFile = FIO_openDstFile(srcFileName, dstFileName); ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
if (ress.dstFile==NULL) return 1; /* could not open dstFileName */ if (ress.dstFile==NULL) return 1; /* could not open dstFileName */
/* Must only be added after FIO_openDstFile() succeeds. /* Must only be added after FIO_openDstFile() succeeds.
* Otherwise we may delete the destination file if it already exists, * Otherwise we may delete the destination file if it already exists,
@ -1170,7 +1254,7 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
transfer_permissions = 1; transfer_permissions = 1;
} }
result = FIO_compressFilename_internal(ress, dstFileName, srcFileName, compressionLevel); result = FIO_compressFilename_internal(prefs, ress, dstFileName, srcFileName, compressionLevel);
if (closeDstFile) { if (closeDstFile) {
FILE* const dstFile = ress.dstFile; FILE* const dstFile = ress.dstFile;
@ -1203,7 +1287,8 @@ static int FIO_compressFilename_dstFile(cRess_t ress,
* 1 : missing or pb opening srcFileName * 1 : missing or pb opening srcFileName
*/ */
static int static int
FIO_compressFilename_srcFile(cRess_t ress, FIO_compressFilename_srcFile(FIO_prefs_t* const prefs,
cRess_t ress,
const char* dstFileName, const char* dstFileName,
const char* srcFileName, const char* srcFileName,
int compressionLevel) int compressionLevel)
@ -1219,11 +1304,11 @@ FIO_compressFilename_srcFile(cRess_t ress,
ress.srcFile = FIO_openSrcFile(srcFileName); ress.srcFile = FIO_openSrcFile(srcFileName);
if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */
result = FIO_compressFilename_dstFile(ress, dstFileName, srcFileName, compressionLevel); result = FIO_compressFilename_dstFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
fclose(ress.srcFile); fclose(ress.srcFile);
ress.srcFile = NULL; ress.srcFile = NULL;
if ( g_removeSrcFile /* --rm */ if ( prefs->removeSrcFile /* --rm */
&& result == 0 /* success */ && result == 0 /* success */
&& strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */ && strcmp(srcFileName, stdinmark) /* exception : don't erase stdin */
) { ) {
@ -1238,7 +1323,8 @@ FIO_compressFilename_srcFile(cRess_t ress,
} }
int FIO_compressFilename(const char* dstFileName, const char* srcFileName, int FIO_compressFilename(FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName,
const char* dictFileName, int compressionLevel, const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams) ZSTD_compressionParameters comprParams)
{ {
@ -1246,8 +1332,8 @@ int FIO_compressFilename(const char* dstFileName, const char* srcFileName,
U64 const fileSize = UTIL_getFileSize(srcFileName); U64 const fileSize = UTIL_getFileSize(srcFileName);
U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize; U64 const srcSize = (fileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : fileSize;
cRess_t const ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); cRess_t const ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams);
int const result = FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); int const result = FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC; double const seconds = (double)(clock() - start) / CLOCKS_PER_SEC;
DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds); DISPLAYLEVEL(4, "Completed in %.2f sec \n", seconds);
@ -1292,7 +1378,8 @@ FIO_determineCompressedName(const char* srcFileName, const char* suffix)
* into one destination (outFileName) * into one destination (outFileName)
* or into one file each (outFileName == NULL, but suffix != NULL). * or into one file each (outFileName == NULL, but suffix != NULL).
*/ */
int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFiles, int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char** inFileNamesTable, unsigned nbFiles,
const char* outFileName, const char* suffix, const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel, const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams) ZSTD_compressionParameters comprParams)
@ -1301,19 +1388,19 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]); U64 const firstFileSize = UTIL_getFileSize(inFileNamesTable[0]);
U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize; U64 const firstSrcSize = (firstFileSize == UTIL_FILESIZE_UNKNOWN) ? ZSTD_CONTENTSIZE_UNKNOWN : firstFileSize;
U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ; U64 const srcSize = (nbFiles != 1) ? ZSTD_CONTENTSIZE_UNKNOWN : firstSrcSize ;
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, comprParams); cRess_t ress = FIO_createCResources(prefs, dictFileName, compressionLevel, srcSize, comprParams);
/* init */ /* init */
assert(outFileName != NULL || suffix != NULL); assert(outFileName != NULL || suffix != NULL);
if (outFileName != NULL) { /* output into a single destination (stdout typically) */ if (outFileName != NULL) { /* output into a single destination (stdout typically) */
ress.dstFile = FIO_openDstFile(NULL, outFileName); ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == NULL) { /* could not open outFileName */ if (ress.dstFile == NULL) { /* could not open outFileName */
error = 1; error = 1;
} else { } else {
unsigned u; unsigned u;
for (u=0; u<nbFiles; u++) for (u=0; u<nbFiles; u++)
error |= FIO_compressFilename_srcFile(ress, outFileName, inFileNamesTable[u], compressionLevel); error |= FIO_compressFilename_srcFile(prefs, ress, outFileName, inFileNamesTable[u], compressionLevel);
if (fclose(ress.dstFile)) if (fclose(ress.dstFile))
EXM_THROW(29, "Write error (%s) : cannot properly close %s", EXM_THROW(29, "Write error (%s) : cannot properly close %s",
strerror(errno), outFileName); strerror(errno), outFileName);
@ -1324,7 +1411,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
for (u=0; u<nbFiles; u++) { for (u=0; u<nbFiles; u++) {
const char* const srcFileName = inFileNamesTable[u]; const char* const srcFileName = inFileNamesTable[u];
const char* const dstFileName = FIO_determineCompressedName(srcFileName, suffix); /* cannot fail */ const char* const dstFileName = FIO_determineCompressedName(srcFileName, suffix); /* cannot fail */
error |= FIO_compressFilename_srcFile(ress, dstFileName, srcFileName, compressionLevel); error |= FIO_compressFilename_srcFile(prefs, ress, dstFileName, srcFileName, compressionLevel);
} } } }
FIO_freeCResources(ress); FIO_freeCResources(ress);
@ -1350,7 +1437,7 @@ typedef struct {
FILE* dstFile; FILE* dstFile;
} dRess_t; } dRess_t;
static dRess_t FIO_createDResources(const char* dictFileName) static dRess_t FIO_createDResources(FIO_prefs_t* const prefs, const char* dictFileName)
{ {
dRess_t ress; dRess_t ress;
memset(&ress, 0, sizeof(ress)); memset(&ress, 0, sizeof(ress));
@ -1359,7 +1446,7 @@ static dRess_t FIO_createDResources(const char* dictFileName)
ress.dctx = ZSTD_createDStream(); ress.dctx = ZSTD_createDStream();
if (ress.dctx==NULL) if (ress.dctx==NULL)
EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno)); EXM_THROW(60, "Error: %s : can't create ZSTD_DStream", strerror(errno));
CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, g_memLimit) ); CHECK( ZSTD_DCtx_setMaxWindowSize(ress.dctx, prefs->memLimit) );
ress.srcBufferSize = ZSTD_DStreamInSize(); ress.srcBufferSize = ZSTD_DStreamInSize();
ress.srcBuffer = malloc(ress.srcBufferSize); ress.srcBuffer = malloc(ress.srcBufferSize);
ress.dstBufferSize = ZSTD_DStreamOutSize(); ress.dstBufferSize = ZSTD_DStreamOutSize();
@ -1387,7 +1474,7 @@ static void FIO_freeDResources(dRess_t ress)
/** FIO_fwriteSparse() : /** FIO_fwriteSparse() :
* @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */ * @return : storedSkips, to be provided to next call to FIO_fwriteSparse() of LZ4IO_fwriteSparseEnd() */
static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips) static unsigned FIO_fwriteSparse(FIO_prefs_t* const prefs, FILE* file, const void* buffer, size_t bufferSize, unsigned storedSkips)
{ {
const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */ const size_t* const bufferT = (const size_t*)buffer; /* Buffer is supposed malloc'ed, hence aligned on size_t */
size_t bufferSizeT = bufferSize / sizeof(size_t); size_t bufferSizeT = bufferSize / sizeof(size_t);
@ -1395,7 +1482,7 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
const size_t* ptrT = bufferT; const size_t* ptrT = bufferT;
static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */ static const size_t segmentSizeT = (32 KB) / sizeof(size_t); /* 0-test re-attempted every 32 KB */
if (!g_sparseFileSupport) { /* normal write */ if (!prefs->sparseFileSupport) { /* normal write */
size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file); size_t const sizeCheck = fwrite(buffer, 1, bufferSize, file);
if (sizeCheck != bufferSize) if (sizeCheck != bufferSize)
EXM_THROW(70, "Write error : %s (cannot write decoded block)", EXM_THROW(70, "Write error : %s (cannot write decoded block)",
@ -1456,10 +1543,10 @@ static unsigned FIO_fwriteSparse(FILE* file, const void* buffer, size_t bufferSi
return storedSkips; return storedSkips;
} }
static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips) static void FIO_fwriteSparseEnd(FIO_prefs_t* const prefs, FILE* file, unsigned storedSkips)
{ {
if (storedSkips>0) { if (storedSkips>0) {
assert(g_sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */ assert(prefs->sparseFileSupport > 0); /* storedSkips>0 implies sparse support is enabled */
if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0) if (LONG_SEEK(file, storedSkips-1, SEEK_CUR) != 0)
EXM_THROW(69, "Final skip error (sparse file support)"); EXM_THROW(69, "Final skip error (sparse file support)");
/* last zero must be explicitly written, /* last zero must be explicitly written,
@ -1473,7 +1560,7 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode /** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
@return : 0 (no error) */ @return : 0 (no error) */
static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded) static unsigned FIO_passThrough(FIO_prefs_t* const prefs, FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded)
{ {
size_t const blockSize = MIN(64 KB, bufferSize); size_t const blockSize = MIN(64 KB, bufferSize);
size_t readFromInput = 1; size_t readFromInput = 1;
@ -1488,10 +1575,10 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
while (readFromInput) { while (readFromInput) {
readFromInput = fread(buffer, 1, blockSize, finput); readFromInput = fread(buffer, 1, blockSize, finput);
storedSkips = FIO_fwriteSparse(foutput, buffer, readFromInput, storedSkips); storedSkips = FIO_fwriteSparse(prefs, foutput, buffer, readFromInput, storedSkips);
} }
FIO_fwriteSparseEnd(foutput, storedSkips); FIO_fwriteSparseEnd(prefs, foutput, storedSkips);
return 0; return 0;
} }
@ -1510,7 +1597,7 @@ static unsigned FIO_highbit64(unsigned long long v)
/* FIO_zstdErrorHelp() : /* FIO_zstdErrorHelp() :
* detailed error message when requested window size is too large */ * detailed error message when requested window size is too large */
static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName) static void FIO_zstdErrorHelp(FIO_prefs_t* const prefs, dRess_t* ress, size_t err, char const* srcFileName)
{ {
ZSTD_frameHeader header; ZSTD_frameHeader header;
@ -1523,9 +1610,9 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName
if (err == 0) { if (err == 0) {
unsigned long long const windowSize = header.windowSize; unsigned long long const windowSize = header.windowSize;
unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0); unsigned const windowLog = FIO_highbit64(windowSize) + ((windowSize & (windowSize - 1)) != 0);
assert(g_memLimit > 0); assert(prefs->memLimit > 0);
DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n", DISPLAYLEVEL(1, "%s : Window size larger than maximum : %llu > %u\n",
srcFileName, windowSize, g_memLimit); srcFileName, windowSize, prefs->memLimit);
if (windowLog <= ZSTD_WINDOWLOG_MAX) { if (windowLog <= ZSTD_WINDOWLOG_MAX) {
unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0)); unsigned const windowMB = (unsigned)((windowSize >> 20) + ((windowSize & ((1 MB) - 1)) != 0));
assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */ assert(windowSize < (U64)(1ULL << 52)); /* ensure now overflow for windowMB */
@ -1542,7 +1629,9 @@ static void FIO_zstdErrorHelp(dRess_t* ress, size_t err, char const* srcFileName
* @return : size of decoded zstd frame, or an error code * @return : size of decoded zstd frame, or an error code
*/ */
#define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2)) #define FIO_ERROR_FRAME_DECODING ((unsigned long long)(-2))
static unsigned long long FIO_decompressZstdFrame(dRess_t* ress, static unsigned long long FIO_decompressZstdFrame(
FIO_prefs_t* const prefs,
dRess_t* ress,
FILE* finput, FILE* finput,
const char* srcFileName, const char* srcFileName,
U64 alreadyDecoded) U64 alreadyDecoded)
@ -1571,12 +1660,12 @@ static unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
if (ZSTD_isError(readSizeHint)) { if (ZSTD_isError(readSizeHint)) {
DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n", DISPLAYLEVEL(1, "%s : Decoding error (36) : %s \n",
srcFileName, ZSTD_getErrorName(readSizeHint)); srcFileName, ZSTD_getErrorName(readSizeHint));
FIO_zstdErrorHelp(ress, readSizeHint, srcFileName); FIO_zstdErrorHelp(prefs, ress, readSizeHint, srcFileName);
return FIO_ERROR_FRAME_DECODING; return FIO_ERROR_FRAME_DECODING;
} }
/* Write block */ /* Write block */
storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips); storedSkips = FIO_fwriteSparse(prefs, ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
frameSize += outBuff.pos; frameSize += outBuff.pos;
DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ", DISPLAYUPDATE(2, "\r%-20.20s : %u MB... ",
srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) ); srcFileName, (unsigned)((alreadyDecoded+frameSize)>>20) );
@ -1607,7 +1696,7 @@ static unsigned long long FIO_decompressZstdFrame(dRess_t* ress,
ress->srcBufferLoaded += readSize; ress->srcBufferLoaded += readSize;
} } } } } }
FIO_fwriteSparseEnd(ress->dstFile, storedSkips); FIO_fwriteSparseEnd(prefs, ress->dstFile, storedSkips);
return frameSize; return frameSize;
} }
@ -1838,7 +1927,7 @@ static unsigned long long FIO_decompressLz4Frame(dRess_t* ress,
* @return : 0 : OK * @return : 0 : OK
* 1 : error * 1 : error
*/ */
static int FIO_decompressFrames(dRess_t ress, FILE* srcFile, static int FIO_decompressFrames(FIO_prefs_t* const prefs, dRess_t ress, FILE* srcFile,
const char* dstFileName, const char* srcFileName) const char* dstFileName, const char* srcFileName)
{ {
unsigned readSomething = 0; unsigned readSomething = 0;
@ -1866,7 +1955,7 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
return 1; return 1;
} }
if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) { if (ZSTD_isFrame(buf, ress.srcBufferLoaded)) {
unsigned long long const frameSize = FIO_decompressZstdFrame(&ress, srcFile, srcFileName, filesize); unsigned long long const frameSize = FIO_decompressZstdFrame(prefs, &ress, srcFile, srcFileName, filesize);
if (frameSize == FIO_ERROR_FRAME_DECODING) return 1; if (frameSize == FIO_ERROR_FRAME_DECODING) return 1;
filesize += frameSize; filesize += frameSize;
} else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */ } else if (buf[0] == 31 && buf[1] == 139) { /* gz magic number */
@ -1897,8 +1986,8 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName); DISPLAYLEVEL(1, "zstd: %s: lz4 file cannot be uncompressed (zstd compiled without HAVE_LZ4) -- ignored \n", srcFileName);
return 1; return 1;
#endif #endif
} else if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */ } else if ((prefs->overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
return FIO_passThrough(ress.dstFile, srcFile, return FIO_passThrough(prefs, ress.dstFile, srcFile,
ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded); ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
} else { } else {
DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName); DISPLAYLEVEL(1, "zstd: %s: unsupported format \n", srcFileName);
@ -1919,7 +2008,8 @@ static int FIO_decompressFrames(dRess_t ress, FILE* srcFile,
@return : 0 : OK @return : 0 : OK
1 : operation aborted 1 : operation aborted
*/ */
static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile, static int FIO_decompressDstFile(FIO_prefs_t* const prefs,
dRess_t ress, FILE* srcFile,
const char* dstFileName, const char* srcFileName) const char* dstFileName, const char* srcFileName)
{ {
int result; int result;
@ -1930,7 +2020,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
if (ress.dstFile == NULL) { if (ress.dstFile == NULL) {
releaseDstFile = 1; releaseDstFile = 1;
ress.dstFile = FIO_openDstFile(srcFileName, dstFileName); ress.dstFile = FIO_openDstFile(prefs, srcFileName, dstFileName);
if (ress.dstFile==0) return 1; if (ress.dstFile==0) return 1;
/* Must only be added after FIO_openDstFile() succeeds. /* Must only be added after FIO_openDstFile() succeeds.
@ -1945,7 +2035,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
} }
result = FIO_decompressFrames(ress, srcFile, dstFileName, srcFileName); result = FIO_decompressFrames(prefs, ress, srcFile, dstFileName, srcFileName);
if (releaseDstFile) { if (releaseDstFile) {
FILE* const dstFile = ress.dstFile; FILE* const dstFile = ress.dstFile;
@ -1978,7 +2068,7 @@ static int FIO_decompressDstFile(dRess_t ress, FILE* srcFile,
@return : 0 : OK @return : 0 : OK
1 : error 1 : error
*/ */
static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const char* srcFileName) static int FIO_decompressSrcFile(FIO_prefs_t* const prefs, dRess_t ress, const char* dstFileName, const char* srcFileName)
{ {
FILE* srcFile; FILE* srcFile;
int result; int result;
@ -1992,14 +2082,14 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
if (srcFile==NULL) return 1; if (srcFile==NULL) return 1;
ress.srcBufferLoaded = 0; ress.srcBufferLoaded = 0;
result = FIO_decompressDstFile(ress, srcFile, dstFileName, srcFileName); result = FIO_decompressDstFile(prefs, ress, srcFile, dstFileName, srcFileName);
/* Close file */ /* Close file */
if (fclose(srcFile)) { if (fclose(srcFile)) {
DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */ DISPLAYLEVEL(1, "zstd: %s: %s \n", srcFileName, strerror(errno)); /* error should not happen */
return 1; return 1;
} }
if ( g_removeSrcFile /* --rm */ if ( prefs->removeSrcFile /* --rm */
&& (result==0) /* decompression successful */ && (result==0) /* decompression successful */
&& strcmp(srcFileName, stdinmark) ) /* not stdin */ { && strcmp(srcFileName, stdinmark) ) /* not stdin */ {
/* We must clear the handler, since after this point calling it would /* We must clear the handler, since after this point calling it would
@ -2016,12 +2106,13 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
int FIO_decompressFilename(const char* dstFileName, const char* srcFileName, int FIO_decompressFilename(FIO_prefs_t* const prefs,
const char* dstFileName, const char* srcFileName,
const char* dictFileName) const char* dictFileName)
{ {
dRess_t const ress = FIO_createDResources(dictFileName); dRess_t const ress = FIO_createDResources(prefs, dictFileName);
int const decodingError = FIO_decompressSrcFile(ress, dstFileName, srcFileName); int const decodingError = FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
FIO_freeDResources(ress); FIO_freeDResources(ress);
return decodingError; return decodingError;
@ -2098,19 +2189,20 @@ FIO_determineDstName(const char* srcFileName)
int int
FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles, FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char* srcNamesTable[], unsigned nbFiles,
const char* outFileName, const char* outFileName,
const char* dictFileName) const char* dictFileName)
{ {
int error = 0; int error = 0;
dRess_t ress = FIO_createDResources(dictFileName); dRess_t ress = FIO_createDResources(prefs, dictFileName);
if (outFileName) { if (outFileName) {
unsigned u; unsigned u;
ress.dstFile = FIO_openDstFile(NULL, outFileName); ress.dstFile = FIO_openDstFile(prefs, NULL, outFileName);
if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName); if (ress.dstFile == 0) EXM_THROW(71, "cannot open %s", outFileName);
for (u=0; u<nbFiles; u++) for (u=0; u<nbFiles; u++)
error |= FIO_decompressSrcFile(ress, outFileName, srcNamesTable[u]); error |= FIO_decompressSrcFile(prefs, ress, outFileName, srcNamesTable[u]);
if (fclose(ress.dstFile)) if (fclose(ress.dstFile))
EXM_THROW(72, "Write error : %s : cannot properly close output file", EXM_THROW(72, "Write error : %s : cannot properly close output file",
strerror(errno)); strerror(errno));
@ -2121,7 +2213,7 @@ FIO_decompressMultipleFilenames(const char* srcNamesTable[], unsigned nbFiles,
const char* const dstFileName = FIO_determineDstName(srcFileName); const char* const dstFileName = FIO_determineDstName(srcFileName);
if (dstFileName == NULL) { error=1; continue; } if (dstFileName == NULL) { error=1; continue; }
error |= FIO_decompressSrcFile(ress, dstFileName, srcFileName); error |= FIO_decompressSrcFile(prefs, ress, dstFileName, srcFileName);
} }
} }

60
programs/fileio.h
View File

@ -42,44 +42,52 @@ extern "C" {
***************************************/ ***************************************/
typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression, FIO_lz4Compression } FIO_compressionType_t; typedef enum { FIO_zstdCompression, FIO_gzipCompression, FIO_xzCompression, FIO_lzmaCompression, FIO_lz4Compression } FIO_compressionType_t;
typedef struct FIO_prefs_s FIO_prefs_t;
FIO_prefs_t* FIO_createPreferences(void);
void FIO_freePreferences(FIO_prefs_t* const prefs);
typedef struct FIO_display_prefs_s FIO_display_prefs_t;
/*-************************************* /*-*************************************
* Parameters * Parameters
***************************************/ ***************************************/
void FIO_setCompressionType(FIO_compressionType_t compressionType); void FIO_setCompressionType(FIO_prefs_t* const prefs, FIO_compressionType_t compressionType);
void FIO_overwriteMode(void); void FIO_overwriteMode(FIO_prefs_t* const prefs);
void FIO_setAdaptiveMode(unsigned adapt); void FIO_setAdaptiveMode(FIO_prefs_t* const prefs, unsigned adapt);
void FIO_setAdaptMin(int minCLevel); void FIO_setAdaptMin(FIO_prefs_t* const prefs, int minCLevel);
void FIO_setAdaptMax(int maxCLevel); void FIO_setAdaptMax(FIO_prefs_t* const prefs, int maxCLevel);
void FIO_setBlockSize(unsigned blockSize); void FIO_setBlockSize(FIO_prefs_t* const prefs, unsigned blockSize);
void FIO_setChecksumFlag(unsigned checksumFlag); void FIO_setChecksumFlag(FIO_prefs_t* const prefs, unsigned checksumFlag);
void FIO_setDictIDFlag(unsigned dictIDFlag); void FIO_setDictIDFlag(FIO_prefs_t* const prefs, unsigned dictIDFlag);
void FIO_setLdmBucketSizeLog(unsigned ldmBucketSizeLog); void FIO_setLdmBucketSizeLog(FIO_prefs_t* const prefs, unsigned ldmBucketSizeLog);
void FIO_setLdmFlag(unsigned ldmFlag); void FIO_setLdmFlag(FIO_prefs_t* const prefs, unsigned ldmFlag);
void FIO_setLdmHashRateLog(unsigned ldmHashRateLog); void FIO_setLdmHashRateLog(FIO_prefs_t* const prefs, unsigned ldmHashRateLog);
void FIO_setLdmHashLog(unsigned ldmHashLog); void FIO_setLdmHashLog(FIO_prefs_t* const prefs, unsigned ldmHashLog);
void FIO_setLdmMinMatch(unsigned ldmMinMatch); void FIO_setLdmMinMatch(FIO_prefs_t* const prefs, unsigned ldmMinMatch);
void FIO_setMemLimit(unsigned memLimit); void FIO_setMemLimit(FIO_prefs_t* const prefs, unsigned memLimit);
void FIO_setNbWorkers(unsigned nbWorkers); void FIO_setNbWorkers(FIO_prefs_t* const prefs, unsigned nbWorkers);
void FIO_setNotificationLevel(unsigned level); void FIO_setOverlapLog(FIO_prefs_t* const prefs, unsigned overlapLog);
void FIO_setOverlapLog(unsigned overlapLog); void FIO_setRemoveSrcFile(FIO_prefs_t* const prefs, unsigned flag);
void FIO_setRemoveSrcFile(unsigned flag); void FIO_setSparseWrite(FIO_prefs_t* const prefs, unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */
void FIO_setSparseWrite(unsigned sparse); /**< 0: no sparse; 1: disable on stdout; 2: always enabled */ void FIO_setRsyncable(FIO_prefs_t* const prefs, unsigned rsyncable);
void FIO_setRsyncable(unsigned rsyncable);
void FIO_setNoProgress(unsigned noProgress);
void FIO_setNoProgress(unsigned noProgress);
void FIO_setNotificationLevel(unsigned level);
/*-************************************* /*-*************************************
* Single File functions * Single File functions
***************************************/ ***************************************/
/** FIO_compressFilename() : /** FIO_compressFilename() :
@return : 0 == ok; 1 == pb with src file. */ @return : 0 == ok; 1 == pb with src file. */
int FIO_compressFilename (const char* outfilename, const char* infilename, const char* dictFileName, int FIO_compressFilename (FIO_prefs_t* const prefs,
const char* outfilename, const char* infilename, const char* dictFileName,
int compressionLevel, ZSTD_compressionParameters comprParams); int compressionLevel, ZSTD_compressionParameters comprParams);
/** FIO_decompressFilename() : /** FIO_decompressFilename() :
@return : 0 == ok; 1 == pb with src file. */ @return : 0 == ok; 1 == pb with src file. */
int FIO_decompressFilename (const char* outfilename, const char* infilename, const char* dictFileName); int FIO_decompressFilename (FIO_prefs_t* const prefs,
const char* outfilename, const char* infilename, const char* dictFileName);
int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel); int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int displayLevel);
@ -89,14 +97,16 @@ int FIO_listMultipleFiles(unsigned numFiles, const char** filenameTable, int dis
***************************************/ ***************************************/
/** FIO_compressMultipleFilenames() : /** FIO_compressMultipleFilenames() :
@return : nb of missing files */ @return : nb of missing files */
int FIO_compressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles, int FIO_compressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles,
const char* outFileName, const char* suffix, const char* outFileName, const char* suffix,
const char* dictFileName, int compressionLevel, const char* dictFileName, int compressionLevel,
ZSTD_compressionParameters comprParams); ZSTD_compressionParameters comprParams);
/** FIO_decompressMultipleFilenames() : /** FIO_decompressMultipleFilenames() :
@return : nb of missing or skipped files */ @return : nb of missing or skipped files */
int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles, int FIO_decompressMultipleFilenames(FIO_prefs_t* const prefs,
const char** srcNamesTable, unsigned nbFiles,
const char* outFileName, const char* outFileName,
const char* dictFileName); const char* dictFileName);

94
programs/zstdcli.c
View File

@ -536,6 +536,8 @@ int main(int argCount, const char* argv[])
double compressibility = 0.5; double compressibility = 0.5;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0; size_t blockSize = 0;
FIO_prefs_t* const prefs = FIO_createPreferences();
zstd_operation_mode operation = zom_compress; zstd_operation_mode operation = zom_compress;
ZSTD_compressionParameters compressionParams; ZSTD_compressionParameters compressionParams;
int cLevel; int cLevel;
@ -582,17 +584,17 @@ int main(int argCount, const char* argv[])
/* preset behaviors */ /* preset behaviors */
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0; if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress; if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */ if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(prefs); outFileName=stdoutmark; g_displayLevel=1; } /* supports multiple formats */
if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(prefs); outFileName=stdoutmark; g_displayLevel=1; } /* behave like zcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); } /* behave like gzip */ if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like gzip */
if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); } /* behave like gunzip, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(prefs, 1); } /* behave like gunzip, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(prefs); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like lzma */ if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like lzma */
if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); } /* behave like unlzma, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lzmaCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unlzma, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like xz */ if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like xz */
if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); } /* behave like unxz, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_xzCompression); FIO_setRemoveSrcFile(prefs, 1); } /* behave like unxz, also supports multiple formats */
if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); } /* behave like lz4 */ if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); } /* behave like lz4 */
if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */ if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(prefs, FIO_lz4Compression); } /* behave like unlz4, also supports multiple formats */
memset(&compressionParams, 0, sizeof(compressionParams)); memset(&compressionParams, 0, sizeof(compressionParams));
/* init crash handler */ /* init crash handler */
@ -623,38 +625,38 @@ int main(int argCount, const char* argv[])
if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
if (!strcmp(argument, "--force")) { FIO_overwriteMode(); forceStdout=1; followLinks=1; continue; } if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; continue; }
if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); } if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; } if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; } if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; } if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
if (!strcmp(argument, "--ultra")) { ultra=1; continue; } if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; } if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(prefs, 2); continue; }
if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; } if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; } if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; } if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(prefs, 0); continue; }
if (!strcmp(argument, "--test")) { operation=zom_test; continue; } if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; } if (!strcmp(argument, "--train")) { operation=zom_train; if (outFileName==NULL) outFileName=g_defaultDictName; continue; }
if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */ if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; } /* kept available for compatibility with old syntax ; will be removed one day */
if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; } if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(prefs, 0); continue; }
if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; } if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(prefs, 0); continue; }
if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; } if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(prefs, 1); continue; }
if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; } if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
if (!strcmp(argument, "--adapt")) { adapt = 1; continue; } if (!strcmp(argument, "--adapt")) { adapt = 1; continue; }
if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; } if (longCommandWArg(&argument, "--adapt=")) { adapt = 1; if (!parseAdaptParameters(argument, &adaptMin, &adaptMax)) CLEAN_RETURN(badusage(programName)); continue; }
if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; } if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(FIO_zstdCompression); continue; } if (!strcmp(argument, "--format=zstd")) { suffix = ZSTD_EXTENSION; FIO_setCompressionType(prefs, FIO_zstdCompression); continue; }
#ifdef ZSTD_GZCOMPRESS #ifdef ZSTD_GZCOMPRESS
if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; } if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(prefs, FIO_gzipCompression); continue; }
#endif #endif
#ifdef ZSTD_LZMACOMPRESS #ifdef ZSTD_LZMACOMPRESS
if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); continue; } if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(prefs, FIO_lzmaCompression); continue; }
if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); continue; } if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(prefs, FIO_xzCompression); continue; }
#endif #endif
#ifdef ZSTD_LZ4COMPRESS #ifdef ZSTD_LZ4COMPRESS
if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); continue; } if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(prefs, FIO_lz4Compression); continue; }
#endif #endif
if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; } if (!strcmp(argument, "--rsyncable")) { rsyncable = 1; continue; }
if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; } if (!strcmp(argument, "--no-progress")) { FIO_setNoProgress(1); continue; }
@ -784,7 +786,7 @@ int main(int argCount, const char* argv[])
case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break; case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
/* Overwrite */ /* Overwrite */
case 'f': FIO_overwriteMode(); forceStdout=1; followLinks=1; argument++; break; case 'f': FIO_overwriteMode(prefs); forceStdout=1; followLinks=1; argument++; break;
/* Verbose mode */ /* Verbose mode */
case 'v': g_displayLevel++; argument++; break; case 'v': g_displayLevel++; argument++; break;
@ -793,10 +795,10 @@ int main(int argCount, const char* argv[])
case 'q': g_displayLevel--; argument++; break; case 'q': g_displayLevel--; argument++; break;
/* keep source file (default) */ /* keep source file (default) */
case 'k': FIO_setRemoveSrcFile(0); argument++; break; case 'k': FIO_setRemoveSrcFile(prefs, 0); argument++; break;
/* Checksum */ /* Checksum */
case 'C': FIO_setChecksumFlag(2); argument++; break; case 'C': FIO_setChecksumFlag(prefs, 2); argument++; break;
/* test compressed file */ /* test compressed file */
case 't': operation=zom_test; argument++; break; case 't': operation=zom_test; argument++; break;
@ -1057,7 +1059,7 @@ int main(int argCount, const char* argv[])
} }
#ifndef ZSTD_NODECOMPRESS #ifndef ZSTD_NODECOMPRESS
if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); } /* test mode */ if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(prefs, 0); } /* test mode */
#endif #endif
/* No input filename ==> use stdin and stdout */ /* No input filename ==> use stdin and stdout */
@ -1092,25 +1094,25 @@ int main(int argCount, const char* argv[])
FIO_setNotificationLevel(g_displayLevel); FIO_setNotificationLevel(g_displayLevel);
if (operation==zom_compress) { if (operation==zom_compress) {
#ifndef ZSTD_NOCOMPRESS #ifndef ZSTD_NOCOMPRESS
FIO_setNbWorkers(nbWorkers); FIO_setNbWorkers(prefs, nbWorkers);
FIO_setBlockSize((U32)blockSize); FIO_setBlockSize(prefs, (U32)blockSize);
if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog); if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(prefs, g_overlapLog);
FIO_setLdmFlag(ldmFlag); FIO_setLdmFlag(prefs, ldmFlag);
FIO_setLdmHashLog(g_ldmHashLog); FIO_setLdmHashLog(prefs, g_ldmHashLog);
FIO_setLdmMinMatch(g_ldmMinMatch); FIO_setLdmMinMatch(prefs, g_ldmMinMatch);
if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog); if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) FIO_setLdmBucketSizeLog(prefs, g_ldmBucketSizeLog);
if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(g_ldmHashRateLog); if (g_ldmHashRateLog != LDM_PARAM_DEFAULT) FIO_setLdmHashRateLog(prefs, g_ldmHashRateLog);
FIO_setAdaptiveMode(adapt); FIO_setAdaptiveMode(prefs, adapt);
FIO_setAdaptMin(adaptMin); FIO_setAdaptMin(prefs, adaptMin);
FIO_setAdaptMax(adaptMax); FIO_setAdaptMax(prefs, adaptMax);
FIO_setRsyncable(rsyncable); FIO_setRsyncable(prefs, rsyncable);
if (adaptMin > cLevel) cLevel = adaptMin; if (adaptMin > cLevel) cLevel = adaptMin;
if (adaptMax < cLevel) cLevel = adaptMax; if (adaptMax < cLevel) cLevel = adaptMax;
if ((filenameIdx==1) && outFileName) if ((filenameIdx==1) && outFileName)
operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, compressionParams); operationResult = FIO_compressFilename(prefs, outFileName, filenameTable[0], dictFileName, cLevel, compressionParams);
else else
operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams); operationResult = FIO_compressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, compressionParams);
#else #else
(void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */ (void)suffix; (void)adapt; (void)rsyncable; (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
DISPLAY("Compression not supported \n"); DISPLAY("Compression not supported \n");
@ -1124,17 +1126,19 @@ int main(int argCount, const char* argv[])
memLimit = (U32)1 << (compressionParams.windowLog & 31); memLimit = (U32)1 << (compressionParams.windowLog & 31);
} }
} }
FIO_setMemLimit(memLimit); FIO_setMemLimit(prefs, memLimit);
if (filenameIdx==1 && outFileName) if (filenameIdx==1 && outFileName)
operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName); operationResult = FIO_decompressFilename(prefs, outFileName, filenameTable[0], dictFileName);
else else
operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName); operationResult = FIO_decompressMultipleFilenames(prefs, filenameTable, filenameIdx, outFileName, dictFileName);
#else #else
DISPLAY("Decompression not supported \n"); DISPLAY("Decompression not supported \n");
#endif #endif
} }
_end: _end:
FIO_freePreferences(prefs);
if (main_pause) waitEnter(); if (main_pause) waitEnter();
#ifdef UTIL_HAS_CREATEFILELIST #ifdef UTIL_HAS_CREATEFILELIST
if (extendedFileList) if (extendedFileList)