diff options
Diffstat (limited to 'src/zstd/tests/paramgrill.c')
| -rw-r--r-- | src/zstd/tests/paramgrill.c | 1049 |
1 files changed, 1049 insertions, 0 deletions
diff --git a/src/zstd/tests/paramgrill.c b/src/zstd/tests/paramgrill.c new file mode 100644 index 00000000..317ec461 --- /dev/null +++ b/src/zstd/tests/paramgrill.c @@ -0,0 +1,1049 @@ +/* + * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*-************************************ +* Dependencies +**************************************/ +#include "util.h" /* Compiler options, UTIL_GetFileSize */ +#include <stdlib.h> /* malloc */ +#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <string.h> /* strcmp */ +#include <math.h> /* log */ +#include <time.h> /* clock_t */ + +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ +#include "zstd.h" +#include "datagen.h" +#include "xxhash.h" + + +/*-************************************ +* Constants +**************************************/ +#define PROGRAM_DESCRIPTION "ZSTD parameters tester" +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR, __DATE__ + + +#define KB *(1<<10) +#define MB *(1<<20) +#define GB *(1ULL<<30) + +#define NBLOOPS 2 +#define TIMELOOP (2 * CLOCKS_PER_SEC) + +#define NB_LEVELS_TRACKED 30 + +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); + +#define COMPRESSIBILITY_DEFAULT 0.50 +static const size_t sampleSize = 10000000; + +static const double g_grillDuration_s = 90000; /* about 24 hours */ +static const clock_t g_maxParamTime = 15 * CLOCKS_PER_SEC; +static const clock_t g_maxVariationTime = 60 * CLOCKS_PER_SEC; +static const int g_maxNbVariations = 64; + + +/*-************************************ +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + +#undef MIN +#undef MAX +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) + + +/*-************************************ +* Benchmark Parameters +**************************************/ +static U32 g_nbIterations = NBLOOPS; +static double g_compressibility = COMPRESSIBILITY_DEFAULT; +static U32 g_blockSize = 0; +static U32 g_rand = 1; +static U32 g_singleRun = 0; +static U32 g_target = 0; +static U32 g_noSeed = 0; +static ZSTD_compressionParameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; + +void BMK_SetNbIterations(int nbLoops) +{ + g_nbIterations = nbLoops; + DISPLAY("- %u iterations -\n", g_nbIterations); +} + + +/*-******************************************************* +* Private functions +*********************************************************/ + +/* works even if overflow ; max span ~ 30 mn */ +static clock_t BMK_clockSpan(clock_t cStart) { return clock() - cStart; } + +/* accuracy in seconds only, span can be multiple years */ +static double BMK_timeSpan(time_t tStart) { return difftime(time(NULL), tStart); } + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t const step = 64 MB; + void* testmem = NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + if (requiredMem > maxMemory) requiredMem = maxMemory; + + requiredMem += 2*step; + while (!testmem) { + requiredMem -= step; + testmem = malloc ((size_t)requiredMem); + } + + free (testmem); + return (size_t) (requiredMem - step); +} + + +static U32 FUZ_rotl32(U32 x, U32 r) +{ + return ((x << r) | (x >> (32 - r))); +} + +U32 FUZ_rand(U32* src) +{ + const U32 prime1 = 2654435761U; + const U32 prime2 = 2246822519U; + U32 rand32 = *src; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + + +/*-******************************************************* +* Bench functions +*********************************************************/ +typedef struct { + size_t cSize; + double cSpeed; /* bytes / sec */ + double dSpeed; +} BMK_result_t; + +typedef struct +{ + const char* srcPtr; + size_t srcSize; + char* cPtr; + size_t cRoom; + size_t cSize; + char* resPtr; + size_t resSize; +} blockParam_t; + + +static size_t BMK_benchParam(BMK_result_t* resultPtr, + const void* srcBuffer, size_t srcSize, + ZSTD_CCtx* ctx, + const ZSTD_compressionParameters cParams) +{ + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + const U32 nbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize); + blockParam_t* const blockTable = (blockParam_t*) malloc(nbBlocks * sizeof(blockParam_t)); + const size_t maxCompressedSize = (size_t)nbBlocks * ZSTD_compressBound(blockSize); + void* const compressedBuffer = malloc(maxCompressedSize); + void* const resultBuffer = malloc(srcSize); + ZSTD_parameters params; + U32 Wlog = cParams.windowLog; + U32 Clog = cParams.chainLog; + U32 Hlog = cParams.hashLog; + U32 Slog = cParams.searchLog; + U32 Slength = cParams.searchLength; + U32 Tlength = cParams.targetLength; + ZSTD_strategy strat = cParams.strategy; + char name[30] = { 0 }; + U64 crcOrig; + + /* init result for early exit */ + resultPtr->cSize = srcSize; + resultPtr->cSpeed = 0.; + resultPtr->dSpeed = 0.; + + /* Memory allocation & restrictions */ + snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%03uS%1u", Wlog, Clog, Hlog, Slog, Slength, Tlength, strat); + if (!compressedBuffer || !resultBuffer || !blockTable) { + DISPLAY("\nError: not enough memory!\n"); + free(compressedBuffer); + free(resultBuffer); + free(blockTable); + return 12; + } + + /* Calculating input Checksum */ + crcOrig = XXH64(srcBuffer, srcSize, 0); + + /* Init blockTable data */ + { + U32 i; + size_t remaining = srcSize; + const char* srcPtr = (const char*)srcBuffer; + char* cPtr = (char*)compressedBuffer; + char* resPtr = (char*)resultBuffer; + for (i=0; i<nbBlocks; i++) { + size_t thisBlockSize = MIN(remaining, blockSize); + blockTable[i].srcPtr = srcPtr; + blockTable[i].cPtr = cPtr; + blockTable[i].resPtr = resPtr; + blockTable[i].srcSize = thisBlockSize; + blockTable[i].cRoom = ZSTD_compressBound(thisBlockSize); + srcPtr += thisBlockSize; + cPtr += blockTable[i].cRoom; + resPtr += thisBlockSize; + remaining -= thisBlockSize; + } } + + /* warmimg up memory */ + RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.10, 1); + + /* Bench */ + { U32 loopNb; + size_t cSize = 0; + double fastestC = 100000000., fastestD = 100000000.; + double ratio = 0.; + clock_t const benchStart = clock(); + + DISPLAY("\r%79s\r", ""); + memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + for (loopNb = 1; loopNb <= g_nbIterations; loopNb++) { + int nbLoops; + U32 blockNb; + clock_t roundStart, roundClock; + + { clock_t const benchTime = BMK_clockSpan(benchStart); + if (benchTime > g_maxParamTime) break; } + + /* Compression */ + DISPLAY("\r%1u-%s : %9u ->", loopNb, name, (U32)srcSize); + memset(compressedBuffer, 0xE5, maxCompressedSize); + + nbLoops = 0; + roundStart = clock(); + while (clock() == roundStart); + roundStart = clock(); + while (BMK_clockSpan(roundStart) < TIMELOOP) { + for (blockNb=0; blockNb<nbBlocks; blockNb++) + blockTable[blockNb].cSize = ZSTD_compress_advanced(ctx, + blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, + blockTable[blockNb].srcPtr, blockTable[blockNb].srcSize, + NULL, 0, + params); + nbLoops++; + } + roundClock = BMK_clockSpan(roundStart); + + cSize = 0; + for (blockNb=0; blockNb<nbBlocks; blockNb++) + cSize += blockTable[blockNb].cSize; + ratio = (double)srcSize / (double)cSize; + if ((double)roundClock < fastestC * CLOCKS_PER_SEC * nbLoops) fastestC = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops; + DISPLAY("\r"); + DISPLAY("%1u-%s : %9u ->", loopNb, name, (U32)srcSize); + DISPLAY(" %9u (%4.3f),%7.1f MB/s", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.); + resultPtr->cSize = cSize; + resultPtr->cSpeed = (double)srcSize / fastestC; + +#if 1 + /* Decompression */ + memset(resultBuffer, 0xD6, srcSize); + + nbLoops = 0; + roundStart = clock(); + while (clock() == roundStart); + roundStart = clock(); + for ( ; BMK_clockSpan(roundStart) < TIMELOOP; nbLoops++) { + for (blockNb=0; blockNb<nbBlocks; blockNb++) + blockTable[blockNb].resSize = ZSTD_decompress(blockTable[blockNb].resPtr, blockTable[blockNb].srcSize, + blockTable[blockNb].cPtr, blockTable[blockNb].cSize); + } + roundClock = BMK_clockSpan(roundStart); + + if ((double)roundClock < fastestD * CLOCKS_PER_SEC * nbLoops) fastestD = ((double)roundClock / CLOCKS_PER_SEC) / nbLoops; + DISPLAY("\r"); + DISPLAY("%1u-%s : %9u -> ", loopNb, name, (U32)srcSize); + DISPLAY("%9u (%4.3f),%7.1f MB/s, ", (U32)cSize, ratio, (double)srcSize / fastestC / 1000000.); + DISPLAY("%7.1f MB/s", (double)srcSize / fastestD / 1000000.); + resultPtr->dSpeed = (double)srcSize / fastestD; + + /* CRC Checking */ + { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + if (crcOrig!=crcCheck) { + unsigned u; + unsigned eBlockSize = (unsigned)(MIN(65536*2, blockSize)); + DISPLAY("\n!!! WARNING !!! Invalid Checksum : %x != %x\n", (unsigned)crcOrig, (unsigned)crcCheck); + for (u=0; u<srcSize; u++) { + if (((const BYTE*)srcBuffer)[u] != ((BYTE*)resultBuffer)[u]) { + printf("Decoding error at pos %u (block %u, pos %u) \n", u, u / eBlockSize, u % eBlockSize); + break; + } } + break; + } } +#endif + } } + + /* End cleaning */ + DISPLAY("\r"); + free(compressedBuffer); + free(resultBuffer); + return 0; +} + + +const char* g_stratName[] = { "ZSTD_fast ", + "ZSTD_dfast ", + "ZSTD_greedy ", + "ZSTD_lazy ", + "ZSTD_lazy2 ", + "ZSTD_btlazy2 ", + "ZSTD_btopt ", + "ZSTD_btultra "}; + +static void BMK_printWinner(FILE* f, U32 cLevel, BMK_result_t result, ZSTD_compressionParameters params, size_t srcSize) +{ + DISPLAY("\r%79s\r", ""); + fprintf(f," {%3u,%3u,%3u,%3u,%3u,%3u, %s }, ", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, + params.targetLength, g_stratName[(U32)(params.strategy)]); + fprintf(f, + "/* level %2u */ /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + cLevel, (double)srcSize / result.cSize, result.cSpeed / 1000000., result.dSpeed / 1000000.); +} + + +static double g_cSpeedTarget[NB_LEVELS_TRACKED] = { 0. }; /* NB_LEVELS_TRACKED : checked at main() */ + +typedef struct { + BMK_result_t result; + ZSTD_compressionParameters params; +} winnerInfo_t; + +static void BMK_printWinners2(FILE* f, const winnerInfo_t* winners, size_t srcSize) +{ + int cLevel; + + fprintf(f, "\n /* Proposed configurations : */ \n"); + fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); + + for (cLevel=0; cLevel <= ZSTD_maxCLevel(); cLevel++) + BMK_printWinner(f, cLevel, winners[cLevel].result, winners[cLevel].params, srcSize); +} + + +static void BMK_printWinners(FILE* f, const winnerInfo_t* winners, size_t srcSize) +{ + fseek(f, 0, SEEK_SET); + BMK_printWinners2(f, winners, srcSize); + fflush(f); + BMK_printWinners2(stdout, winners, srcSize); +} + +static int BMK_seed(winnerInfo_t* winners, const ZSTD_compressionParameters params, + const void* srcBuffer, size_t srcSize, + ZSTD_CCtx* ctx) +{ + BMK_result_t testResult; + int better = 0; + int cLevel; + + BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); + + for (cLevel = 1; cLevel <= ZSTD_maxCLevel(); cLevel++) { + if (testResult.cSpeed < g_cSpeedTarget[cLevel]) + continue; /* not fast enough for this level */ + if (winners[cLevel].result.cSize==0) { + /* first solution for this cLevel */ + winners[cLevel].result = testResult; + winners[cLevel].params = params; + BMK_printWinner(stdout, cLevel, testResult, params, srcSize); + better = 1; + continue; + } + + if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) { + /* Validate solution is "good enough" */ + double W_ratio = (double)srcSize / testResult.cSize; + double O_ratio = (double)srcSize / winners[cLevel].result.cSize; + double W_ratioNote = log (W_ratio); + double O_ratioNote = log (O_ratio); + size_t W_DMemUsed = (1 << params.windowLog) + (16 KB); + size_t O_DMemUsed = (1 << winners[cLevel].params.windowLog) + (16 KB); + double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); + double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); + + size_t W_CMemUsed = (1 << params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(params); + size_t O_CMemUsed = (1 << winners[cLevel].params.windowLog) + ZSTD_estimateCCtxSize_usingCParams(winners[cLevel].params); + double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); + double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); + + double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); + double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); + + double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); + double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); + + if (W_DMemUsed_note < O_DMemUsed_note) { + /* uses too much Decompression memory for too little benefit */ + if (W_ratio > O_ratio) + DISPLAY ("Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", + W_ratio, (double)(W_DMemUsed) / 1024 / 1024, + O_ratio, (double)(O_DMemUsed) / 1024 / 1024, cLevel); + continue; + } + if (W_CMemUsed_note < O_CMemUsed_note) { + /* uses too much memory for compression for too little benefit */ + if (W_ratio > O_ratio) + DISPLAY ("Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", + W_ratio, (double)(W_CMemUsed) / 1024 / 1024, + O_ratio, (double)(O_CMemUsed) / 1024 / 1024, cLevel); + continue; + } + if (W_CSpeed_note < O_CSpeed_note ) { + /* too large compression speed difference for the compression benefit */ + if (W_ratio > O_ratio) + DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", + W_ratio, testResult.cSpeed / 1000000, + O_ratio, winners[cLevel].result.cSpeed / 1000000., cLevel); + continue; + } + if (W_DSpeed_note < O_DSpeed_note ) { + /* too large decompression speed difference for the compression benefit */ + if (W_ratio > O_ratio) + DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", + W_ratio, testResult.dSpeed / 1000000., + O_ratio, winners[cLevel].result.dSpeed / 1000000., cLevel); + continue; + } + + if (W_ratio < O_ratio) + DISPLAY("Solution %4.3f selected over %4.3f at level %i, due to better secondary statistics \n", W_ratio, O_ratio, cLevel); + + winners[cLevel].result = testResult; + winners[cLevel].params = params; + BMK_printWinner(stdout, cLevel, testResult, params, srcSize); + + better = 1; + } } + + return better; +} + + +/* nullified useless params, to ensure count stats */ +static ZSTD_compressionParameters* sanitizeParams(ZSTD_compressionParameters params) +{ + g_params = params; + if (params.strategy == ZSTD_fast) + g_params.chainLog = 0, g_params.searchLog = 0; + if (params.strategy == ZSTD_dfast) + g_params.searchLog = 0; + if (params.strategy != ZSTD_btopt && params.strategy != ZSTD_btultra) + g_params.targetLength = 0; + return &g_params; +} + + +static void paramVariation(ZSTD_compressionParameters* ptr) +{ + ZSTD_compressionParameters p; + U32 validated = 0; + while (!validated) { + U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; + p = *ptr; + for ( ; nbChanges ; nbChanges--) { + const U32 changeID = FUZ_rand(&g_rand) % 14; + switch(changeID) + { + case 0: + p.chainLog++; break; + case 1: + p.chainLog--; break; + case 2: + p.hashLog++; break; + case 3: + p.hashLog--; break; + case 4: + p.searchLog++; break; + case 5: + p.searchLog--; break; + case 6: + p.windowLog++; break; + case 7: + p.windowLog--; break; + case 8: + p.searchLength++; break; + case 9: + p.searchLength--; break; + case 10: + p.strategy = (ZSTD_strategy)(((U32)p.strategy)+1); break; + case 11: + p.strategy = (ZSTD_strategy)(((U32)p.strategy)-1); break; + case 12: + p.targetLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; + case 13: + p.targetLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; + } + } + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + } + *ptr = p; +} + + +#define PARAMTABLELOG 25 +#define PARAMTABLESIZE (1<<PARAMTABLELOG) +#define PARAMTABLEMASK (PARAMTABLESIZE-1) +static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ + +#define NB_TESTS_PLAYED(p) \ + g_alreadyTested[(XXH64(sanitizeParams(p), sizeof(p), 0) >> 3) & PARAMTABLEMASK] + + +static void playAround(FILE* f, winnerInfo_t* winners, + ZSTD_compressionParameters params, + const void* srcBuffer, size_t srcSize, + ZSTD_CCtx* ctx) +{ + int nbVariations = 0; + clock_t const clockStart = clock(); + + while (BMK_clockSpan(clockStart) < g_maxVariationTime) { + ZSTD_compressionParameters p = params; + + if (nbVariations++ > g_maxNbVariations) break; + paramVariation(&p); + + /* exclude faster if already played params */ + if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(p))-1)) + continue; + + /* test */ + NB_TESTS_PLAYED(p)++; + if (!BMK_seed(winners, p, srcBuffer, srcSize, ctx)) continue; + + /* improvement found => search more */ + BMK_printWinners(f, winners, srcSize); + playAround(f, winners, p, srcBuffer, srcSize, ctx); + } + +} + + +static ZSTD_compressionParameters randomParams(void) +{ + ZSTD_compressionParameters p; + U32 validated = 0; + while (!validated) { + /* totally random entry */ + p.chainLog = FUZ_rand(&g_rand) % (ZSTD_CHAINLOG_MAX+1 - ZSTD_CHAINLOG_MIN) + ZSTD_CHAINLOG_MIN; + p.hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN; + p.searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN; + p.windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN; + p.searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN; + p.targetLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN; + p.strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btultra +1)); + validated = !ZSTD_isError(ZSTD_checkCParams(p)); + } + return p; +} + +static void BMK_selectRandomStart( + FILE* f, winnerInfo_t* winners, + const void* srcBuffer, size_t srcSize, + ZSTD_CCtx* ctx) +{ + U32 const id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1)); + if ((id==0) || (winners[id].params.windowLog==0)) { + /* totally random entry */ + ZSTD_compressionParameters const p = ZSTD_adjustCParams(randomParams(), srcSize, 0); + playAround(f, winners, p, srcBuffer, srcSize, ctx); + } + else + playAround(f, winners, winners[id].params, srcBuffer, srcSize, ctx); +} + + +static void BMK_benchMem(void* srcBuffer, size_t srcSize) +{ + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_compressionParameters params; + winnerInfo_t winners[NB_LEVELS_TRACKED]; + const char* const rfName = "grillResults.txt"; + FILE* const f = fopen(rfName, "w"); + const size_t blockSize = g_blockSize ? g_blockSize : srcSize; + + /* init */ + if (ctx==NULL) { DISPLAY("ZSTD_createCCtx() failed \n"); exit(1); } + memset(winners, 0, sizeof(winners)); + if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } + + if (g_singleRun) { + BMK_result_t testResult; + g_params = ZSTD_adjustCParams(g_params, srcSize, 0); + BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, g_params); + DISPLAY("\n"); + return; + } + + if (g_target) + g_cSpeedTarget[1] = g_target * 1000000; + else { + /* baseline config for level 1 */ + BMK_result_t testResult; + params = ZSTD_getCParams(1, blockSize, 0); + BMK_benchParam(&testResult, srcBuffer, srcSize, ctx, params); + g_cSpeedTarget[1] = (testResult.cSpeed * 31) / 32; + } + + /* establish speed objectives (relative to level 1) */ + { int i; + for (i=2; i<=ZSTD_maxCLevel(); i++) + g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) / 32; + } + + /* populate initial solution */ + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + int i; + for (i=0; i<=maxSeeds; i++) { + params = ZSTD_getCParams(i, blockSize, 0); + BMK_seed(winners, params, srcBuffer, srcSize, ctx); + } } + BMK_printWinners(f, winners, srcSize); + + /* start tests */ + { const time_t grillStart = time(NULL); + do { + BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx); + } while (BMK_timeSpan(grillStart) < g_grillDuration_s); + } + + /* end summary */ + BMK_printWinners(f, winners, srcSize); + DISPLAY("grillParams operations completed \n"); + + /* clean up*/ + fclose(f); + ZSTD_freeCCtx(ctx); +} + + +static int benchSample(void) +{ + void* origBuff; + size_t const benchedSize = sampleSize; + const char* const name = "Sample 10MiB"; + + /* Allocation */ + origBuff = malloc(benchedSize); + if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; } + + /* Fill buffer */ + RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("using %s %i%%: \n", name, (int)(g_compressibility*100)); + BMK_benchMem(origBuff, benchedSize); + + free(origBuff); + return 0; +} + + +int benchFiles(const char** fileNamesTable, int nbFiles) +{ + int fileIdx=0; + + /* Loop for each file */ + while (fileIdx<nbFiles) { + const char* const inFileName = fileNamesTable[fileIdx++]; + FILE* const inFile = fopen( inFileName, "rb" ); + U64 const inFileSize = UTIL_getFileSize(inFileName); + size_t benchedSize; + void* origBuff; + + /* Check file existence */ + if (inFile==NULL) { + DISPLAY( "Pb opening %s\n", inFileName); + return 11; + } + + /* Memory allocation */ + benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) + DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); + origBuff = malloc(benchedSize); + if (origBuff==NULL) { + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + return 12; + } + + /* Fill input buffer */ + DISPLAY("Loading %s... \r", inFileName); + { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if(readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("using %s : \n", inFileName); + BMK_benchMem(origBuff, benchedSize); + + /* clean */ + free(origBuff); + } + + return 0; +} + + +static void BMK_translateAdvancedParams(ZSTD_compressionParameters params) +{ + DISPLAY("--zstd=windowLog=%u,chainLog=%u,hashLog=%u,searchLog=%u,searchLength=%u,targetLength=%u,strategy=%u \n", + params.windowLog, params.chainLog, params.hashLog, params.searchLog, params.searchLength, params.targetLength, (U32)(params.strategy)); +} + +/* optimizeForSize(): + * targetSpeed : expressed in MB/s */ +int optimizeForSize(const char* inFileName, U32 targetSpeed) +{ + FILE* const inFile = fopen( inFileName, "rb" ); + U64 const inFileSize = UTIL_getFileSize(inFileName); + size_t benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + void* origBuff; + + /* Init */ + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } + + /* Memory allocation & restrictions */ + if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) { + DISPLAY("Not enough memory for '%s' \n", inFileName); + fclose(inFile); + return 11; + } + + /* Alloc */ + origBuff = malloc(benchedSize); + if(!origBuff) { + DISPLAY("\nError: not enough memory!\n"); + fclose(inFile); + return 12; + } + + /* Fill input buffer */ + DISPLAY("Loading %s... \r", inFileName); + { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if(readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("optimizing for %s - limit speed %u MB/s \n", inFileName, targetSpeed); + targetSpeed *= 1000000; + + { ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + winnerInfo_t winner; + BMK_result_t candidate; + const size_t blockSize = g_blockSize ? g_blockSize : benchedSize; + + /* init */ + if (ctx==NULL) { DISPLAY("\n ZSTD_createCCtx error \n"); free(origBuff); return 14;} + memset(&winner, 0, sizeof(winner)); + winner.result.cSize = (size_t)(-1); + + /* find best solution from default params */ + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + int i; + for (i=1; i<=maxSeeds; i++) { + ZSTD_compressionParameters const CParams = ZSTD_getCParams(i, blockSize, 0); + BMK_benchParam(&candidate, origBuff, benchedSize, ctx, CParams); + if (candidate.cSpeed < targetSpeed) + break; + if ( (candidate.cSize < winner.result.cSize) + | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) + { + winner.params = CParams; + winner.result = candidate; + BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); + } } + } + BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); + BMK_translateAdvancedParams(winner.params); + + /* start tests */ + { time_t const grillStart = time(NULL); + do { + ZSTD_compressionParameters params = winner.params; + paramVariation(¶ms); + if ((FUZ_rand(&g_rand) & 31) == 3) params = randomParams(); /* totally random config to improve search space */ + params = ZSTD_adjustCParams(params, blockSize, 0); + + /* exclude faster if already played set of params */ + if (FUZ_rand(&g_rand) & ((1 << NB_TESTS_PLAYED(params))-1)) continue; + + /* test */ + NB_TESTS_PLAYED(params)++; + BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params); + + /* improvement found => new winner */ + if ( (candidate.cSpeed > targetSpeed) + & ( (candidate.cSize < winner.result.cSize) + | ((candidate.cSize == winner.result.cSize) & (candidate.cSpeed > winner.result.cSpeed)) ) ) + { + winner.params = params; + winner.result = candidate; + BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); + BMK_translateAdvancedParams(winner.params); + } + } while (BMK_timeSpan(grillStart) < g_grillDuration_s); + } + + /* end summary */ + BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); + DISPLAY("grillParams size - optimizer completed \n"); + + /* clean up*/ + ZSTD_freeCCtx(ctx); + } + + free(origBuff); + return 0; +} + + +static int usage(const char* exename) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] file\n", exename); + DISPLAY( "Arguments :\n"); + DISPLAY( " file : path to the file used as reference (if none, generates a compressible sample)\n"); + DISPLAY( " -H/-h : Help (this text + advanced options)\n"); + return 0; +} + +static int usage_advanced(void) +{ + DISPLAY( "\nAdvanced options :\n"); + DISPLAY( " -T# : set level 1 speed objective \n"); + DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); + DISPLAY( " -i# : iteration loops [1-9](default : %i) \n", NBLOOPS); + DISPLAY( " -O# : find Optimized parameters for # MB/s compression speed (default : 0) \n"); + DISPLAY( " -S : Single run \n"); + DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); + return 0; +} + +static int badusage(const char* exename) +{ + DISPLAY("Wrong parameters\n"); + usage(exename); + return 1; +} + +int main(int argc, const char** argv) +{ + int i, + filenamesStart=0, + result; + const char* exename=argv[0]; + const char* input_filename=0; + U32 optimizer = 0; + U32 main_pause = 0; + U32 targetSpeed = 0; + + /* checks */ + if (NB_LEVELS_TRACKED <= ZSTD_maxCLevel()) { + DISPLAY("Error : NB_LEVELS_TRACKED <= ZSTD_maxCLevel() \n"); + exit(1); + } + + /* Welcome message */ + DISPLAY(WELCOME_MESSAGE); + + if (argc<1) { badusage(exename); return 1; } + + for(i=1; i<argc; i++) { + const char* argument = argv[i]; + + if(!argument) continue; /* Protection if argument empty */ + + if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } + + /* Decode command (note : aggregated commands are allowed) */ + if (argument[0]=='-') { + argument++; + + while (argument[0]!=0) { + + switch(argument[0]) + { + /* Display help on usage */ + case 'h' : + case 'H': usage(exename); usage_advanced(); return 0; + + /* Pause at the end (hidden option) */ + case 'p': main_pause = 1; argument++; break; + + /* Modify Nb Iterations */ + case 'i': + argument++; + if ((argument[0] >='0') & (argument[0] <='9')) + g_nbIterations = *argument++ - '0'; + break; + + /* Sample compressibility (when no file provided) */ + case 'P': + argument++; + { U32 proba32 = 0; + while ((argument[0]>= '0') & (argument[0]<= '9')) + proba32 = (proba32*10) + (*argument++ - '0'); + g_compressibility = (double)proba32 / 100.; + } + break; + + case 'O': + argument++; + optimizer=1; + targetSpeed = 0; + while ((*argument >= '0') & (*argument <= '9')) + targetSpeed = (targetSpeed*10) + (*argument++ - '0'); + break; + + /* Run Single conf */ + case 'S': + g_singleRun = 1; + argument++; + g_params = ZSTD_getCParams(2, g_blockSize, 0); + for ( ; ; ) { + switch(*argument) + { + case 'w': + g_params.windowLog = 0; + argument++; + while ((*argument>= '0') && (*argument<='9')) + g_params.windowLog *= 10, g_params.windowLog += *argument++ - '0'; + continue; + case 'c': + g_params.chainLog = 0; + argument++; + while ((*argument>= '0') && (*argument<='9')) + g_params.chainLog *= 10, g_params.chainLog += *argument++ - '0'; + continue; + case 'h': + g_params.hashLog = 0; + argument++; + while ((*argument>= '0') && (*argument<='9')) + g_params.hashLog *= 10, g_params.hashLog += *argument++ - '0'; + continue; + case 's': + g_params.searchLog = 0; + argument++; + while ((*argument>= '0') && (*argument<='9')) + g_params.searchLog *= 10, g_params.searchLog += *argument++ - '0'; + continue; + case 'l': /* search length */ + g_params.searchLength = 0; + argument++; + while ((*argument>= '0') && (*argument<='9')) + g_params.searchLength *= 10, g_params.searchLength += *argument++ - '0'; + continue; + case 't': /* target length */ + g_params.targetLength = 0; + argument++; + while ((*argument>= '0') && (*argument<='9')) + g_params.targetLength *= 10, g_params.targetLength += *argument++ - '0'; + continue; + case 'S': /* strategy */ + argument++; + while ((*argument>= '0') && (*argument<='9')) + g_params.strategy = (ZSTD_strategy)(*argument++ - '0'); + continue; + case 'L': + { int cLevel = 0; + argument++; + while ((*argument>= '0') && (*argument<='9')) + cLevel *= 10, cLevel += *argument++ - '0'; + g_params = ZSTD_getCParams(cLevel, g_blockSize, 0); + continue; + } + default : ; + } + break; + } + break; + + /* target level1 speed objective, in MB/s */ + case 'T': + argument++; + g_target = 0; + while ((*argument >= '0') && (*argument <= '9')) + g_target = (g_target*10) + (*argument++ - '0'); + break; + + /* cut input into blocks */ + case 'B': + g_blockSize = 0; + argument++; + while ((*argument >='0') & (*argument <='9')) + g_blockSize = (g_blockSize*10) + (*argument++ - '0'); + if (*argument=='K') g_blockSize<<=10, argument++; /* allows using KB notation */ + if (*argument=='M') g_blockSize<<=20, argument++; + if (*argument=='B') argument++; + DISPLAY("using %u KB block size \n", g_blockSize>>10); + break; + + /* Unknown command */ + default : return badusage(exename); + } + } + continue; + } /* if (argument[0]=='-') */ + + /* first provided filename is input */ + if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } + } + + if (filenamesStart==0) + result = benchSample(); + else { + if (optimizer) + result = optimizeForSize(input_filename, targetSpeed); + else + result = benchFiles(argv+filenamesStart, argc-filenamesStart); + } + + if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } + + return result; +} |