2 * Copyright (c) Meta Platforms, Inc. and affiliates.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
12 /*-************************************
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
22 /*-************************************
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #include <time.h> /* time(), time_t */
29 #undef NDEBUG /* always enable assert() */
31 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
32 #include "debug.h" /* DEBUG_STATIC_ASSERT */
34 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
35 #include "zstd.h" /* ZSTD_VERSION_STRING */
36 #include "zstd_errors.h" /* ZSTD_getErrorCode */
37 #define ZDICT_STATIC_LINKING_ONLY
38 #include "zdict.h" /* ZDICT_trainFromBuffer */
40 #include "datagen.h" /* RDG_genBuffer */
41 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
42 #include "xxhash.h" /* XXH64 */
44 #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
45 /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
46 #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
47 #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */
50 /*-************************************
52 **************************************/
55 static const int FUZ_compressibility_default = 50;
56 static const int nbTestsDefault = 30000;
59 /*-************************************
61 **************************************/
62 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
63 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
64 static U32 g_displayLevel = 2;
66 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
67 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
69 #define DISPLAYUPDATE(l, ...) \
70 if (g_displayLevel>=l) { \
71 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
72 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
73 if (g_displayLevel>=4) fflush(stderr); } \
77 /*-*******************************************************
79 *********************************************************/
82 /* Declaring the function, to avoid -Wmissing-prototype */
83 void FUZ_bug976(void);
85 { /* these constants shall not depend on MIN() macro */
86 DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
87 DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
91 /*-*******************************************************
93 *********************************************************/
94 #define MIN(a,b) ((a)<(b)?(a):(b))
95 #define MAX(a,b) ((a)>(b)?(a):(b))
97 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
98 static U32 FUZ_rand(U32* src)
100 static const U32 prime1 = 2654435761U;
101 static const U32 prime2 = 2246822519U;
105 rand32 = FUZ_rotl32(rand32, 13);
110 static U32 FUZ_highbit32(U32 v32)
113 if (v32==0) return 0;
114 while (v32) v32 >>= 1, nbBits++;
119 /*=============================================
121 =============================================*/
122 #define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
124 #define CHECK_Z(f) { \
125 size_t const err = f; \
126 if (ZSTD_isError(err)) { \
127 DISPLAY("Error => %s : %s ", \
128 #f, ZSTD_getErrorName(err)); \
132 #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
133 #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
134 #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
136 #define CHECK_OP(op, lhs, rhs) { \
137 if (!((lhs) op (rhs))) { \
138 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
142 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
143 #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
146 /*=============================================
148 =============================================*/
149 #if defined(__APPLE__) && defined(__MACH__)
151 #include <malloc/malloc.h> /* malloc_size */
154 unsigned long long totalMalloc;
155 size_t currentMalloc;
161 static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };
163 static void* FUZ_mallocDebug(void* counter, size_t size)
165 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
166 void* const ptr = malloc(size);
167 if (ptr==NULL) return NULL;
168 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
169 (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */
170 mcPtr->totalMalloc += size;
171 mcPtr->currentMalloc += size;
172 if (mcPtr->currentMalloc > mcPtr->peakMalloc)
173 mcPtr->peakMalloc = mcPtr->currentMalloc;
174 mcPtr->nbMalloc += 1;
178 static void FUZ_freeDebug(void* counter, void* address)
180 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
181 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10));
183 mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */
187 static void FUZ_displayMallocStats(mallocCounter_t count)
189 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n",
190 (unsigned)(count.peakMalloc >> 10),
192 (unsigned)(count.totalMalloc >> 10));
195 static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
196 void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
198 /* test only played in verbose mode, as they are long */
199 if (g_displayLevel<3) return 0;
201 /* Create compressible noise */
202 if (!inBuffer || !outBuffer) {
203 DISPLAY("Not enough memory, aborting\n");
206 RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
208 /* simple compression tests */
210 { int compressionLevel;
211 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
212 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
213 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
214 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
215 CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
217 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
218 FUZ_displayMallocStats(malcount);
221 /* streaming compression tests */
223 { int compressionLevel;
224 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
225 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
226 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
227 ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
228 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
229 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
230 CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
231 CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
232 CHECK_Z( ZSTD_endStream(cstream, &out) );
233 ZSTD_freeCStream(cstream);
234 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
235 FUZ_displayMallocStats(malcount);
238 /* advanced MT API test */
241 for (nbThreads=1; nbThreads<=4; nbThreads++) {
242 int compressionLevel;
243 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
244 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
245 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
246 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
247 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
248 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
249 CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
251 DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
252 nbThreads, compressionLevel);
253 FUZ_displayMallocStats(malcount);
256 /* advanced MT streaming API test */
259 for (nbThreads=1; nbThreads<=4; nbThreads++) {
260 int compressionLevel;
261 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
262 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
263 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
264 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
265 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
266 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
267 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
268 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
269 CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
270 while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
272 DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ",
273 nbThreads, compressionLevel);
274 FUZ_displayMallocStats(malcount);
280 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
282 size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
283 size_t const outSize = ZSTD_compressBound(inSize);
284 void* const inBuffer = malloc(inSize);
285 void* const outBuffer = malloc(outSize);
288 /* Create compressible noise */
289 if (!inBuffer || !outBuffer) {
290 DISPLAY("Not enough memory, aborting \n");
294 result = FUZ_mallocTests_internal(seed, compressibility, part,
295 inBuffer, inSize, outBuffer, outSize);
304 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
306 (void)seed; (void)compressibility; (void)part;
312 static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
313 BYTE* src, size_t size, ZSTD_sequenceFormat_e format)
317 for(i = 0; i < seqsSize; ++i) {
318 assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size);
319 assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size);
320 if (format == ZSTD_sf_noBlockDelimiters) {
321 assert(seqs[i].matchLength != 0 || seqs[i].offset != 0);
324 memcpy(dst, src, seqs[i].litLength);
325 dst += seqs[i].litLength;
326 src += seqs[i].litLength;
327 size -= seqs[i].litLength;
329 if (seqs[i].offset != 0) {
330 for (j = 0; j < seqs[i].matchLength; ++j)
331 dst[j] = dst[(ptrdiff_t)(j - seqs[i].offset)];
332 dst += seqs[i].matchLength;
333 src += seqs[i].matchLength;
334 size -= seqs[i].matchLength;
337 if (format == ZSTD_sf_noBlockDelimiters) {
338 memcpy(dst, src, size);
342 #ifdef ZSTD_MULTITHREAD
346 ZSTD_threadPool* pool;
349 void* compressedBuffer;
350 size_t compressedBufferSize;
353 } threadPoolTests_compressionJob_payload;
355 static void* threadPoolTests_compressionJob(void* payload) {
356 threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload;
358 if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1;
359 cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize);
360 if (ZSTD_isError(cSize)) args->err = 1;
361 if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1;
365 static int threadPoolTests(void) {
369 size_t const CNBuffSize = 5 MB;
370 void* const CNBuffer = malloc(CNBuffSize);
371 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
372 void* const compressedBuffer = malloc(compressedBufferSize);
373 void* const decodedBuffer = malloc(CNBuffSize);
375 size_t const kPoolNumThreads = 8;
377 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0);
379 DISPLAYLEVEL(3, "thread pool test : threadPool reuse roundtrips: ");
381 ZSTD_CCtx* cctx = ZSTD_createCCtx();
382 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
384 size_t nbThreads = 1;
385 for (; nbThreads <= kPoolNumThreads; ++nbThreads) {
386 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
387 ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads);
388 err = ZSTD_CCtx_refThreadPool(cctx, pool);
389 if (ZSTD_isError(err)) {
390 DISPLAYLEVEL(3, "refThreadPool error!\n");
394 err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
395 if (ZSTD_isError(err)) {
396 DISPLAYLEVEL(3, "Compression error!\n");
400 err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err);
401 if (ZSTD_isError(err)) {
402 DISPLAYLEVEL(3, "Decompression error!\n");
409 ZSTD_freeThreadPool(pool);
411 DISPLAYLEVEL(3, "OK \n");
413 DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: ");
415 void* const decodedBuffer2 = malloc(CNBuffSize);
416 void* const compressedBuffer2 = malloc(compressedBufferSize);
417 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
418 ZSTD_CCtx* cctx1 = ZSTD_createCCtx();
419 ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
423 threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize,
424 compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */};
425 threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize,
426 compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */};
428 ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2);
429 ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2);
430 ZSTD_CCtx_refThreadPool(cctx1, pool);
431 ZSTD_CCtx_refThreadPool(cctx2, pool);
433 ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
434 ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
435 ZSTD_pthread_join(t1);
436 ZSTD_pthread_join(t2);
438 assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
439 free(decodedBuffer2);
440 free(compressedBuffer2);
442 ZSTD_freeThreadPool(pool);
443 ZSTD_freeCCtx(cctx1);
444 ZSTD_freeCCtx(cctx2);
446 if (p1.err || p2.err) goto _output_error;
448 DISPLAYLEVEL(3, "OK \n");
452 free(compressedBuffer);
458 DISPLAY("Error detected in Unit tests ! \n");
461 #endif /* ZSTD_MULTITHREAD */
463 /*=============================================
465 =============================================*/
467 static void test_compressBound(unsigned tnb)
469 DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
471 /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
472 * for a large range of known valid values */
473 DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
475 for (s=0; s<30; s++) {
476 size_t const w = (size_t)1 << s;
477 CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
480 /* Ensure error if srcSize too big */
481 { size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
482 CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
483 CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
486 DISPLAYLEVEL(3, "OK \n");
489 static void test_decompressBound(unsigned tnb)
491 DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
493 /* Simple compression, with size : should provide size; */
494 { const char example[] = "abcd";
495 char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
496 size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
498 CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
501 /* Simple small compression without size : should provide 1 block size */
502 { char cBuffer[ZSTD_COMPRESSBOUND(0)];
503 ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
504 ZSTD_inBuffer in = { NULL, 0, 0 };
505 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
507 CHECK_Z( ZSTD_initCStream(cctx, 0) );
508 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
509 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
510 CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
514 /* Attempt to overflow 32-bit intermediate multiplication result
515 * This requires dBound >= 4 GB, aka 2^32.
516 * This requires 2^32 / 2^17 = 2^15 blocks
517 * => create 2^15 blocks (can be empty, or just 1 byte). */
518 { const char input[] = "a";
519 size_t const nbBlocks = (1 << 15) + 1;
521 size_t const outCapacity = 1 << 18; /* large margin */
522 char* const outBuffer = malloc (outCapacity);
523 ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
524 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
527 CHECK_Z( ZSTD_initCStream(cctx, 0) );
528 for (blockNb=0; blockNb<nbBlocks; blockNb++) {
529 ZSTD_inBuffer in = { input, sizeof(input), 0 };
530 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
531 CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
533 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
534 CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000ULL /* 4 GB */ );
539 DISPLAYLEVEL(3, "OK \n");
542 static void test_setCParams(unsigned tnb)
544 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
545 ZSTD_compressionParameters cparams;
548 DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
551 cparams = ZSTD_getCParams(1, 0, 0);
552 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
554 /* invalid cparams (must fail) */
555 cparams.windowLog = 99;
556 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
559 DISPLAYLEVEL(3, "OK \n");
562 static int basicUnitTests(U32 const seed, double compressibility)
564 size_t const CNBuffSize = 5 MB;
565 void* const CNBuffer = malloc(CNBuffSize);
566 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
567 void* const compressedBuffer = malloc(compressedBufferSize);
568 void* const decodedBuffer = malloc(CNBuffSize);
573 /* Create compressible noise */
574 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
575 DISPLAY("Not enough memory, aborting\n");
579 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
582 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++);
583 { const char* errorString = ZSTD_getErrorName(0);
584 DISPLAYLEVEL(3, "OK : %s \n", errorString);
587 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++);
588 { const char* errorString = ZSTD_getErrorName(499);
589 DISPLAYLEVEL(3, "OK : %s \n", errorString);
592 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++);
593 { int const mcl = ZSTD_minCLevel();
594 DISPLAYLEVEL(3, "%i (OK) \n", mcl);
597 DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++);
598 { int const defaultCLevel = ZSTD_defaultCLevel();
599 if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error;
600 DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel);
603 DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
604 { unsigned const vn = ZSTD_versionNumber();
605 DISPLAYLEVEL(3, "%u (OK) \n", vn);
608 test_compressBound(testNb++);
610 test_decompressBound(testNb++);
612 test_setCParams(testNb++);
614 DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
616 ZSTD_compressionParameters params;
617 memset(¶ms, 0, sizeof(params));
618 params.windowLog = 10;
620 params.chainLog = 19;
621 params = ZSTD_adjustCParams(params, 1000, 100000);
622 if (params.hashLog != 18) goto _output_error;
623 if (params.chainLog != 17) goto _output_error;
625 DISPLAYLEVEL(3, "OK \n");
627 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
628 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
629 if (cctx==NULL) goto _output_error;
630 CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
631 compressedBuffer, compressedBufferSize,
632 CNBuffer, CNBuffSize, 1) );
633 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
635 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
636 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
637 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize);
642 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++);
644 char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
645 size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8);
646 if (!ZSTD_isError(size)) goto _output_error;
648 DISPLAYLEVEL(3, "OK \n");
650 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
651 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
652 if (rSize != CNBuffSize) goto _output_error;
654 DISPLAYLEVEL(3, "OK \n");
656 DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++);
657 { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
658 if (rSize != CNBuffSize) goto _output_error;
660 DISPLAYLEVEL(3, "OK \n");
662 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
663 { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
664 if (rSize != CNBuffSize) goto _output_error;
666 DISPLAYLEVEL(3, "OK \n");
668 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
670 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
671 if (bound != CNBuffSize) goto _output_error;
673 DISPLAYLEVEL(3, "OK \n");
675 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
677 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
678 if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
680 DISPLAYLEVEL(3, "OK \n");
682 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
683 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
684 if (r != CNBuffSize) goto _output_error; }
685 DISPLAYLEVEL(3, "OK \n");
687 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
689 ZSTD_DCtx* dctx = ZSTD_createDCtx();
691 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
692 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
693 if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
696 DISPLAYLEVEL(3, "OK \n");
698 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
700 for (u=0; u<CNBuffSize; u++) {
701 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
703 DISPLAYLEVEL(3, "OK \n");
705 DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++);
706 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
707 ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 };
708 ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 };
709 if (cctx==NULL) goto _output_error;
710 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */
711 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */
714 DISPLAYLEVEL(3, "OK \n");
716 DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++);
718 ZSTD_parameters params = ZSTD_getParams(3, 0, 0);
719 assert(!ZSTD_checkCParams(params.cParams));
721 DISPLAYLEVEL(3, "OK \n");
723 DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++);
725 ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
726 assert(dctx != NULL);
727 assert(ZSTD_sizeof_DCtx(dctx) != 0);
730 DISPLAYLEVEL(3, "OK \n");
732 DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++);
734 /* %p takes a void*. In ISO C, it's illegal to cast a function pointer
735 * to a data pointer. (Although in POSIX you're required to be allowed
736 * to do it...) So we have to fall back to our trusty friend memcpy. */
737 unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) =
738 ZSTD_getDictID_fromDDict;
739 ZSTD_DStream* (* const funcptr_createDStream)(
740 ZSTD_customMem customMem) = ZSTD_createDStream_advanced;
741 void (* const funcptr_copyDCtx)(
742 ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx;
743 ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) =
745 const void *voidptr_getDictID;
746 const void *voidptr_createDStream;
747 const void *voidptr_copyDCtx;
748 const void *voidptr_nextInputType;
749 DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID));
751 (void*)&voidptr_getDictID,
752 (const void*)&funcptr_getDictID,
755 (void*)&voidptr_createDStream,
756 (const void*)&funcptr_createDStream,
759 (void*)&voidptr_copyDCtx,
760 (const void*)&funcptr_copyDCtx,
763 (void*)&voidptr_nextInputType,
764 (const void*)&funcptr_nextInputType,
766 DISPLAYLEVEL(3, "%p ", voidptr_getDictID);
767 DISPLAYLEVEL(3, "%p ", voidptr_createDStream);
768 DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx);
769 DISPLAYLEVEL(3, "%p ", voidptr_nextInputType);
771 DISPLAYLEVEL(3, ": OK \n");
773 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
774 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
775 { size_t const r = ZSTD_decompress_usingDict(dctx,
776 decodedBuffer, CNBuffSize,
777 compressedBuffer, cSize,
779 if (r != CNBuffSize) goto _output_error;
783 DISPLAYLEVEL(3, "OK \n");
785 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
786 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
787 { size_t const r = ZSTD_decompress_usingDDict(dctx,
788 decodedBuffer, CNBuffSize,
789 compressedBuffer, cSize,
791 if (r != CNBuffSize) goto _output_error;
795 DISPLAYLEVEL(3, "OK \n");
797 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
798 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
799 if (!ZSTD_isError(r)) goto _output_error;
800 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
801 DISPLAYLEVEL(3, "OK \n");
803 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
804 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
805 if (!ZSTD_isError(r)) goto _output_error;
806 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
807 DISPLAYLEVEL(3, "OK \n");
809 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++);
810 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize);
811 if (!ZSTD_isError(r)) goto _output_error;
812 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
813 DISPLAYLEVEL(3, "OK \n");
815 DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++);
816 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize);
817 if (!ZSTD_isError(r)) goto _output_error;
818 if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; }
819 DISPLAYLEVEL(3, "OK \n");
821 DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++);
822 { /* create compressed buffer with checksumming enabled */
823 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
825 DISPLAY("Not enough memory, aborting\n");
829 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
830 CHECK_VAR(cSize, ZSTD_compress2(cctx,
831 compressedBuffer, compressedBufferSize,
832 CNBuffer, CNBuffSize) );
835 { /* copy the compressed buffer and corrupt the checksum */
837 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
839 DISPLAY("Not enough memory, aborting\n");
844 ((char*)compressedBuffer)[cSize-1] += 1;
845 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
846 if (!ZSTD_isError(r)) goto _output_error;
847 if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error;
849 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum));
850 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
851 if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */
852 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
853 if (ZSTD_isError(r)) goto _output_error;
857 DISPLAYLEVEL(3, "OK \n");
860 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
861 { /* create compressed buffer with content size missing */
862 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
863 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
864 CHECK_VAR(cSize, ZSTD_compress2(cctx,
865 compressedBuffer, compressedBufferSize,
866 CNBuffer, CNBuffSize) );
869 { /* ensure frame content size is missing */
870 ZSTD_frameHeader zfh;
871 size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
872 if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
874 { /* ensure CNBuffSize <= decompressBound */
875 unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
876 if (CNBuffSize > bound) goto _output_error;
878 DISPLAYLEVEL(3, "OK \n");
880 DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++);
882 size_t const largeFrameSrcSize = 200;
883 size_t const smallFrameSrcSize = 10;
884 size_t const nbFrames = 256;
886 size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0;
889 BYTE* const dst = (BYTE*)compressedBuffer;
890 ZSTD_DCtx* dctx = ZSTD_createDCtx();
892 /* create a large frame and then a bunch of small frames */
893 size_t srcSize = ZSTD_compress((void*)dst,
894 compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
895 for (i = 0; i < nbFrames; i++)
896 srcSize += ZSTD_compress((void*)(dst + srcSize),
897 compressedBufferSize - srcSize, CNBuffer,
898 smallFrameSrcSize, 3);
900 /* decompressStream and make sure that dctx size was reduced at least once */
901 while (consumed < srcSize) {
902 ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
903 ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0};
904 ZSTD_decompressStream(dctx, &out, &in);
908 /* success! size was reduced from the previous frame */
909 if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx))
912 prevDCtxSize = ZSTD_sizeof_DCtx(dctx);
919 DISPLAYLEVEL(3, "OK \n");
922 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
923 ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1);
924 ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
925 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
927 DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++);
928 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1));
929 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
930 DISPLAYLEVEL(3, "OK \n");
932 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++);
933 CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1));
934 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
935 DISPLAYLEVEL(3, "OK \n");
937 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++);
938 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict));
939 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
940 DISPLAYLEVEL(3, "OK \n");
942 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++);
943 CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params));
944 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
945 DISPLAYLEVEL(3, "OK \n");
947 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++);
948 CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams));
949 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
950 DISPLAYLEVEL(3, "OK \n");
952 ZSTD_freeCDict(cdict);
956 DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2K", testNb++);
958 ZSTD_CCtx* cctx = ZSTD_createCCtx();
959 ZSTD_DCtx* dctx = ZSTD_createDCtx();
960 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
961 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048));
962 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048));
964 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
966 CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize));
968 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024));
969 CHECK(ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize)));
975 DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
977 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
979 size_t const size = (1U << 10);
980 size_t const dstCapacity = ZSTD_compressBound(size);
981 void* dict = (void*)malloc(size);
982 void* src = (void*)malloc(size);
983 void* dst = (void*)malloc(dstCapacity);
985 RDG_genBuffer(dict, size, 0.5, 0.5, seed);
986 RDG_genBuffer(src, size, 0.5, 0.5, seed);
988 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
989 assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
996 DISPLAYLEVEL(3, "OK \n");
998 DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
1000 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1001 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1002 void* dict = (void*)malloc(CNBuffSize);
1005 for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) {
1006 RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
1007 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
1009 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
1010 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1011 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
1012 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1013 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
1014 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1016 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize));
1019 ZSTD_freeCCtx(cctx);
1020 ZSTD_freeDCtx(dctx);
1023 DISPLAYLEVEL(3, "OK \n");
1025 DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++);
1027 size_t const testSize = 1024;
1028 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1029 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1030 char* dict = (char*)malloc(2 * testSize);
1031 int ldmEnabled, level;
1033 RDG_genBuffer(dict, testSize, 0.5, 0.5, seed);
1034 RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
1035 memcpy(dict + testSize, CNBuffer, testSize);
1036 for (level = 1; level <= 5; ++level) {
1037 for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
1039 XXH64_hash_t compressedChecksum0;
1041 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1042 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level));
1043 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled));
1044 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1));
1046 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1047 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize);
1049 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize));
1052 compressedChecksum0 = XXH64(compressedBuffer, cSize, 0);
1054 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1055 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize);
1058 if (cSize != cSize0) goto _output_error;
1059 if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error;
1063 ZSTD_freeCCtx(cctx);
1064 ZSTD_freeDCtx(dctx);
1067 DISPLAYLEVEL(3, "OK \n");
1069 DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
1070 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1071 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1072 size_t const srcSize = 300 KB;
1073 size_t const flushSize = 128 KB + 5;
1074 size_t const dstSize = ZSTD_compressBound(srcSize);
1075 char* src = (char*)CNBuffer;
1076 char* dst = (char*)compressedBuffer;
1078 ZSTD_outBuffer out = { dst, dstSize, 0 };
1079 ZSTD_inBuffer in = { src, flushSize, 0 };
1081 if (!cctx || !dctx) {
1082 DISPLAY("Not enough memory, aborting\n");
1087 RDG_genBuffer(src, srcSize, 0.5, 0.5, seed);
1088 /* Force an LDM to exist that crosses block boundary into uncompressible block */
1089 memcpy(src + 125 KB, src, 3 KB + 5);
1091 /* Enable MT, LDM, and opt parser */
1092 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
1093 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1094 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1095 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1097 /* Flushes a block of 128 KB and block of 5 bytes */
1098 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1100 /* Compress the rest */
1102 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1104 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos));
1106 ZSTD_freeCCtx(cctx);
1107 ZSTD_freeDCtx(dctx);
1109 DISPLAYLEVEL(3, "OK \n");
1111 DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++);
1113 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1114 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1115 void* dict = (void*)malloc(CNBuffSize);
1116 size_t const kWindowLog = 10;
1117 size_t const kWindowSize = (size_t)1 << kWindowLog;
1118 size_t const dictSize = kWindowSize * 10;
1119 size_t const srcSize1 = kWindowSize / 2;
1120 size_t const srcSize2 = kWindowSize * 10;
1125 if (CNBuffSize < dictSize) goto _output_error;
1127 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
1128 RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed);
1130 /* Enable checksum to verify round trip. */
1131 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1132 /* Disable content size to skip single-pass decompression. */
1133 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
1134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
1135 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1136 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
1137 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
1138 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
1139 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3));
1141 /* Round trip once with a dictionary. */
1142 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1143 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
1145 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1147 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
1148 /* Streaming decompression to catch out of bounds offsets. */
1150 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1151 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1152 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1154 if (dSize != 0) goto _output_error;
1157 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
1158 /* Round trip once with a dictionary. */
1159 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1160 { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
1161 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1162 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1163 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1166 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1168 { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
1169 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1170 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1171 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1174 /* Streaming decompression to catch out of bounds offsets. */
1175 { ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1176 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1177 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1179 if (dSize != 0) goto _output_error;
1182 ZSTD_freeCCtx(cctx);
1183 ZSTD_freeDCtx(dctx);
1186 DISPLAYLEVEL(3, "OK \n");
1188 /* Note: this test takes 0.5 seconds to run */
1189 DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
1191 /* test a big buffer so that ldm can take effect */
1192 size_t const size = 100 MB;
1193 int const windowLog = 27;
1194 size_t const dstSize = ZSTD_compressBound(size);
1196 void* dict = (void*)malloc(size);
1197 void* src = (void*)malloc(size);
1198 void* dst = (void*)malloc(dstSize);
1199 void* recon = (void*)malloc(size);
1201 size_t refPrefixCompressedSize = 0;
1202 size_t refPrefixLdmCompressedSize = 0;
1203 size_t reconSize = 0;
1205 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1206 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1208 /* make dict and src the same uncompressible data */
1209 RDG_genBuffer(src, size, 0, 0, seed);
1210 memcpy(dict, src, size);
1211 assert(!memcmp(dict, src, size));
1213 /* set level 1 and windowLog to cover src */
1214 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1));
1215 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog));
1217 /* compress on level 1 using just refPrefix and no ldm */
1218 ZSTD_CCtx_refPrefix(cctx, dict, size);
1219 refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1220 assert(!ZSTD_isError(refPrefixCompressedSize));
1222 /* test round trip just refPrefix */
1223 ZSTD_DCtx_refPrefix(dctx, dict, size);
1224 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize);
1225 assert(!ZSTD_isError(reconSize));
1226 assert(reconSize == size);
1227 assert(!memcmp(recon, src, size));
1229 /* compress on level 1 using refPrefix and ldm */
1230 ZSTD_CCtx_refPrefix(cctx, dict, size);;
1231 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
1232 refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1233 assert(!ZSTD_isError(refPrefixLdmCompressedSize));
1235 /* test round trip refPrefix + ldm*/
1236 ZSTD_DCtx_refPrefix(dctx, dict, size);
1237 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize);
1238 assert(!ZSTD_isError(reconSize));
1239 assert(reconSize == size);
1240 assert(!memcmp(recon, src, size));
1242 /* make sure that refPrefixCompressedSize is significantly greater */
1243 assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize);
1244 /* make sure the ldm compressed size is less than 1% of original */
1245 assert((double)refPrefixLdmCompressedSize / (double)size < 0.01);
1247 ZSTD_freeDCtx(dctx);
1248 ZSTD_freeCCtx(cctx);
1254 DISPLAYLEVEL(3, "OK \n");
1256 DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++);
1257 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX);
1259 CHECK_LT(CNBuffSize, cSize);
1261 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1262 size_t const outputSize = (CNBuffSize + margin);
1263 char* output = malloc(outputSize);
1264 char* input = output + outputSize - cSize;
1265 CHECK_LT(cSize, CNBuffSize + margin);
1266 CHECK(output != NULL);
1268 CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX));
1269 memcpy(input, compressedBuffer, cSize);
1272 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1274 CHECK_EQ(dSize, CNBuffSize);
1276 CHECK(!memcmp(output, CNBuffer, CNBuffSize));
1279 DISPLAYLEVEL(3, "OK \n");
1281 DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++);
1282 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1285 size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1290 size_t const srcSize = (CNBuffSize / 3) * 2;
1291 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1292 size_t const outputSize = (CNBuffSize + margin);
1293 char* output = malloc(outputSize);
1294 char* input = output + outputSize - cSize;
1295 CHECK_LT(cSize, CNBuffSize + margin);
1296 CHECK(output != NULL);
1298 memcpy(input, compressedBuffer, cSize);
1301 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1303 CHECK_EQ(dSize, srcSize);
1305 CHECK(!memcmp(output, CNBuffer, srcSize));
1308 DISPLAYLEVEL(3, "OK \n");
1310 DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++);
1311 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1312 size_t const srcSize = 256 * 1024;
1313 U32 const compressibleLenU32 = 32 * 1024 / 4;
1314 U32 const blockSizeU32 = 128 * 1024 / 4;
1315 U32 const litLenU32 = 64 * 1024 / 4;
1316 U32* data = (U32*)malloc(srcSize);
1319 if (data == NULL || cctx == NULL) goto _output_error;
1321 /* Generate data without any matches */
1322 RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U);
1323 /* Generate 32K of compressible data */
1324 RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe);
1326 /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */
1327 data[compressibleLenU32 + 0] = 0xFFFFFFFF;
1328 data[compressibleLenU32 + 1] = 0xEEEEEEEE;
1329 data[compressibleLenU32 + 4] = 0xFFFFFFFF;
1330 data[compressibleLenU32 + 5] = 0xEEEEEEEE;
1332 /* Add a match of offset=16, length=8 at idx=64K + 64.
1333 * This generates a sequence with llen=64K, and repeat code 1.
1334 * The block splitter thought this was ll0, and corrupted the
1335 * repeat offset history.
1337 data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD;
1338 data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC;
1339 data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD;
1340 data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC;
1342 /* Add a match of offset=16, length=8 at idx=128K + 16.
1343 * This should generate a sequence with repeat code = 1.
1344 * But the block splitters mistake caused zstd to generate
1345 * repeat code = 2, corrupting the data.
1347 data[blockSizeU32] = 0xBBBBBBBB;
1348 data[blockSizeU32 + 1] = 0xAAAAAAAA;
1349 data[blockSizeU32 + 4] = 0xBBBBBBBB;
1350 data[blockSizeU32 + 5] = 0xAAAAAAAA;
1352 /* Generate a golden file from this data in case datagen changes and
1353 * doesn't generate the exact same data. We will also test this golden file.
1356 FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb");
1357 fwrite(data, 1, srcSize, f);
1361 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1362 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7));
1363 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable));
1365 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize);
1367 dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1369 CHECK_EQ(dSize, srcSize);
1370 CHECK(!memcmp(decodedBuffer, data, srcSize));
1373 ZSTD_freeCCtx(cctx);
1375 DISPLAYLEVEL(3, "OK \n");
1377 DISPLAYLEVEL(3, "test%3d : superblock uncompressible data: too many nocompress superblocks : ", testNb++);
1379 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1380 const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;
1381 size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize);
1383 /* This is the number of bytes to stream before ending. This value
1384 * was obtained by trial and error :/. */
1386 const size_t streamCompressThreshold = 161792;
1387 const size_t streamCompressDelta = 1024;
1389 /* The first 1/5 of the buffer is compressible and the last 4/5 is
1390 * uncompressible. This is an approximation of the type of data
1391 * the fuzzer generated to catch this bug. Streams like this were making
1392 * zstd generate noCompress superblocks (which are larger than the src
1393 * they come from). Do this enough times, and we'll run out of room
1394 * and throw a dstSize_tooSmall error. */
1396 const size_t compressiblePartSize = srcSize/5;
1397 const size_t uncompressiblePartSize = srcSize-compressiblePartSize;
1398 RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed);
1399 RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed);
1401 /* Setting target block size so that superblock is used */
1403 assert(cctx != NULL);
1404 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81);
1407 for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) {
1408 ZSTD_inBuffer in = {src, streamCompressDelta, 0};
1409 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1410 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1411 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1412 src += streamCompressDelta; srcSize -= streamCompressDelta;
1413 dst += out.pos; dstCapacity -= out.pos;
1416 /* This is trying to catch a dstSize_tooSmall error */
1418 { ZSTD_inBuffer in = {src, srcSize, 0};
1419 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1420 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1422 ZSTD_freeCCtx(cctx);
1424 DISPLAYLEVEL(3, "OK \n");
1426 DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++);
1427 /* Generate the same data 20 times over */
1428 { size_t const avgChunkSize = CNBuffSize / 20;
1430 for (b = 0; b < CNBuffSize; b += avgChunkSize) {
1431 size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize);
1432 RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed);
1434 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1435 size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1436 size_t const allowedExpansion = (CNBuffSize * 3 / 1000);
1438 CHECK_Z(normalCSize);
1439 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
1440 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000);
1441 superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1442 CHECK_Z(superCSize);
1443 if (superCSize > normalCSize + allowedExpansion) {
1444 DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion);
1447 ZSTD_freeCCtx(cctx);
1449 DISPLAYLEVEL(3, "OK \n");
1451 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed);
1452 DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++)
1453 /* This tests whether or not we leave enough room for the checksum at the end
1454 * of the dst buffer. The bug that motivated this test was found by the
1455 * stream_round_trip fuzzer but this crashes for the same reason and is
1456 * far more compact than re-creating the stream_round_trip fuzzer's code path */
1457 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1458 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64);
1459 assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278)));
1460 ZSTD_freeCCtx(cctx);
1462 DISPLAYLEVEL(3, "OK \n");
1464 DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
1466 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1467 if (!cctx) goto _output_error;
1468 for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
1469 CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
1470 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
1471 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
1473 ZSTD_freeCCtx(cctx);
1475 DISPLAYLEVEL(3, "OK \n");
1477 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
1478 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1479 size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
1480 if (ZSTD_isError(r)) goto _output_error;
1481 if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
1482 ZSTD_freeCCtx(cctx);
1485 DISPLAYLEVEL(3, "OK \n");
1487 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++);
1488 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize);
1489 if (ZSTD_isError(r)) goto _output_error;
1490 if (r != 0) goto _output_error;
1492 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1493 ZSTD_outBuffer output;
1494 if (cctx==NULL) goto _output_error;
1495 output.dst = compressedBuffer;
1496 output.size = compressedBufferSize;
1498 CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */
1499 CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */
1500 CHECK_Z( ZSTD_endStream(cctx, &output) );
1501 ZSTD_freeCCtx(cctx);
1502 /* single scan decompression */
1503 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos);
1504 if (ZSTD_isError(r)) goto _output_error;
1505 if (r != 0) goto _output_error;
1507 /* streaming decompression */
1508 { ZSTD_DCtx* const dstream = ZSTD_createDStream();
1509 ZSTD_inBuffer dinput;
1510 ZSTD_outBuffer doutput;
1512 if (dstream==NULL) goto _output_error;
1513 dinput.src = compressedBuffer;
1519 CHECK_Z ( ZSTD_initDStream(dstream) );
1520 for (ipos=1; ipos<=output.pos; ipos++) {
1522 CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) );
1524 if (doutput.pos != 0) goto _output_error;
1525 ZSTD_freeDStream(dstream);
1528 DISPLAYLEVEL(3, "OK \n");
1530 DISPLAYLEVEL(3, "test%3d : reuse CCtx with expanding block size : ", testNb++);
1531 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1532 ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1533 assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
1534 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
1535 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
1537 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */
1538 { size_t const inSize = 2* 128 KB;
1539 size_t const outSize = ZSTD_compressBound(inSize);
1540 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
1541 /* will fail if blockSize is not resized */
1543 ZSTD_freeCCtx(cctx);
1545 DISPLAYLEVEL(3, "OK \n");
1547 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++);
1548 { size_t const sampleSize = 30;
1550 for (i=0; i<20; i++)
1551 ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */
1552 memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
1553 for (i=1; i<=19; i++) {
1554 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1555 size_t size1, size2;
1556 DISPLAYLEVEL(5, "l%i ", i);
1557 size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1560 size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1562 CHECK_EQ(size1, size2);
1564 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) );
1565 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize);
1567 CHECK_EQ(size1, size2);
1569 size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */
1571 CHECK_EQ(size1, size2);
1573 { ZSTD_inBuffer inb;
1574 ZSTD_outBuffer outb;
1577 inb.size = sampleSize;
1578 outb.dst = compressedBuffer;
1580 outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */
1581 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );
1582 assert(inb.pos == inb.size);
1583 CHECK_EQ(size1, outb.pos);
1586 ZSTD_freeCCtx(cctx);
1589 DISPLAYLEVEL(3, "OK \n");
1591 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++);
1592 { size_t const sampleSize = 1024;
1593 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1595 ZSTD_outBuffer outb;
1599 outb.dst = compressedBuffer;
1601 outb.size = compressedBufferSize;
1602 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) );
1604 inb.size = sampleSize; /* start with something, so that context is already used */
1605 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1606 assert(inb.pos == inb.size);
1607 outb.pos = 0; /* cancel output */
1609 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) );
1610 inb.size = 4; /* too small size : compression will be skipped */
1612 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1613 assert(inb.pos == inb.size);
1615 inb.size += 5; /* too small size : compression will be skipped */
1616 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1617 assert(inb.pos == inb.size);
1619 inb.size += 11; /* small enough to attempt compression */
1620 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1621 assert(inb.pos == inb.size);
1623 assert(inb.pos < sampleSize);
1624 inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */
1625 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1626 assert(inb.pos == inb.size);
1627 ZSTD_freeCCtx(cctx);
1629 DISPLAYLEVEL(3, "OK \n");
1631 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
1632 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1633 ZSTD_outBuffer out = {NULL, 0, 0};
1634 ZSTD_inBuffer in = {NULL, 0, 0};
1637 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1639 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1641 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
1642 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1644 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1645 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1646 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
1647 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1649 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1650 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1651 /* Start a compression job */
1652 ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1653 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1655 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1656 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1657 /* Reset the CCtx */
1658 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
1659 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1661 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1662 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1663 /* Reset the parameters */
1664 ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
1665 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1667 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1670 ZSTD_freeCCtx(cctx);
1672 DISPLAYLEVEL(3, "OK \n");
1674 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++);
1675 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1677 ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0);
1678 cparams.strategy = -1;
1679 /* Set invalid cParams == no change. */
1680 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
1682 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1684 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1686 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1688 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1690 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1692 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1694 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1697 cparams = ZSTD_getCParams(12, 0, 0);
1698 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
1700 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1701 CHECK_EQ(value, (int)cparams.windowLog);
1702 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1703 CHECK_EQ(value, (int)cparams.chainLog);
1704 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1705 CHECK_EQ(value, (int)cparams.hashLog);
1706 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1707 CHECK_EQ(value, (int)cparams.searchLog);
1708 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1709 CHECK_EQ(value, (int)cparams.minMatch);
1710 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1711 CHECK_EQ(value, (int)cparams.targetLength);
1712 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1713 CHECK_EQ(value, (int)cparams.strategy);
1715 ZSTD_freeCCtx(cctx);
1718 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++);
1719 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1721 ZSTD_frameParameters fparams = {0, 1, 1};
1723 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1725 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1727 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1730 CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams));
1732 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1733 CHECK_EQ(value, fparams.contentSizeFlag);
1734 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1735 CHECK_EQ(value, fparams.checksumFlag);
1736 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1737 CHECK_EQ(value, !fparams.noDictIDFlag);
1739 ZSTD_freeCCtx(cctx);
1742 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCarams() : ", testNb++);
1743 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1745 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
1746 params.cParams.strategy = -1;
1747 /* Set invalid params == no change. */
1748 CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params)));
1750 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1752 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1754 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1756 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1758 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1760 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1762 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1764 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1766 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1768 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1771 params = ZSTD_getParams(12, 0, 0);
1772 params.fParams.contentSizeFlag = 0;
1773 params.fParams.checksumFlag = 1;
1774 params.fParams.noDictIDFlag = 1;
1775 CHECK_Z(ZSTD_CCtx_setParams(cctx, params));
1777 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1778 CHECK_EQ(value, (int)params.cParams.windowLog);
1779 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1780 CHECK_EQ(value, (int)params.cParams.chainLog);
1781 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1782 CHECK_EQ(value, (int)params.cParams.hashLog);
1783 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1784 CHECK_EQ(value, (int)params.cParams.searchLog);
1785 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1786 CHECK_EQ(value, (int)params.cParams.minMatch);
1787 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1788 CHECK_EQ(value, (int)params.cParams.targetLength);
1789 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1790 CHECK_EQ(value, (int)params.cParams.strategy);
1791 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1792 CHECK_EQ(value, params.fParams.contentSizeFlag);
1793 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1794 CHECK_EQ(value, params.fParams.checksumFlag);
1795 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1796 CHECK_EQ(value, !params.fParams.noDictIDFlag);
1798 ZSTD_freeCCtx(cctx);
1801 DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
1802 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1803 ZSTD_outBuffer out = {NULL, 0, 0};
1804 ZSTD_inBuffer in = {NULL, 0, 0};
1807 /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt),
1808 * we should not modify the actual parameter specified by the user within the CCtx
1810 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27));
1811 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt));
1813 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1814 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value));
1817 ZSTD_freeCCtx(cctx);
1819 DISPLAYLEVEL(3, "OK \n");
1821 /* this test is really too long, and should be made faster */
1822 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
1823 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1824 ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1825 size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
1827 assert(cctx != NULL);
1828 params.fParams.contentSizeFlag = 0;
1829 params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
1830 for (cnb = 0; cnb < nbCompressions; ++cnb) {
1831 DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions);
1832 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */
1833 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
1835 ZSTD_freeCCtx(cctx);
1837 DISPLAYLEVEL(3, "OK \n");
1839 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
1840 { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
1841 assert(largeCCtx != NULL);
1842 CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
1843 CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
1844 { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
1845 { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
1846 assert(smallCCtx != NULL);
1847 CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1848 { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
1849 DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
1850 largeCCtxSize>>10, smallCCtxSize>>10);
1851 assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
1852 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
1854 ZSTD_freeCCtx(smallCCtx);
1856 { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
1857 * currently defined as 128x, but could be adjusted in the future.
1858 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
1860 for (u=0; u<maxNbAttempts; u++) {
1861 CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1862 if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
1864 DISPLAYLEVEL(5, "size down after %u attempts : ", u);
1865 if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
1868 ZSTD_freeCCtx(largeCCtx);
1870 DISPLAYLEVEL(3, "OK \n");
1872 /* Static CCtx tests */
1873 #define STATIC_CCTX_LEVEL 4
1874 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1875 { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
1876 void* const staticCCtxBuffer = malloc(staticCStreamSize);
1877 size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
1878 void* const staticDCtxBuffer = malloc(staticDCtxSize);
1879 DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize);
1880 if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
1881 free(staticCCtxBuffer);
1882 free(staticDCtxBuffer);
1883 DISPLAY("Not enough memory, aborting\n");
1887 { size_t const smallInSize = 32 KB;
1888 ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0);
1889 size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small);
1890 size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL);
1891 ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize);
1892 ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
1893 DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize);
1894 DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize);
1895 if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
1896 DISPLAYLEVEL(3, "OK \n");
1898 DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++);
1899 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1900 compressedBuffer, compressedBufferSize,
1901 CNBuffer, smallInSize, STATIC_CCTX_LEVEL) );
1902 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1903 (unsigned)cSize, (double)cSize/smallInSize*100);
1905 DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++);
1906 { size_t const r = ZSTD_compressCCtx(staticCCtx,
1907 compressedBuffer, compressedBufferSize,
1908 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL);
1909 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error;
1911 DISPLAYLEVEL(3, "OK \n");
1913 DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
1914 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
1915 DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx);
1916 if (staticCCtx == NULL) goto _output_error;
1917 DISPLAYLEVEL(3, "OK \n");
1919 DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++);
1920 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1921 compressedBuffer, compressedBufferSize,
1922 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1923 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1924 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1926 DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++);
1928 assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */
1929 assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3);
1930 for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) {
1931 CHECK_Z(ZSTD_compressCCtx(staticCCtx,
1932 compressedBuffer, compressedBufferSize,
1933 (char*)CNBuffer + nbc, smallInSize,
1934 STATIC_CCTX_LEVEL) );
1936 DISPLAYLEVEL(3, "OK \n")
1938 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1939 CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) );
1940 DISPLAYLEVEL(3, "OK \n");
1942 DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++);
1943 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1944 compressedBuffer, compressedBufferSize,
1945 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1946 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1947 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1949 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
1950 { size_t const r = ZSTD_decompressDCtx(staticDCtx,
1951 decodedBuffer, CNBuffSize,
1952 compressedBuffer, cSize);
1953 if (r != CNBuffSize) goto _output_error; }
1954 DISPLAYLEVEL(3, "OK \n");
1956 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1957 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
1958 DISPLAYLEVEL(3, "OK \n");
1960 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
1961 { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
1962 if (!ZSTD_isError(r)) goto _output_error; }
1963 DISPLAYLEVEL(3, "OK \n");
1965 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
1966 CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
1967 DISPLAYLEVEL(3, "OK \n");
1969 DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
1970 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1971 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1972 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1973 size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */
1974 if (!ZSTD_isError(r)) goto _output_error;
1976 DISPLAYLEVEL(3, "OK \n");
1978 DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++);
1979 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize);
1980 assert(staticCCtx != NULL);
1981 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1982 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1983 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1984 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1986 DISPLAYLEVEL(3, "OK \n");
1988 DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1);
1989 CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */
1990 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1991 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1992 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1994 DISPLAYLEVEL(3, "OK \n");
1996 DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++);
1997 { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
1998 if (!ZSTD_isError(r)) goto _output_error; }
1999 DISPLAYLEVEL(3, "OK \n");
2001 DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++);
2002 CHECK_Z( ZSTD_initDStream(staticDCtx) );
2003 { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
2004 ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
2005 size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
2006 if (!ZSTD_isError(r)) goto _output_error;
2008 DISPLAYLEVEL(3, "OK \n");
2010 DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
2012 // Test ZSTD_estimateCCtxSize_usingCCtxParams
2014 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2015 size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2016 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2017 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2018 compressedBuffer, compressedBufferSize,
2019 CNBuffer, CNBuffSize, 3));
2022 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2023 decodedBuffer, CNBuffSize,
2024 compressedBuffer, cSize);
2025 if (r != CNBuffSize) goto _output_error;
2026 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2028 ZSTD_freeCCtxParams(params);
2031 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2033 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2034 size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2035 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2036 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2037 compressedBuffer, compressedBufferSize,
2038 CNBuffer, CNBuffSize, 3) );
2041 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2042 decodedBuffer, CNBuffSize,
2043 compressedBuffer, cSize);
2044 if (r != CNBuffSize) goto _output_error;
2045 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2047 ZSTD_freeCCtxParams(params);
2050 DISPLAYLEVEL(3, "OK \n");
2052 DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
2054 // Test ZSTD_estimateCCtxSize_usingCCtxParams
2056 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2057 size_t cctxSizeDefault;
2058 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2059 cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2060 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2061 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2062 compressedBuffer, compressedBufferSize,
2063 CNBuffer, CNBuffSize, 3) );
2066 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2067 decodedBuffer, CNBuffSize,
2068 compressedBuffer, cSize);
2069 if (r != CNBuffSize) goto _output_error;
2070 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2072 ZSTD_freeCCtxParams(params);
2075 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2077 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2078 size_t cctxSizeDefault;
2079 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2080 cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2081 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2082 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2083 compressedBuffer, compressedBufferSize,
2084 CNBuffer, CNBuffSize, 3) );
2087 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2088 decodedBuffer, CNBuffSize,
2089 compressedBuffer, cSize);
2090 if (r != CNBuffSize) goto _output_error;
2091 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2093 ZSTD_freeCCtxParams(params);
2096 DISPLAYLEVEL(3, "OK \n");
2098 free(staticCCtxBuffer);
2099 free(staticDCtxBuffer);
2102 DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++);
2103 { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1);
2104 size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1);
2105 size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1);
2106 size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1);
2108 if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error;
2109 if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error;
2111 DISPLAYLEVEL(3, "OK \n");
2114 /* ZSTDMT simple MT compression test */
2115 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
2116 { ZSTD_CCtx* const mtctx = ZSTD_createCCtx();
2118 DISPLAY("mtctx : not enough memory, aborting \n");
2122 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
2123 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
2124 DISPLAYLEVEL(3, "OK \n");
2126 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
2127 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2128 compressedBuffer, compressedBufferSize,
2129 CNBuffer, CNBuffSize) );
2130 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2132 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
2133 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2134 if (rSize != CNBuffSize) {
2135 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize);
2138 DISPLAYLEVEL(3, "OK \n");
2140 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2141 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2142 if (r != CNBuffSize) goto _output_error; }
2143 DISPLAYLEVEL(3, "OK \n");
2145 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2147 for (u=0; u<CNBuffSize; u++) {
2148 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
2150 DISPLAYLEVEL(3, "OK \n");
2152 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
2153 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
2154 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
2155 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
2156 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2157 compressedBuffer, compressedBufferSize,
2158 CNBuffer, CNBuffSize) );
2159 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2161 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2162 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2163 if (r != CNBuffSize) goto _output_error; }
2164 DISPLAYLEVEL(3, "OK \n");
2166 ZSTD_freeCCtx(mtctx);
2169 DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++);
2170 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2171 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2173 if (cctx == NULL || dctx == NULL) goto _output_error;
2174 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
2175 CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
2176 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
2178 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
2179 { char const* outPtr = out;
2180 ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
2181 ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
2183 CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) );
2184 if (dSize != 0) goto _output_error;
2187 ZSTD_freeDCtx(dctx);
2188 ZSTD_freeCCtx(cctx);
2191 DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
2192 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2193 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
2194 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2196 ZSTD_freeCCtx(cctx);
2198 DISPLAYLEVEL(3, "OK \n");
2200 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
2201 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2202 size_t cSize1, cSize2;
2203 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2204 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
2205 cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2207 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
2208 cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2210 CHECK_LT(cSize1, cSize2);
2211 ZSTD_freeCCtx(cctx);
2213 DISPLAYLEVEL(3, "OK \n");
2215 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++)
2216 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2217 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
2218 * ZSTDMT is forced to not take the shortcut.
2220 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2221 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
2222 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
2223 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
2224 ZSTD_freeCCtx(cctx);
2226 DISPLAYLEVEL(3, "OK \n");
2228 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
2229 { ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2230 int const jobSize = 512 KB;
2232 /* Check that the overlap log and job size are unset. */
2233 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2235 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2237 /* Set and check the overlap log and job size. */
2238 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
2239 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
2240 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2242 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2243 CHECK_EQ(value, jobSize);
2244 /* Set the number of workers and check the overlap log and job size. */
2245 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
2246 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2248 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2249 CHECK_EQ(value, jobSize);
2250 ZSTD_freeCCtxParams(params);
2252 DISPLAYLEVEL(3, "OK \n");
2254 /* Simple API multiframe test */
2255 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);
2259 /* only use the first half so we don't push against size limit of compressedBuffer */
2260 size_t const segSize = (CNBuffSize / 2) / segs;
2262 const U32 skipLen = 129 KB;
2263 char* const skipBuff = (char*)malloc(skipLen);
2264 assert(skipBuff != NULL);
2265 memset(skipBuff, 0, skipLen);
2266 for (i = 0; i < segs; i++) {
2267 CHECK_NEWV(r, ZSTD_compress(
2268 (BYTE*)compressedBuffer + off, CNBuffSize - off,
2269 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
2273 /* insert skippable frame */
2274 size_t const skippableSize =
2275 ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize,
2276 skipBuff, skipLen, seed % 15);
2277 CHECK_Z(skippableSize);
2278 off += skippableSize;
2284 DISPLAYLEVEL(3, "OK \n");
2286 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++);
2287 { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
2288 if (r != CNBuffSize / 2) goto _output_error; }
2289 DISPLAYLEVEL(3, "OK \n");
2291 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
2292 { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize);
2293 if (bound != CNBuffSize / 2) goto _output_error; }
2294 DISPLAYLEVEL(3, "OK \n");
2296 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
2297 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
2298 if (r != CNBuffSize / 2) goto _output_error; }
2299 DISPLAYLEVEL(3, "OK \n");
2301 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2302 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
2303 DISPLAYLEVEL(3, "OK \n");
2305 /* Simple API skippable frame test */
2306 DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++);
2309 unsigned long long receivedSize;
2310 size_t skippableSize;
2311 const U32 skipLen = 129 KB;
2312 char* const skipBuff = (char*)malloc(skipLen);
2313 assert(skipBuff != NULL);
2314 for (i = 0; i < skipLen; i++)
2315 skipBuff[i] = (char) ((seed + i) % 256);
2316 skippableSize = ZSTD_writeSkippableFrame(
2317 compressedBuffer, compressedBufferSize,
2318 skipBuff, skipLen, seed % 15);
2319 CHECK_Z(skippableSize);
2320 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2321 receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize);
2322 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2323 CHECK_EQ(seed % 15, readMagic);
2324 if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error;
2328 DISPLAYLEVEL(3, "OK \n");
2330 DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++);
2333 unsigned long long receivedSize;
2334 size_t skippableSize;
2335 skippableSize = ZSTD_writeSkippableFrame(
2336 compressedBuffer, compressedBufferSize,
2337 CNBuffer, 0, seed % 15);
2338 CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize);
2339 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2340 receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize);
2341 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2342 CHECK_EQ(seed % 15, readMagic);
2344 DISPLAYLEVEL(3, "OK \n");
2346 /* Dictionary and CCtx Duplication tests */
2347 { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
2348 ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
2349 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2350 static const size_t dictSize = 551;
2351 assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
2353 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
2354 { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
2355 if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
2356 DISPLAYLEVEL(3, "OK \n");
2358 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
2359 CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
2360 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
2361 DISPLAYLEVEL(3, "OK \n");
2363 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
2365 CHECKPLUS(r, ZSTD_compressEnd(ctxOrig,
2366 compressedBuffer, compressedBufferSize,
2367 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2369 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2371 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
2372 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2373 decodedBuffer, CNBuffSize,
2374 compressedBuffer, cSize,
2375 CNBuffer, dictSize),
2376 if (r != CNBuffSize - dictSize) goto _output_error);
2377 DISPLAYLEVEL(3, "OK \n");
2379 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
2380 { size_t const cSizeOrig = cSize;
2382 CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated,
2383 compressedBuffer, compressedBufferSize,
2384 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2386 if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
2388 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2390 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
2391 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2392 decodedBuffer, CNBuffSize,
2393 compressedBuffer, cSize,
2394 CNBuffer, dictSize),
2395 if (r != CNBuffSize - dictSize) goto _output_error);
2396 DISPLAYLEVEL(3, "OK \n");
2398 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++);
2399 { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
2400 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2401 if (r != CNBuffSize - dictSize) goto _output_error;
2402 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict));
2403 ZSTD_freeDDict(ddict);
2406 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
2407 { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
2408 void* const ddictBuffer = malloc(ddictBufferSize);
2409 if (ddictBuffer == NULL) goto _output_error;
2410 { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
2411 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2412 if (r != CNBuffSize - dictSize) goto _output_error;
2415 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize);
2418 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
2419 { size_t const testSize = CNBuffSize / 3;
2420 CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
2421 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
2423 CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
2424 (const char*)CNBuffer + dictSize, testSize) );
2425 { ZSTD_frameHeader zfh;
2426 if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
2427 if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
2429 DISPLAYLEVEL(3, "OK \n");
2431 #if !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \
2432 && !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \
2433 && !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \
2434 && !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \
2435 && !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \
2436 && !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \
2437 && !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \
2438 && !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR)
2439 /* Note : these tests should be replaced by proper regression tests,
2440 * but existing ones do not focus on small data + dictionary + all levels.
2442 if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
2443 size_t const flatdictSize = 22 KB;
2444 size_t const contentSize = 9 KB;
2445 const void* const dict = (const char*)CNBuffer;
2446 const void* const contentStart = (const char*)dict + flatdictSize;
2447 /* These upper bounds are generally within a few bytes of the compressed size */
2448 size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
2449 3770, 3770, 3770, 3750, 3750,
2450 3742, 3675, 3674, 3665, 3664,
2451 3663, 3662, 3661, 3660, 3660,
2453 size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940,
2454 2950, 2950, 2925, 2900, 2892,
2455 2910, 2910, 2910, 2780, 2775,
2456 2765, 2760, 2755, 2754, 2753,
2459 int const maxLevel = ZSTD_maxCLevel();
2460 /* clevels with strategies that support rowhash on small inputs */
2462 int const rowLevelEnd = 8;
2464 DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
2465 assert(maxLevel == 22);
2466 RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
2467 DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n",
2468 (unsigned long long)XXH64(contentStart, contentSize, 0),
2469 (unsigned long long)XXH64(dict, flatdictSize, 0));
2471 for ( ; l <= maxLevel; l++) {
2472 size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize,
2473 contentStart, contentSize, l);
2474 if (nodict_cSize > target_nodict_cSize[l]) {
2475 DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n",
2476 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2479 DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n",
2480 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2482 for ( l=1 ; l <= maxLevel; l++) {
2483 size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig,
2484 compressedBuffer, compressedBufferSize,
2485 contentStart, contentSize,
2488 if (wdict_cSize > target_wdict_cSize[l]) {
2489 DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n",
2490 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2493 DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
2494 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2496 /* Compression with ZSTD_compress2 and row match finder force enabled.
2497 * Give some slack for force-enabled row matchfinder since we're on a small input (9KB)
2499 for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5;
2500 for (l=1 ; l <= maxLevel; l++) {
2501 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2502 size_t nodict_cSize;
2503 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l);
2504 ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable);
2505 nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize,
2506 contentStart, contentSize);
2507 if (nodict_cSize > target_nodict_cSize[l]) {
2508 DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n",
2509 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2510 ZSTD_freeCCtx(cctx);
2513 DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n",
2514 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2515 ZSTD_freeCCtx(cctx);
2517 /* Dict compression with DMS */
2518 for ( l=1 ; l <= maxLevel; l++) {
2520 CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) );
2521 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) );
2522 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) );
2523 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) );
2524 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) );
2525 wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize);
2526 if (wdict_cSize > target_wdict_cSize[l]) {
2527 DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n",
2528 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2531 DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n",
2532 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2535 DISPLAYLEVEL(4, "compression efficiency tests OK \n");
2539 ZSTD_freeCCtx(ctxOrig);
2540 ZSTD_freeCCtx(ctxDuplicated);
2541 ZSTD_freeDCtx(dctx);
2544 /* Dictionary and dictBuilder tests */
2545 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2546 size_t const dictBufferCapacity = 16 KB;
2547 void* const dictBuffer = malloc(dictBufferCapacity);
2548 size_t const totalSampleSize = 1 MB;
2549 size_t const sampleUnitSize = 8 KB;
2550 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
2551 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
2554 size_t dictHeaderSize;
2555 size_t dictBufferFixedSize = 144;
2556 unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
2557 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2558 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
2559 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
2560 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2561 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2562 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
2563 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
2564 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
2565 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
2566 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
2567 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69};
2569 if (dictBuffer==NULL || samplesSizes==NULL) {
2575 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++);
2576 assert(compressedBufferSize >= totalSampleSize);
2577 { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; }
2578 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2579 { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2580 decodedBuffer, samplesSizes, nbSamples);
2581 if (ZDICT_isError(sDictSize)) goto _output_error;
2582 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize);
2585 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++);
2586 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2587 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2588 CNBuffer, samplesSizes, nbSamples);
2589 if (ZDICT_isError(dictSize)) goto _output_error;
2590 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2592 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++);
2593 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2594 { ZDICT_cover_params_t coverParams;
2595 memset(&coverParams, 0, sizeof(coverParams));
2596 coverParams.steps = 8;
2597 coverParams.nbThreads = 4;
2598 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2599 dictBuffer, dictBufferCapacity,
2600 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2602 if (ZDICT_isError(dictSize)) goto _output_error;
2604 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2606 DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++);
2607 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2608 { ZDICT_cover_params_t coverParams;
2609 memset(&coverParams, 0, sizeof(coverParams));
2610 coverParams.steps = 8;
2611 coverParams.nbThreads = 4;
2612 coverParams.shrinkDict = 1;
2613 coverParams.shrinkDictMaxRegression = 1;
2614 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2615 dictBuffer, dictBufferCapacity,
2616 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2618 if (ZDICT_isError(dictSize)) goto _output_error;
2620 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2622 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++);
2623 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2624 { ZDICT_fastCover_params_t fastCoverParams;
2625 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2626 fastCoverParams.steps = 8;
2627 fastCoverParams.nbThreads = 4;
2628 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2629 dictBuffer, dictBufferCapacity,
2630 CNBuffer, samplesSizes, nbSamples,
2632 if (ZDICT_isError(dictSize)) goto _output_error;
2634 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2636 DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++);
2637 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2638 { ZDICT_fastCover_params_t fastCoverParams;
2639 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2640 fastCoverParams.steps = 8;
2641 fastCoverParams.nbThreads = 4;
2642 fastCoverParams.shrinkDict = 1;
2643 fastCoverParams.shrinkDictMaxRegression = 1;
2644 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2645 dictBuffer, dictBufferCapacity,
2646 CNBuffer, samplesSizes, nbSamples,
2648 if (ZDICT_isError(dictSize)) goto _output_error;
2650 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2652 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2653 dictID = ZDICT_getDictID(dictBuffer, dictSize);
2654 if (dictID==0) goto _output_error;
2655 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2657 DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++);
2658 dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize);
2659 if (dictHeaderSize==0) goto _output_error;
2660 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2662 DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
2663 { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize);
2664 if (dictHeaderSize != 115) goto _output_error;
2666 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2668 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
2669 cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
2670 CNBuffer, CNBuffSize,
2671 dictBuffer, dictSize, 4);
2672 if (ZSTD_isError(cSize)) goto _output_error;
2673 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2675 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++);
2676 { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
2677 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2679 DISPLAYLEVEL(3, "OK \n");
2681 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2682 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2683 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2685 DISPLAYLEVEL(3, "OK \n");
2687 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2688 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2689 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2690 decodedBuffer, CNBuffSize,
2691 compressedBuffer, cSize,
2692 dictBuffer, dictSize),
2693 if (r != CNBuffSize) goto _output_error);
2694 ZSTD_freeDCtx(dctx);
2696 DISPLAYLEVEL(3, "OK \n");
2698 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
2699 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2700 size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef);
2701 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize);
2704 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
2705 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2706 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
2707 ZSTD_dlm_byRef, ZSTD_dct_auto,
2708 cParams, ZSTD_defaultCMem);
2709 assert(cdict != NULL);
2710 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict));
2711 assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict));
2712 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
2713 CNBuffer, CNBuffSize, cdict);
2714 ZSTD_freeCDict(cdict);
2715 if (ZSTD_isError(cSize)) goto _output_error;
2717 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2719 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2720 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2721 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2723 DISPLAYLEVEL(3, "OK \n");
2725 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2726 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2727 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2728 decodedBuffer, CNBuffSize,
2729 compressedBuffer, cSize,
2730 dictBuffer, dictSize),
2731 if (r != CNBuffSize) goto _output_error);
2732 ZSTD_freeDCtx(dctx);
2734 DISPLAYLEVEL(3, "OK \n");
2736 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
2737 { int const maxLevel = ZSTD_maxCLevel();
2739 for (level = 1; level <= maxLevel; ++level) {
2740 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize);
2741 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
2742 void* const cdictBuffer = malloc(cdictSize);
2743 if (cdictBuffer==NULL) goto _output_error;
2744 { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict(
2745 cdictBuffer, cdictSize,
2746 dictBuffer, dictSize,
2747 ZSTD_dlm_byCopy, ZSTD_dct_auto,
2749 if (cdict == NULL) {
2750 DISPLAY("ZSTD_initStaticCDict failed ");
2753 cSize = ZSTD_compress_usingCDict(cctx,
2754 compressedBuffer, compressedBufferSize,
2755 CNBuffer, MIN(10 KB, CNBuffSize), cdict);
2756 if (ZSTD_isError(cSize)) {
2757 DISPLAY("ZSTD_compress_usingCDict failed ");
2762 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2764 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
2765 { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
2766 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2767 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
2768 assert(cdict != NULL);
2769 cSize = ZSTD_compress_usingCDict_advanced(cctx,
2770 compressedBuffer, compressedBufferSize,
2771 CNBuffer, CNBuffSize,
2773 ZSTD_freeCDict(cdict);
2774 if (ZSTD_isError(cSize)) goto _output_error;
2776 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2778 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++);
2779 { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2780 if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
2782 DISPLAYLEVEL(3, "OK (unknown)\n");
2784 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2785 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2786 assert(dctx != NULL);
2787 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2788 decodedBuffer, CNBuffSize,
2789 compressedBuffer, cSize,
2790 dictBuffer, dictSize),
2791 if (r != CNBuffSize) goto _output_error);
2792 ZSTD_freeDCtx(dctx);
2794 DISPLAYLEVEL(3, "OK \n");
2796 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
2797 { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
2798 p.fParams.noDictIDFlag = 1;
2799 cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
2800 CNBuffer, CNBuffSize,
2801 dictBuffer, dictSize, p);
2802 if (ZSTD_isError(cSize)) goto _output_error;
2804 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2806 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2807 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2808 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2809 decodedBuffer, CNBuffSize,
2810 compressedBuffer, cSize,
2811 dictBuffer, dictSize),
2812 if (r != CNBuffSize) goto _output_error);
2813 ZSTD_freeDCtx(dctx);
2815 DISPLAYLEVEL(3, "OK \n");
2817 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
2818 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2819 assert(dctx != NULL);
2820 { const size_t ret = ZSTD_decompress_usingDict(
2821 dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
2822 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
2823 if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
2826 ZSTD_freeDCtx(dctx);
2828 DISPLAYLEVEL(3, "OK \n");
2830 DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++);
2831 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2832 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2833 ZSTD_frameParameters const fParams = { 0, 1, 0 };
2836 CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN));
2837 cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000);
2838 CHECK_Z(cBlockSize);
2839 cSize += cBlockSize;
2840 cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000);
2841 CHECK_Z(cBlockSize);
2842 cSize += cBlockSize;
2844 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
2846 ZSTD_freeCDict(cdict);
2847 ZSTD_freeDCtx(dctx);
2849 DISPLAYLEVEL(3, "OK \n");
2851 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
2852 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2853 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2854 if (cdict==NULL) goto _output_error;
2855 ZSTD_freeCDict(cdict);
2857 DISPLAYLEVEL(3, "OK \n");
2859 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
2860 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2861 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2862 if (cdict!=NULL) goto _output_error;
2863 ZSTD_freeCDict(cdict);
2865 DISPLAYLEVEL(3, "OK \n");
2867 { char* rawDictBuffer = (char*)malloc(dictSize);
2868 assert(rawDictBuffer);
2869 memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2);
2870 memset(rawDictBuffer + dictSize - 2, 0, 2);
2871 MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY);
2873 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
2876 /* Either operation is allowed to fail, but one must fail. */
2877 ret = ZSTD_CCtx_loadDictionary_advanced(
2878 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
2879 if (!ZSTD_isError(ret)) {
2880 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2881 if (!ZSTD_isError(ret)) goto _output_error;
2884 DISPLAYLEVEL(3, "OK \n");
2886 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
2889 ret = ZSTD_CCtx_loadDictionary_advanced(
2890 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
2891 if (ZSTD_isError(ret)) goto _output_error;
2892 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2893 if (ZSTD_isError(ret)) goto _output_error;
2895 DISPLAYLEVEL(3, "OK \n");
2897 DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++);
2898 { size_t const srcSize = MIN(CNBuffSize, 100);
2899 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2900 /* Force the dictionary to be reloaded in raw content mode */
2901 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad));
2902 CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent));
2903 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
2906 DISPLAYLEVEL(3, "OK \n");
2908 free(rawDictBuffer);
2911 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
2912 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
2913 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2914 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2915 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2916 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2917 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2918 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2919 ZSTD_freeCDict(cdict);
2921 DISPLAYLEVEL(3, "OK \n");
2923 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++);
2925 size_t size1, size2;
2926 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2927 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2928 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2929 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2930 if (ZSTD_isError(size1)) goto _output_error;
2932 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2933 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2934 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2935 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2936 if (ZSTD_isError(size2)) goto _output_error;
2938 if (size1 != size2) goto _output_error;
2940 DISPLAYLEVEL(3, "OK \n");
2942 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++);
2944 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2945 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2946 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2948 DISPLAYLEVEL(3, "OK \n");
2950 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++);
2952 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2953 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2954 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2955 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2956 ZSTD_freeCDict(cdict);
2958 DISPLAYLEVEL(3, "OK \n");
2960 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++);
2962 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2963 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2964 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2965 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2966 ZSTD_freeCDict(cdict);
2968 DISPLAYLEVEL(3, "OK \n");
2970 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++);
2972 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2973 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2974 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2975 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2976 ZSTD_freeCDict(cdict);
2978 DISPLAYLEVEL(3, "OK \n");
2980 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++);
2982 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2983 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2984 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2986 DISPLAYLEVEL(3, "OK \n");
2988 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++);
2990 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2991 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2992 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2993 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2994 ZSTD_freeCDict(cdict);
2996 DISPLAYLEVEL(3, "OK \n");
2998 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++);
3000 size_t size1, size2;
3001 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3002 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
3003 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3004 if (ZSTD_isError(size1)) goto _output_error;
3006 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
3007 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3008 if (ZSTD_isError(size2)) goto _output_error;
3010 if (size1 != size2) goto _output_error;
3012 DISPLAYLEVEL(3, "OK \n");
3014 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++);
3016 size_t size1, size2;
3017 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3018 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
3019 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3020 if (ZSTD_isError(size1)) goto _output_error;
3022 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3023 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3024 if (ZSTD_isError(size2)) goto _output_error;
3026 if (size1 == size2) goto _output_error;
3028 DISPLAYLEVEL(3, "OK \n");
3030 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3031 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) );
3032 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3034 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++);
3036 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3038 /* We should fail to decompress without a dictionary. */
3039 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3040 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3041 if (!ZSTD_isError(ret)) goto _output_error;
3042 /* We should succeed to decompress with the dictionary. */
3043 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3044 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) );
3045 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3046 /* The dictionary should persist across calls. */
3047 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3048 /* When we reset the context the dictionary is cleared. */
3049 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3050 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3051 if (!ZSTD_isError(ret)) goto _output_error;
3052 ZSTD_freeDCtx(dctx);
3054 DISPLAYLEVEL(3, "OK \n");
3056 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
3058 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3059 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
3061 /* We should succeed to decompress with the ddict. */
3062 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3063 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
3064 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3065 /* The ddict should persist across calls. */
3066 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3067 /* When we reset the context the ddict is cleared. */
3068 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3069 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3070 if (!ZSTD_isError(ret)) goto _output_error;
3071 ZSTD_freeDCtx(dctx);
3072 ZSTD_freeDDict(ddict);
3074 DISPLAYLEVEL(3, "OK \n");
3076 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
3078 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3080 /* We should succeed to decompress with the prefix. */
3081 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3082 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
3083 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3084 /* The prefix should be cleared after the first compression. */
3085 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3086 if (!ZSTD_isError(ret)) goto _output_error;
3087 ZSTD_freeDCtx(dctx);
3089 DISPLAYLEVEL(3, "OK \n");
3091 DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3093 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3094 ZSTD_customMem customMem = {NULL, NULL, NULL};
3095 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3097 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast));
3098 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3099 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3100 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3101 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3102 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3103 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3104 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3105 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3106 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3107 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3108 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3109 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3111 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3112 ZSTD_freeCDict(cdict);
3113 ZSTD_freeDCtx(dctx);
3114 ZSTD_freeCCtxParams(cctxParams);
3116 DISPLAYLEVEL(3, "OK \n");
3118 DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3120 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3121 ZSTD_customMem customMem = {NULL, NULL, NULL};
3122 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3124 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast));
3125 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3126 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3127 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3128 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3129 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3130 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3131 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3132 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3133 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3135 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3136 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3138 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3139 ZSTD_freeCDict(cdict);
3140 ZSTD_freeDCtx(dctx);
3141 ZSTD_freeCCtxParams(cctxParams);
3143 DISPLAYLEVEL(3, "OK \n");
3145 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++);
3147 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3148 ZSTD_customMem customMem = {NULL, NULL, NULL};
3149 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3151 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy));
3152 /* Force enable row based match finder, and disable dedicated dict search. */
3153 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
3154 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0));
3155 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4));
3156 /* Set windowLog to 29 so hash/chain logs don't get sized down */
3157 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29));
3158 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29));
3159 /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */
3160 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29));
3161 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3162 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3163 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3164 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3165 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3166 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3168 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3169 ZSTD_freeCDict(cdict);
3170 ZSTD_freeDCtx(dctx);
3171 ZSTD_freeCCtxParams(cctxParams);
3173 DISPLAYLEVEL(3, "OK \n");
3175 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
3176 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3177 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
3178 CNBuffer, samplesSizes, nbSamples);
3179 if (ZDICT_isError(dictSize)) goto _output_error;
3180 /* Set all the repcodes to non-default */
3182 BYTE* dictPtr = (BYTE*)dictBuffer;
3183 BYTE* dictLimit = dictPtr + dictSize - 12;
3184 /* Find the repcodes */
3185 while (dictPtr < dictLimit &&
3186 (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
3187 MEM_readLE32(dictPtr + 8) != 8)) {
3190 if (dictPtr >= dictLimit) goto _output_error;
3191 MEM_writeLE32(dictPtr + 0, 10);
3192 MEM_writeLE32(dictPtr + 4, 10);
3193 MEM_writeLE32(dictPtr + 8, 10);
3194 /* Set the last 8 bytes to 'x' */
3195 memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
3197 /* The optimal parser checks all the repcodes.
3198 * Make sure at least one is a match >= targetLength so that it is
3199 * immediately chosen. This will make sure that the compressor and
3200 * decompressor agree on at least one of the repcodes.
3204 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3205 ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
3206 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
3207 ZSTD_dlm_byRef, ZSTD_dct_auto,
3208 cParams, ZSTD_defaultCMem);
3209 assert(dctx != NULL); assert(cdict != NULL);
3210 memset(data, 'x', sizeof(data));
3211 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
3212 data, sizeof(data), cdict);
3213 ZSTD_freeCDict(cdict);
3214 if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
3215 dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
3216 if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
3217 if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
3218 ZSTD_freeDCtx(dctx);
3220 DISPLAYLEVEL(3, "OK \n");
3222 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++);
3224 const size_t numDicts = 128;
3225 const size_t numFrames = 4;
3227 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3228 ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts);
3229 ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts);
3230 U32 dictIDSeed = seed;
3231 /* Create new compressed buffer that will hold frames with differing dictIDs */
3232 char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */
3234 ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize);
3235 /* Create a bunch of DDicts with random dict IDs */
3236 for (i = 0; i < numDicts; ++i) {
3237 U32 currDictID = FUZ_rand(&dictIDSeed);
3238 MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID);
3239 ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize);
3240 cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3);
3241 if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) {
3245 /* Compress a few frames using random CDicts */
3248 /* only use the first half so we don't push against size limit of compressedBuffer */
3249 size_t const segSize = (CNBuffSize / 2) / numFrames;
3250 for (i = 0; i < numFrames; i++) {
3251 size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts;
3252 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3253 { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx,
3254 (BYTE*)compressedBuffer + off, CNBuffSize - off,
3255 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
3256 cdictTable[dictIdx]));
3263 /* We should succeed to decompression even though different dicts were used on different frames */
3264 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3265 ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts);
3266 /* Reference every single ddict we made */
3267 for (i = 0; i < numDicts; ++i) {
3268 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i]));
3270 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3271 /* Streaming decompression should also work */
3273 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
3274 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
3275 while (in.pos < in.size) {
3276 CHECK_Z(ZSTD_decompressStream(dctx, &out, &in));
3279 ZSTD_freeDCtx(dctx);
3280 for (i = 0; i < numDicts; ++i) {
3281 ZSTD_freeCDict(cdictTable[i]);
3282 ZSTD_freeDDict(ddictTable[i]);
3284 free(dictBufferMulti);
3288 DISPLAYLEVEL(3, "OK \n");
3290 ZSTD_freeCCtx(cctx);
3295 /* COVER dictionary builder tests */
3296 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3297 size_t dictSize = 16 KB;
3298 size_t optDictSize = dictSize;
3299 void* dictBuffer = malloc(dictSize);
3300 size_t const totalSampleSize = 1 MB;
3301 size_t const sampleUnitSize = 8 KB;
3302 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3303 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3305 ZDICT_cover_params_t params;
3308 if (dictBuffer==NULL || samplesSizes==NULL) {
3314 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
3315 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3316 memset(¶ms, 0, sizeof(params));
3317 params.d = 1 + (FUZ_rand(&seed32) % 16);
3318 params.k = params.d + (FUZ_rand(&seed32) % 256);
3319 dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize,
3320 CNBuffer, samplesSizes, nbSamples,
3322 if (ZDICT_isError(dictSize)) goto _output_error;
3323 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
3325 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3326 dictID = ZDICT_getDictID(dictBuffer, dictSize);
3327 if (dictID==0) goto _output_error;
3328 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3330 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
3331 memset(¶ms, 0, sizeof(params));
3333 optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize,
3334 CNBuffer, samplesSizes,
3335 nbSamples / 4, ¶ms);
3336 if (ZDICT_isError(optDictSize)) goto _output_error;
3337 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize);
3339 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3340 dictID = ZDICT_getDictID(dictBuffer, optDictSize);
3341 if (dictID==0) goto _output_error;
3342 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3344 ZSTD_freeCCtx(cctx);
3349 /* Decompression defense tests */
3350 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++);
3351 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
3352 if (!ZSTD_isError(r)) goto _output_error;
3353 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
3354 DISPLAYLEVEL(3, "OK \n");
3356 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++);
3357 ((char*)(CNBuffer))[0] = 1;
3358 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
3359 if (!ZSTD_isError(r)) goto _output_error; }
3360 DISPLAYLEVEL(3, "OK \n");
3362 /* content size verification test */
3363 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++);
3364 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3365 size_t const srcSize = 5000;
3366 size_t const wrongSrcSize = (srcSize + 1000);
3367 ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
3368 params.fParams.contentSizeFlag = 1;
3369 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
3370 { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
3371 if (!ZSTD_isError(result)) goto _output_error;
3372 if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
3373 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result));
3375 ZSTD_freeCCtx(cctx);
3378 /* negative compression level test : ensure simple API and advanced API produce same result */
3379 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++);
3380 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3381 size_t const srcSize = CNBuffSize / 5;
3382 int const compressionLevel = -1;
3384 assert(cctx != NULL);
3385 { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize,
3386 CNBuffer, srcSize, compressionLevel);
3387 if (ZSTD_isError(cSize_1pass)) goto _output_error;
3389 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
3390 { size_t const compressionResult = ZSTD_compress2(cctx,
3391 compressedBuffer, compressedBufferSize,
3393 DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
3394 if (ZSTD_isError(compressionResult)) goto _output_error;
3395 if (compressionResult != cSize_1pass) goto _output_error;
3397 ZSTD_freeCCtx(cctx);
3399 DISPLAYLEVEL(3, "OK \n");
3401 /* parameters order test */
3402 { size_t const inputSize = CNBuffSize / 2;
3405 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3406 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
3407 assert(cctx != NULL);
3408 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3409 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3410 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3411 { size_t const compressedSize = ZSTD_compress2(cctx,
3412 compressedBuffer, ZSTD_compressBound(inputSize),
3413 CNBuffer, inputSize);
3414 CHECK_Z(compressedSize);
3415 cSize = compressedSize;
3416 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
3418 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3419 ZSTD_freeCCtx(cctx);
3422 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
3423 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
3424 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3425 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3426 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3427 { size_t const result = ZSTD_compress2(cctx,
3428 compressedBuffer, ZSTD_compressBound(inputSize),
3429 CNBuffer, inputSize);
3431 if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
3432 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
3433 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
3435 ZSTD_freeCCtx(cctx);
3439 /* advanced parameters for decompression */
3440 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3441 assert(dctx != NULL);
3443 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
3444 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
3445 CHECK_Z(bounds.error);
3447 DISPLAYLEVEL(3, "OK \n");
3449 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
3450 { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
3451 if (!ZSTD_isError(sr)) goto _output_error;
3453 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
3454 if (!ZSTD_isError(bounds.error)) goto _output_error;
3456 DISPLAYLEVEL(3, "OK \n");
3458 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
3459 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
3460 if (!ZSTD_isError(sr)) goto _output_error;
3462 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
3463 if (!ZSTD_isError(sr)) goto _output_error;
3465 DISPLAYLEVEL(3, "OK \n");
3467 ZSTD_freeDCtx(dctx);
3471 /* custom formats tests */
3472 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3473 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3474 size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
3475 assert(dctx != NULL); assert(cctx != NULL);
3477 /* basic block compression */
3478 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
3479 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3480 { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
3481 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
3482 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3483 if (result != 0) goto _output_error;
3484 if (in.pos != in.size) goto _output_error;
3487 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3489 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++);
3490 { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3491 if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
3492 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult));
3495 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
3496 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3497 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3498 { ZSTD_frameHeader zfh;
3499 size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
3500 if (zfhrt != 0) goto _output_error;
3503 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3504 if (result != inputSize) goto _output_error;
3505 DISPLAYLEVEL(3, "one-shot OK, ");
3508 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3509 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3510 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3511 if (result != 0) goto _output_error;
3512 if (in.pos != in.size) goto _output_error;
3513 if (out.pos != inputSize) goto _output_error;
3514 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3517 /* basic block compression */
3518 DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
3519 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3520 { ZSTD_inBuffer in = { CNBuffer, 0, 0 };
3521 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
3522 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3523 if (result != 0) goto _output_error;
3524 if (in.pos != in.size) goto _output_error;
3527 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
3529 DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
3530 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3531 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3533 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3534 if (result != 0) goto _output_error;
3535 DISPLAYLEVEL(3, "one-shot OK, ");
3538 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3539 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3540 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3541 if (result != 0) goto _output_error;
3542 if (in.pos != in.size) goto _output_error;
3543 if (out.pos != 0) goto _output_error;
3544 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3547 ZSTD_freeCCtx(cctx);
3548 ZSTD_freeDCtx(dctx);
3551 DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++);
3553 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3554 /* Attempt to future proof this to new parameters. */
3555 int const maxParam = 2000;
3557 if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error;
3558 for (param = 0; param < maxParam; ++param) {
3559 ZSTD_dParameter dParam = (ZSTD_dParameter)param;
3560 ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam);
3564 if (ZSTD_isError(bounds.error))
3566 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
3567 value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
3568 CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
3569 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3570 if (check != value2) goto _output_error;
3571 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
3572 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3573 if (check != value1) goto _output_error;
3575 ZSTD_freeDCtx(dctx);
3577 DISPLAYLEVEL(3, "OK \n");
3579 /* block API tests */
3580 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3581 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3582 static const size_t dictSize = 65 KB;
3583 static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
3585 assert(cctx != NULL); assert(dctx != NULL);
3587 /* basic block compression */
3588 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
3589 CHECK_Z( ZSTD_compressBegin(cctx, 5) );
3590 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3591 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
3592 DISPLAYLEVEL(3, "OK \n");
3594 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
3595 CHECK_Z( ZSTD_decompressBegin(dctx) );
3596 { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3597 if (r != blockSize) goto _output_error; }
3598 DISPLAYLEVEL(3, "OK \n");
3600 /* very long stream of block compression */
3601 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
3602 CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
3603 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3604 { U64 const toCompress = 5000000000ULL; /* > 4 GB */
3606 while (compressed < toCompress) {
3607 size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
3608 assert(blockCSize != 0);
3609 if (ZSTD_isError(blockCSize)) goto _output_error;
3610 compressed += blockCSize;
3612 DISPLAYLEVEL(3, "OK \n");
3614 /* dictionary block compression */
3615 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
3616 CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
3617 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
3618 RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
3619 { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
3620 assert(r == 0); /* non-compressible block */ }
3621 memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */
3622 CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
3623 (char*)CNBuffer+dictSize+2*blockSize, blockSize));
3624 DISPLAYLEVEL(3, "OK \n");
3626 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
3627 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
3628 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
3629 if (r != blockSize) {
3630 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3633 memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize);
3634 ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
3635 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
3636 if (r != blockSize) {
3637 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3640 assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */
3641 DISPLAYLEVEL(3, "OK \n");
3643 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
3644 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
3645 if (cdict==NULL) goto _output_error;
3646 CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
3647 CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
3648 ZSTD_freeCDict(cdict);
3650 DISPLAYLEVEL(3, "OK \n");
3652 ZSTD_freeCCtx(cctx);
3653 ZSTD_freeDCtx(dctx);
3657 { size_t sampleSize = 0;
3658 size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
3659 DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
3660 memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
3661 sampleSize += 256 KB - 1;
3662 memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
3663 sampleSize += 96 KB;
3664 cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
3665 if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
3666 { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
3667 if (regenSize!=sampleSize) goto _output_error; }
3668 DISPLAYLEVEL(3, "OK \n");
3671 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++);
3673 size_t srcSize = 150 KB;
3674 BYTE* src = (BYTE*)CNBuffer;
3675 BYTE* decoded = (BYTE*)compressedBuffer;
3677 ZSTD_CCtx* cctx = ZSTD_createCCtx();
3678 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3681 if (seqs == NULL) goto _output_error;
3682 assert(cctx != NULL);
3683 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
3684 /* Populate src with random data */
3685 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
3687 /* Test with block delimiters roundtrip */
3688 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3690 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters);
3691 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3693 /* Test no block delimiters roundtrip */
3694 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
3696 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters);
3697 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3699 ZSTD_freeCCtx(cctx);
3702 DISPLAYLEVEL(3, "OK \n");
3704 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences too small output buffer : ", testNb++);
3706 const size_t seqsCapacity = 10;
3707 const size_t srcSize = 150 KB;
3708 const BYTE* src = (BYTE*)CNBuffer;
3710 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3711 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(seqsCapacity * sizeof(ZSTD_Sequence));
3713 if (seqs == NULL) goto _output_error;
3714 if (cctx == NULL) goto _output_error;
3715 /* Populate src with random data */
3716 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
3718 /* Test with block delimiters roundtrip */
3720 size_t const seqsSize = ZSTD_generateSequences(cctx, seqs, seqsCapacity, src, srcSize);
3721 if (!ZSTD_isError(seqsSize)) goto _output_error;
3724 ZSTD_freeCCtx(cctx);
3727 DISPLAYLEVEL(3, "OK \n");
3729 DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
3731 const size_t srcSize = 500 KB;
3732 const BYTE* const src = (BYTE*)CNBuffer;
3733 BYTE* const dst = (BYTE*)compressedBuffer;
3734 const size_t dstCapacity = ZSTD_compressBound(srcSize);
3735 const size_t decompressSize = srcSize;
3736 char* const decompressBuffer = (char*)malloc(decompressSize);
3737 size_t compressedSize;
3739 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3740 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3743 if (seqs == NULL) goto _output_error;
3744 assert(cctx != NULL);
3746 /* Populate src with random data */
3747 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
3749 /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */
3750 nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3751 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3752 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
3753 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize);
3754 if (ZSTD_isError(compressedSize)) {
3755 DISPLAY("Error in sequence compression with block delims\n");
3758 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3759 if (ZSTD_isError(dSize)) {
3760 DISPLAY("Error in sequence compression roundtrip with block delims\n");
3763 assert(!memcmp(decompressBuffer, src, srcSize));
3765 /* Roundtrip Test with no block delimiters */
3766 { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs);
3767 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3768 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
3769 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize);
3771 if (ZSTD_isError(compressedSize)) {
3772 DISPLAY("Error in sequence compression with no block delims\n");
3775 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3776 if (ZSTD_isError(dSize)) {
3777 DISPLAY("Error in sequence compression roundtrip with no block delims\n");
3780 assert(!memcmp(decompressBuffer, src, srcSize));
3782 ZSTD_freeCCtx(cctx);
3783 free(decompressBuffer);
3786 DISPLAYLEVEL(3, "OK \n");
3788 /* Multiple blocks of zeros test */
3789 #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
3790 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3791 memset(CNBuffer, 0, LONGZEROSLENGTH);
3792 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) );
3793 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100);
3795 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3796 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) );
3797 if (r != LONGZEROSLENGTH) goto _output_error; }
3798 DISPLAYLEVEL(3, "OK \n");
3800 /* All zeroes test (test bug #137) */
3801 #define ZEROESLENGTH 100
3802 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
3803 memset(CNBuffer, 0, ZEROESLENGTH);
3804 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
3805 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100);
3807 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
3808 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
3809 if (r != ZEROESLENGTH) goto _output_error; }
3810 DISPLAYLEVEL(3, "OK \n");
3812 /* nbSeq limit test */
3813 #define _3BYTESTESTLENGTH 131000
3814 #define NB3BYTESSEQLOG 9
3815 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
3816 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
3817 /* creates a buffer full of 3-bytes sequences */
3818 { BYTE _3BytesSeqs[NB3BYTESSEQ][3];
3821 /* create batch of 3-bytes sequences */
3823 for (i=0; i < NB3BYTESSEQ; i++) {
3824 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
3825 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
3826 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
3829 /* randomly fills CNBuffer with prepared 3-bytes sequences */
3831 for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
3832 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
3833 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
3834 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
3835 ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
3837 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++);
3838 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3839 size_t const maxNbSeq = _3BYTESTESTLENGTH / 3;
3840 size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
3842 while (nbSeq <= maxNbSeq) {
3843 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
3844 /* Check every sequence for the first 100, then skip more rapidly. */
3848 nbSeq += (nbSeq >> 2);
3851 ZSTD_freeCCtx(cctx);
3853 DISPLAYLEVEL(3, "OK \n");
3855 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
3856 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
3857 CNBuffer, _3BYTESTESTLENGTH, 19) );
3858 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
3860 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
3861 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
3862 if (r != _3BYTESTESTLENGTH) goto _output_error; }
3863 DISPLAYLEVEL(3, "OK \n");
3866 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++);
3867 RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
3868 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3869 size_t const bound = ZSTD_compressBound(CNBuffSize);
3871 while (size <= CNBuffSize) {
3872 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
3873 /* Check every size for the first 100, then skip more rapidly. */
3877 size += (size >> 2);
3880 ZSTD_freeCCtx(cctx);
3882 DISPLAYLEVEL(3, "OK \n");
3884 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
3885 { /* Train a dictionary on low characters */
3886 size_t dictSize = 16 KB;
3887 void* const dictBuffer = malloc(dictSize);
3888 size_t const totalSampleSize = 1 MB;
3889 size_t const sampleUnitSize = 8 KB;
3890 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3891 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3892 if (!dictBuffer || !samplesSizes) goto _output_error;
3893 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3894 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
3895 if (ZDICT_isError(dictSize)) goto _output_error;
3896 /* Reverse the characters to make the dictionary ill suited */
3898 for (u = 0; u < CNBuffSize; ++u) {
3899 ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
3901 { /* Compress the data */
3902 size_t const inputSize = 500;
3903 size_t const outputSize = ZSTD_compressBound(inputSize);
3904 void* const outputBuffer = malloc(outputSize);
3905 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3906 if (!outputBuffer || !cctx) goto _output_error;
3907 CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
3909 ZSTD_freeCCtx(cctx);
3915 DISPLAYLEVEL(3, "OK \n");
3918 /* findFrameCompressedSize on skippable frames */
3919 DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++);
3920 { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
3921 size_t const frameSrcSize = 13;
3922 if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
3923 DISPLAYLEVEL(3, "OK \n");
3925 /* error string tests */
3926 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
3927 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
3928 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
3929 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
3930 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
3931 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
3932 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
3933 DISPLAYLEVEL(3, "OK \n");
3935 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
3936 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
3938 size_t const size = MIN(128 KB, CNBuffSize);
3939 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3940 ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
3941 ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
3942 ZSTD_frameHeader lgHeader;
3943 ZSTD_frameHeader smHeader;
3945 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
3946 CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
3947 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
3948 CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
3950 if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
3952 ZSTD_freeCDict(smCDict);
3953 ZSTD_freeCDict(lgCDict);
3954 ZSTD_freeCCtx(cctx);
3956 DISPLAYLEVEL(3, "OK \n");
3958 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++);
3962 unsigned const tableLog = 5;
3963 size_t const nbSeq = 32;
3964 unsigned const maxSymbolValue = 31;
3967 for (i = 0; i < 32; ++i)
3969 /* Calling FSE_normalizeCount() on a uniform distribution should not
3970 * cause a division by zero.
3972 FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1);
3974 DISPLAYLEVEL(3, "OK \n");
3976 DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++);
3978 size_t const outBufSize = 9;
3979 short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18};
3980 unsigned const tableLog = 5;
3981 unsigned const maxSymbolValue = 10;
3982 BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE));
3984 /* Ensure that this write doesn't write out of bounds, and that
3985 * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1.
3987 FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog);
3990 DISPLAYLEVEL(3, "OK \n");
3992 DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
3994 U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
3995 U32 rand32 = FUZ_rand(&seed_copy);
3996 U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
3997 U32 lowbit_only_32 = 1;
3998 U64 lowbit_only_64 = 1;
3999 U32 highbit_only_32 = (U32)1 << 31;
4000 U64 highbit_only_64 = (U64)1 << 63;
4002 if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
4003 if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
4005 /* Test ZSTD_countTrailingZeros32 */
4006 CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
4007 CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u);
4008 CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32));
4010 /* Test ZSTD_countLeadingZeros32 */
4011 CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u);
4012 CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u);
4013 CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32));
4015 /* Test ZSTD_countTrailingZeros64 */
4016 CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u);
4017 CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u);
4019 /* Test ZSTD_countLeadingZeros64 */
4020 CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u);
4021 CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u);
4023 /* Test ZSTD_highbit32 */
4024 CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u);
4025 CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u);
4027 /* Test ZSTD_NbCommonBytes */
4028 if (MEM_isLittleEndian()) {
4030 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
4031 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
4033 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
4034 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
4038 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u);
4039 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u);
4041 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u);
4042 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u);
4046 /* Test MEM_ intrinsics */
4047 CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32));
4048 CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64));
4050 /* Test fallbacks vs intrinsics on a range of small integers */
4051 for (i=1; i <= 1000; i++) {
4052 CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i));
4053 CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i));
4054 CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i));
4055 CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i));
4058 DISPLAYLEVEL(3, "OK \n");
4060 #ifdef ZSTD_MULTITHREAD
4061 DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++);
4062 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4063 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4064 size_t const dstSize = ZSTD_compressBound(srcSize);
4065 void* const src = CNBuffer;
4066 void* const dst = compressedBuffer;
4067 void* dict = (void*)malloc(srcSize);
4069 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4070 RDG_genBuffer(dict, srcSize, compressibility, 0., seed);
4072 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4073 memset(dict, 0, sizeof(U32));
4075 /* something more than 1 */
4076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4077 /* lie and claim this is a full dict */
4078 CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict));
4080 { ZSTD_outBuffer out = {dst, dstSize, 0};
4081 ZSTD_inBuffer in = {src, srcSize, 0};
4082 /* should fail because its not a full dict like we said it was */
4083 assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)));
4086 ZSTD_freeCCtx(cctx);
4089 DISPLAYLEVEL(3, "OK \n");
4091 DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++);
4092 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4093 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4094 size_t const dictSize = 10;
4095 size_t const dstSize = ZSTD_compressBound(srcSize);
4096 void* const src = CNBuffer;
4097 void* const dst = compressedBuffer;
4098 void* dict = (void*)malloc(dictSize);
4100 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4101 RDG_genBuffer(dict, dictSize, compressibility, 0., seed);
4103 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4104 memset(dict, 0, sizeof(U32));
4106 /* Enable MT, LDM, and use refPrefix() for a small dict */
4107 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4108 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4109 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
4111 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
4113 ZSTD_freeCCtx(cctx);
4116 DISPLAYLEVEL(3, "OK \n");
4118 DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++);
4120 ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0);
4121 ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0);
4122 ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400);
4123 ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000);
4124 ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000);
4126 assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict)));
4127 assert(!memcmp(&medium, &mediumDict, sizeof(medium)));
4128 assert(!memcmp(&large, &largeDict, sizeof(large)));
4130 DISPLAYLEVEL(3, "OK \n");
4132 DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++);
4134 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0);
4135 ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400);
4136 ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400);
4138 assert(smallSrcAndDict.windowLog == 10);
4139 assert(!memcmp(&cParams, &smallDict, sizeof(cParams)));
4141 DISPLAYLEVEL(3, "OK \n");
4143 DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
4146 size_t prevSize = 0;
4147 for (; level < ZSTD_maxCLevel(); ++level) {
4148 size_t const currSize = ZSTD_estimateCCtxSize(level);
4149 if (prevSize > currSize) {
4150 DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
4151 prevSize, level-1, currSize, level);
4154 prevSize = currSize;
4157 DISPLAYLEVEL(3, "OK \n");
4159 DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
4161 size_t const kSizeIncrement = 2 KB;
4164 for (; level <= ZSTD_maxCLevel(); ++level) {
4165 size_t dictSize = 0;
4166 for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
4167 size_t srcSize = 2 KB;
4168 for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
4169 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
4170 size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
4171 size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
4172 if (cctxSizeUsingLevel < cctxSizeUsingCParams
4173 || ZSTD_isError(cctxSizeUsingCParams)
4174 || ZSTD_isError(cctxSizeUsingLevel)) {
4175 DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
4176 level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
4179 DISPLAYLEVEL(3, "OK \n");
4181 DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++)
4183 int const threadPoolTestResult = threadPoolTests();
4184 if (threadPoolTestResult) {
4188 DISPLAYLEVEL(3, "thread pool tests OK \n");
4190 #endif /* ZSTD_MULTITHREAD */
4194 free(compressedBuffer);
4195 free(decodedBuffer);
4200 DISPLAY("Error detected in Unit tests ! \n");
4204 static int longUnitTests(U32 const seed, double compressibility)
4206 size_t const CNBuffSize = 5 MB;
4207 void* const CNBuffer = malloc(CNBuffSize);
4208 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
4209 void* const compressedBuffer = malloc(compressedBufferSize);
4210 void* const decodedBuffer = malloc(CNBuffSize);
4215 /* Create compressible noise */
4216 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
4217 DISPLAY("Not enough memory, aborting\n");
4221 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
4223 /* note : this test is rather long, it would be great to find a way to speed up its execution */
4224 DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++);
4226 size_t approxIndex = 0;
4227 size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */
4229 /* Provision enough space in a static context so that we can do all
4230 * this without ever reallocating, which would reset the indices. */
4231 size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22);
4232 void* const staticCCtxBuffer = malloc(staticCCtxSize);
4233 ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
4235 /* bump the indices so the following compressions happen at high
4237 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4238 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4239 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4240 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4241 while (approxIndex <= (maxIndex / 4) * 3) {
4242 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4243 approxIndex += in.pos;
4244 CHECK_Z(in.pos == in.size);
4248 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4251 /* spew a bunch of stuff into the table area */
4252 for (cLevel = 1; cLevel <= 22; cLevel++) {
4253 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4254 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4255 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4256 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4257 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4258 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4259 approxIndex += in.pos;
4262 /* now crank the indices so we overflow */
4263 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4264 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4265 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4266 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4267 while (approxIndex <= maxIndex) {
4268 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4269 approxIndex += in.pos;
4270 CHECK_Z(in.pos == in.size);
4274 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4277 /* do a bunch of compressions again in low indices and ensure we don't
4278 * hit untracked invalid indices */
4279 for (cLevel = 1; cLevel <= 22; cLevel++) {
4280 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4281 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4282 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4283 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4284 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4285 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4286 approxIndex += in.pos;
4289 free(staticCCtxBuffer);
4291 DISPLAYLEVEL(3, "OK \n");
4293 DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++);
4296 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4298 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed);
4300 /* Enable checksum to verify round trip. */
4301 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4302 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4303 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4305 /* Round trip once with ldm. */
4306 cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4308 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm));
4310 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4311 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4312 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
4313 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4315 /* Round trip once without ldm. */
4316 cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4317 CHECK_Z(cSizeNoLdm);
4318 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm));
4320 if (cSizeLdm > cSizeNoLdm) {
4321 DISPLAY("Using long mode should not cause regressions for btopt+\n");
4326 ZSTD_freeCCtx(cctx);
4328 DISPLAYLEVEL(3, "OK \n");
4330 DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++);
4331 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4332 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4333 size_t dictSize = CNBuffSize;
4334 void* dict = (void*)malloc(dictSize);
4335 ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams();
4336 ZSTD_dictAttachPref_e const attachPrefs[] = {
4337 ZSTD_dictDefaultAttach,
4338 ZSTD_dictForceAttach,
4341 ZSTD_dictDefaultAttach,
4342 ZSTD_dictForceAttach,
4346 int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1};
4350 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
4351 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
4353 CHECK_Z(cctx_params != NULL);
4355 for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
4356 DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
4357 for (cLevel = 4; cLevel < 13; cLevel++) {
4358 for (i = 0; i < 8; ++i) {
4359 ZSTD_dictAttachPref_e const attachPref = attachPrefs[i];
4360 int const enableDDS = enableDedicatedDictSearch[i];
4363 DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i);
4365 ZSTD_CCtxParams_init(cctx_params, cLevel);
4366 CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS));
4368 cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem);
4369 CHECK(cdict != NULL);
4371 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
4372 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref));
4374 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4376 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
4378 DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize);
4380 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
4381 ZSTD_freeCDict(cdict);
4384 ZSTD_freeCCtx(cctx);
4385 ZSTD_freeDCtx(dctx);
4386 ZSTD_freeCCtxParams(cctx_params);
4389 DISPLAYLEVEL(3, "OK \n");
4393 free(compressedBuffer);
4394 free(decodedBuffer);
4399 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
4401 const BYTE* b1 = (const BYTE*)buf1;
4402 const BYTE* b2 = (const BYTE*)buf2;
4404 for (u=0; u<max; u++) {
4405 if (b1[u] != b2[u]) break;
4411 static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
4413 ZSTD_parameters params;
4414 params.cParams = cParams;
4415 params.fParams = fParams;
4419 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
4421 size_t const lengthMask = ((size_t)1 << logLength) - 1;
4422 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
4425 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
4427 U32 const logLength = FUZ_rand(seed) % maxLog;
4428 return FUZ_rLogLength(seed, logLength);
4432 #define CHECK(cond, ...) { \
4434 DISPLAY("Error => "); \
4435 DISPLAY(__VA_ARGS__); \
4436 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4437 goto _output_error; \
4441 #define CHECK_Z(f) { \
4442 size_t const err = f; \
4443 if (ZSTD_isError(err)) { \
4444 DISPLAY("Error => %s : %s ", \
4445 #f, ZSTD_getErrorName(err)); \
4446 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4447 goto _output_error; \
4451 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
4453 static const U32 maxSrcLog = 23;
4454 static const U32 maxSampleLog = 22;
4455 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
4456 size_t const dstBufferSize = (size_t)1<<maxSampleLog;
4457 size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
4458 BYTE* cNoiseBuffer[5];
4459 BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
4460 BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
4461 BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
4462 ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
4463 ZSTD_CCtx* const ctx = ZSTD_createCCtx();
4464 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4466 unsigned testNb = 0;
4467 U32 coreSeed = seed;
4468 UTIL_time_t const startClock = UTIL_getTime();
4469 U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
4470 int const cLevelLimiter = bigTests ? 3 : 2;
4473 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
4474 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
4475 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
4476 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
4477 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
4478 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
4479 || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
4480 "Not enough memory, fuzzer tests cancelled");
4482 /* Create initial samples */
4483 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
4484 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
4485 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
4486 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
4487 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
4489 /* catch up testNb */
4490 for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
4492 /* main test loop */
4493 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
4494 BYTE* srcBuffer; /* jumping pointer */
4496 size_t sampleSize, maxTestSize, totalTestSize;
4497 size_t cSize, totalCSize, totalGenSize;
4504 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
4505 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
4507 FUZ_rand(&coreSeed);
4508 { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
4510 /* srcBuffer selection [0-4] */
4511 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
4512 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
4516 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
4517 buffNb = tnb[buffNb >> 3];
4519 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
4520 buffNb = tnb[buffNb >> 3];
4522 srcBuffer = cNoiseBuffer[buffNb];
4525 /* select src segment */
4526 sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
4528 /* create sample buffer (to catch read error with valgrind & sanitizers) */
4529 sampleBuffer = (BYTE*)malloc(sampleSize);
4530 CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
4531 { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
4532 memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
4533 crcOrig = XXH64(sampleBuffer, sampleSize, 0);
4535 /* compression tests */
4536 { int const cLevelPositive = (int)
4537 ( FUZ_rand(&lseed) %
4538 ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) )
4540 int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ?
4541 - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */
4543 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel);
4544 cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
4545 CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
4547 /* compression failure test : too small dest buffer */
4549 { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;
4550 const size_t tooSmallSize = cSize - missing;
4551 const unsigned endMark = 0x4DC2B1A9;
4552 memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark));
4553 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
4554 testNb, (unsigned)tooSmallSize, (unsigned)missing);
4555 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
4556 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
4557 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck));
4558 CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); }
4561 /* frame header decompression test */
4562 { ZSTD_frameHeader zfh;
4563 CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
4564 CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
4567 /* Decompressed size test */
4568 { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
4569 CHECK(rSize != sampleSize, "decompressed size incorrect");
4572 /* successful decompression test */
4573 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb);
4574 { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
4575 size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
4576 CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize);
4577 { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
4578 CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize);
4581 free(sampleBuffer); /* no longer useful after this point */
4583 /* truncated src decompression test */
4584 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb);
4585 { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4586 size_t const tooSmallSize = cSize - missing;
4587 void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
4588 CHECK(cBufferTooSmall == NULL, "not enough memory !");
4589 memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
4590 { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
4591 CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
4592 free(cBufferTooSmall);
4595 /* too small dst decompression test */
4596 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb);
4597 if (sampleSize > 3) {
4598 size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4599 size_t const tooSmallSize = sampleSize - missing;
4600 static const BYTE token = 0xA9;
4601 dstBuffer[tooSmallSize] = token;
4602 { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
4603 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
4604 CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
4607 /* noisy src decompression test */
4609 /* insert noise into src */
4610 { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
4611 size_t pos = 4; /* preserve magic number (too easy to detect) */
4613 /* keep some original src */
4614 { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
4615 size_t const mask = (1<<nbBits) - 1;
4616 size_t const skipLength = FUZ_rand(&lseed) & mask;
4619 if (pos >= cSize) break;
4621 { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
4622 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
4623 size_t const mask = (1<<nbBits) - 1;
4624 size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
4625 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
4626 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
4627 memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
4631 /* decompress noisy source */
4632 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb);
4633 { U32 const endMark = 0xA9B1C3D6;
4634 memcpy(dstBuffer+sampleSize, &endMark, 4);
4635 { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
4636 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
4637 CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
4638 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize);
4640 { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
4641 CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
4642 } } } /* noisy src decompression test */
4644 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
4645 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
4646 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
4647 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
4648 int const cLevel = (int)(FUZ_rand(&lseed) %
4649 ((U32)ZSTD_maxCLevel() -
4650 (MAX(testLog, dictLog) / (U32)cLevelLimiter))) +
4652 maxTestSize = FUZ_rLogLength(&lseed, testLog);
4653 if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
4655 dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
4656 dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
4658 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
4659 testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize);
4661 if (FUZ_rand(&lseed) & 0xF) {
4662 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
4664 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
4665 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
4666 !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
4667 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
4668 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
4669 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
4671 CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
4674 { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
4676 XXH64_state_t xxhState;
4677 XXH64_reset(&xxhState, 0);
4678 for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
4679 size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
4680 size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
4682 if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
4683 if (totalTestSize+segmentSize > maxTestSize) break;
4685 { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
4686 CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
4687 cSize += compressResult;
4689 XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
4690 memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
4691 totalTestSize += segmentSize;
4694 { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
4695 CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
4696 cSize += flushResult;
4698 crcOrig = XXH64_digest(&xxhState);
4701 /* streaming decompression test */
4702 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
4703 /* ensure memory requirement is good enough (should always be true) */
4704 { ZSTD_frameHeader zfh;
4705 CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
4706 "ZSTD_getFrameHeader(): error retrieving frame information");
4707 { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
4708 CHECK_Z(roundBuffSize);
4709 CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN),
4710 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)",
4711 (unsigned)roundBuffSize, (unsigned)totalTestSize );
4713 if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
4714 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
4717 while (totalCSize < cSize) {
4718 size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
4719 size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
4720 CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize));
4721 totalGenSize += genSize;
4722 totalCSize += inSize;
4724 CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
4725 CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
4726 CHECK (totalCSize != cSize, "compressed data should be fully read")
4727 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
4728 CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
4729 (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize);
4731 } /* for ( ; (testNb <= nbTests) */
4732 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
4735 ZSTD_freeCCtx(refCtx);
4737 ZSTD_freeDCtx(dctx);
4738 free(cNoiseBuffer[0]);
4739 free(cNoiseBuffer[1]);
4740 free(cNoiseBuffer[2]);
4741 free(cNoiseBuffer[3]);
4742 free(cNoiseBuffer[4]);
4754 /*_*******************************************************
4756 *********************************************************/
4757 static int FUZ_usage(const char* programName)
4759 DISPLAY( "Usage :\n");
4760 DISPLAY( " %s [args]\n", programName);
4762 DISPLAY( "Arguments :\n");
4763 DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault);
4764 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
4765 DISPLAY( " -s# : Select seed (default:prompt user)\n");
4766 DISPLAY( " -t# : Select starting test number (default:0)\n");
4767 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default);
4768 DISPLAY( " -v : verbose\n");
4769 DISPLAY( " -p : pause at the end\n");
4770 DISPLAY( " -h : display help and exit\n");
4774 /*! readU32FromChar() :
4775 @return : unsigned integer value read from input in `char` format
4776 allows and interprets K, KB, KiB, M, MB and MiB suffix.
4777 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
4778 Note : function result can overflow if digit string > MAX_UINT */
4779 static unsigned readU32FromChar(const char** stringPtr)
4781 unsigned result = 0;
4782 while ((**stringPtr >='0') && (**stringPtr <='9'))
4783 result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ;
4784 if ((**stringPtr=='K') || (**stringPtr=='M')) {
4786 if (**stringPtr=='M') result <<= 10;
4788 if (**stringPtr=='i') (*stringPtr)++;
4789 if (**stringPtr=='B') (*stringPtr)++;
4794 /** longCommandWArg() :
4795 * check if *stringPtr is the same as longCommand.
4796 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
4797 * @return 0 and doesn't modify *stringPtr otherwise.
4799 static int longCommandWArg(const char** stringPtr, const char* longCommand)
4801 size_t const comSize = strlen(longCommand);
4802 int const result = !strncmp(*stringPtr, longCommand, comSize);
4803 if (result) *stringPtr += comSize;
4807 int main(int argc, const char** argv)
4812 int nbTests = nbTestsDefault;
4814 int proba = FUZ_compressibility_default;
4818 U32 maxDuration = 0;
4821 U32 memTestsOnly = 0;
4822 const char* const programName = argv[0];
4824 /* Check command line */
4825 for (argNb=1; argNb<argc; argNb++) {
4826 const char* argument = argv[argNb];
4827 if(!argument) continue; /* Protection if argument empty */
4829 /* Handle commands. Aggregated commands are allowed */
4830 if (argument[0]=='-') {
4832 if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; }
4834 if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
4835 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
4836 if (!strcmp(argument, "--long-tests")) { longTests=1; continue; }
4837 if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; }
4840 while (*argument!=0) {
4844 return FUZ_usage(programName);
4856 case 'p': /* pause at the end */
4862 argument++; maxDuration = 0;
4863 nbTests = (int)readU32FromChar(&argument);
4869 maxDuration = readU32FromChar(&argument);
4870 if (*argument=='s') argument++; /* seconds */
4871 if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
4872 if (*argument=='n') argument++;
4878 seed = readU32FromChar(&argument);
4883 testNb = (int)readU32FromChar(&argument);
4886 case 'P': /* compressibility % */
4888 proba = (int)readU32FromChar(&argument);
4889 if (proba>100) proba = 100;
4893 return (FUZ_usage(programName), 1);
4894 } } } } /* for (argNb=1; argNb<argc; argNb++) */
4897 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
4900 time_t const t = time(NULL);
4901 U32 const h = XXH32(&t, sizeof(t), 1);
4905 DISPLAY("Seed = %u\n", (unsigned)seed);
4906 if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba);
4908 probfloat = ((double)proba) / 100;
4911 g_displayLevel = MAX(3, g_displayLevel);
4912 return FUZ_mallocTests(seed, probfloat, memTestsOnly);
4915 if (nbTests < testNb) nbTests = testNb;
4918 result = basicUnitTests(0, probfloat); /* constant seed for predictability */
4920 if (!result && longTests) {
4921 result = longUnitTests(0, probfloat);
4925 result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests);
4928 DISPLAY("Press Enter \n");