2 * Copyright (c) Meta Platforms, Inc. and affiliates.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
12 /*-************************************
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
22 /*-************************************
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #include <time.h> /* time(), time_t */
29 #undef NDEBUG /* always enable assert() */
31 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
32 #include "debug.h" /* DEBUG_STATIC_ASSERT */
34 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
35 #include "zstd.h" /* ZSTD_VERSION_STRING */
36 #include "zstd_errors.h" /* ZSTD_getErrorCode */
37 #define ZDICT_STATIC_LINKING_ONLY
38 #include "zdict.h" /* ZDICT_trainFromBuffer */
40 #include "datagen.h" /* RDG_genBuffer */
41 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
42 #include "xxhash.h" /* XXH64 */
44 #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
45 /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
46 #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
47 #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */
50 /*-************************************
52 **************************************/
55 static const int FUZ_compressibility_default = 50;
56 static const int nbTestsDefault = 30000;
59 /*-************************************
61 **************************************/
62 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
63 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
64 static U32 g_displayLevel = 2;
66 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
67 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
69 #define DISPLAYUPDATE(l, ...) \
70 if (g_displayLevel>=l) { \
71 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
72 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
73 if (g_displayLevel>=4) fflush(stderr); } \
77 /*-*******************************************************
79 *********************************************************/
82 /* Declaring the function, to avoid -Wmissing-prototype */
83 void FUZ_bug976(void);
85 { /* these constants shall not depend on MIN() macro */
86 DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
87 DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
91 /*-*******************************************************
93 *********************************************************/
94 #define MIN(a,b) ((a)<(b)?(a):(b))
95 #define MAX(a,b) ((a)>(b)?(a):(b))
97 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
98 static U32 FUZ_rand(U32* src)
100 static const U32 prime1 = 2654435761U;
101 static const U32 prime2 = 2246822519U;
105 rand32 = FUZ_rotl32(rand32, 13);
110 static U32 FUZ_highbit32(U32 v32)
113 if (v32==0) return 0;
114 while (v32) v32 >>= 1, nbBits++;
119 /*=============================================
121 =============================================*/
122 #define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
124 #define CHECK_Z(f) { \
125 size_t const err = f; \
126 if (ZSTD_isError(err)) { \
127 DISPLAY("Error => %s : %s ", \
128 #f, ZSTD_getErrorName(err)); \
132 #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
133 #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
134 #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
136 #define CHECK_OP(op, lhs, rhs) { \
137 if (!((lhs) op (rhs))) { \
138 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
142 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
143 #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
146 /*=============================================
148 =============================================*/
149 #if defined(__APPLE__) && defined(__MACH__)
151 #include <malloc/malloc.h> /* malloc_size */
154 unsigned long long totalMalloc;
155 size_t currentMalloc;
161 static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };
163 static void* FUZ_mallocDebug(void* counter, size_t size)
165 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
166 void* const ptr = malloc(size);
167 if (ptr==NULL) return NULL;
168 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
169 (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */
170 mcPtr->totalMalloc += size;
171 mcPtr->currentMalloc += size;
172 if (mcPtr->currentMalloc > mcPtr->peakMalloc)
173 mcPtr->peakMalloc = mcPtr->currentMalloc;
174 mcPtr->nbMalloc += 1;
178 static void FUZ_freeDebug(void* counter, void* address)
180 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
181 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10));
183 mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */
187 static void FUZ_displayMallocStats(mallocCounter_t count)
189 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n",
190 (unsigned)(count.peakMalloc >> 10),
192 (unsigned)(count.totalMalloc >> 10));
195 static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
196 void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
198 /* test only played in verbose mode, as they are long */
199 if (g_displayLevel<3) return 0;
201 /* Create compressible noise */
202 if (!inBuffer || !outBuffer) {
203 DISPLAY("Not enough memory, aborting\n");
206 RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
208 /* simple compression tests */
210 { int compressionLevel;
211 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
212 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
213 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
214 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
215 CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
217 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
218 FUZ_displayMallocStats(malcount);
221 /* streaming compression tests */
223 { int compressionLevel;
224 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
225 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
226 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
227 ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
228 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
229 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
230 CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
231 CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
232 CHECK_Z( ZSTD_endStream(cstream, &out) );
233 ZSTD_freeCStream(cstream);
234 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
235 FUZ_displayMallocStats(malcount);
238 /* advanced MT API test */
241 for (nbThreads=1; nbThreads<=4; nbThreads++) {
242 int compressionLevel;
243 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
244 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
245 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
246 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
247 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
248 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
249 CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
251 DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
252 nbThreads, compressionLevel);
253 FUZ_displayMallocStats(malcount);
256 /* advanced MT streaming API test */
259 for (nbThreads=1; nbThreads<=4; nbThreads++) {
260 int compressionLevel;
261 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
262 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
263 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
264 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
265 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
266 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
267 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
268 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
269 CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
270 while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
272 DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ",
273 nbThreads, compressionLevel);
274 FUZ_displayMallocStats(malcount);
280 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
282 size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
283 size_t const outSize = ZSTD_compressBound(inSize);
284 void* const inBuffer = malloc(inSize);
285 void* const outBuffer = malloc(outSize);
288 /* Create compressible noise */
289 if (!inBuffer || !outBuffer) {
290 DISPLAY("Not enough memory, aborting \n");
294 result = FUZ_mallocTests_internal(seed, compressibility, part,
295 inBuffer, inSize, outBuffer, outSize);
304 static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
306 (void)seed; (void)compressibility; (void)part;
312 static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
313 BYTE* src, size_t size, ZSTD_sequenceFormat_e format)
317 for(i = 0; i < seqsSize; ++i) {
318 assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size);
319 assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size);
320 if (format == ZSTD_sf_noBlockDelimiters) {
321 assert(seqs[i].matchLength != 0 || seqs[i].offset != 0);
324 memcpy(dst, src, seqs[i].litLength);
325 dst += seqs[i].litLength;
326 src += seqs[i].litLength;
327 size -= seqs[i].litLength;
329 if (seqs[i].offset != 0) {
330 for (j = 0; j < seqs[i].matchLength; ++j)
331 dst[j] = dst[j - seqs[i].offset];
332 dst += seqs[i].matchLength;
333 src += seqs[i].matchLength;
334 size -= seqs[i].matchLength;
337 if (format == ZSTD_sf_noBlockDelimiters) {
338 memcpy(dst, src, size);
342 #ifdef ZSTD_MULTITHREAD
346 ZSTD_threadPool* pool;
349 void* compressedBuffer;
350 size_t compressedBufferSize;
353 } threadPoolTests_compressionJob_payload;
355 static void* threadPoolTests_compressionJob(void* payload) {
356 threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload;
358 if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1;
359 cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize);
360 if (ZSTD_isError(cSize)) args->err = 1;
361 if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1;
365 static int threadPoolTests(void) {
369 size_t const CNBuffSize = 5 MB;
370 void* const CNBuffer = malloc(CNBuffSize);
371 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
372 void* const compressedBuffer = malloc(compressedBufferSize);
373 void* const decodedBuffer = malloc(CNBuffSize);
375 size_t const kPoolNumThreads = 8;
377 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0);
379 DISPLAYLEVEL(3, "thread pool test : threadPool re-use roundtrips: ");
381 ZSTD_CCtx* cctx = ZSTD_createCCtx();
382 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
384 size_t nbThreads = 1;
385 for (; nbThreads <= kPoolNumThreads; ++nbThreads) {
386 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
387 ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads);
388 err = ZSTD_CCtx_refThreadPool(cctx, pool);
389 if (ZSTD_isError(err)) {
390 DISPLAYLEVEL(3, "refThreadPool error!\n");
394 err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
395 if (ZSTD_isError(err)) {
396 DISPLAYLEVEL(3, "Compression error!\n");
400 err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err);
401 if (ZSTD_isError(err)) {
402 DISPLAYLEVEL(3, "Decompression error!\n");
409 ZSTD_freeThreadPool(pool);
411 DISPLAYLEVEL(3, "OK \n");
413 DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: ");
415 void* const decodedBuffer2 = malloc(CNBuffSize);
416 void* const compressedBuffer2 = malloc(compressedBufferSize);
417 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
418 ZSTD_CCtx* cctx1 = ZSTD_createCCtx();
419 ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
423 threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize,
424 compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */};
425 threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize,
426 compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */};
428 ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2);
429 ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2);
430 ZSTD_CCtx_refThreadPool(cctx1, pool);
431 ZSTD_CCtx_refThreadPool(cctx2, pool);
433 ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
434 ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
435 ZSTD_pthread_join(t1);
436 ZSTD_pthread_join(t2);
438 assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
439 free(decodedBuffer2);
440 free(compressedBuffer2);
442 ZSTD_freeThreadPool(pool);
443 ZSTD_freeCCtx(cctx1);
444 ZSTD_freeCCtx(cctx2);
446 if (p1.err || p2.err) goto _output_error;
448 DISPLAYLEVEL(3, "OK \n");
452 free(compressedBuffer);
458 DISPLAY("Error detected in Unit tests ! \n");
461 #endif /* ZSTD_MULTITHREAD */
463 /*=============================================
465 =============================================*/
467 static void test_compressBound(unsigned tnb)
469 DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
471 /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
472 * for a large range of known valid values */
473 DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
475 for (s=0; s<30; s++) {
476 size_t const w = (size_t)1 << s;
477 CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
480 /* Ensure error if srcSize too big */
481 { size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
482 CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
483 CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
486 DISPLAYLEVEL(3, "OK \n");
489 static void test_decompressBound(unsigned tnb)
491 DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
493 /* Simple compression, with size : should provide size; */
494 { const char example[] = "abcd";
495 char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
496 size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
498 CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
501 /* Simple small compression without size : should provide 1 block size */
502 { char cBuffer[ZSTD_COMPRESSBOUND(0)];
503 ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
504 ZSTD_inBuffer in = { NULL, 0, 0 };
505 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
507 CHECK_Z( ZSTD_initCStream(cctx, 0) );
508 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
509 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
510 CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
514 /* Attempt to overflow 32-bit intermediate multiplication result
515 * This requires dBound >= 4 GB, aka 2^32.
516 * This requires 2^32 / 2^17 = 2^15 blocks
517 * => create 2^15 blocks (can be empty, or just 1 byte). */
518 { const char input[] = "a";
519 size_t const nbBlocks = (1 << 15) + 1;
521 size_t const outCapacity = 1 << 18; /* large margin */
522 char* const outBuffer = malloc (outCapacity);
523 ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
524 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
527 CHECK_Z( ZSTD_initCStream(cctx, 0) );
528 for (blockNb=0; blockNb<nbBlocks; blockNb++) {
529 ZSTD_inBuffer in = { input, sizeof(input), 0 };
530 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
531 CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
533 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
534 CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000LLU /* 4 GB */ );
539 DISPLAYLEVEL(3, "OK \n");
542 static void test_setCParams(unsigned tnb)
544 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
545 ZSTD_compressionParameters cparams;
548 DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
551 cparams = ZSTD_getCParams(1, 0, 0);
552 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
554 /* invalid cparams (must fail) */
555 cparams.windowLog = 99;
556 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
559 DISPLAYLEVEL(3, "OK \n");
562 static int basicUnitTests(U32 const seed, double compressibility)
564 size_t const CNBuffSize = 5 MB;
565 void* const CNBuffer = malloc(CNBuffSize);
566 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
567 void* const compressedBuffer = malloc(compressedBufferSize);
568 void* const decodedBuffer = malloc(CNBuffSize);
573 /* Create compressible noise */
574 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
575 DISPLAY("Not enough memory, aborting\n");
579 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
582 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++);
583 { const char* errorString = ZSTD_getErrorName(0);
584 DISPLAYLEVEL(3, "OK : %s \n", errorString);
587 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++);
588 { const char* errorString = ZSTD_getErrorName(499);
589 DISPLAYLEVEL(3, "OK : %s \n", errorString);
592 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++);
593 { int const mcl = ZSTD_minCLevel();
594 DISPLAYLEVEL(3, "%i (OK) \n", mcl);
597 DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++);
598 { int const defaultCLevel = ZSTD_defaultCLevel();
599 if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error;
600 DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel);
603 DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
604 { unsigned const vn = ZSTD_versionNumber();
605 DISPLAYLEVEL(3, "%u (OK) \n", vn);
608 test_compressBound(testNb++);
610 test_decompressBound(testNb++);
612 test_setCParams(testNb++);
614 DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
616 ZSTD_compressionParameters params;
617 memset(¶ms, 0, sizeof(params));
618 params.windowLog = 10;
620 params.chainLog = 19;
621 params = ZSTD_adjustCParams(params, 1000, 100000);
622 if (params.hashLog != 18) goto _output_error;
623 if (params.chainLog != 17) goto _output_error;
625 DISPLAYLEVEL(3, "OK \n");
627 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
628 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
629 if (cctx==NULL) goto _output_error;
630 CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
631 compressedBuffer, compressedBufferSize,
632 CNBuffer, CNBuffSize, 1) );
633 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
635 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
636 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
637 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize);
642 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++);
644 char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
645 size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8);
646 if (!ZSTD_isError(size)) goto _output_error;
648 DISPLAYLEVEL(3, "OK \n");
650 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
651 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
652 if (rSize != CNBuffSize) goto _output_error;
654 DISPLAYLEVEL(3, "OK \n");
656 DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++);
657 { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
658 if (rSize != CNBuffSize) goto _output_error;
660 DISPLAYLEVEL(3, "OK \n");
662 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
663 { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
664 if (rSize != CNBuffSize) goto _output_error;
666 DISPLAYLEVEL(3, "OK \n");
668 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
670 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
671 if (bound != CNBuffSize) goto _output_error;
673 DISPLAYLEVEL(3, "OK \n");
675 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
677 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
678 if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
680 DISPLAYLEVEL(3, "OK \n");
682 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
683 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
684 if (r != CNBuffSize) goto _output_error; }
685 DISPLAYLEVEL(3, "OK \n");
687 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
689 ZSTD_DCtx* dctx = ZSTD_createDCtx();
691 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
692 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
693 if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
696 DISPLAYLEVEL(3, "OK \n");
698 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
700 for (u=0; u<CNBuffSize; u++) {
701 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
703 DISPLAYLEVEL(3, "OK \n");
705 DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++);
706 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
707 ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 };
708 ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 };
709 if (cctx==NULL) goto _output_error;
710 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */
711 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */
714 DISPLAYLEVEL(3, "OK \n");
716 DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++);
718 ZSTD_parameters params = ZSTD_getParams(3, 0, 0);
719 assert(!ZSTD_checkCParams(params.cParams));
721 DISPLAYLEVEL(3, "OK \n");
723 DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++);
725 ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
726 assert(dctx != NULL);
727 assert(ZSTD_sizeof_DCtx(dctx) != 0);
730 DISPLAYLEVEL(3, "OK \n");
732 DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++);
734 /* %p takes a void*. In ISO C, it's illegal to cast a function pointer
735 * to a data pointer. (Although in POSIX you're required to be allowed
736 * to do it...) So we have to fall back to our trusty friend memcpy. */
737 unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) =
738 ZSTD_getDictID_fromDDict;
739 ZSTD_DStream* (* const funcptr_createDStream)(
740 ZSTD_customMem customMem) = ZSTD_createDStream_advanced;
741 void (* const funcptr_copyDCtx)(
742 ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx;
743 ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) =
745 const void *voidptr_getDictID;
746 const void *voidptr_createDStream;
747 const void *voidptr_copyDCtx;
748 const void *voidptr_nextInputType;
749 DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID));
751 (void*)&voidptr_getDictID,
752 (const void*)&funcptr_getDictID,
755 (void*)&voidptr_createDStream,
756 (const void*)&funcptr_createDStream,
759 (void*)&voidptr_copyDCtx,
760 (const void*)&funcptr_copyDCtx,
763 (void*)&voidptr_nextInputType,
764 (const void*)&funcptr_nextInputType,
766 DISPLAYLEVEL(3, "%p ", voidptr_getDictID);
767 DISPLAYLEVEL(3, "%p ", voidptr_createDStream);
768 DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx);
769 DISPLAYLEVEL(3, "%p ", voidptr_nextInputType);
771 DISPLAYLEVEL(3, ": OK \n");
773 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
774 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
775 { size_t const r = ZSTD_decompress_usingDict(dctx,
776 decodedBuffer, CNBuffSize,
777 compressedBuffer, cSize,
779 if (r != CNBuffSize) goto _output_error;
783 DISPLAYLEVEL(3, "OK \n");
785 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
786 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
787 { size_t const r = ZSTD_decompress_usingDDict(dctx,
788 decodedBuffer, CNBuffSize,
789 compressedBuffer, cSize,
791 if (r != CNBuffSize) goto _output_error;
795 DISPLAYLEVEL(3, "OK \n");
797 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
798 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
799 if (!ZSTD_isError(r)) goto _output_error;
800 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
801 DISPLAYLEVEL(3, "OK \n");
803 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
804 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
805 if (!ZSTD_isError(r)) goto _output_error;
806 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
807 DISPLAYLEVEL(3, "OK \n");
809 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++);
810 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize);
811 if (!ZSTD_isError(r)) goto _output_error;
812 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
813 DISPLAYLEVEL(3, "OK \n");
815 DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++);
816 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize);
817 if (!ZSTD_isError(r)) goto _output_error;
818 if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; }
819 DISPLAYLEVEL(3, "OK \n");
821 DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++);
822 { /* create compressed buffer with checksumming enabled */
823 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
825 DISPLAY("Not enough memory, aborting\n");
829 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
830 CHECK_VAR(cSize, ZSTD_compress2(cctx,
831 compressedBuffer, compressedBufferSize,
832 CNBuffer, CNBuffSize) );
835 { /* copy the compressed buffer and corrupt the checksum */
837 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
839 DISPLAY("Not enough memory, aborting\n");
844 ((char*)compressedBuffer)[cSize-1] += 1;
845 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
846 if (!ZSTD_isError(r)) goto _output_error;
847 if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error;
849 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum));
850 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
851 if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */
852 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
853 if (ZSTD_isError(r)) goto _output_error;
857 DISPLAYLEVEL(3, "OK \n");
860 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
861 { /* create compressed buffer with content size missing */
862 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
863 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
864 CHECK_VAR(cSize, ZSTD_compress2(cctx,
865 compressedBuffer, compressedBufferSize,
866 CNBuffer, CNBuffSize) );
869 { /* ensure frame content size is missing */
870 ZSTD_frameHeader zfh;
871 size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
872 if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
874 { /* ensure CNBuffSize <= decompressBound */
875 unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
876 if (CNBuffSize > bound) goto _output_error;
878 DISPLAYLEVEL(3, "OK \n");
880 DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++);
882 size_t const largeFrameSrcSize = 200;
883 size_t const smallFrameSrcSize = 10;
884 size_t const nbFrames = 256;
886 size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0;
889 BYTE* const dst = (BYTE*)compressedBuffer;
890 ZSTD_DCtx* dctx = ZSTD_createDCtx();
892 /* create a large frame and then a bunch of small frames */
893 size_t srcSize = ZSTD_compress((void*)dst,
894 compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
895 for (i = 0; i < nbFrames; i++)
896 srcSize += ZSTD_compress((void*)(dst + srcSize),
897 compressedBufferSize - srcSize, CNBuffer,
898 smallFrameSrcSize, 3);
900 /* decompressStream and make sure that dctx size was reduced at least once */
901 while (consumed < srcSize) {
902 ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
903 ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0};
904 ZSTD_decompressStream(dctx, &out, &in);
908 /* success! size was reduced from the previous frame */
909 if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx))
912 prevDCtxSize = ZSTD_sizeof_DCtx(dctx);
919 DISPLAYLEVEL(3, "OK \n");
922 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
923 ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1);
924 ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
925 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
927 DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++);
928 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1));
929 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
930 DISPLAYLEVEL(3, "OK \n");
932 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++);
933 CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1));
934 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
935 DISPLAYLEVEL(3, "OK \n");
937 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++);
938 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict));
939 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
940 DISPLAYLEVEL(3, "OK \n");
942 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++);
943 CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params));
944 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
945 DISPLAYLEVEL(3, "OK \n");
947 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++);
948 CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams));
949 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
950 DISPLAYLEVEL(3, "OK \n");
952 ZSTD_freeCDict(cdict);
956 DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
958 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
960 size_t const size = (1U << 10);
961 size_t const dstCapacity = ZSTD_compressBound(size);
962 void* dict = (void*)malloc(size);
963 void* src = (void*)malloc(size);
964 void* dst = (void*)malloc(dstCapacity);
966 RDG_genBuffer(dict, size, 0.5, 0.5, seed);
967 RDG_genBuffer(src, size, 0.5, 0.5, seed);
969 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
970 assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
977 DISPLAYLEVEL(3, "OK \n");
979 DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
981 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
982 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
983 void* dict = (void*)malloc(CNBuffSize);
986 for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) {
987 RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
988 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
990 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
991 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
992 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
993 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
994 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
995 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
997 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize));
1000 ZSTD_freeCCtx(cctx);
1001 ZSTD_freeDCtx(dctx);
1004 DISPLAYLEVEL(3, "OK \n");
1006 DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++);
1008 size_t const testSize = 1024;
1009 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1010 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1011 char* dict = (char*)malloc(2 * testSize);
1012 int ldmEnabled, level;
1014 RDG_genBuffer(dict, testSize, 0.5, 0.5, seed);
1015 RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
1016 memcpy(dict + testSize, CNBuffer, testSize);
1017 for (level = 1; level <= 5; ++level) {
1018 for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
1020 XXH64_hash_t compressedChecksum0;
1022 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1023 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level));
1024 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled));
1025 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1));
1027 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1028 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize);
1030 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize));
1033 compressedChecksum0 = XXH64(compressedBuffer, cSize, 0);
1035 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1036 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize);
1039 if (cSize != cSize0) goto _output_error;
1040 if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error;
1044 ZSTD_freeCCtx(cctx);
1045 ZSTD_freeDCtx(dctx);
1048 DISPLAYLEVEL(3, "OK \n");
1050 DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
1051 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1052 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1053 size_t const srcSize = 300 KB;
1054 size_t const flushSize = 128 KB + 5;
1055 size_t const dstSize = ZSTD_compressBound(srcSize);
1056 char* src = (char*)CNBuffer;
1057 char* dst = (char*)compressedBuffer;
1059 ZSTD_outBuffer out = { dst, dstSize, 0 };
1060 ZSTD_inBuffer in = { src, flushSize, 0 };
1062 if (!cctx || !dctx) {
1063 DISPLAY("Not enough memory, aborting\n");
1068 RDG_genBuffer(src, srcSize, 0.5, 0.5, seed);
1069 /* Force an LDM to exist that crosses block boundary into uncompressible block */
1070 memcpy(src + 125 KB, src, 3 KB + 5);
1072 /* Enable MT, LDM, and opt parser */
1073 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
1074 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1075 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1078 /* Flushes a block of 128 KB and block of 5 bytes */
1079 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1081 /* Compress the rest */
1083 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1085 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos));
1087 ZSTD_freeCCtx(cctx);
1088 ZSTD_freeDCtx(dctx);
1090 DISPLAYLEVEL(3, "OK \n");
1092 DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++);
1094 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1095 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1096 void* dict = (void*)malloc(CNBuffSize);
1097 size_t const kWindowLog = 10;
1098 size_t const kWindowSize = (size_t)1 << kWindowLog;
1099 size_t const dictSize = kWindowSize * 10;
1100 size_t const srcSize1 = kWindowSize / 2;
1101 size_t const srcSize2 = kWindowSize * 10;
1103 if (CNBuffSize < dictSize) goto _output_error;
1105 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
1106 RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed);
1108 /* Enable checksum to verify round trip. */
1109 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1110 /* Disable content size to skip single-pass decompression. */
1111 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
1112 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
1113 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1114 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
1115 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
1116 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
1117 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3));
1119 /* Round trip once with a dictionary. */
1120 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1121 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
1123 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1124 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
1125 /* Streaming decompression to catch out of bounds offsets. */
1127 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1128 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1129 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1131 if (dSize != 0) goto _output_error;
1134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
1135 /* Round trip once with a dictionary. */
1136 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1138 ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
1139 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1140 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1141 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1144 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1146 ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
1147 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1148 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1149 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1152 /* Streaming decompression to catch out of bounds offsets. */
1154 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1155 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1156 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1158 if (dSize != 0) goto _output_error;
1161 ZSTD_freeCCtx(cctx);
1162 ZSTD_freeDCtx(dctx);
1165 DISPLAYLEVEL(3, "OK \n");
1167 /* Note: this test takes 0.5 seconds to run */
1168 DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
1170 /* test a big buffer so that ldm can take effect */
1171 size_t const size = 100 MB;
1172 int const windowLog = 27;
1173 size_t const dstSize = ZSTD_compressBound(size);
1175 void* dict = (void*)malloc(size);
1176 void* src = (void*)malloc(size);
1177 void* dst = (void*)malloc(dstSize);
1178 void* recon = (void*)malloc(size);
1180 size_t refPrefixCompressedSize = 0;
1181 size_t refPrefixLdmCompressedSize = 0;
1182 size_t reconSize = 0;
1184 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1185 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1187 /* make dict and src the same uncompressible data */
1188 RDG_genBuffer(src, size, 0, 0, seed);
1189 memcpy(dict, src, size);
1190 assert(!memcmp(dict, src, size));
1192 /* set level 1 and windowLog to cover src */
1193 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1));
1194 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog));
1196 /* compress on level 1 using just refPrefix and no ldm */
1197 ZSTD_CCtx_refPrefix(cctx, dict, size);
1198 refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1199 assert(!ZSTD_isError(refPrefixCompressedSize));
1201 /* test round trip just refPrefix */
1202 ZSTD_DCtx_refPrefix(dctx, dict, size);
1203 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize);
1204 assert(!ZSTD_isError(reconSize));
1205 assert(reconSize == size);
1206 assert(!memcmp(recon, src, size));
1208 /* compress on level 1 using refPrefix and ldm */
1209 ZSTD_CCtx_refPrefix(cctx, dict, size);;
1210 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
1211 refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1212 assert(!ZSTD_isError(refPrefixLdmCompressedSize));
1214 /* test round trip refPrefix + ldm*/
1215 ZSTD_DCtx_refPrefix(dctx, dict, size);
1216 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize);
1217 assert(!ZSTD_isError(reconSize));
1218 assert(reconSize == size);
1219 assert(!memcmp(recon, src, size));
1221 /* make sure that refPrefixCompressedSize is significantly greater */
1222 assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize);
1223 /* make sure the ldm compressed size is less than 1% of original */
1224 assert((double)refPrefixLdmCompressedSize / (double)size < 0.01);
1226 ZSTD_freeDCtx(dctx);
1227 ZSTD_freeCCtx(cctx);
1233 DISPLAYLEVEL(3, "OK \n");
1235 DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++);
1236 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX);
1238 CHECK_LT(CNBuffSize, cSize);
1240 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1241 size_t const outputSize = (CNBuffSize + margin);
1242 char* output = malloc(outputSize);
1243 char* input = output + outputSize - cSize;
1244 CHECK_LT(cSize, CNBuffSize + margin);
1245 CHECK(output != NULL);
1247 CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX));
1248 memcpy(input, compressedBuffer, cSize);
1251 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1253 CHECK_EQ(dSize, CNBuffSize);
1255 CHECK(!memcmp(output, CNBuffer, CNBuffSize));
1258 DISPLAYLEVEL(3, "OK \n");
1260 DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++);
1261 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1264 size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1269 size_t const srcSize = (CNBuffSize / 3) * 2;
1270 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1271 size_t const outputSize = (CNBuffSize + margin);
1272 char* output = malloc(outputSize);
1273 char* input = output + outputSize - cSize;
1274 CHECK_LT(cSize, CNBuffSize + margin);
1275 CHECK(output != NULL);
1277 memcpy(input, compressedBuffer, cSize);
1280 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1282 CHECK_EQ(dSize, srcSize);
1284 CHECK(!memcmp(output, CNBuffer, srcSize));
1287 DISPLAYLEVEL(3, "OK \n");
1289 DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++);
1290 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1291 size_t const srcSize = 256 * 1024;
1292 U32 const compressibleLenU32 = 32 * 1024 / 4;
1293 U32 const blockSizeU32 = 128 * 1024 / 4;
1294 U32 const litLenU32 = 64 * 1024 / 4;
1295 U32* data = (U32*)malloc(srcSize);
1298 if (data == NULL || cctx == NULL) goto _output_error;
1300 /* Generate data without any matches */
1301 RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U);
1302 /* Generate 32K of compressible data */
1303 RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe);
1305 /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */
1306 data[compressibleLenU32 + 0] = 0xFFFFFFFF;
1307 data[compressibleLenU32 + 1] = 0xEEEEEEEE;
1308 data[compressibleLenU32 + 4] = 0xFFFFFFFF;
1309 data[compressibleLenU32 + 5] = 0xEEEEEEEE;
1311 /* Add a match of offset=16, length=8 at idx=64K + 64.
1312 * This generates a sequence with llen=64K, and repeat code 1.
1313 * The block splitter thought this was ll0, and corrupted the
1314 * repeat offset history.
1316 data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD;
1317 data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC;
1318 data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD;
1319 data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC;
1321 /* Add a match of offset=16, length=8 at idx=128K + 16.
1322 * This should generate a sequence with repeat code = 1.
1323 * But the block splitters mistake caused zstd to generate
1324 * repeat code = 2, corrupting the data.
1326 data[blockSizeU32] = 0xBBBBBBBB;
1327 data[blockSizeU32 + 1] = 0xAAAAAAAA;
1328 data[blockSizeU32 + 4] = 0xBBBBBBBB;
1329 data[blockSizeU32 + 5] = 0xAAAAAAAA;
1331 /* Generate a golden file from this data in case datagen changes and
1332 * doesn't generate the exact same data. We will also test this golden file.
1335 FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb");
1336 fwrite(data, 1, srcSize, f);
1340 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1341 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7));
1342 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable));
1344 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize);
1346 dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1348 CHECK_EQ(dSize, srcSize);
1349 CHECK(!memcmp(decodedBuffer, data, srcSize));
1352 ZSTD_freeCCtx(cctx);
1354 DISPLAYLEVEL(3, "OK \n");
1356 DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
1358 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1359 const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;
1360 size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize);
1362 /* This is the number of bytes to stream before ending. This value
1363 * was obtained by trial and error :/. */
1365 const size_t streamCompressThreshold = 161792;
1366 const size_t streamCompressDelta = 1024;
1368 /* The first 1/5 of the buffer is compressible and the last 4/5 is
1369 * uncompressible. This is an approximation of the type of data
1370 * the fuzzer generated to catch this bug. Streams like this were making
1371 * zstd generate noCompress superblocks (which are larger than the src
1372 * they come from). Do this enough times, and we'll run out of room
1373 * and throw a dstSize_tooSmall error. */
1375 const size_t compressiblePartSize = srcSize/5;
1376 const size_t uncompressiblePartSize = srcSize-compressiblePartSize;
1377 RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed);
1378 RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed);
1380 /* Setting target block size so that superblock is used */
1382 assert(cctx != NULL);
1383 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81);
1386 for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) {
1387 ZSTD_inBuffer in = {src, streamCompressDelta, 0};
1388 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1389 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1390 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1391 src += streamCompressDelta; srcSize -= streamCompressDelta;
1392 dst += out.pos; dstCapacity -= out.pos;
1395 /* This is trying to catch a dstSize_tooSmall error */
1397 { ZSTD_inBuffer in = {src, srcSize, 0};
1398 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1399 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1401 ZSTD_freeCCtx(cctx);
1403 DISPLAYLEVEL(3, "OK \n");
1405 DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++);
1406 /* Generate the same data 20 times over */
1407 { size_t const avgChunkSize = CNBuffSize / 20;
1409 for (b = 0; b < CNBuffSize; b += avgChunkSize) {
1410 size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize);
1411 RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed);
1413 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1414 size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1415 size_t const allowedExpansion = (CNBuffSize * 3 / 1000);
1417 CHECK_Z(normalCSize);
1418 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
1419 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000);
1420 superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1421 CHECK_Z(superCSize);
1422 if (superCSize > normalCSize + allowedExpansion) {
1423 DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion);
1426 ZSTD_freeCCtx(cctx);
1428 DISPLAYLEVEL(3, "OK \n");
1430 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed);
1431 DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++)
1432 /* This tests whether or not we leave enough room for the checksum at the end
1433 * of the dst buffer. The bug that motivated this test was found by the
1434 * stream_round_trip fuzzer but this crashes for the same reason and is
1435 * far more compact than re-creating the stream_round_trip fuzzer's code path */
1436 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1437 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64);
1438 assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278)));
1439 ZSTD_freeCCtx(cctx);
1441 DISPLAYLEVEL(3, "OK \n");
1443 DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
1445 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1446 if (!cctx) goto _output_error;
1447 for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
1448 CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
1449 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
1450 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
1452 ZSTD_freeCCtx(cctx);
1454 DISPLAYLEVEL(3, "OK \n");
1456 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
1457 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1458 size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
1459 if (ZSTD_isError(r)) goto _output_error;
1460 if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
1461 ZSTD_freeCCtx(cctx);
1464 DISPLAYLEVEL(3, "OK \n");
1466 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++);
1467 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize);
1468 if (ZSTD_isError(r)) goto _output_error;
1469 if (r != 0) goto _output_error;
1471 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1472 ZSTD_outBuffer output;
1473 if (cctx==NULL) goto _output_error;
1474 output.dst = compressedBuffer;
1475 output.size = compressedBufferSize;
1477 CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */
1478 CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */
1479 CHECK_Z( ZSTD_endStream(cctx, &output) );
1480 ZSTD_freeCCtx(cctx);
1481 /* single scan decompression */
1482 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos);
1483 if (ZSTD_isError(r)) goto _output_error;
1484 if (r != 0) goto _output_error;
1486 /* streaming decompression */
1487 { ZSTD_DCtx* const dstream = ZSTD_createDStream();
1488 ZSTD_inBuffer dinput;
1489 ZSTD_outBuffer doutput;
1491 if (dstream==NULL) goto _output_error;
1492 dinput.src = compressedBuffer;
1498 CHECK_Z ( ZSTD_initDStream(dstream) );
1499 for (ipos=1; ipos<=output.pos; ipos++) {
1501 CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) );
1503 if (doutput.pos != 0) goto _output_error;
1504 ZSTD_freeDStream(dstream);
1507 DISPLAYLEVEL(3, "OK \n");
1509 DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++);
1510 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1511 ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1512 assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
1513 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
1514 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
1516 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1517 { size_t const inSize = 2* 128 KB;
1518 size_t const outSize = ZSTD_compressBound(inSize);
1519 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
1520 /* will fail if blockSize is not resized */
1522 ZSTD_freeCCtx(cctx);
1524 DISPLAYLEVEL(3, "OK \n");
1526 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++);
1527 { size_t const sampleSize = 30;
1529 for (i=0; i<20; i++)
1530 ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */
1531 memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
1532 for (i=1; i<=19; i++) {
1533 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1534 size_t size1, size2;
1535 DISPLAYLEVEL(5, "l%i ", i);
1536 size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1539 size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1541 CHECK_EQ(size1, size2);
1543 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) );
1544 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize);
1546 CHECK_EQ(size1, size2);
1548 size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */
1550 CHECK_EQ(size1, size2);
1552 { ZSTD_inBuffer inb;
1553 ZSTD_outBuffer outb;
1556 inb.size = sampleSize;
1557 outb.dst = compressedBuffer;
1559 outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */
1560 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );
1561 assert(inb.pos == inb.size);
1562 CHECK_EQ(size1, outb.pos);
1565 ZSTD_freeCCtx(cctx);
1568 DISPLAYLEVEL(3, "OK \n");
1570 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++);
1571 { size_t const sampleSize = 1024;
1572 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1574 ZSTD_outBuffer outb;
1578 outb.dst = compressedBuffer;
1580 outb.size = compressedBufferSize;
1581 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) );
1583 inb.size = sampleSize; /* start with something, so that context is already used */
1584 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1585 assert(inb.pos == inb.size);
1586 outb.pos = 0; /* cancel output */
1588 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) );
1589 inb.size = 4; /* too small size : compression will be skipped */
1591 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1592 assert(inb.pos == inb.size);
1594 inb.size += 5; /* too small size : compression will be skipped */
1595 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1596 assert(inb.pos == inb.size);
1598 inb.size += 11; /* small enough to attempt compression */
1599 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1600 assert(inb.pos == inb.size);
1602 assert(inb.pos < sampleSize);
1603 inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */
1604 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1605 assert(inb.pos == inb.size);
1606 ZSTD_freeCCtx(cctx);
1608 DISPLAYLEVEL(3, "OK \n");
1610 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
1611 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1612 ZSTD_outBuffer out = {NULL, 0, 0};
1613 ZSTD_inBuffer in = {NULL, 0, 0};
1616 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1618 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1620 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
1621 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1623 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1624 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1625 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
1626 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1628 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1629 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1630 /* Start a compression job */
1631 ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1632 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1634 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1635 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1636 /* Reset the CCtx */
1637 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
1638 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1640 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1641 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1642 /* Reset the parameters */
1643 ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
1644 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1646 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1649 ZSTD_freeCCtx(cctx);
1651 DISPLAYLEVEL(3, "OK \n");
1653 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++);
1654 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1656 ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0);
1657 cparams.strategy = -1;
1658 /* Set invalid cParams == no change. */
1659 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
1661 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1663 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1665 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1667 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1669 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1671 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1673 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1676 cparams = ZSTD_getCParams(12, 0, 0);
1677 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
1679 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1680 CHECK_EQ(value, (int)cparams.windowLog);
1681 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1682 CHECK_EQ(value, (int)cparams.chainLog);
1683 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1684 CHECK_EQ(value, (int)cparams.hashLog);
1685 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1686 CHECK_EQ(value, (int)cparams.searchLog);
1687 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1688 CHECK_EQ(value, (int)cparams.minMatch);
1689 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1690 CHECK_EQ(value, (int)cparams.targetLength);
1691 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1692 CHECK_EQ(value, (int)cparams.strategy);
1694 ZSTD_freeCCtx(cctx);
1697 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++);
1698 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1700 ZSTD_frameParameters fparams = {0, 1, 1};
1702 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1704 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1706 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1709 CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams));
1711 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1712 CHECK_EQ(value, fparams.contentSizeFlag);
1713 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1714 CHECK_EQ(value, fparams.checksumFlag);
1715 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1716 CHECK_EQ(value, !fparams.noDictIDFlag);
1718 ZSTD_freeCCtx(cctx);
1721 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCarams() : ", testNb++);
1722 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1724 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
1725 params.cParams.strategy = -1;
1726 /* Set invalid params == no change. */
1727 CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params)));
1729 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1731 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1733 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1735 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1737 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1739 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1741 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1743 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1745 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1747 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1750 params = ZSTD_getParams(12, 0, 0);
1751 params.fParams.contentSizeFlag = 0;
1752 params.fParams.checksumFlag = 1;
1753 params.fParams.noDictIDFlag = 1;
1754 CHECK_Z(ZSTD_CCtx_setParams(cctx, params));
1756 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1757 CHECK_EQ(value, (int)params.cParams.windowLog);
1758 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1759 CHECK_EQ(value, (int)params.cParams.chainLog);
1760 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1761 CHECK_EQ(value, (int)params.cParams.hashLog);
1762 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1763 CHECK_EQ(value, (int)params.cParams.searchLog);
1764 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1765 CHECK_EQ(value, (int)params.cParams.minMatch);
1766 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1767 CHECK_EQ(value, (int)params.cParams.targetLength);
1768 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1769 CHECK_EQ(value, (int)params.cParams.strategy);
1770 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1771 CHECK_EQ(value, params.fParams.contentSizeFlag);
1772 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1773 CHECK_EQ(value, params.fParams.checksumFlag);
1774 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1775 CHECK_EQ(value, !params.fParams.noDictIDFlag);
1777 ZSTD_freeCCtx(cctx);
1780 DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
1781 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1782 ZSTD_outBuffer out = {NULL, 0, 0};
1783 ZSTD_inBuffer in = {NULL, 0, 0};
1786 /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt),
1787 * we should not modify the actual parameter specified by the user within the CCtx
1789 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27));
1790 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt));
1792 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1793 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value));
1796 ZSTD_freeCCtx(cctx);
1798 DISPLAYLEVEL(3, "OK \n");
1800 /* this test is really too long, and should be made faster */
1801 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
1802 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1803 ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1804 size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
1806 assert(cctx != NULL);
1807 params.fParams.contentSizeFlag = 0;
1808 params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
1809 for (cnb = 0; cnb < nbCompressions; ++cnb) {
1810 DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions);
1811 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1812 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
1814 ZSTD_freeCCtx(cctx);
1816 DISPLAYLEVEL(3, "OK \n");
1818 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
1819 { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
1820 assert(largeCCtx != NULL);
1821 CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
1822 CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
1823 { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
1824 { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
1825 assert(smallCCtx != NULL);
1826 CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1827 { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
1828 DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
1829 largeCCtxSize>>10, smallCCtxSize>>10);
1830 assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
1831 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
1833 ZSTD_freeCCtx(smallCCtx);
1835 { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
1836 * currently defined as 128x, but could be adjusted in the future.
1837 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
1839 for (u=0; u<maxNbAttempts; u++) {
1840 CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1841 if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
1843 DISPLAYLEVEL(5, "size down after %u attempts : ", u);
1844 if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
1847 ZSTD_freeCCtx(largeCCtx);
1849 DISPLAYLEVEL(3, "OK \n");
1851 /* Static CCtx tests */
1852 #define STATIC_CCTX_LEVEL 4
1853 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1854 { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
1855 void* const staticCCtxBuffer = malloc(staticCStreamSize);
1856 size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
1857 void* const staticDCtxBuffer = malloc(staticDCtxSize);
1858 DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize);
1859 if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
1860 free(staticCCtxBuffer);
1861 free(staticDCtxBuffer);
1862 DISPLAY("Not enough memory, aborting\n");
1866 { size_t const smallInSize = 32 KB;
1867 ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0);
1868 size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small);
1869 size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL);
1870 ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize);
1871 ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
1872 DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize);
1873 DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize);
1874 if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
1875 DISPLAYLEVEL(3, "OK \n");
1877 DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++);
1878 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1879 compressedBuffer, compressedBufferSize,
1880 CNBuffer, smallInSize, STATIC_CCTX_LEVEL) );
1881 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1882 (unsigned)cSize, (double)cSize/smallInSize*100);
1884 DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++);
1885 { size_t const r = ZSTD_compressCCtx(staticCCtx,
1886 compressedBuffer, compressedBufferSize,
1887 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL);
1888 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error;
1890 DISPLAYLEVEL(3, "OK \n");
1892 DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
1893 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
1894 DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx);
1895 if (staticCCtx == NULL) goto _output_error;
1896 DISPLAYLEVEL(3, "OK \n");
1898 DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++);
1899 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1900 compressedBuffer, compressedBufferSize,
1901 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1902 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1903 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1905 DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++);
1907 assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */
1908 assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3);
1909 for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) {
1910 CHECK_Z(ZSTD_compressCCtx(staticCCtx,
1911 compressedBuffer, compressedBufferSize,
1912 (char*)CNBuffer + nbc, smallInSize,
1913 STATIC_CCTX_LEVEL) );
1915 DISPLAYLEVEL(3, "OK \n")
1917 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1918 CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) );
1919 DISPLAYLEVEL(3, "OK \n");
1921 DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++);
1922 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1923 compressedBuffer, compressedBufferSize,
1924 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1925 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1926 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1928 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
1929 { size_t const r = ZSTD_decompressDCtx(staticDCtx,
1930 decodedBuffer, CNBuffSize,
1931 compressedBuffer, cSize);
1932 if (r != CNBuffSize) goto _output_error; }
1933 DISPLAYLEVEL(3, "OK \n");
1935 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1936 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
1937 DISPLAYLEVEL(3, "OK \n");
1939 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
1940 { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
1941 if (!ZSTD_isError(r)) goto _output_error; }
1942 DISPLAYLEVEL(3, "OK \n");
1944 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
1945 CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
1946 DISPLAYLEVEL(3, "OK \n");
1948 DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
1949 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1950 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1951 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1952 size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */
1953 if (!ZSTD_isError(r)) goto _output_error;
1955 DISPLAYLEVEL(3, "OK \n");
1957 DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++);
1958 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize);
1959 assert(staticCCtx != NULL);
1960 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1961 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1962 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1963 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1965 DISPLAYLEVEL(3, "OK \n");
1967 DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1);
1968 CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */
1969 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1970 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1971 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1973 DISPLAYLEVEL(3, "OK \n");
1975 DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++);
1976 { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
1977 if (!ZSTD_isError(r)) goto _output_error; }
1978 DISPLAYLEVEL(3, "OK \n");
1980 DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++);
1981 CHECK_Z( ZSTD_initDStream(staticDCtx) );
1982 { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
1983 ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
1984 size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
1985 if (!ZSTD_isError(r)) goto _output_error;
1987 DISPLAYLEVEL(3, "OK \n");
1989 DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
1991 // Test ZSTD_estimateCCtxSize_usingCCtxParams
1993 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
1994 size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
1995 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
1996 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1997 compressedBuffer, compressedBufferSize,
1998 CNBuffer, CNBuffSize, 3));
2001 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2002 decodedBuffer, CNBuffSize,
2003 compressedBuffer, cSize);
2004 if (r != CNBuffSize) goto _output_error;
2005 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2007 ZSTD_freeCCtxParams(params);
2010 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2012 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2013 size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2014 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2015 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2016 compressedBuffer, compressedBufferSize,
2017 CNBuffer, CNBuffSize, 3) );
2020 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2021 decodedBuffer, CNBuffSize,
2022 compressedBuffer, cSize);
2023 if (r != CNBuffSize) goto _output_error;
2024 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2026 ZSTD_freeCCtxParams(params);
2029 DISPLAYLEVEL(3, "OK \n");
2031 DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
2033 // Test ZSTD_estimateCCtxSize_usingCCtxParams
2035 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2036 size_t cctxSizeDefault;
2037 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2038 cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2039 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2040 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2041 compressedBuffer, compressedBufferSize,
2042 CNBuffer, CNBuffSize, 3) );
2045 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2046 decodedBuffer, CNBuffSize,
2047 compressedBuffer, cSize);
2048 if (r != CNBuffSize) goto _output_error;
2049 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2051 ZSTD_freeCCtxParams(params);
2054 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2056 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2057 size_t cctxSizeDefault;
2058 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2059 cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2060 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2061 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2062 compressedBuffer, compressedBufferSize,
2063 CNBuffer, CNBuffSize, 3) );
2066 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2067 decodedBuffer, CNBuffSize,
2068 compressedBuffer, cSize);
2069 if (r != CNBuffSize) goto _output_error;
2070 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2072 ZSTD_freeCCtxParams(params);
2075 DISPLAYLEVEL(3, "OK \n");
2077 free(staticCCtxBuffer);
2078 free(staticDCtxBuffer);
2081 DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++);
2082 { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1);
2083 size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1);
2084 size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1);
2085 size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1);
2087 if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error;
2088 if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error;
2090 DISPLAYLEVEL(3, "OK \n");
2093 /* ZSTDMT simple MT compression test */
2094 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
2095 { ZSTD_CCtx* const mtctx = ZSTD_createCCtx();
2097 DISPLAY("mtctx : not enough memory, aborting \n");
2101 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
2102 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
2103 DISPLAYLEVEL(3, "OK \n");
2105 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
2106 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2107 compressedBuffer, compressedBufferSize,
2108 CNBuffer, CNBuffSize) );
2109 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2111 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
2112 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2113 if (rSize != CNBuffSize) {
2114 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize);
2117 DISPLAYLEVEL(3, "OK \n");
2119 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2120 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2121 if (r != CNBuffSize) goto _output_error; }
2122 DISPLAYLEVEL(3, "OK \n");
2124 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2126 for (u=0; u<CNBuffSize; u++) {
2127 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
2129 DISPLAYLEVEL(3, "OK \n");
2131 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
2132 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
2133 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
2134 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
2135 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2136 compressedBuffer, compressedBufferSize,
2137 CNBuffer, CNBuffSize) );
2138 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2140 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2141 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2142 if (r != CNBuffSize) goto _output_error; }
2143 DISPLAYLEVEL(3, "OK \n");
2145 ZSTD_freeCCtx(mtctx);
2148 DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++);
2149 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2150 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2152 if (cctx == NULL || dctx == NULL) goto _output_error;
2153 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
2154 CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
2155 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
2157 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
2158 { char const* outPtr = out;
2159 ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
2160 ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
2162 CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) );
2163 if (dSize != 0) goto _output_error;
2166 ZSTD_freeDCtx(dctx);
2167 ZSTD_freeCCtx(cctx);
2170 DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
2171 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2172 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
2173 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2175 ZSTD_freeCCtx(cctx);
2177 DISPLAYLEVEL(3, "OK \n");
2179 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
2180 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2181 size_t cSize1, cSize2;
2182 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2183 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
2184 cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2186 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
2187 cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2189 CHECK_LT(cSize1, cSize2);
2190 ZSTD_freeCCtx(cctx);
2192 DISPLAYLEVEL(3, "OK \n");
2194 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++)
2195 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2196 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
2197 * ZSTDMT is forced to not take the shortcut.
2199 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2200 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
2201 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
2202 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
2203 ZSTD_freeCCtx(cctx);
2205 DISPLAYLEVEL(3, "OK \n");
2207 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
2208 { ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2209 int const jobSize = 512 KB;
2211 /* Check that the overlap log and job size are unset. */
2212 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2214 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2216 /* Set and check the overlap log and job size. */
2217 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
2218 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
2219 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2221 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2222 CHECK_EQ(value, jobSize);
2223 /* Set the number of workers and check the overlap log and job size. */
2224 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
2225 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2227 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2228 CHECK_EQ(value, jobSize);
2229 ZSTD_freeCCtxParams(params);
2231 DISPLAYLEVEL(3, "OK \n");
2233 /* Simple API multiframe test */
2234 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);
2238 /* only use the first half so we don't push against size limit of compressedBuffer */
2239 size_t const segSize = (CNBuffSize / 2) / segs;
2241 const U32 skipLen = 129 KB;
2242 char* const skipBuff = (char*)malloc(skipLen);
2243 assert(skipBuff != NULL);
2244 memset(skipBuff, 0, skipLen);
2245 for (i = 0; i < segs; i++) {
2246 CHECK_NEWV(r, ZSTD_compress(
2247 (BYTE*)compressedBuffer + off, CNBuffSize - off,
2248 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
2252 /* insert skippable frame */
2253 size_t const skippableSize =
2254 ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize,
2255 skipBuff, skipLen, seed % 15);
2256 CHECK_Z(skippableSize);
2257 off += skippableSize;
2263 DISPLAYLEVEL(3, "OK \n");
2265 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++);
2266 { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
2267 if (r != CNBuffSize / 2) goto _output_error; }
2268 DISPLAYLEVEL(3, "OK \n");
2270 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
2271 { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize);
2272 if (bound != CNBuffSize / 2) goto _output_error; }
2273 DISPLAYLEVEL(3, "OK \n");
2275 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
2276 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
2277 if (r != CNBuffSize / 2) goto _output_error; }
2278 DISPLAYLEVEL(3, "OK \n");
2280 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2281 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
2282 DISPLAYLEVEL(3, "OK \n");
2284 /* Simple API skippable frame test */
2285 DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++);
2288 unsigned long long receivedSize;
2289 size_t skippableSize;
2290 const U32 skipLen = 129 KB;
2291 char* const skipBuff = (char*)malloc(skipLen);
2292 assert(skipBuff != NULL);
2293 for (i = 0; i < skipLen; i++)
2294 skipBuff[i] = (char) ((seed + i) % 256);
2295 skippableSize = ZSTD_writeSkippableFrame(
2296 compressedBuffer, compressedBufferSize,
2297 skipBuff, skipLen, seed % 15);
2298 CHECK_Z(skippableSize);
2299 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2300 receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize);
2301 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2302 CHECK_EQ(seed % 15, readMagic);
2303 if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error;
2307 DISPLAYLEVEL(3, "OK \n");
2309 DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++);
2312 unsigned long long receivedSize;
2313 size_t skippableSize;
2314 skippableSize = ZSTD_writeSkippableFrame(
2315 compressedBuffer, compressedBufferSize,
2316 CNBuffer, 0, seed % 15);
2317 CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize);
2318 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2319 receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize);
2320 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2321 CHECK_EQ(seed % 15, readMagic);
2323 DISPLAYLEVEL(3, "OK \n");
2325 /* Dictionary and CCtx Duplication tests */
2326 { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
2327 ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
2328 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2329 static const size_t dictSize = 551;
2330 assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
2332 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
2333 { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
2334 if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
2335 DISPLAYLEVEL(3, "OK \n");
2337 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
2338 CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
2339 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
2340 DISPLAYLEVEL(3, "OK \n");
2342 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
2344 CHECKPLUS(r, ZSTD_compressEnd(ctxOrig,
2345 compressedBuffer, compressedBufferSize,
2346 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2348 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2350 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
2351 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2352 decodedBuffer, CNBuffSize,
2353 compressedBuffer, cSize,
2354 CNBuffer, dictSize),
2355 if (r != CNBuffSize - dictSize) goto _output_error);
2356 DISPLAYLEVEL(3, "OK \n");
2358 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
2359 { size_t const cSizeOrig = cSize;
2361 CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated,
2362 compressedBuffer, compressedBufferSize,
2363 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2365 if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
2367 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2369 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
2370 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2371 decodedBuffer, CNBuffSize,
2372 compressedBuffer, cSize,
2373 CNBuffer, dictSize),
2374 if (r != CNBuffSize - dictSize) goto _output_error);
2375 DISPLAYLEVEL(3, "OK \n");
2377 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++);
2378 { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
2379 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2380 if (r != CNBuffSize - dictSize) goto _output_error;
2381 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict));
2382 ZSTD_freeDDict(ddict);
2385 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
2386 { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
2387 void* const ddictBuffer = malloc(ddictBufferSize);
2388 if (ddictBuffer == NULL) goto _output_error;
2389 { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
2390 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2391 if (r != CNBuffSize - dictSize) goto _output_error;
2394 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize);
2397 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
2398 { size_t const testSize = CNBuffSize / 3;
2399 CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
2400 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
2402 CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
2403 (const char*)CNBuffer + dictSize, testSize) );
2404 { ZSTD_frameHeader zfh;
2405 if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
2406 if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
2408 DISPLAYLEVEL(3, "OK \n");
2410 /* Note : these tests should be replaced by proper regression tests,
2411 * but existing ones do not focus on small data + dictionary + all levels.
2413 if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
2414 size_t const flatdictSize = 22 KB;
2415 size_t const contentSize = 9 KB;
2416 const void* const dict = (const char*)CNBuffer;
2417 const void* const contentStart = (const char*)dict + flatdictSize;
2418 /* These upper bounds are generally within a few bytes of the compressed size */
2419 size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
2420 3770, 3770, 3770, 3750, 3750,
2421 3742, 3675, 3674, 3665, 3664,
2422 3663, 3662, 3661, 3660, 3660,
2424 size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940,
2425 2950, 2950, 2925, 2900, 2892,
2426 2910, 2910, 2910, 2780, 2775,
2427 2765, 2760, 2755, 2754, 2753,
2430 int const maxLevel = ZSTD_maxCLevel();
2431 /* clevels with strategies that support rowhash on small inputs */
2433 int const rowLevelEnd = 8;
2435 DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
2436 assert(maxLevel == 22);
2437 RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
2438 DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n",
2439 (unsigned long long)XXH64(contentStart, contentSize, 0),
2440 (unsigned long long)XXH64(dict, flatdictSize, 0));
2442 for ( ; l <= maxLevel; l++) {
2443 size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize,
2444 contentStart, contentSize, l);
2445 if (nodict_cSize > target_nodict_cSize[l]) {
2446 DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n",
2447 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2450 DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n",
2451 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2453 for ( l=1 ; l <= maxLevel; l++) {
2454 size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig,
2455 compressedBuffer, compressedBufferSize,
2456 contentStart, contentSize,
2459 if (wdict_cSize > target_wdict_cSize[l]) {
2460 DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n",
2461 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2464 DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
2465 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2467 /* Compression with ZSTD_compress2 and row match finder force enabled.
2468 * Give some slack for force-enabled row matchfinder since we're on a small input (9KB)
2470 for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5;
2471 for (l=1 ; l <= maxLevel; l++) {
2472 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2473 size_t nodict_cSize;
2474 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l);
2475 ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable);
2476 nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize,
2477 contentStart, contentSize);
2478 if (nodict_cSize > target_nodict_cSize[l]) {
2479 DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n",
2480 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2481 ZSTD_freeCCtx(cctx);
2484 DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n",
2485 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2486 ZSTD_freeCCtx(cctx);
2488 /* Dict compression with DMS */
2489 for ( l=1 ; l <= maxLevel; l++) {
2491 CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) );
2492 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) );
2493 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) );
2494 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) );
2495 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) );
2496 wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize);
2497 if (wdict_cSize > target_wdict_cSize[l]) {
2498 DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n",
2499 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2502 DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n",
2503 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2506 DISPLAYLEVEL(4, "compression efficiency tests OK \n");
2509 ZSTD_freeCCtx(ctxOrig);
2510 ZSTD_freeCCtx(ctxDuplicated);
2511 ZSTD_freeDCtx(dctx);
2514 /* Dictionary and dictBuilder tests */
2515 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2516 size_t const dictBufferCapacity = 16 KB;
2517 void* const dictBuffer = malloc(dictBufferCapacity);
2518 size_t const totalSampleSize = 1 MB;
2519 size_t const sampleUnitSize = 8 KB;
2520 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
2521 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
2524 size_t dictHeaderSize;
2525 size_t dictBufferFixedSize = 144;
2526 unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
2527 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2528 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
2529 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
2530 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2531 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2532 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
2533 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
2534 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
2535 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
2536 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
2537 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69};
2539 if (dictBuffer==NULL || samplesSizes==NULL) {
2545 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++);
2546 assert(compressedBufferSize >= totalSampleSize);
2547 { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; }
2548 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2549 { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2550 decodedBuffer, samplesSizes, nbSamples);
2551 if (ZDICT_isError(sDictSize)) goto _output_error;
2552 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize);
2555 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++);
2556 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2557 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2558 CNBuffer, samplesSizes, nbSamples);
2559 if (ZDICT_isError(dictSize)) goto _output_error;
2560 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2562 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++);
2563 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2564 { ZDICT_cover_params_t coverParams;
2565 memset(&coverParams, 0, sizeof(coverParams));
2566 coverParams.steps = 8;
2567 coverParams.nbThreads = 4;
2568 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2569 dictBuffer, dictBufferCapacity,
2570 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2572 if (ZDICT_isError(dictSize)) goto _output_error;
2574 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2576 DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++);
2577 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2578 { ZDICT_cover_params_t coverParams;
2579 memset(&coverParams, 0, sizeof(coverParams));
2580 coverParams.steps = 8;
2581 coverParams.nbThreads = 4;
2582 coverParams.shrinkDict = 1;
2583 coverParams.shrinkDictMaxRegression = 1;
2584 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2585 dictBuffer, dictBufferCapacity,
2586 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2588 if (ZDICT_isError(dictSize)) goto _output_error;
2590 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2592 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++);
2593 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2594 { ZDICT_fastCover_params_t fastCoverParams;
2595 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2596 fastCoverParams.steps = 8;
2597 fastCoverParams.nbThreads = 4;
2598 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2599 dictBuffer, dictBufferCapacity,
2600 CNBuffer, samplesSizes, nbSamples,
2602 if (ZDICT_isError(dictSize)) goto _output_error;
2604 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2606 DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++);
2607 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2608 { ZDICT_fastCover_params_t fastCoverParams;
2609 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2610 fastCoverParams.steps = 8;
2611 fastCoverParams.nbThreads = 4;
2612 fastCoverParams.shrinkDict = 1;
2613 fastCoverParams.shrinkDictMaxRegression = 1;
2614 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2615 dictBuffer, dictBufferCapacity,
2616 CNBuffer, samplesSizes, nbSamples,
2618 if (ZDICT_isError(dictSize)) goto _output_error;
2620 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2622 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2623 dictID = ZDICT_getDictID(dictBuffer, dictSize);
2624 if (dictID==0) goto _output_error;
2625 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2627 DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++);
2628 dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize);
2629 if (dictHeaderSize==0) goto _output_error;
2630 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2632 DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
2633 { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize);
2634 if (dictHeaderSize != 115) goto _output_error;
2636 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2638 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
2639 cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
2640 CNBuffer, CNBuffSize,
2641 dictBuffer, dictSize, 4);
2642 if (ZSTD_isError(cSize)) goto _output_error;
2643 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2645 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++);
2646 { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
2647 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2649 DISPLAYLEVEL(3, "OK \n");
2651 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2652 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2653 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2655 DISPLAYLEVEL(3, "OK \n");
2657 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2658 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2659 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2660 decodedBuffer, CNBuffSize,
2661 compressedBuffer, cSize,
2662 dictBuffer, dictSize),
2663 if (r != CNBuffSize) goto _output_error);
2664 ZSTD_freeDCtx(dctx);
2666 DISPLAYLEVEL(3, "OK \n");
2668 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
2669 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2670 size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef);
2671 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize);
2674 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
2675 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2676 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
2677 ZSTD_dlm_byRef, ZSTD_dct_auto,
2678 cParams, ZSTD_defaultCMem);
2679 assert(cdict != NULL);
2680 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict));
2681 assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict));
2682 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
2683 CNBuffer, CNBuffSize, cdict);
2684 ZSTD_freeCDict(cdict);
2685 if (ZSTD_isError(cSize)) goto _output_error;
2687 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2689 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2690 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2691 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2693 DISPLAYLEVEL(3, "OK \n");
2695 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2696 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2697 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2698 decodedBuffer, CNBuffSize,
2699 compressedBuffer, cSize,
2700 dictBuffer, dictSize),
2701 if (r != CNBuffSize) goto _output_error);
2702 ZSTD_freeDCtx(dctx);
2704 DISPLAYLEVEL(3, "OK \n");
2706 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
2707 { int const maxLevel = ZSTD_maxCLevel();
2709 for (level = 1; level <= maxLevel; ++level) {
2710 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize);
2711 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
2712 void* const cdictBuffer = malloc(cdictSize);
2713 if (cdictBuffer==NULL) goto _output_error;
2714 { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict(
2715 cdictBuffer, cdictSize,
2716 dictBuffer, dictSize,
2717 ZSTD_dlm_byCopy, ZSTD_dct_auto,
2719 if (cdict == NULL) {
2720 DISPLAY("ZSTD_initStaticCDict failed ");
2723 cSize = ZSTD_compress_usingCDict(cctx,
2724 compressedBuffer, compressedBufferSize,
2725 CNBuffer, MIN(10 KB, CNBuffSize), cdict);
2726 if (ZSTD_isError(cSize)) {
2727 DISPLAY("ZSTD_compress_usingCDict failed ");
2732 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2734 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
2735 { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
2736 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2737 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
2738 assert(cdict != NULL);
2739 cSize = ZSTD_compress_usingCDict_advanced(cctx,
2740 compressedBuffer, compressedBufferSize,
2741 CNBuffer, CNBuffSize,
2743 ZSTD_freeCDict(cdict);
2744 if (ZSTD_isError(cSize)) goto _output_error;
2746 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2748 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++);
2749 { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2750 if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
2752 DISPLAYLEVEL(3, "OK (unknown)\n");
2754 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2755 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2756 assert(dctx != NULL);
2757 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2758 decodedBuffer, CNBuffSize,
2759 compressedBuffer, cSize,
2760 dictBuffer, dictSize),
2761 if (r != CNBuffSize) goto _output_error);
2762 ZSTD_freeDCtx(dctx);
2764 DISPLAYLEVEL(3, "OK \n");
2766 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
2767 { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
2768 p.fParams.noDictIDFlag = 1;
2769 cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
2770 CNBuffer, CNBuffSize,
2771 dictBuffer, dictSize, p);
2772 if (ZSTD_isError(cSize)) goto _output_error;
2774 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2776 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2777 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2778 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2779 decodedBuffer, CNBuffSize,
2780 compressedBuffer, cSize,
2781 dictBuffer, dictSize),
2782 if (r != CNBuffSize) goto _output_error);
2783 ZSTD_freeDCtx(dctx);
2785 DISPLAYLEVEL(3, "OK \n");
2787 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
2788 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2789 assert(dctx != NULL);
2790 { const size_t ret = ZSTD_decompress_usingDict(
2791 dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
2792 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
2793 if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
2796 ZSTD_freeDCtx(dctx);
2798 DISPLAYLEVEL(3, "OK \n");
2800 DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++);
2801 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2802 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2803 ZSTD_frameParameters const fParams = { 0, 1, 0 };
2806 CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN));
2807 cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000);
2808 CHECK_Z(cBlockSize);
2809 cSize += cBlockSize;
2810 cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000);
2811 CHECK_Z(cBlockSize);
2812 cSize += cBlockSize;
2814 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
2816 ZSTD_freeCDict(cdict);
2817 ZSTD_freeDCtx(dctx);
2819 DISPLAYLEVEL(3, "OK \n");
2821 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
2822 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2823 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2824 if (cdict==NULL) goto _output_error;
2825 ZSTD_freeCDict(cdict);
2827 DISPLAYLEVEL(3, "OK \n");
2829 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
2830 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2831 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2832 if (cdict!=NULL) goto _output_error;
2833 ZSTD_freeCDict(cdict);
2835 DISPLAYLEVEL(3, "OK \n");
2837 { char* rawDictBuffer = (char*)malloc(dictSize);
2838 assert(rawDictBuffer);
2839 memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2);
2840 memset(rawDictBuffer + dictSize - 2, 0, 2);
2841 MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY);
2843 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
2846 /* Either operation is allowed to fail, but one must fail. */
2847 ret = ZSTD_CCtx_loadDictionary_advanced(
2848 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
2849 if (!ZSTD_isError(ret)) {
2850 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2851 if (!ZSTD_isError(ret)) goto _output_error;
2854 DISPLAYLEVEL(3, "OK \n");
2856 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
2859 ret = ZSTD_CCtx_loadDictionary_advanced(
2860 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
2861 if (ZSTD_isError(ret)) goto _output_error;
2862 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2863 if (ZSTD_isError(ret)) goto _output_error;
2865 DISPLAYLEVEL(3, "OK \n");
2867 DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++);
2868 { size_t const srcSize = MIN(CNBuffSize, 100);
2869 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2870 /* Force the dictionary to be reloaded in raw content mode */
2871 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad));
2872 CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent));
2873 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
2876 DISPLAYLEVEL(3, "OK \n");
2878 free(rawDictBuffer);
2881 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
2882 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
2883 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2884 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2885 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2886 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2887 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2888 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2889 ZSTD_freeCDict(cdict);
2891 DISPLAYLEVEL(3, "OK \n");
2893 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++);
2895 size_t size1, size2;
2896 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2897 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2898 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2899 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2900 if (ZSTD_isError(size1)) goto _output_error;
2902 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2903 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2904 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2905 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2906 if (ZSTD_isError(size2)) goto _output_error;
2908 if (size1 != size2) goto _output_error;
2910 DISPLAYLEVEL(3, "OK \n");
2912 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++);
2914 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2915 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2916 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2918 DISPLAYLEVEL(3, "OK \n");
2920 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++);
2922 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2923 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2924 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2925 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2926 ZSTD_freeCDict(cdict);
2928 DISPLAYLEVEL(3, "OK \n");
2930 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++);
2932 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2933 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2934 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2935 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2936 ZSTD_freeCDict(cdict);
2938 DISPLAYLEVEL(3, "OK \n");
2940 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++);
2942 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2943 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2944 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2945 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2946 ZSTD_freeCDict(cdict);
2948 DISPLAYLEVEL(3, "OK \n");
2950 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++);
2952 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2953 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2954 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2956 DISPLAYLEVEL(3, "OK \n");
2958 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++);
2960 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2961 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2962 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2963 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2964 ZSTD_freeCDict(cdict);
2966 DISPLAYLEVEL(3, "OK \n");
2968 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++);
2970 size_t size1, size2;
2971 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2972 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2973 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2974 if (ZSTD_isError(size1)) goto _output_error;
2976 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
2977 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2978 if (ZSTD_isError(size2)) goto _output_error;
2980 if (size1 != size2) goto _output_error;
2982 DISPLAYLEVEL(3, "OK \n");
2984 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++);
2986 size_t size1, size2;
2987 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2988 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2989 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2990 if (ZSTD_isError(size1)) goto _output_error;
2992 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2993 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2994 if (ZSTD_isError(size2)) goto _output_error;
2996 if (size1 == size2) goto _output_error;
2998 DISPLAYLEVEL(3, "OK \n");
3000 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3001 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) );
3002 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3004 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++);
3006 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3008 /* We should fail to decompress without a dictionary. */
3009 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3010 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3011 if (!ZSTD_isError(ret)) goto _output_error;
3012 /* We should succeed to decompress with the dictionary. */
3013 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3014 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) );
3015 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3016 /* The dictionary should persist across calls. */
3017 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3018 /* When we reset the context the dictionary is cleared. */
3019 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3020 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3021 if (!ZSTD_isError(ret)) goto _output_error;
3022 ZSTD_freeDCtx(dctx);
3024 DISPLAYLEVEL(3, "OK \n");
3026 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
3028 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3029 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
3031 /* We should succeed to decompress with the ddict. */
3032 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3033 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
3034 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3035 /* The ddict should persist across calls. */
3036 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3037 /* When we reset the context the ddict is cleared. */
3038 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3039 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3040 if (!ZSTD_isError(ret)) goto _output_error;
3041 ZSTD_freeDCtx(dctx);
3042 ZSTD_freeDDict(ddict);
3044 DISPLAYLEVEL(3, "OK \n");
3046 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
3048 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3050 /* We should succeed to decompress with the prefix. */
3051 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3052 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
3053 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3054 /* The prefix should be cleared after the first compression. */
3055 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3056 if (!ZSTD_isError(ret)) goto _output_error;
3057 ZSTD_freeDCtx(dctx);
3059 DISPLAYLEVEL(3, "OK \n");
3061 DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3063 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3064 ZSTD_customMem customMem = {NULL, NULL, NULL};
3065 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3067 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast));
3068 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3069 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3070 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3071 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3072 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3073 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3074 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3075 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3077 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3078 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3079 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3081 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3082 ZSTD_freeCDict(cdict);
3083 ZSTD_freeDCtx(dctx);
3084 ZSTD_freeCCtxParams(cctxParams);
3086 DISPLAYLEVEL(3, "OK \n");
3088 DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3090 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3091 ZSTD_customMem customMem = {NULL, NULL, NULL};
3092 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3094 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast));
3095 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3096 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3097 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3098 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3099 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3100 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3101 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3102 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3103 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3104 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3105 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3106 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3108 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3109 ZSTD_freeCDict(cdict);
3110 ZSTD_freeDCtx(dctx);
3111 ZSTD_freeCCtxParams(cctxParams);
3113 DISPLAYLEVEL(3, "OK \n");
3115 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++);
3117 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3118 ZSTD_customMem customMem = {NULL, NULL, NULL};
3119 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3121 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy));
3122 /* Force enable row based match finder, and disable dedicated dict search. */
3123 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
3124 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0));
3125 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4));
3126 /* Set windowLog to 29 so hash/chain logs don't get sized down */
3127 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29));
3128 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29));
3129 /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */
3130 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29));
3131 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3132 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3133 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3135 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3136 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3138 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3139 ZSTD_freeCDict(cdict);
3140 ZSTD_freeDCtx(dctx);
3141 ZSTD_freeCCtxParams(cctxParams);
3143 DISPLAYLEVEL(3, "OK \n");
3145 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
3146 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3147 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
3148 CNBuffer, samplesSizes, nbSamples);
3149 if (ZDICT_isError(dictSize)) goto _output_error;
3150 /* Set all the repcodes to non-default */
3152 BYTE* dictPtr = (BYTE*)dictBuffer;
3153 BYTE* dictLimit = dictPtr + dictSize - 12;
3154 /* Find the repcodes */
3155 while (dictPtr < dictLimit &&
3156 (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
3157 MEM_readLE32(dictPtr + 8) != 8)) {
3160 if (dictPtr >= dictLimit) goto _output_error;
3161 MEM_writeLE32(dictPtr + 0, 10);
3162 MEM_writeLE32(dictPtr + 4, 10);
3163 MEM_writeLE32(dictPtr + 8, 10);
3164 /* Set the last 8 bytes to 'x' */
3165 memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
3167 /* The optimal parser checks all the repcodes.
3168 * Make sure at least one is a match >= targetLength so that it is
3169 * immediately chosen. This will make sure that the compressor and
3170 * decompressor agree on at least one of the repcodes.
3174 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3175 ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
3176 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
3177 ZSTD_dlm_byRef, ZSTD_dct_auto,
3178 cParams, ZSTD_defaultCMem);
3179 assert(dctx != NULL); assert(cdict != NULL);
3180 memset(data, 'x', sizeof(data));
3181 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
3182 data, sizeof(data), cdict);
3183 ZSTD_freeCDict(cdict);
3184 if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
3185 dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
3186 if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
3187 if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
3188 ZSTD_freeDCtx(dctx);
3190 DISPLAYLEVEL(3, "OK \n");
3192 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++);
3194 const size_t numDicts = 128;
3195 const size_t numFrames = 4;
3197 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3198 ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts);
3199 ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts);
3200 U32 dictIDSeed = seed;
3201 /* Create new compressed buffer that will hold frames with differing dictIDs */
3202 char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */
3204 ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize);
3205 /* Create a bunch of DDicts with random dict IDs */
3206 for (i = 0; i < numDicts; ++i) {
3207 U32 currDictID = FUZ_rand(&dictIDSeed);
3208 MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID);
3209 ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize);
3210 cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3);
3211 if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) {
3215 /* Compress a few frames using random CDicts */
3218 /* only use the first half so we don't push against size limit of compressedBuffer */
3219 size_t const segSize = (CNBuffSize / 2) / numFrames;
3220 for (i = 0; i < numFrames; i++) {
3221 size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts;
3222 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3223 { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx,
3224 (BYTE*)compressedBuffer + off, CNBuffSize - off,
3225 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
3226 cdictTable[dictIdx]));
3233 /* We should succeed to decompression even though different dicts were used on different frames */
3234 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3235 ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts);
3236 /* Reference every single ddict we made */
3237 for (i = 0; i < numDicts; ++i) {
3238 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i]));
3240 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3241 /* Streaming decompression should also work */
3243 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
3244 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
3245 while (in.pos < in.size) {
3246 CHECK_Z(ZSTD_decompressStream(dctx, &out, &in));
3249 ZSTD_freeDCtx(dctx);
3250 for (i = 0; i < numDicts; ++i) {
3251 ZSTD_freeCDict(cdictTable[i]);
3252 ZSTD_freeDDict(ddictTable[i]);
3254 free(dictBufferMulti);
3258 DISPLAYLEVEL(3, "OK \n");
3260 ZSTD_freeCCtx(cctx);
3265 /* COVER dictionary builder tests */
3266 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3267 size_t dictSize = 16 KB;
3268 size_t optDictSize = dictSize;
3269 void* dictBuffer = malloc(dictSize);
3270 size_t const totalSampleSize = 1 MB;
3271 size_t const sampleUnitSize = 8 KB;
3272 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3273 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3275 ZDICT_cover_params_t params;
3278 if (dictBuffer==NULL || samplesSizes==NULL) {
3284 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
3285 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3286 memset(¶ms, 0, sizeof(params));
3287 params.d = 1 + (FUZ_rand(&seed32) % 16);
3288 params.k = params.d + (FUZ_rand(&seed32) % 256);
3289 dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize,
3290 CNBuffer, samplesSizes, nbSamples,
3292 if (ZDICT_isError(dictSize)) goto _output_error;
3293 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
3295 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3296 dictID = ZDICT_getDictID(dictBuffer, dictSize);
3297 if (dictID==0) goto _output_error;
3298 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3300 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
3301 memset(¶ms, 0, sizeof(params));
3303 optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize,
3304 CNBuffer, samplesSizes,
3305 nbSamples / 4, ¶ms);
3306 if (ZDICT_isError(optDictSize)) goto _output_error;
3307 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize);
3309 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3310 dictID = ZDICT_getDictID(dictBuffer, optDictSize);
3311 if (dictID==0) goto _output_error;
3312 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3314 ZSTD_freeCCtx(cctx);
3319 /* Decompression defense tests */
3320 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++);
3321 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
3322 if (!ZSTD_isError(r)) goto _output_error;
3323 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
3324 DISPLAYLEVEL(3, "OK \n");
3326 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++);
3327 ((char*)(CNBuffer))[0] = 1;
3328 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
3329 if (!ZSTD_isError(r)) goto _output_error; }
3330 DISPLAYLEVEL(3, "OK \n");
3332 /* content size verification test */
3333 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++);
3334 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3335 size_t const srcSize = 5000;
3336 size_t const wrongSrcSize = (srcSize + 1000);
3337 ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
3338 params.fParams.contentSizeFlag = 1;
3339 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
3340 { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
3341 if (!ZSTD_isError(result)) goto _output_error;
3342 if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
3343 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result));
3345 ZSTD_freeCCtx(cctx);
3348 /* negative compression level test : ensure simple API and advanced API produce same result */
3349 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++);
3350 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3351 size_t const srcSize = CNBuffSize / 5;
3352 int const compressionLevel = -1;
3354 assert(cctx != NULL);
3355 { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize,
3356 CNBuffer, srcSize, compressionLevel);
3357 if (ZSTD_isError(cSize_1pass)) goto _output_error;
3359 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
3360 { size_t const compressionResult = ZSTD_compress2(cctx,
3361 compressedBuffer, compressedBufferSize,
3363 DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
3364 if (ZSTD_isError(compressionResult)) goto _output_error;
3365 if (compressionResult != cSize_1pass) goto _output_error;
3367 ZSTD_freeCCtx(cctx);
3369 DISPLAYLEVEL(3, "OK \n");
3371 /* parameters order test */
3372 { size_t const inputSize = CNBuffSize / 2;
3375 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3376 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
3377 assert(cctx != NULL);
3378 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3379 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3380 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3381 { size_t const compressedSize = ZSTD_compress2(cctx,
3382 compressedBuffer, ZSTD_compressBound(inputSize),
3383 CNBuffer, inputSize);
3384 CHECK_Z(compressedSize);
3385 cSize = compressedSize;
3386 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
3388 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3389 ZSTD_freeCCtx(cctx);
3392 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
3393 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
3394 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3395 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3396 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3397 { size_t const result = ZSTD_compress2(cctx,
3398 compressedBuffer, ZSTD_compressBound(inputSize),
3399 CNBuffer, inputSize);
3401 if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
3402 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
3403 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
3405 ZSTD_freeCCtx(cctx);
3409 /* advanced parameters for decompression */
3410 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3411 assert(dctx != NULL);
3413 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
3414 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
3415 CHECK_Z(bounds.error);
3417 DISPLAYLEVEL(3, "OK \n");
3419 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
3420 { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
3421 if (!ZSTD_isError(sr)) goto _output_error;
3423 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
3424 if (!ZSTD_isError(bounds.error)) goto _output_error;
3426 DISPLAYLEVEL(3, "OK \n");
3428 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
3429 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
3430 if (!ZSTD_isError(sr)) goto _output_error;
3432 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
3433 if (!ZSTD_isError(sr)) goto _output_error;
3435 DISPLAYLEVEL(3, "OK \n");
3437 ZSTD_freeDCtx(dctx);
3441 /* custom formats tests */
3442 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3443 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3444 size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
3445 assert(dctx != NULL); assert(cctx != NULL);
3447 /* basic block compression */
3448 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
3449 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3450 { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
3451 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
3452 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3453 if (result != 0) goto _output_error;
3454 if (in.pos != in.size) goto _output_error;
3457 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3459 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++);
3460 { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3461 if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
3462 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult));
3465 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
3466 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3467 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3468 { ZSTD_frameHeader zfh;
3469 size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
3470 if (zfhrt != 0) goto _output_error;
3473 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3474 if (result != inputSize) goto _output_error;
3475 DISPLAYLEVEL(3, "one-shot OK, ");
3478 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3479 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3480 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3481 if (result != 0) goto _output_error;
3482 if (in.pos != in.size) goto _output_error;
3483 if (out.pos != inputSize) goto _output_error;
3484 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3487 /* basic block compression */
3488 DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
3489 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3490 { ZSTD_inBuffer in = { CNBuffer, 0, 0 };
3491 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
3492 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3493 if (result != 0) goto _output_error;
3494 if (in.pos != in.size) goto _output_error;
3497 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
3499 DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
3500 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3501 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3503 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3504 if (result != 0) goto _output_error;
3505 DISPLAYLEVEL(3, "one-shot OK, ");
3508 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3509 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3510 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3511 if (result != 0) goto _output_error;
3512 if (in.pos != in.size) goto _output_error;
3513 if (out.pos != 0) goto _output_error;
3514 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3517 ZSTD_freeCCtx(cctx);
3518 ZSTD_freeDCtx(dctx);
3521 DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++);
3523 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3524 /* Attempt to future proof this to new parameters. */
3525 int const maxParam = 2000;
3527 if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error;
3528 for (param = 0; param < maxParam; ++param) {
3529 ZSTD_dParameter dParam = (ZSTD_dParameter)param;
3530 ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam);
3534 if (ZSTD_isError(bounds.error))
3536 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
3537 value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
3538 CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
3539 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3540 if (check != value2) goto _output_error;
3541 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
3542 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3543 if (check != value1) goto _output_error;
3545 ZSTD_freeDCtx(dctx);
3547 DISPLAYLEVEL(3, "OK \n");
3549 /* block API tests */
3550 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3551 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3552 static const size_t dictSize = 65 KB;
3553 static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
3555 assert(cctx != NULL); assert(dctx != NULL);
3557 /* basic block compression */
3558 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
3559 CHECK_Z( ZSTD_compressBegin(cctx, 5) );
3560 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3561 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
3562 DISPLAYLEVEL(3, "OK \n");
3564 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
3565 CHECK_Z( ZSTD_decompressBegin(dctx) );
3566 { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3567 if (r != blockSize) goto _output_error; }
3568 DISPLAYLEVEL(3, "OK \n");
3570 /* very long stream of block compression */
3571 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
3572 CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
3573 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3574 { U64 const toCompress = 5000000000ULL; /* > 4 GB */
3576 while (compressed < toCompress) {
3577 size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
3578 assert(blockCSize != 0);
3579 if (ZSTD_isError(blockCSize)) goto _output_error;
3580 compressed += blockCSize;
3582 DISPLAYLEVEL(3, "OK \n");
3584 /* dictionary block compression */
3585 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
3586 CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
3587 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
3588 RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
3589 { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
3590 assert(r == 0); /* non-compressible block */ }
3591 memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */
3592 CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
3593 (char*)CNBuffer+dictSize+2*blockSize, blockSize));
3594 DISPLAYLEVEL(3, "OK \n");
3596 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
3597 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
3598 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
3599 if (r != blockSize) {
3600 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3603 memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize);
3604 ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
3605 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
3606 if (r != blockSize) {
3607 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3610 assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */
3611 DISPLAYLEVEL(3, "OK \n");
3613 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
3614 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
3615 if (cdict==NULL) goto _output_error;
3616 CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
3617 CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
3618 ZSTD_freeCDict(cdict);
3620 DISPLAYLEVEL(3, "OK \n");
3622 ZSTD_freeCCtx(cctx);
3623 ZSTD_freeDCtx(dctx);
3627 { size_t sampleSize = 0;
3628 size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
3629 DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
3630 memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
3631 sampleSize += 256 KB - 1;
3632 memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
3633 sampleSize += 96 KB;
3634 cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
3635 if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
3636 { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
3637 if (regenSize!=sampleSize) goto _output_error; }
3638 DISPLAYLEVEL(3, "OK \n");
3641 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++);
3643 size_t srcSize = 150 KB;
3644 BYTE* src = (BYTE*)CNBuffer;
3645 BYTE* decoded = (BYTE*)compressedBuffer;
3647 ZSTD_CCtx* cctx = ZSTD_createCCtx();
3648 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3651 if (seqs == NULL) goto _output_error;
3652 assert(cctx != NULL);
3653 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
3654 /* Populate src with random data */
3655 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
3657 /* Test with block delimiters roundtrip */
3658 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3659 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters);
3660 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3662 /* Test no block delimiters roundtrip */
3663 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
3664 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters);
3665 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3667 ZSTD_freeCCtx(cctx);
3670 DISPLAYLEVEL(3, "OK \n");
3672 DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
3674 const size_t srcSize = 500 KB;
3675 const BYTE* const src = (BYTE*)CNBuffer;
3676 BYTE* const dst = (BYTE*)compressedBuffer;
3677 const size_t dstCapacity = ZSTD_compressBound(srcSize);
3678 const size_t decompressSize = srcSize;
3679 char* const decompressBuffer = (char*)malloc(decompressSize);
3680 size_t compressedSize;
3682 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3683 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3686 if (seqs == NULL) goto _output_error;
3687 assert(cctx != NULL);
3689 /* Populate src with random data */
3690 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
3692 /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */
3693 nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3694 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3695 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
3696 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize);
3697 if (ZSTD_isError(compressedSize)) {
3698 DISPLAY("Error in sequence compression with block delims\n");
3701 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3702 if (ZSTD_isError(dSize)) {
3703 DISPLAY("Error in sequence compression roundtrip with block delims\n");
3706 assert(!memcmp(decompressBuffer, src, srcSize));
3708 /* Roundtrip Test with no block delimiters */
3709 { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs);
3710 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3711 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
3712 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize);
3714 if (ZSTD_isError(compressedSize)) {
3715 DISPLAY("Error in sequence compression with no block delims\n");
3718 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3719 if (ZSTD_isError(dSize)) {
3720 DISPLAY("Error in sequence compression roundtrip with no block delims\n");
3723 assert(!memcmp(decompressBuffer, src, srcSize));
3725 ZSTD_freeCCtx(cctx);
3726 free(decompressBuffer);
3729 DISPLAYLEVEL(3, "OK \n");
3731 /* Multiple blocks of zeros test */
3732 #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
3733 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3734 memset(CNBuffer, 0, LONGZEROSLENGTH);
3735 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) );
3736 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100);
3738 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3739 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) );
3740 if (r != LONGZEROSLENGTH) goto _output_error; }
3741 DISPLAYLEVEL(3, "OK \n");
3743 /* All zeroes test (test bug #137) */
3744 #define ZEROESLENGTH 100
3745 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
3746 memset(CNBuffer, 0, ZEROESLENGTH);
3747 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
3748 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100);
3750 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
3751 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
3752 if (r != ZEROESLENGTH) goto _output_error; }
3753 DISPLAYLEVEL(3, "OK \n");
3755 /* nbSeq limit test */
3756 #define _3BYTESTESTLENGTH 131000
3757 #define NB3BYTESSEQLOG 9
3758 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
3759 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
3760 /* creates a buffer full of 3-bytes sequences */
3761 { BYTE _3BytesSeqs[NB3BYTESSEQ][3];
3764 /* create batch of 3-bytes sequences */
3766 for (i=0; i < NB3BYTESSEQ; i++) {
3767 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
3768 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
3769 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
3772 /* randomly fills CNBuffer with prepared 3-bytes sequences */
3774 for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
3775 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
3776 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
3777 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
3778 ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
3780 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++);
3781 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3782 size_t const maxNbSeq = _3BYTESTESTLENGTH / 3;
3783 size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
3785 while (nbSeq <= maxNbSeq) {
3786 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
3787 /* Check every sequence for the first 100, then skip more rapidly. */
3791 nbSeq += (nbSeq >> 2);
3794 ZSTD_freeCCtx(cctx);
3796 DISPLAYLEVEL(3, "OK \n");
3798 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
3799 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
3800 CNBuffer, _3BYTESTESTLENGTH, 19) );
3801 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
3803 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
3804 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
3805 if (r != _3BYTESTESTLENGTH) goto _output_error; }
3806 DISPLAYLEVEL(3, "OK \n");
3809 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++);
3810 RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
3811 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3812 size_t const bound = ZSTD_compressBound(CNBuffSize);
3814 while (size <= CNBuffSize) {
3815 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
3816 /* Check every size for the first 100, then skip more rapidly. */
3820 size += (size >> 2);
3823 ZSTD_freeCCtx(cctx);
3825 DISPLAYLEVEL(3, "OK \n");
3827 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
3828 { /* Train a dictionary on low characters */
3829 size_t dictSize = 16 KB;
3830 void* const dictBuffer = malloc(dictSize);
3831 size_t const totalSampleSize = 1 MB;
3832 size_t const sampleUnitSize = 8 KB;
3833 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3834 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3835 if (!dictBuffer || !samplesSizes) goto _output_error;
3836 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3837 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
3838 if (ZDICT_isError(dictSize)) goto _output_error;
3839 /* Reverse the characters to make the dictionary ill suited */
3841 for (u = 0; u < CNBuffSize; ++u) {
3842 ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
3844 { /* Compress the data */
3845 size_t const inputSize = 500;
3846 size_t const outputSize = ZSTD_compressBound(inputSize);
3847 void* const outputBuffer = malloc(outputSize);
3848 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3849 if (!outputBuffer || !cctx) goto _output_error;
3850 CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
3852 ZSTD_freeCCtx(cctx);
3858 DISPLAYLEVEL(3, "OK \n");
3861 /* findFrameCompressedSize on skippable frames */
3862 DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++);
3863 { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
3864 size_t const frameSrcSize = 13;
3865 if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
3866 DISPLAYLEVEL(3, "OK \n");
3868 /* error string tests */
3869 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
3870 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
3871 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
3872 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
3873 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
3874 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
3875 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
3876 DISPLAYLEVEL(3, "OK \n");
3878 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
3879 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
3881 size_t const size = MIN(128 KB, CNBuffSize);
3882 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3883 ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
3884 ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
3885 ZSTD_frameHeader lgHeader;
3886 ZSTD_frameHeader smHeader;
3888 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
3889 CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
3890 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
3891 CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
3893 if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
3895 ZSTD_freeCDict(smCDict);
3896 ZSTD_freeCDict(lgCDict);
3897 ZSTD_freeCCtx(cctx);
3899 DISPLAYLEVEL(3, "OK \n");
3901 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++);
3905 unsigned const tableLog = 5;
3906 size_t const nbSeq = 32;
3907 unsigned const maxSymbolValue = 31;
3910 for (i = 0; i < 32; ++i)
3912 /* Calling FSE_normalizeCount() on a uniform distribution should not
3913 * cause a division by zero.
3915 FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1);
3917 DISPLAYLEVEL(3, "OK \n");
3919 DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++);
3921 size_t const outBufSize = 9;
3922 short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18};
3923 unsigned const tableLog = 5;
3924 unsigned const maxSymbolValue = 10;
3925 BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE));
3927 /* Ensure that this write doesn't write out of bounds, and that
3928 * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1.
3930 FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog);
3933 DISPLAYLEVEL(3, "OK \n");
3935 DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
3937 U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
3938 U32 rand32 = FUZ_rand(&seed_copy);
3939 U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
3940 U32 lowbit_only_32 = 1;
3941 U64 lowbit_only_64 = 1;
3942 U32 highbit_only_32 = (U32)1 << 31;
3943 U64 highbit_only_64 = (U64)1 << 63;
3945 if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
3946 if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
3948 /* Test ZSTD_countTrailingZeros32 */
3949 CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
3950 CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u);
3951 CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32));
3953 /* Test ZSTD_countLeadingZeros32 */
3954 CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u);
3955 CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u);
3956 CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32));
3958 /* Test ZSTD_countTrailingZeros64 */
3959 CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u);
3960 CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u);
3962 /* Test ZSTD_countLeadingZeros64 */
3963 CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u);
3964 CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u);
3966 /* Test ZSTD_highbit32 */
3967 CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u);
3968 CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u);
3970 /* Test ZSTD_NbCommonBytes */
3971 if (MEM_isLittleEndian()) {
3973 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
3974 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
3976 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
3977 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
3981 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u);
3982 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u);
3984 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u);
3985 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u);
3989 /* Test MEM_ intrinsics */
3990 CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32));
3991 CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64));
3993 /* Test fallbacks vs intrinsics on a range of small integers */
3994 for (i=1; i <= 1000; i++) {
3995 CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i));
3996 CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i));
3997 CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i));
3998 CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i));
4001 DISPLAYLEVEL(3, "OK \n");
4003 #ifdef ZSTD_MULTITHREAD
4004 DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++);
4005 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4006 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4007 size_t const dstSize = ZSTD_compressBound(srcSize);
4008 void* const src = CNBuffer;
4009 void* const dst = compressedBuffer;
4010 void* dict = (void*)malloc(srcSize);
4012 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4013 RDG_genBuffer(dict, srcSize, compressibility, 0., seed);
4015 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4016 memset(dict, 0, sizeof(U32));
4018 /* something more than 1 */
4019 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4020 /* lie and claim this is a full dict */
4021 CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict));
4023 { ZSTD_outBuffer out = {dst, dstSize, 0};
4024 ZSTD_inBuffer in = {src, srcSize, 0};
4025 /* should fail because its not a full dict like we said it was */
4026 assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)));
4029 ZSTD_freeCCtx(cctx);
4032 DISPLAYLEVEL(3, "OK \n");
4034 DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++);
4035 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4036 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4037 size_t const dictSize = 10;
4038 size_t const dstSize = ZSTD_compressBound(srcSize);
4039 void* const src = CNBuffer;
4040 void* const dst = compressedBuffer;
4041 void* dict = (void*)malloc(dictSize);
4043 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4044 RDG_genBuffer(dict, dictSize, compressibility, 0., seed);
4046 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4047 memset(dict, 0, sizeof(U32));
4049 /* Enable MT, LDM, and use refPrefix() for a small dict */
4050 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4051 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4052 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
4054 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
4056 ZSTD_freeCCtx(cctx);
4059 DISPLAYLEVEL(3, "OK \n");
4061 DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++);
4063 ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0);
4064 ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0);
4065 ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400);
4066 ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000);
4067 ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000);
4069 assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict)));
4070 assert(!memcmp(&medium, &mediumDict, sizeof(medium)));
4071 assert(!memcmp(&large, &largeDict, sizeof(large)));
4073 DISPLAYLEVEL(3, "OK \n");
4075 DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++);
4077 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0);
4078 ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400);
4079 ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400);
4081 assert(smallSrcAndDict.windowLog == 10);
4082 assert(!memcmp(&cParams, &smallDict, sizeof(cParams)));
4084 DISPLAYLEVEL(3, "OK \n");
4086 DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
4089 size_t prevSize = 0;
4090 for (; level < ZSTD_maxCLevel(); ++level) {
4091 size_t const currSize = ZSTD_estimateCCtxSize(level);
4092 if (prevSize > currSize) {
4093 DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
4094 prevSize, level-1, currSize, level);
4097 prevSize = currSize;
4100 DISPLAYLEVEL(3, "OK \n");
4102 DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
4104 size_t const kSizeIncrement = 2 KB;
4107 for (; level <= ZSTD_maxCLevel(); ++level) {
4108 size_t dictSize = 0;
4109 for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
4110 size_t srcSize = 2 KB;
4111 for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
4112 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
4113 size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
4114 size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
4115 if (cctxSizeUsingLevel < cctxSizeUsingCParams
4116 || ZSTD_isError(cctxSizeUsingCParams)
4117 || ZSTD_isError(cctxSizeUsingLevel)) {
4118 DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
4119 level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
4122 DISPLAYLEVEL(3, "OK \n");
4124 DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++)
4126 int const threadPoolTestResult = threadPoolTests();
4127 if (threadPoolTestResult) {
4131 DISPLAYLEVEL(3, "thread pool tests OK \n");
4133 #endif /* ZSTD_MULTITHREAD */
4137 free(compressedBuffer);
4138 free(decodedBuffer);
4143 DISPLAY("Error detected in Unit tests ! \n");
4147 static int longUnitTests(U32 const seed, double compressibility)
4149 size_t const CNBuffSize = 5 MB;
4150 void* const CNBuffer = malloc(CNBuffSize);
4151 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
4152 void* const compressedBuffer = malloc(compressedBufferSize);
4153 void* const decodedBuffer = malloc(CNBuffSize);
4158 /* Create compressible noise */
4159 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
4160 DISPLAY("Not enough memory, aborting\n");
4164 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
4166 /* note : this test is rather long, it would be great to find a way to speed up its execution */
4167 DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++);
4169 size_t approxIndex = 0;
4170 size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */
4172 /* Provision enough space in a static context so that we can do all
4173 * this without ever reallocating, which would reset the indices. */
4174 size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22);
4175 void* const staticCCtxBuffer = malloc(staticCCtxSize);
4176 ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
4178 /* bump the indices so the following compressions happen at high
4180 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4181 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4182 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4183 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4184 while (approxIndex <= (maxIndex / 4) * 3) {
4185 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4186 approxIndex += in.pos;
4187 CHECK_Z(in.pos == in.size);
4191 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4194 /* spew a bunch of stuff into the table area */
4195 for (cLevel = 1; cLevel <= 22; cLevel++) {
4196 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4197 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4198 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4199 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4200 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4201 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4202 approxIndex += in.pos;
4205 /* now crank the indices so we overflow */
4206 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4207 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4208 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4209 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4210 while (approxIndex <= maxIndex) {
4211 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4212 approxIndex += in.pos;
4213 CHECK_Z(in.pos == in.size);
4217 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4220 /* do a bunch of compressions again in low indices and ensure we don't
4221 * hit untracked invalid indices */
4222 for (cLevel = 1; cLevel <= 22; cLevel++) {
4223 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4224 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4225 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4226 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4227 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4228 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4229 approxIndex += in.pos;
4232 free(staticCCtxBuffer);
4234 DISPLAYLEVEL(3, "OK \n");
4236 DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++);
4239 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4241 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed);
4243 /* Enable checksum to verify round trip. */
4244 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4245 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4246 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4248 /* Round trip once with ldm. */
4249 cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4251 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm));
4253 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4254 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4255 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
4256 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4258 /* Round trip once without ldm. */
4259 cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4260 CHECK_Z(cSizeNoLdm);
4261 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm));
4263 if (cSizeLdm > cSizeNoLdm) {
4264 DISPLAY("Using long mode should not cause regressions for btopt+\n");
4269 ZSTD_freeCCtx(cctx);
4271 DISPLAYLEVEL(3, "OK \n");
4273 DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++);
4274 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4275 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4276 size_t dictSize = CNBuffSize;
4277 void* dict = (void*)malloc(dictSize);
4278 ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams();
4279 ZSTD_dictAttachPref_e const attachPrefs[] = {
4280 ZSTD_dictDefaultAttach,
4281 ZSTD_dictForceAttach,
4284 ZSTD_dictDefaultAttach,
4285 ZSTD_dictForceAttach,
4289 int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1};
4293 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
4294 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
4296 CHECK_Z(cctx_params != NULL);
4298 for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
4299 DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
4300 for (cLevel = 4; cLevel < 13; cLevel++) {
4301 for (i = 0; i < 8; ++i) {
4302 ZSTD_dictAttachPref_e const attachPref = attachPrefs[i];
4303 int const enableDDS = enableDedicatedDictSearch[i];
4306 DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i);
4308 ZSTD_CCtxParams_init(cctx_params, cLevel);
4309 CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS));
4311 cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem);
4312 CHECK(cdict != NULL);
4314 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
4315 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref));
4317 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4319 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
4321 DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize);
4323 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
4324 ZSTD_freeCDict(cdict);
4327 ZSTD_freeCCtx(cctx);
4328 ZSTD_freeDCtx(dctx);
4329 ZSTD_freeCCtxParams(cctx_params);
4332 DISPLAYLEVEL(3, "OK \n");
4336 free(compressedBuffer);
4337 free(decodedBuffer);
4342 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
4344 const BYTE* b1 = (const BYTE*)buf1;
4345 const BYTE* b2 = (const BYTE*)buf2;
4347 for (u=0; u<max; u++) {
4348 if (b1[u] != b2[u]) break;
4354 static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
4356 ZSTD_parameters params;
4357 params.cParams = cParams;
4358 params.fParams = fParams;
4362 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
4364 size_t const lengthMask = ((size_t)1 << logLength) - 1;
4365 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
4368 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
4370 U32 const logLength = FUZ_rand(seed) % maxLog;
4371 return FUZ_rLogLength(seed, logLength);
4375 #define CHECK(cond, ...) { \
4377 DISPLAY("Error => "); \
4378 DISPLAY(__VA_ARGS__); \
4379 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4380 goto _output_error; \
4384 #define CHECK_Z(f) { \
4385 size_t const err = f; \
4386 if (ZSTD_isError(err)) { \
4387 DISPLAY("Error => %s : %s ", \
4388 #f, ZSTD_getErrorName(err)); \
4389 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4390 goto _output_error; \
4394 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
4396 static const U32 maxSrcLog = 23;
4397 static const U32 maxSampleLog = 22;
4398 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
4399 size_t const dstBufferSize = (size_t)1<<maxSampleLog;
4400 size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
4401 BYTE* cNoiseBuffer[5];
4402 BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
4403 BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
4404 BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
4405 ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
4406 ZSTD_CCtx* const ctx = ZSTD_createCCtx();
4407 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4409 unsigned testNb = 0;
4410 U32 coreSeed = seed;
4411 UTIL_time_t const startClock = UTIL_getTime();
4412 U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
4413 int const cLevelLimiter = bigTests ? 3 : 2;
4416 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
4417 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
4418 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
4419 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
4420 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
4421 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
4422 || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
4423 "Not enough memory, fuzzer tests cancelled");
4425 /* Create initial samples */
4426 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
4427 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
4428 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
4429 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
4430 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
4432 /* catch up testNb */
4433 for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
4435 /* main test loop */
4436 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
4437 BYTE* srcBuffer; /* jumping pointer */
4439 size_t sampleSize, maxTestSize, totalTestSize;
4440 size_t cSize, totalCSize, totalGenSize;
4447 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
4448 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
4450 FUZ_rand(&coreSeed);
4451 { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
4453 /* srcBuffer selection [0-4] */
4454 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
4455 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
4459 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
4460 buffNb = tnb[buffNb >> 3];
4462 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
4463 buffNb = tnb[buffNb >> 3];
4465 srcBuffer = cNoiseBuffer[buffNb];
4468 /* select src segment */
4469 sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
4471 /* create sample buffer (to catch read error with valgrind & sanitizers) */
4472 sampleBuffer = (BYTE*)malloc(sampleSize);
4473 CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
4474 { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
4475 memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
4476 crcOrig = XXH64(sampleBuffer, sampleSize, 0);
4478 /* compression tests */
4479 { int const cLevelPositive = (int)
4480 ( FUZ_rand(&lseed) %
4481 ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) )
4483 int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ?
4484 - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */
4486 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel);
4487 cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
4488 CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
4490 /* compression failure test : too small dest buffer */
4492 { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;
4493 const size_t tooSmallSize = cSize - missing;
4494 const unsigned endMark = 0x4DC2B1A9;
4495 memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark));
4496 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
4497 testNb, (unsigned)tooSmallSize, (unsigned)missing);
4498 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
4499 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
4500 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck));
4501 CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); }
4504 /* frame header decompression test */
4505 { ZSTD_frameHeader zfh;
4506 CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
4507 CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
4510 /* Decompressed size test */
4511 { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
4512 CHECK(rSize != sampleSize, "decompressed size incorrect");
4515 /* successful decompression test */
4516 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb);
4517 { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
4518 size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
4519 CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize);
4520 { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
4521 CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize);
4524 free(sampleBuffer); /* no longer useful after this point */
4526 /* truncated src decompression test */
4527 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb);
4528 { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4529 size_t const tooSmallSize = cSize - missing;
4530 void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
4531 CHECK(cBufferTooSmall == NULL, "not enough memory !");
4532 memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
4533 { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
4534 CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
4535 free(cBufferTooSmall);
4538 /* too small dst decompression test */
4539 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb);
4540 if (sampleSize > 3) {
4541 size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4542 size_t const tooSmallSize = sampleSize - missing;
4543 static const BYTE token = 0xA9;
4544 dstBuffer[tooSmallSize] = token;
4545 { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
4546 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
4547 CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
4550 /* noisy src decompression test */
4552 /* insert noise into src */
4553 { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
4554 size_t pos = 4; /* preserve magic number (too easy to detect) */
4556 /* keep some original src */
4557 { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
4558 size_t const mask = (1<<nbBits) - 1;
4559 size_t const skipLength = FUZ_rand(&lseed) & mask;
4562 if (pos >= cSize) break;
4564 { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
4565 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
4566 size_t const mask = (1<<nbBits) - 1;
4567 size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
4568 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
4569 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
4570 memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
4574 /* decompress noisy source */
4575 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb);
4576 { U32 const endMark = 0xA9B1C3D6;
4577 memcpy(dstBuffer+sampleSize, &endMark, 4);
4578 { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
4579 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
4580 CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
4581 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize);
4583 { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
4584 CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
4585 } } } /* noisy src decompression test */
4587 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
4588 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
4589 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
4590 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
4591 int const cLevel = (int)(FUZ_rand(&lseed) %
4592 ((U32)ZSTD_maxCLevel() -
4593 (MAX(testLog, dictLog) / (U32)cLevelLimiter))) +
4595 maxTestSize = FUZ_rLogLength(&lseed, testLog);
4596 if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
4598 dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
4599 dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
4601 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
4602 testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize);
4604 if (FUZ_rand(&lseed) & 0xF) {
4605 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
4607 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
4608 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
4609 !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
4610 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
4611 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
4612 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
4614 CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
4617 { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
4619 XXH64_state_t xxhState;
4620 XXH64_reset(&xxhState, 0);
4621 for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
4622 size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
4623 size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
4625 if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
4626 if (totalTestSize+segmentSize > maxTestSize) break;
4628 { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
4629 CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
4630 cSize += compressResult;
4632 XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
4633 memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
4634 totalTestSize += segmentSize;
4637 { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
4638 CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
4639 cSize += flushResult;
4641 crcOrig = XXH64_digest(&xxhState);
4644 /* streaming decompression test */
4645 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
4646 /* ensure memory requirement is good enough (should always be true) */
4647 { ZSTD_frameHeader zfh;
4648 CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
4649 "ZSTD_getFrameHeader(): error retrieving frame information");
4650 { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
4651 CHECK_Z(roundBuffSize);
4652 CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN),
4653 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)",
4654 (unsigned)roundBuffSize, (unsigned)totalTestSize );
4656 if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
4657 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
4660 while (totalCSize < cSize) {
4661 size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
4662 size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
4663 CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize));
4664 totalGenSize += genSize;
4665 totalCSize += inSize;
4667 CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
4668 CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
4669 CHECK (totalCSize != cSize, "compressed data should be fully read")
4670 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
4671 CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
4672 (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize);
4674 } /* for ( ; (testNb <= nbTests) */
4675 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
4678 ZSTD_freeCCtx(refCtx);
4680 ZSTD_freeDCtx(dctx);
4681 free(cNoiseBuffer[0]);
4682 free(cNoiseBuffer[1]);
4683 free(cNoiseBuffer[2]);
4684 free(cNoiseBuffer[3]);
4685 free(cNoiseBuffer[4]);
4697 /*_*******************************************************
4699 *********************************************************/
4700 static int FUZ_usage(const char* programName)
4702 DISPLAY( "Usage :\n");
4703 DISPLAY( " %s [args]\n", programName);
4705 DISPLAY( "Arguments :\n");
4706 DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault);
4707 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
4708 DISPLAY( " -s# : Select seed (default:prompt user)\n");
4709 DISPLAY( " -t# : Select starting test number (default:0)\n");
4710 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default);
4711 DISPLAY( " -v : verbose\n");
4712 DISPLAY( " -p : pause at the end\n");
4713 DISPLAY( " -h : display help and exit\n");
4717 /*! readU32FromChar() :
4718 @return : unsigned integer value read from input in `char` format
4719 allows and interprets K, KB, KiB, M, MB and MiB suffix.
4720 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
4721 Note : function result can overflow if digit string > MAX_UINT */
4722 static unsigned readU32FromChar(const char** stringPtr)
4724 unsigned result = 0;
4725 while ((**stringPtr >='0') && (**stringPtr <='9'))
4726 result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ;
4727 if ((**stringPtr=='K') || (**stringPtr=='M')) {
4729 if (**stringPtr=='M') result <<= 10;
4731 if (**stringPtr=='i') (*stringPtr)++;
4732 if (**stringPtr=='B') (*stringPtr)++;
4737 /** longCommandWArg() :
4738 * check if *stringPtr is the same as longCommand.
4739 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
4740 * @return 0 and doesn't modify *stringPtr otherwise.
4742 static int longCommandWArg(const char** stringPtr, const char* longCommand)
4744 size_t const comSize = strlen(longCommand);
4745 int const result = !strncmp(*stringPtr, longCommand, comSize);
4746 if (result) *stringPtr += comSize;
4750 int main(int argc, const char** argv)
4755 int nbTests = nbTestsDefault;
4757 int proba = FUZ_compressibility_default;
4761 U32 maxDuration = 0;
4764 U32 memTestsOnly = 0;
4765 const char* const programName = argv[0];
4767 /* Check command line */
4768 for (argNb=1; argNb<argc; argNb++) {
4769 const char* argument = argv[argNb];
4770 if(!argument) continue; /* Protection if argument empty */
4772 /* Handle commands. Aggregated commands are allowed */
4773 if (argument[0]=='-') {
4775 if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; }
4777 if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
4778 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
4779 if (!strcmp(argument, "--long-tests")) { longTests=1; continue; }
4780 if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; }
4783 while (*argument!=0) {
4787 return FUZ_usage(programName);
4799 case 'p': /* pause at the end */
4805 argument++; maxDuration = 0;
4806 nbTests = (int)readU32FromChar(&argument);
4812 maxDuration = readU32FromChar(&argument);
4813 if (*argument=='s') argument++; /* seconds */
4814 if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
4815 if (*argument=='n') argument++;
4821 seed = readU32FromChar(&argument);
4826 testNb = (int)readU32FromChar(&argument);
4829 case 'P': /* compressibility % */
4831 proba = (int)readU32FromChar(&argument);
4832 if (proba>100) proba = 100;
4836 return (FUZ_usage(programName), 1);
4837 } } } } /* for (argNb=1; argNb<argc; argNb++) */
4840 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
4843 time_t const t = time(NULL);
4844 U32 const h = XXH32(&t, sizeof(t), 1);
4848 DISPLAY("Seed = %u\n", (unsigned)seed);
4849 if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba);
4851 probfloat = ((double)proba) / 100;
4854 g_displayLevel = MAX(3, g_displayLevel);
4855 return FUZ_mallocTests(seed, probfloat, memTestsOnly);
4858 if (nbTests < testNb) nbTests = testNb;
4861 result = basicUnitTests(0, probfloat); /* constant seed for predictability */
4863 if (!result && longTests) {
4864 result = longUnitTests(0, probfloat);
4868 result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests);
4871 DISPLAY("Press Enter \n");