2 * Copyright (c) Meta Platforms, Inc. and affiliates.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
12 /*-************************************
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
22 /*-************************************
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #include <time.h> /* time_t, time(), to randomize seed */
29 #include <assert.h> /* assert */
30 #include "timefn.h" /* UTIL_time_t, UTIL_getTime */
32 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
33 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
34 #include "zstd.h" /* ZSTD_compressBound */
35 #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
36 #include "zdict.h" /* ZDICT_trainFromBuffer */
37 #include "datagen.h" /* RDG_genBuffer */
38 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
39 #include "xxhash.h" /* XXH64_* */
42 #include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
43 #include "external_matchfinder.h" /* zstreamSequenceProducer, EMF_testCase */
45 /*-************************************
47 **************************************/
52 static const int nbTestsDefault = 10000;
53 static const U32 g_cLevelMax_smallTests = 10;
54 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
55 #define FUZ_COMPRESSIBILITY_DEFAULT 50
56 static const U32 prime32 = 2654435761U;
59 /*-************************************
61 **************************************/
62 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
63 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \
64 DISPLAY(__VA_ARGS__); \
65 if (g_displayLevel>=4) fflush(stderr); }
66 static U32 g_displayLevel = 2;
68 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
69 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
71 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
72 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
73 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
74 if (g_displayLevel>=4) fflush(stderr); } }
76 static U64 g_clockTime = 0;
79 /*-*******************************************************
81 *********************************************************/
84 #define MIN(a,b) ((a)<(b)?(a):(b))
85 #define MAX(a,b) ((a)>(b)?(a):(b))
87 @return : a 27 bits random value, from a 32-bits `seed`.
88 `seed` is also modified */
89 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
90 static U32 FUZ_rand(U32* seedPtr)
92 static const U32 prime2 = 2246822519U;
93 U32 rand32 = *seedPtr;
96 rand32 = FUZ_rotl32(rand32, 13);
101 #define CHECK(cond, ...) { \
103 DISPLAY("Error => "); \
104 DISPLAY(__VA_ARGS__); \
105 DISPLAY(" (seed %u, test nb %u, line %u) \n", \
106 (unsigned)seed, testNb, __LINE__); \
107 goto _output_error; \
110 #define CHECK_Z(f) { \
111 size_t const err = f; \
112 CHECK(ZSTD_isError(err), "%s : %s ", \
113 #f, ZSTD_getErrorName(err)); \
116 #define CHECK_RET(ret, cond, ...) { \
118 DISPLAY("Error %llu => ", (unsigned long long)ret); \
119 DISPLAY(__VA_ARGS__); \
120 DISPLAY(" (line %u)\n", __LINE__); \
124 #define CHECK_RET_Z(f) { \
125 size_t const err = f; \
126 CHECK_RET(err, ZSTD_isError(err), "%s : %s ", \
127 #f, ZSTD_getErrorName(err)); \
131 /*======================================================
133 *======================================================*/
141 static const buffer_t kBuffNull = { NULL, 0 , 0 };
143 static void FUZ_freeDictionary(buffer_t dict)
148 static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
150 buffer_t dict = kBuffNull;
151 size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
152 size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
153 if (!blockSizes) return kBuffNull;
154 dict.start = malloc(requestedDictSize);
155 if (!dict.start) { free(blockSizes); return kBuffNull; }
157 for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
158 blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
160 { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
162 if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; }
163 dict.size = requestedDictSize;
164 dict.filled = dictSize;
169 /* Round trips data and updates xxh with the decompressed data produced */
170 static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
171 XXH64_state_t* xxh, void* data, size_t size,
172 ZSTD_EndDirective endOp)
174 static BYTE compressed[1024];
175 static BYTE uncompressed[1024];
177 ZSTD_inBuffer cin = {data, size, 0};
181 ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
182 ZSTD_inBuffer din = { compressed, 0, 0 };
183 ZSTD_outBuffer dout = { uncompressed, 0, 0 };
185 cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
186 if (ZSTD_isError(cret))
190 while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
194 dout.size = sizeof(uncompressed);
195 dret = ZSTD_decompressStream(dctx, &dout, &din);
196 if (ZSTD_isError(dret))
198 XXH64_update(xxh, dout.dst, dout.pos);
202 } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
206 /* Generates some data and round trips it */
207 static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
208 XXH64_state_t* xxh, SEQ_stream* seq,
209 SEQ_gen_type type, unsigned value)
211 static BYTE data[1024];
215 SEQ_outBuffer sout = {data, sizeof(data), 0};
217 gen = SEQ_gen(seq, type, value, &sout);
219 ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
220 if (ZSTD_isError(ret))
227 static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
230 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
231 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
232 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
233 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
234 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
235 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
236 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
237 savedParams->cParams.strategy = value;
239 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
240 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
241 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
242 savedParams->fParams.noDictIDFlag = !value;
246 static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
248 ZSTD_parameters params;
249 if (ZSTD_isError(getCCtxParams(zc, ¶ms))) return 10;
250 CHECK_RET(1, params.cParams.windowLog != savedParams.cParams.windowLog, "windowLog");
251 CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
252 CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
253 CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
254 CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
255 CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
257 CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
258 CHECK_RET(8, params.fParams.contentSizeFlag != savedParams.fParams.contentSizeFlag, "contentSizeFlag");
259 CHECK_RET(9, params.fParams.noDictIDFlag != savedParams.fParams.noDictIDFlag, "noDictIDFlag");
263 static int basicUnitTests(U32 seed, double compressibility, int bigTests)
265 size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
266 void* CNBuffer = malloc(CNBufferSize);
267 size_t const skippableFrameSize = 200 KB;
268 size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
269 void* compressedBuffer = malloc(compressedBufferSize);
270 size_t const decodedBufferSize = CNBufferSize;
271 void* decodedBuffer = malloc(decodedBufferSize);
275 U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */
276 ZSTD_CStream* zc = ZSTD_createCStream();
277 ZSTD_DStream* zd = ZSTD_createDStream();
278 ZSTD_CCtx* mtctx = ZSTD_createCCtx();
280 ZSTD_inBuffer inBuff, inBuff2;
281 ZSTD_outBuffer outBuff;
282 buffer_t dictionary = kBuffNull;
283 size_t const dictSize = 128 KB;
286 /* Create compressible test buffer */
287 if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd || !mtctx) {
288 DISPLAY("Not enough memory, aborting \n");
291 RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
293 CHECK_Z(ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2));
295 /* Create dictionary */
296 DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
297 dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB);
298 if (!dictionary.start) {
299 DISPLAY("Error creating dictionary, aborting \n");
302 dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
304 /* Basic compression test */
305 DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
306 CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
307 outBuff.dst = (char*)(compressedBuffer);
308 outBuff.size = compressedBufferSize;
310 inBuff.src = CNBuffer;
311 inBuff.size = CNBufferSize;
313 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
314 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
315 { size_t const r = ZSTD_endStream(zc, &outBuff);
316 if (r != 0) goto _output_error; } /* error, or some data not flushed */
317 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos);
319 /* generate skippable frame */
320 MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
321 MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
322 cSize = skippableFrameSize + 8;
324 /* Basic compression test using dict */
325 DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
326 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
327 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
328 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) );
329 outBuff.dst = (char*)(compressedBuffer)+cSize;
330 assert(compressedBufferSize > cSize);
331 outBuff.size = compressedBufferSize - cSize;
333 inBuff.src = CNBuffer;
334 inBuff.size = CNBufferSize;
336 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
337 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
338 { size_t const r = ZSTD_endStream(zc, &outBuff);
339 if (r != 0) goto _output_error; } /* error, or some data not flushed */
340 cSize += outBuff.pos;
341 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
342 (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
344 /* context size functions */
345 DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
346 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
347 size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams);
348 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */
349 if (ZSTD_isError(cstreamSize)) goto _output_error;
350 if (ZSTD_isError(cdictSize)) goto _output_error;
351 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize));
354 /* context size functions */
355 DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++);
356 { ZSTD_CCtx_params* const params = ZSTD_createCCtxParams();
357 size_t cstreamSize, cctxSize;
358 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) );
359 cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
360 CHECK_Z(cstreamSize);
361 cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
363 if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error;
364 ZSTD_freeCCtxParams(params);
365 DISPLAYLEVEL(3, "OK \n");
368 DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
369 { size_t const s = ZSTD_sizeof_CStream(zc);
370 if (ZSTD_isError(s)) goto _output_error;
371 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
374 /* Attempt bad compression parameters */
375 DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++);
377 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
378 params.cParams.minMatch = 2;
379 r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
380 if (!ZSTD_isError(r)) goto _output_error;
381 DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
384 /* skippable frame test */
385 DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
386 CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) );
387 inBuff.src = compressedBuffer;
390 outBuff.dst = decodedBuffer;
391 outBuff.size = CNBufferSize;
393 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
394 DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r);
395 if (r != 0) goto _output_error;
397 if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */
398 DISPLAYLEVEL(3, "OK \n");
400 /* Basic decompression test */
402 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
403 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
404 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */
405 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
406 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
407 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
408 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
409 DISPLAYLEVEL(3, "OK \n");
411 /* Re-use without init */
412 DISPLAYLEVEL(3, "test%3i : decompress again without init (re-use previous settings): ", testNb++);
414 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
415 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
416 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
417 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
418 DISPLAYLEVEL(3, "OK \n");
420 /* check regenerated data is byte exact */
421 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
423 for (i=0; i<CNBufferSize; i++) {
424 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
426 DISPLAYLEVEL(3, "OK \n");
428 /* check decompression fails early if first bytes are wrong */
429 DISPLAYLEVEL(3, "test%3i : early decompression error if first bytes are incorrect : ", testNb++);
430 { const char buf[3] = { 0 }; /* too short, not enough to start decoding header */
431 ZSTD_inBuffer inb = { buf, sizeof(buf), 0 };
432 size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inb);
433 if (!ZSTD_isError(remaining)) goto _output_error; /* should have errored out immediately (note: this does not test the exact error code) */
435 DISPLAYLEVEL(3, "OK \n");
437 /* context size functions */
438 DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
439 { ZSTD_frameHeader fhi;
440 const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
441 size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
442 if (gfhError!=0) goto _output_error;
443 DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize);
444 { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
445 /* uses ZSTD_initDStream_usingDict() */
446 + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
447 if (ZSTD_isError(s)) goto _output_error;
448 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
451 DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
452 { size_t const s = ZSTD_sizeof_DStream(zd);
453 if (ZSTD_isError(s)) goto _output_error;
454 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
457 /* Decompression by small increment */
458 DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
459 { /* skippable frame */
461 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
462 inBuff.src = compressedBuffer;
463 outBuff.dst = decodedBuffer;
466 while (r) { /* skippable frame */
467 size_t const inSize = (FUZ_rand(&coreSeed) & 15) + 1;
468 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + 1;
469 inBuff.size = inBuff.pos + inSize;
470 outBuff.size = outBuff.pos + outSize;
471 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
472 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream on skippable frame error : %s \n", ZSTD_getErrorName(r));
473 if (ZSTD_isError(r)) goto _output_error;
476 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
479 size_t const inSize = FUZ_rand(&coreSeed) & 15;
480 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + (!inSize); /* avoid having both sizes at 0 => would trigger a no_forward_progress error */
481 inBuff.size = inBuff.pos + inSize;
482 outBuff.size = outBuff.pos + outSize;
483 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
484 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(r));
485 if (ZSTD_isError(r)) goto _output_error;
488 if (outBuff.pos != CNBufferSize) DISPLAYLEVEL(4, "outBuff.pos != CNBufferSize : should have regenerated same amount ! \n");
489 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
490 if (inBuff.pos != cSize) DISPLAYLEVEL(4, "inBuff.pos != cSize : should have real all input ! \n");
491 if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
492 DISPLAYLEVEL(3, "OK \n");
494 /* check regenerated data is byte exact */
495 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
497 for (i=0; i<CNBufferSize; i++) {
498 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
500 DISPLAYLEVEL(3, "OK \n");
502 /* Decompression forward progress */
503 DISPLAYLEVEL(3, "test%3i : generate error when ZSTD_decompressStream() doesn't progress : ", testNb++);
504 { /* skippable frame */
507 int const maxDec = 100;
508 inBuff.src = compressedBuffer;
512 outBuff.dst = decodedBuffer;
514 outBuff.size = CNBufferSize-1; /* 1 byte missing */
516 for (decNb=0; decNb<maxDec; decNb++) {
517 if (r==0) ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
518 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
519 if (ZSTD_isError(r)) break;
521 if (!ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream should have triggered a no_forward_progress error \n");
522 if (!ZSTD_isError(r)) goto _output_error; /* should have triggered no_forward_progress error */
524 DISPLAYLEVEL(3, "OK \n");
526 DISPLAYLEVEL(3, "test%3i : NULL output and NULL input : ", testNb++);
533 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
534 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
535 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
536 outBuff.dst = (char*)(compressedBuffer);
537 outBuff.size = compressedBufferSize;
539 { size_t const r = ZSTD_endStream(zc, &outBuff);
540 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
542 inBuff.src = outBuff.dst;
543 inBuff.size = outBuff.pos;
548 CHECK_Z( ZSTD_initDStream(zd) );
549 { size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
550 if (ret != 0) goto _output_error;
552 DISPLAYLEVEL(3, "OK\n");
554 DISPLAYLEVEL(3, "test%3i : NULL output buffer with non-NULL input : ", testNb++);
556 const char* test = "aa";
563 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
564 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
565 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
566 outBuff.dst = (char*)(compressedBuffer);
567 outBuff.size = compressedBufferSize;
569 { size_t const r = ZSTD_endStream(zc, &outBuff);
570 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
572 inBuff.src = outBuff.dst;
573 inBuff.size = outBuff.pos;
578 CHECK_Z( ZSTD_initDStream(zd) );
579 CHECK_Z(ZSTD_decompressStream(zd, &outBuff, &inBuff));
582 DISPLAYLEVEL(3, "OK\n");
583 /* _srcSize compression test */
584 DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
585 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
586 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
587 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
588 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) );
589 outBuff.dst = (char*)(compressedBuffer);
590 outBuff.size = compressedBufferSize;
592 inBuff.src = CNBuffer;
593 inBuff.size = CNBufferSize;
595 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
596 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
597 { size_t const r = ZSTD_endStream(zc, &outBuff);
598 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
600 { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
601 CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
602 CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
604 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
606 /* wrong _srcSize compression test */
607 DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
608 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
609 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
610 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
611 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) );
612 outBuff.dst = (char*)(compressedBuffer);
613 outBuff.size = compressedBufferSize;
615 inBuff.src = CNBuffer;
616 inBuff.size = CNBufferSize;
618 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
619 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
620 { size_t const r = ZSTD_endStream(zc, &outBuff);
621 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
622 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
624 /* wrong _srcSize compression test */
625 DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
626 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
627 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
628 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
629 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) );
630 outBuff.dst = (char*)(compressedBuffer);
631 outBuff.size = compressedBufferSize;
633 inBuff.src = CNBuffer;
634 inBuff.size = CNBufferSize;
636 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
637 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
638 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
641 DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
642 { CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
643 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) );
644 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) );
645 outBuff.dst = (char*)compressedBuffer;
646 outBuff.size = compressedBufferSize;
648 inBuff.src = CNBuffer;
649 inBuff.size = CNBufferSize;
651 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
652 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
653 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
656 /* Compression state re-use scenario */
657 DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
658 ZSTD_freeCStream(zc);
659 zc = ZSTD_createCStream();
660 if (zc==NULL) goto _output_error; /* memory allocation issue */
662 { size_t const inSize = 513;
663 DISPLAYLEVEL(5, "use1 ");
664 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
665 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
666 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
667 inBuff.src = CNBuffer;
668 inBuff.size = inSize;
670 outBuff.dst = (char*)(compressedBuffer)+cSize;
671 outBuff.size = ZSTD_compressBound(inSize);
673 DISPLAYLEVEL(5, "compress1 ");
674 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
675 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
676 DISPLAYLEVEL(5, "end1 ");
677 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
680 { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
681 DISPLAYLEVEL(5, "use2 ");
682 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
683 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
684 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
685 inBuff.src = CNBuffer;
686 inBuff.size = inSize;
688 outBuff.dst = (char*)(compressedBuffer)+cSize;
689 outBuff.size = ZSTD_compressBound(inSize);
691 DISPLAYLEVEL(5, "compress2 ");
692 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
693 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
694 DISPLAYLEVEL(5, "end2 ");
695 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
697 DISPLAYLEVEL(3, "OK \n");
699 /* Decompression single pass with empty frame */
700 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, 1);
702 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass on empty frame : ", testNb++);
703 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
704 size_t const dctxSize = ZSTD_sizeof_DCtx(dctx);
705 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
707 outBuff.dst = decodedBuffer;
709 outBuff.size = CNBufferSize;
711 inBuff.src = compressedBuffer;
714 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
716 CHECK(r != 0, "Entire frame must be decompressed");
717 CHECK(outBuff.pos != 0, "Wrong size!");
718 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
720 CHECK(dctxSize != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
723 DISPLAYLEVEL(3, "OK \n");
725 /* Decompression with ZSTD_d_stableOutBuffer */
726 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1);
728 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
729 size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
731 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
733 outBuff.dst = decodedBuffer;
735 outBuff.size = CNBufferSize;
737 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass : ", testNb++);
738 inBuff.src = compressedBuffer;
741 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
743 CHECK(r != 0, "Entire frame must be decompressed");
744 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
745 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
747 CHECK(dctxSize0 != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
748 DISPLAYLEVEL(3, "OK \n");
750 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer : ", testNb++);
754 while (inBuff.pos < cSize) {
755 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
756 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
758 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
759 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
760 dctxSize1 = ZSTD_sizeof_DCtx(dctx);
761 CHECK(!(dctxSize0 < dctxSize1), "Input buffer allocated");
762 DISPLAYLEVEL(3, "OK \n");
764 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer too small : ", testNb++);
765 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
766 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
767 inBuff.src = compressedBuffer;
771 outBuff.size = CNBufferSize - 1;
772 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
773 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall, "Must error but got %s", ZSTD_getErrorName(r));
775 DISPLAYLEVEL(3, "OK \n");
777 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer modified : ", testNb++);
778 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
779 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
780 inBuff.src = compressedBuffer;
781 inBuff.size = cSize - 1;
784 outBuff.size = CNBufferSize;
785 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
788 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
789 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r));
791 DISPLAYLEVEL(3, "OK \n");
793 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++);
794 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
795 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0));
799 while (inBuff.pos < cSize) {
800 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
801 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
803 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
804 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
805 CHECK(!(dctxSize1 < ZSTD_sizeof_DCtx(dctx)), "Output buffer allocated");
806 DISPLAYLEVEL(3, "OK \n");
811 /* Compression with ZSTD_c_stable{In,Out}Buffer */
812 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
817 assert(cctx != NULL);
819 in.size = CNBufferSize;
820 out.dst = compressedBuffer;
821 out.size = compressedBufferSize;
822 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
823 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++);
824 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
825 CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
826 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
827 CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
828 /* @cctxSize2 : sizeof_CCtx when doing full streaming (no stable in/out) */
829 { ZSTD_CCtx* const cctx2 = ZSTD_createCCtx();
830 assert(cctx2 != NULL);
831 in.pos = out.pos = 0;
832 CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
833 CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
834 CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
835 ZSTD_freeCCtx(cctx2);
837 /* @cctxSize1 : sizeof_CCtx when doing single-shot compression (no streaming) */
838 { ZSTD_CCtx* const cctx1 = ZSTD_createCCtx();
839 ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
841 assert(cctx1 != NULL);
842 params.fParams.checksumFlag = 1;
843 cSize3 = ZSTD_compress_advanced(cctx1, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
845 CHECK(!(cSize == cSize3), "Must be same compressed size");
846 CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx1)), "Must be same CCtx size");
847 ZSTD_freeCCtx(cctx1);
849 CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
850 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
851 DISPLAYLEVEL(3, "OK \n");
853 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
854 { int stableInBuffer;
856 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
857 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
858 CHECK(!(stableInBuffer == 0), "Modified");
859 CHECK(!(stableOutBuffer == 0), "Modified");
860 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
861 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
862 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
863 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
864 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
865 CHECK(!(stableInBuffer == 1), "Modified");
866 CHECK(!(stableOutBuffer == 1), "Modified");
868 DISPLAYLEVEL(3, "OK \n");
870 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer : ", testNb++);
871 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
872 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
873 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
874 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
875 in.pos = out.pos = 0;
876 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
877 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
878 DISPLAYLEVEL(3, "OK \n");
880 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer allocated size : ", testNb++);
881 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
882 CHECK(!(cctxSize1 == cctxSize), "Must be the same size as single pass");
884 DISPLAYLEVEL(3, "OK \n");
886 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer only : ", testNb++);
887 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
888 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
889 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
890 in.pos = out.pos = 0;
891 out.size = cSize / 4;
893 size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
897 out.size = MIN(out.size + cSize / 4, compressedBufferSize);
899 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
900 DISPLAYLEVEL(3, "OK \n");
902 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++);
903 in.pos = out.pos = 0;
904 out.size = cSize / 4;
905 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
906 in.src = (char const*)in.src + in.pos;
909 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
910 CHECK(!ZSTD_isError(ret), "Must error");
911 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
913 DISPLAYLEVEL(3, "OK \n");
915 /* stableSrc + streaming */
916 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream, flushStream and endStream : ", testNb++);
917 CHECK_Z( ZSTD_initCStream(cctx, 1) );
918 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
919 { ZSTD_inBuffer inBuf;
920 ZSTD_outBuffer outBuf;
921 const size_t nonZeroStartPos = 18;
922 const size_t inputSize = 500;
923 inBuf.src = CNBuffer;
925 inBuf.pos = nonZeroStartPos;
926 outBuf.dst = (char*)(compressedBuffer)+cSize;
927 outBuf.size = ZSTD_compressBound(inputSize);
929 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
931 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
932 CHECK_Z( ZSTD_flushStream(cctx, &outBuf) );
933 inBuf.size = nonZeroStartPos + inputSize;
934 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
935 CHECK(ZSTD_endStream(cctx, &outBuf) != 0, "compression should be successful and fully flushed");
936 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
937 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
938 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
940 CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
941 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
943 DISPLAYLEVEL(3, "OK \n");
945 /* stableSrc + streaming */
946 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream2, using different end directives : ", testNb++);
947 CHECK_Z( ZSTD_initCStream(cctx, 1) );
948 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
949 { ZSTD_inBuffer inBuf;
950 ZSTD_outBuffer outBuf;
951 const size_t nonZeroStartPos = 18;
952 const size_t inputSize = 500;
953 inBuf.src = CNBuffer;
955 inBuf.pos = nonZeroStartPos;
956 outBuf.dst = (char*)(compressedBuffer)+cSize;
957 outBuf.size = ZSTD_compressBound(inputSize);
959 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
961 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
962 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_flush) );
963 inBuf.size = nonZeroStartPos + inputSize;
964 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
965 CHECK( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_end) != 0, "compression should be successful and fully flushed");
966 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
967 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
968 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
970 CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
971 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
973 DISPLAYLEVEL(3, "OK \n");
975 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer: context size : ", testNb++);
976 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
977 DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
978 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
979 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
980 cctxSize1 = cctxSize;
982 DISPLAYLEVEL(3, "OK \n");
984 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer only : ", testNb++);
985 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
986 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
987 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
989 in.pos = out.pos = 0;
990 in.size = MIN(CNBufferSize, 10);
991 out.size = compressedBufferSize;
992 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
994 in.size = CNBufferSize - in.size;
995 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
996 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, out.pos));
997 DISPLAYLEVEL(3, "OK \n");
999 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer modify buffer : ", testNb++);
1000 in.pos = out.pos = 0;
1001 in.size = CNBufferSize;
1002 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1003 in.pos = out.pos = 0;
1004 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1005 CHECK(!ZSTD_isError(ret), "Must have errored");
1006 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
1008 DISPLAYLEVEL(3, "OK \n");
1010 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer: context size : ", testNb++);
1011 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
1012 DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
1013 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
1014 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
1016 DISPLAYLEVEL(3, "OK \n");
1018 ZSTD_freeCCtx(cctx);
1021 /* CDict scenario */
1022 DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
1023 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
1024 size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
1025 DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError);
1026 if (ZSTD_isError(initError)) goto _output_error;
1027 outBuff.dst = compressedBuffer;
1028 outBuff.size = compressedBufferSize;
1030 inBuff.src = CNBuffer;
1031 inBuff.size = CNBufferSize;
1033 DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
1034 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1035 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1036 { size_t const r = ZSTD_endStream(zc, &outBuff);
1037 DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r);
1038 if (r != 0) goto _output_error; /* error, or some data not flushed */
1040 cSize = outBuff.pos;
1041 ZSTD_freeCDict(cdict);
1042 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1045 DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
1046 { size_t const s = ZSTD_sizeof_CStream(zc);
1047 if (ZSTD_isError(s)) goto _output_error;
1048 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
1051 DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
1052 { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1053 if (dID != dictID) goto _output_error;
1054 DISPLAYLEVEL(4, "OK (%u) \n", dID);
1057 /* DDict scenario */
1058 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize);
1059 { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1060 size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
1061 if (ZSTD_isError(initError)) goto _output_error;
1062 outBuff.dst = decodedBuffer;
1063 outBuff.size = CNBufferSize;
1065 inBuff.src = compressedBuffer;
1066 inBuff.size = cSize;
1068 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1069 if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
1070 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
1071 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
1072 ZSTD_freeDDict(ddict);
1073 DISPLAYLEVEL(3, "OK \n");
1076 /* Memory restriction */
1077 DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
1078 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
1079 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */
1080 outBuff.dst = decodedBuffer;
1081 outBuff.size = CNBufferSize;
1083 inBuff.src = compressedBuffer;
1084 inBuff.size = cSize;
1086 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1087 if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
1088 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
1089 ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */
1091 DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
1092 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1093 int const maxLevel = 16; /* first level with zstd_opt */
1095 assert(maxLevel < ZSTD_maxCLevel());
1096 CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) );
1097 for (level = 1; level <= maxLevel; ++level) {
1098 ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level);
1099 size_t const maxSize = MIN(1 MB, CNBufferSize);
1101 for (size = 512; size <= maxSize; size <<= 1) {
1102 U64 const crcOrig = XXH64(CNBuffer, size, 0);
1103 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1104 ZSTD_parameters savedParams;
1105 getCCtxParams(cctx, &savedParams);
1106 outBuff.dst = compressedBuffer;
1107 outBuff.size = compressedBufferSize;
1109 inBuff.src = CNBuffer;
1112 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
1113 CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
1114 CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
1115 if (inBuff.pos != inBuff.size) goto _output_error;
1116 { ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
1117 ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
1118 CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
1119 if (decIn.pos != decIn.size) goto _output_error;
1120 if (decOut.pos != size) goto _output_error;
1121 { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
1122 if (crcDec != crcOrig) goto _output_error;
1124 ZSTD_freeCCtx(cctx);
1126 ZSTD_freeCDict(cdict);
1128 ZSTD_freeDCtx(dctx);
1130 DISPLAYLEVEL(3, "OK\n");
1132 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1133 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) );
1134 cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB));
1136 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++);
1138 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1139 /* We should fail to decompress without a dictionary. */
1140 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1141 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1142 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1143 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1144 if (!ZSTD_isError(ret)) goto _output_error;
1146 /* We should succeed to decompress with the dictionary. */
1147 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1148 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1149 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1150 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1151 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1152 if (in.pos != in.size) goto _output_error;
1154 /* The dictionary should persist across calls. */
1155 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1156 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1157 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1158 if (in.pos != in.size) goto _output_error;
1160 /* The dictionary should not be cleared by ZSTD_reset_session_only. */
1161 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
1162 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1163 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1164 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1165 if (in.pos != in.size) goto _output_error;
1167 /* When we reset the context the dictionary is cleared. */
1168 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1169 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1170 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1171 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1172 if (!ZSTD_isError(ret)) goto _output_error;
1174 ZSTD_freeDCtx(dctx);
1176 DISPLAYLEVEL(3, "OK \n");
1178 DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
1180 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1181 /* We should succeed to decompress with the dictionary. */
1182 ZSTD_resetDStream(dctx);
1183 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1184 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1185 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1186 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1187 if (in.pos != in.size) goto _output_error;
1189 /* The dictionary should not be cleared by ZSTD_resetDStream(). */
1190 ZSTD_resetDStream(dctx);
1191 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1192 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1193 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1194 if (in.pos != in.size) goto _output_error;
1196 /* The dictionary should be cleared by ZSTD_initDStream(). */
1197 CHECK_Z( ZSTD_initDStream(dctx) );
1198 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1199 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1200 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1201 if (!ZSTD_isError(ret)) goto _output_error;
1203 ZSTD_freeDCtx(dctx);
1205 DISPLAYLEVEL(3, "OK \n");
1207 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++);
1209 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1210 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1211 /* We should succeed to decompress with the ddict. */
1212 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1213 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
1214 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1215 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1216 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1217 if (in.pos != in.size) goto _output_error;
1219 /* The ddict should persist across calls. */
1220 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1221 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1222 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1223 if (in.pos != in.size) goto _output_error;
1225 /* When we reset the context the ddict is cleared. */
1226 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1227 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1228 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1229 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1230 if (!ZSTD_isError(ret)) goto _output_error;
1232 ZSTD_freeDCtx(dctx);
1233 ZSTD_freeDDict(ddict);
1235 DISPLAYLEVEL(3, "OK \n");
1237 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
1239 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1240 /* We should succeed to decompress with the prefix. */
1241 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1242 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) );
1243 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1244 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1245 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1246 if (in.pos != in.size) goto _output_error;
1248 /* The prefix should be cleared after the first compression. */
1249 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1250 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1251 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1252 if (!ZSTD_isError(ret)) goto _output_error;
1254 ZSTD_freeDCtx(dctx);
1256 DISPLAYLEVEL(3, "OK \n");
1258 DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++);
1260 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1261 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1263 /* We should succeed to decompress with the dictionary. */
1264 CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) );
1265 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1266 /* The dictionary should persist across calls. */
1267 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1268 /* We should succeed to decompress with the ddict. */
1269 CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) );
1270 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1271 /* The ddict should persist across calls. */
1272 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1273 /* When we reset the context the ddict is cleared. */
1274 CHECK_Z( ZSTD_initDStream(dctx) );
1275 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize);
1276 if (!ZSTD_isError(ret)) goto _output_error;
1277 ZSTD_freeDCtx(dctx);
1278 ZSTD_freeDDict(ddict);
1280 DISPLAYLEVEL(3, "OK \n");
1282 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
1283 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
1284 ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
1285 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
1286 size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
1287 if (ZSTD_isError(initError)) goto _output_error;
1288 outBuff.dst = compressedBuffer;
1289 outBuff.size = compressedBufferSize;
1291 inBuff.src = CNBuffer;
1292 inBuff.size = CNBufferSize;
1294 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1295 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1296 { size_t const r = ZSTD_endStream(zc, &outBuff);
1297 if (r != 0) goto _output_error; } /* error, or some data not flushed */
1298 cSize = outBuff.pos;
1299 ZSTD_freeCDict(cdict);
1300 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1303 DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
1304 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1305 if (did != 0) goto _output_error;
1307 DISPLAYLEVEL(3, "OK (not detected) \n");
1309 DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
1310 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1311 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1312 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1315 DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
1316 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) );
1317 outBuff.dst = compressedBuffer;
1318 outBuff.size = compressedBufferSize;
1320 inBuff.src = CNBuffer;
1321 inBuff.size = CNBufferSize;
1323 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1324 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1325 cSize = outBuff.pos;
1326 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1328 DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++);
1329 CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) );
1330 outBuff.dst = decodedBuffer;
1331 outBuff.size = CNBufferSize;
1333 inBuff.src = compressedBuffer;
1334 inBuff.size = cSize;
1336 CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
1337 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1338 if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
1339 DISPLAYLEVEL(3, "OK \n");
1341 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
1342 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1343 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1344 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1347 DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
1348 outBuff.dst = compressedBuffer;
1349 outBuff.size = compressedBufferSize;
1351 inBuff.src = CNBuffer;
1352 inBuff.size = CNBufferSize;
1354 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1355 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1356 cSize = outBuff.pos;
1357 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1359 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
1360 CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) );
1361 DISPLAYLEVEL(3, "OK \n");
1364 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
1365 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1366 params.fParams.contentSizeFlag = 1;
1367 CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
1368 } /* cstream advanced shall write content size = 0 */
1369 outBuff.dst = compressedBuffer;
1370 outBuff.size = compressedBufferSize;
1372 inBuff.src = CNBuffer;
1375 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1376 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1377 cSize = outBuff.pos;
1378 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1379 DISPLAYLEVEL(3, "OK \n");
1381 DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++);
1382 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1383 params.fParams.contentSizeFlag = 1;
1384 CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
1385 } /* cstream advanced shall write content size = 0 */
1386 inBuff.src = CNBuffer;
1389 outBuff.dst = compressedBuffer;
1390 outBuff.size = compressedBufferSize;
1392 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1393 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1394 cSize = outBuff.pos;
1395 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1397 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1398 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1399 outBuff.dst = compressedBuffer;
1400 outBuff.size = compressedBufferSize;
1402 inBuff.src = CNBuffer;
1405 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1406 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1407 cSize = outBuff.pos;
1408 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
1409 DISPLAYLEVEL(3, "OK \n");
1411 /* Basic multithreading compression test */
1412 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
1414 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1415 CHECK(jobSize != 0, "job size non-zero");
1416 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1417 CHECK(jobSize != 0, "job size non-zero");
1419 outBuff.dst = compressedBuffer;
1420 outBuff.size = compressedBufferSize;
1422 inBuff.src = CNBuffer;
1423 inBuff.size = CNBufferSize;
1425 { size_t const compressResult = ZSTD_compressStream2(mtctx, &outBuff, &inBuff, ZSTD_e_end);
1426 if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */
1428 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1429 { size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos);
1430 if (compressedSize != outBuff.pos) goto _output_error; /* must be a full valid frame */
1432 DISPLAYLEVEL(3, "OK \n");
1434 /* Complex multithreading + dictionary test */
1435 { U32 const nbWorkers = 2;
1436 size_t const jobSize = 4 * 1 MB;
1437 size_t const srcSize = jobSize * nbWorkers; /* we want each job to have predictable size */
1438 size_t const segLength = 2 KB;
1439 size_t const offset = 600 KB; /* must be larger than window defined in cdict */
1440 size_t const start = jobSize + (offset-1);
1441 const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
1442 BYTE* const dst = (BYTE*)CNBuffer + start - offset;
1443 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize);
1444 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
1445 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
1446 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
1447 assert(start > offset);
1448 assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
1449 memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
1450 outBuff.dst = compressedBuffer;
1451 outBuff.size = compressedBufferSize;
1453 inBuff.src = CNBuffer;
1454 inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
1457 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
1458 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1459 DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
1460 CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
1461 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1462 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
1463 ZSTD_freeCDict(cdict);
1465 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1466 cSize = outBuff.pos;
1467 DISPLAYLEVEL(3, "OK \n");
1469 DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
1470 { ZSTD_DStream* const dstream = ZSTD_createDCtx();
1471 ZSTD_frameHeader zfh;
1472 ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
1473 DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize);
1474 outBuff.dst = decodedBuffer;
1475 outBuff.size = CNBufferSize;
1477 inBuff.src = compressedBuffer;
1479 CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
1480 inBuff.size = 1; /* avoid shortcut to single-pass mode */
1481 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1482 inBuff.size = cSize;
1483 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1484 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1485 ZSTD_freeDStream(dstream);
1487 DISPLAYLEVEL(3, "OK \n");
1489 DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
1490 { unsigned const kMaxWindowLog = 24;
1492 ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
1495 SEQ_stream seq = SEQ_initStream(0x87654321);
1499 XXH64_reset(&xxh, 0);
1500 cParams.windowLog = kMaxWindowLog;
1501 cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1502 ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1504 if (!cdict || !ddict) goto _output_error;
1506 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1507 ZSTD_resetDStream(zd);
1508 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1509 CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
1510 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
1511 /* Test all values < 300 */
1512 for (value = 0; value < 300; ++value) {
1513 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1514 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1517 /* Test values 2^8 to 2^17 */
1518 for (value = (1 << 8); value < (1 << 17); value <<= 1) {
1519 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1520 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1521 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
1524 /* Test offset values up to the max window log */
1525 for (value = 8; value <= kMaxWindowLog; ++value) {
1526 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
1529 CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
1530 CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
1532 ZSTD_freeCDict(cdict);
1533 ZSTD_freeDDict(ddict);
1535 DISPLAYLEVEL(3, "OK \n");
1537 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
1539 CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
1540 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1541 CHECK(level != 11, "Compression level does not match");
1542 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1543 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1544 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1545 CHECK(level != 11, "Compression level does not match");
1547 DISPLAYLEVEL(3, "OK \n");
1549 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++);
1550 { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0);
1551 CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN));
1552 CHECK(badParameters(zc, params), "Compression parameters do not match");
1553 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1554 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1555 CHECK(badParameters(zc, params), "Compression parameters do not match");
1557 DISPLAYLEVEL(3, "OK \n");
1559 DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++);
1560 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1561 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX));
1563 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint));
1564 CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match");
1566 CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
1567 DISPLAYLEVEL(3, "OK \n");
1569 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy compress with hashLog = 29 and searchLog = 4 : ", testNb++);
1571 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
1572 ZSTD_inBuffer in = { CNBuffer, CNBufferSize, 0 };
1573 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1574 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_strategy, ZSTD_lazy));
1575 /* Force enable the row based match finder */
1576 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
1577 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_searchLog, 4));
1578 /* Set windowLog to 29 so the hashLog doesn't get sized down */
1579 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, 29));
1580 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_hashLog, 29));
1581 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1582 /* Compress with continue first so the hashLog doesn't get sized down */
1583 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_continue));
1584 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_end));
1586 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
1588 DISPLAYLEVEL(3, "OK \n");
1590 DISPLAYLEVEL(3, "test%3i : Test offset == windowSize : ", testNb++);
1593 int const kMaxWindowLog = bigTests ? 29 : 26;
1594 size_t const kNbSequences = 10000;
1595 size_t const kMaxSrcSize = (1u << kMaxWindowLog) + 10 * kNbSequences;
1596 char* src = calloc(kMaxSrcSize, 1);
1597 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
1598 for (windowLog = ZSTD_WINDOWLOG_MIN; windowLog <= kMaxWindowLog; ++windowLog) {
1599 size_t const srcSize = ((size_t)1 << windowLog) + 10 * (kNbSequences - 1);
1601 sequences[0].offset = 32;
1602 sequences[0].litLength = 32;
1603 sequences[0].matchLength = (1u << windowLog) - 32;
1604 sequences[0].rep = 0;
1607 for (i = 1; i < kNbSequences; ++i) {
1608 sequences[i].offset = (1u << windowLog) - (FUZ_rand(&seed) % 8);
1609 sequences[i].litLength = FUZ_rand(&seed) & 7;
1610 sequences[i].matchLength = 10 - sequences[i].litLength;
1611 sequences[i].rep = 0;
1615 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1616 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1617 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_minMatch, 3));
1618 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, 1));
1619 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, windowLog));
1620 assert(srcSize <= kMaxSrcSize);
1621 cSize = ZSTD_compressSequences(zc, compressedBuffer, compressedBufferSize, sequences, kNbSequences, src, srcSize);
1623 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
1624 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, windowLog))
1626 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1627 size_t decompressedBytes = 0;
1629 ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1630 size_t const ret = ZSTD_decompressStream(zd, &out, &in);
1632 CHECK(decompressedBytes + out.pos > srcSize, "Output too large");
1633 CHECK(memcmp(out.dst, src + decompressedBytes, out.pos), "Corrupted");
1634 decompressedBytes += out.pos;
1639 CHECK(decompressedBytes != srcSize, "Output wrong size");
1645 DISPLAYLEVEL(3, "OK \n");
1647 /* Overlen overwriting window data bug */
1648 DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
1649 { /* This test has a window size of 1024 bytes and consists of 3 blocks:
1650 1. 'a' repeated 517 times
1651 2. 'b' repeated 516 times
1652 3. a compressed block with no literals and 3 sequence commands:
1653 litlength = 0, offset = 24, match length = 24
1654 litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
1655 litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
1657 const char* testCase =
1658 "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
1659 "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
1660 "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
1661 ZSTD_DStream* const zds = ZSTD_createDStream();
1662 if (zds==NULL) goto _output_error;
1664 CHECK_Z( ZSTD_initDStream(zds) );
1665 inBuff.src = testCase;
1668 outBuff.dst = decodedBuffer;
1669 outBuff.size = CNBufferSize;
1672 while (inBuff.pos < inBuff.size) {
1673 CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) );
1676 ZSTD_freeDStream(zds);
1678 DISPLAYLEVEL(3, "OK \n");
1680 /* Small Sequence Section bug */
1681 DISPLAYLEVEL(3, "test%3i : decompress blocks with small sequences section : ", testNb++);
1682 { /* This test consists of 3 blocks. Each block has one sequence.
1683 The sequence has literal length of 10, match length of 10 and offset of 10.
1684 The sequence value and compression mode for the blocks are following:
1685 The order of values are ll, ml, of.
1686 - First block : (10, 7, 13) (rle, rle, rle)
1687 - size of sequences section: 6 bytes (1 byte for nbSeq, 1 byte for encoding mode, 3 bytes for rle, 1 byte bitstream)
1688 - Second block : (10, 7, 1) (repeat, repeat, rle)
1689 - size of sequences section: 4 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 bytes for rle, 1 byte bitstream)
1690 - Third block : (10, 7, 1) (repeat, repeat, repeat)
1691 - size of sequences section: 3 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 byte bitstream) */
1693 unsigned char compressed[] = {
1694 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x3c, 0x35, 0x01, 0x00, 0xf0, 0x85, 0x08,
1695 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1696 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac,
1697 0x69, 0x94, 0x89, 0x1c, 0x03, 0x44, 0x0a, 0x07, 0x00, 0xb4, 0x04, 0x80,
1700 unsigned int compressedSize = 51;
1701 unsigned char decompressed[] = {
1702 0x85, 0x08, 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x85, 0x08,
1703 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1704 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0x4c, 0x6b, 0xa9, 0x8b, 0xbc, 0xc5,
1705 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94,
1706 0x89, 0x1c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, 0x89, 0x1c
1708 unsigned int decompressedSize = 60;
1710 ZSTD_DStream* const zds = ZSTD_createDStream();
1711 if (zds==NULL) goto _output_error;
1713 CHECK_Z( ZSTD_initDStream(zds) );
1714 inBuff.src = compressed;
1715 inBuff.size = compressedSize;
1717 outBuff.dst = decodedBuffer;
1718 outBuff.size = CNBufferSize;
1721 CHECK(ZSTD_decompressStream(zds, &outBuff, &inBuff) != 0,
1722 "Decompress did not reach the end of frame");
1723 CHECK(inBuff.pos != inBuff.size, "Decompress did not fully consume input");
1724 CHECK(outBuff.pos != decompressedSize, "Decompressed size does not match");
1725 CHECK(memcmp(outBuff.dst, decompressed, decompressedSize) != 0,
1726 "Decompressed data does not match");
1728 ZSTD_freeDStream(zds);
1730 DISPLAYLEVEL(3, "OK \n");
1732 DISPLAYLEVEL(3, "test%3i : raw block can be streamed: ", testNb++);
1733 { size_t const inputSize = 10000;
1734 size_t const compCapacity = ZSTD_compressBound(inputSize);
1735 BYTE* const input = (BYTE*)malloc(inputSize);
1736 BYTE* const comp = (BYTE*)malloc(compCapacity);
1737 BYTE* const decomp = (BYTE*)malloc(inputSize);
1739 CHECK(input == NULL || comp == NULL || decomp == NULL, "failed to alloc buffers");
1741 RDG_genBuffer(input, inputSize, 0.0, 0.0, seed);
1742 { size_t const compSize = ZSTD_compress(comp, compCapacity, input, inputSize, -(int)inputSize);
1743 ZSTD_inBuffer in = { comp, 0, 0 };
1744 ZSTD_outBuffer out = { decomp, 0, 0 };
1746 CHECK_Z( ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters) );
1747 while (in.size < compSize) {
1748 in.size = MIN(in.size + 100, compSize);
1749 while (in.pos < in.size) {
1750 size_t const outPos = out.pos;
1751 if (out.pos == out.size) {
1752 out.size = MIN(out.size + 10, inputSize);
1754 CHECK_Z( ZSTD_decompressStream(zd, &out, &in) );
1755 CHECK(!(out.pos > outPos), "We are not streaming (no output generated)");
1758 CHECK(in.pos != compSize, "Not all input consumed!");
1759 CHECK(out.pos != inputSize, "Not all output produced!");
1761 CHECK(memcmp(input, decomp, inputSize), "round trip failed!");
1767 DISPLAYLEVEL(3, "OK \n");
1769 DISPLAYLEVEL(3, "test%3i : dictionary + uncompressible block + reusing tables checks offset table validity: ", testNb++);
1770 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1771 dictionary.start, dictionary.filled,
1772 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1773 ZSTD_getCParams(3, 0, dictionary.filled),
1775 const size_t inbufsize = 2 * 128 * 1024; /* 2 blocks */
1776 const size_t outbufsize = ZSTD_compressBound(inbufsize);
1777 size_t inbufpos = 0;
1778 size_t cursegmentlen;
1779 BYTE *inbuf = (BYTE *)malloc(inbufsize);
1780 BYTE *outbuf = (BYTE *)malloc(outbufsize);
1781 BYTE *checkbuf = (BYTE *)malloc(inbufsize);
1784 CHECK(cdict == NULL, "failed to alloc cdict");
1785 CHECK(inbuf == NULL, "failed to alloc input buffer");
1787 /* first block is uncompressible */
1788 cursegmentlen = 128 * 1024;
1789 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0., 0., seed);
1790 inbufpos += cursegmentlen;
1792 /* second block is compressible */
1793 cursegmentlen = 128 * 1024 - 256;
1794 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0.05, 0., seed);
1795 inbufpos += cursegmentlen;
1797 /* and includes a very long backref */
1798 cursegmentlen = 128;
1799 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 256, cursegmentlen);
1800 inbufpos += cursegmentlen;
1802 /* and includes a very long backref */
1803 cursegmentlen = 128;
1804 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 128, cursegmentlen);
1805 inbufpos += cursegmentlen;
1807 ret = ZSTD_compress_usingCDict(zc, outbuf, outbufsize, inbuf, inbufpos, cdict);
1810 ret = ZSTD_decompress_usingDict(zd, checkbuf, inbufsize, outbuf, ret, dictionary.start, dictionary.filled);
1813 CHECK(memcmp(inbuf, checkbuf, inbufpos), "start and finish buffers don't match");
1815 ZSTD_freeCDict(cdict);
1820 DISPLAYLEVEL(3, "OK \n");
1822 DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++);
1823 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1824 dictionary.start, dictionary.filled,
1825 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1826 ZSTD_getCParams(3, 0, dictionary.filled),
1828 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1829 int remainingInput = 256 * 1024;
1832 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1833 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1834 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1835 /* Write a bunch of 6 byte blocks */
1836 while (remainingInput > 0) {
1837 char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
1838 const size_t kSmallBlockSize = sizeof(testBuffer);
1839 ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
1841 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
1842 CHECK(in.pos != in.size, "input not fully consumed");
1843 remainingInput -= kSmallBlockSize;
1845 /* Write several very long offset matches into the dictionary */
1846 for (offset = 1024; offset >= 0; offset -= 128) {
1847 ZSTD_inBuffer in = {(BYTE*)dictionary.start + offset, 128, 0};
1848 ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
1849 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
1850 CHECK(in.pos != in.size, "input not fully consumed");
1852 /* Ensure decompression works */
1853 CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled));
1855 ZSTD_freeCDict(cdict);
1857 DISPLAYLEVEL(3, "OK \n");
1859 DISPLAYLEVEL(3, "test%3i : Block-Level External Sequence Producer API: ", testNb++);
1861 size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
1862 BYTE* const dstBuf = (BYTE*)malloc(ZSTD_compressBound(dstBufSize));
1863 size_t const checkBufSize = CNBufferSize;
1864 BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
1866 EMF_testCase sequenceProducerState;
1868 CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
1870 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1872 /* Reference external matchfinder outside the test loop to
1873 * check that the reference is preserved across compressions */
1874 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1876 for (enableFallback = 0; enableFallback <= 1; enableFallback++) {
1878 size_t const numTestCases = 9;
1880 EMF_testCase const testCases[] = {
1887 EMF_INVALID_MATCHLEN,
1889 EMF_INVALID_LAST_LITS
1892 ZSTD_ErrorCode const errorCodes[] = {
1893 ZSTD_error_no_error,
1894 ZSTD_error_no_error,
1895 ZSTD_error_sequenceProducer_failed,
1896 ZSTD_error_sequenceProducer_failed,
1897 ZSTD_error_sequenceProducer_failed,
1898 ZSTD_error_externalSequences_invalid,
1899 ZSTD_error_externalSequences_invalid,
1900 ZSTD_error_externalSequences_invalid,
1901 ZSTD_error_externalSequences_invalid
1904 for (testCaseId = 0; testCaseId < numTestCases; testCaseId++) {
1907 int const compressionShouldSucceed = (
1908 (errorCodes[testCaseId] == ZSTD_error_no_error) ||
1909 (enableFallback && errorCodes[testCaseId] == ZSTD_error_sequenceProducer_failed)
1912 int const testWithSequenceValidation = (
1913 testCases[testCaseId] == EMF_INVALID_OFFSET
1916 sequenceProducerState = testCases[testCaseId];
1918 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1919 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, testWithSequenceValidation));
1920 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
1921 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
1923 if (compressionShouldSucceed) {
1924 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
1925 CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
1926 CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
1928 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
1930 ZSTD_getErrorCode(res) != errorCodes[testCaseId],
1931 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
1936 /* Test compression with external matchfinder + empty src buffer */
1939 sequenceProducerState = EMF_ZERO_SEQS;
1940 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1941 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
1942 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
1943 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
1944 CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
1948 /* Test that reset clears the external matchfinder */
1949 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1950 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
1951 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
1952 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
1954 /* Test that registering mFinder == NULL clears the external matchfinder */
1955 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1956 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1957 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
1958 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
1959 ZSTD_registerSequenceProducer(zc, NULL, NULL); /* clear the external matchfinder */
1960 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
1962 /* Test that external matchfinder doesn't interact with older APIs */
1963 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1964 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1965 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
1966 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
1967 CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
1969 /* Test that compression returns the correct error with LDM */
1970 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1973 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1974 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1975 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
1976 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
1978 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
1979 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
1983 #ifdef ZSTD_MULTITHREAD
1984 /* Test that compression returns the correct error with nbWorkers > 0 */
1985 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1988 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1989 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
1990 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
1991 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
1993 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
1994 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
2002 DISPLAYLEVEL(3, "OK \n");
2005 /* Test maxBlockSize cctx param functionality */
2006 DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
2008 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2010 /* Quick test to make sure maxBlockSize bounds are enforced */
2011 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
2012 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
2014 /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
2016 size_t srcSize = 2 << 10;
2017 void* const src = CNBuffer;
2018 size_t dstSize = ZSTD_compressBound(srcSize);
2019 void* const dst1 = compressedBuffer;
2020 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2021 size_t size1, size2;
2022 void* const checkBuf = malloc(srcSize);
2023 memset(src, 'x', srcSize);
2025 /* maxBlockSize = 1KB */
2026 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2027 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2029 if (ZSTD_isError(size1)) goto _output_error;
2030 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2031 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2033 /* maxBlockSize = 3KB */
2034 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2035 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2037 if (ZSTD_isError(size2)) goto _output_error;
2038 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2039 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2041 assert(size1 - size2 == 4); /* We add another RLE block with header + character */
2042 assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
2044 /* maxBlockSize = 1KB, windowLog = 10 */
2045 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2046 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2047 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2049 if (ZSTD_isError(size1)) goto _output_error;
2050 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2051 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2053 /* maxBlockSize = 3KB, windowLog = 10 */
2054 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2055 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2056 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2058 if (ZSTD_isError(size2)) goto _output_error;
2059 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2060 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2062 assert(size1 == size2);
2063 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2068 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2070 /* Test maxBlockSize = 0 is valid */
2071 { size_t srcSize = 256 << 10;
2072 void* const src = CNBuffer;
2073 size_t dstSize = ZSTD_compressBound(srcSize);
2074 void* const dst1 = compressedBuffer;
2075 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2076 size_t size1, size2;
2077 void* const checkBuf = malloc(srcSize);
2079 /* maxBlockSize = 0 */
2080 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
2081 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2083 if (ZSTD_isError(size1)) goto _output_error;
2084 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2085 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2087 /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
2088 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
2089 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2091 if (ZSTD_isError(size2)) goto _output_error;
2092 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2093 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2095 assert(size1 == size2);
2096 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2099 ZSTD_freeCCtx(cctx);
2101 DISPLAYLEVEL(3, "OK \n");
2103 /* Test Sequence Validation */
2104 DISPLAYLEVEL(3, "test%3i : Testing sequence validation: ", testNb++);
2106 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2108 /* Test minMatch >= 4, matchLength < 4 */
2110 size_t srcSize = 11;
2111 void* const src = CNBuffer;
2112 size_t dstSize = ZSTD_compressBound(srcSize);
2113 void* const dst = compressedBuffer;
2114 size_t const kNbSequences = 4;
2115 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2117 memset(src, 'x', srcSize);
2119 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2120 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2121 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2122 sequences[3] = (ZSTD_Sequence) {0, 1, 0, 0};
2124 /* Test with sequence validation */
2125 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2126 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2127 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2129 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2130 sequences, kNbSequences,
2133 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2134 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2136 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2138 /* Test without sequence validation */
2139 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2140 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2141 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2143 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2144 sequences, kNbSequences,
2147 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2148 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2153 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2156 /* Test with no block delim */
2159 void* const src = CNBuffer;
2160 size_t dstSize = ZSTD_compressBound(srcSize);
2161 void* const dst = compressedBuffer;
2162 size_t const kNbSequences = 1;
2163 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2164 void* const checkBuf = malloc(srcSize);
2166 memset(src, 'x', srcSize);
2168 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2170 /* Test with sequence validation */
2171 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 3));
2172 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters));
2173 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2175 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2176 sequences, kNbSequences,
2179 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2180 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst, cSize));
2181 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2187 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2189 { /* Test case with two additional sequences */
2190 size_t srcSize = 19;
2191 void* const src = CNBuffer;
2192 size_t dstSize = ZSTD_compressBound(srcSize);
2193 void* const dst = compressedBuffer;
2194 size_t const kNbSequences = 7;
2195 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2197 memset(src, 'x', srcSize);
2199 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2200 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2201 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2202 sequences[3] = (ZSTD_Sequence) {1, 0, 3, 0};
2203 sequences[4] = (ZSTD_Sequence) {1, 0, 3, 0};
2204 sequences[5] = (ZSTD_Sequence) {1, 0, 3, 0};
2205 sequences[6] = (ZSTD_Sequence) {0, 0, 0, 0};
2207 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2208 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2209 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2211 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2212 sequences, kNbSequences,
2215 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2216 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2218 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2220 /* Test without sequence validation */
2221 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2222 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2223 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2225 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2226 sequences, kNbSequences,
2229 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2230 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2234 ZSTD_freeCCtx(cctx);
2236 DISPLAYLEVEL(3, "OK \n");
2239 DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
2241 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2242 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2244 /* Test large offset, small window size*/
2246 size_t srcSize = 21;
2247 void* const src = CNBuffer;
2248 size_t dstSize = ZSTD_compressBound(srcSize);
2249 void* const dst = compressedBuffer;
2250 size_t const kNbSequences = 4;
2251 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2252 void* const checkBuf = malloc(srcSize);
2253 const size_t largeDictSize = 1 << 25;
2254 ZSTD_CDict* cdict = NULL;
2255 ZSTD_DDict* ddict = NULL;
2257 /* Generate large dictionary */
2258 void* dictBuffer = calloc(largeDictSize, 1);
2259 ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, largeDictSize);
2260 cParams.minMatch = ZSTD_MINMATCH_MIN;
2261 cParams.hashLog = ZSTD_HASHLOG_MIN;
2262 cParams.chainLog = ZSTD_CHAINLOG_MIN;
2264 cdict = ZSTD_createCDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
2265 ddict = ZSTD_createDDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
2267 ZSTD_CCtx_refCDict(cctx, cdict);
2268 ZSTD_DCtx_refDDict(dctx, ddict);
2270 sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0};
2271 sequences[1] = (ZSTD_Sequence) {1 << 25, 0, 3, 0};
2272 sequences[2] = (ZSTD_Sequence) {1 << 25, 0, 9, 0};
2273 sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0};
2275 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2276 sequences, kNbSequences,
2279 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2282 size_t dSize = ZSTD_decompressDCtx(dctx, checkBuf, srcSize, dst, cSize);
2283 CHECK(ZSTD_isError(dSize), "Should not throw an error");
2284 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2290 ZSTD_freeCDict(cdict);
2291 ZSTD_freeDDict(ddict);
2293 ZSTD_freeCCtx(cctx);
2294 ZSTD_freeDCtx(dctx);
2296 DISPLAYLEVEL(3, "OK \n");
2299 FUZ_freeDictionary(dictionary);
2300 ZSTD_freeCStream(zc);
2301 ZSTD_freeDStream(zd);
2302 ZSTD_freeCCtx(mtctx);
2304 free(compressedBuffer);
2305 free(decodedBuffer);
2310 DISPLAY("Error detected in Unit tests ! \n");
2315 /* ====== Fuzzer tests ====== */
2317 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
2319 const BYTE* b1 = (const BYTE*)buf1;
2320 const BYTE* b2 = (const BYTE*)buf2;
2322 for (u=0; u<max; u++) {
2323 if (b1[u] != b2[u]) break;
2326 DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max);
2329 DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max);
2331 DISPLAY(" %02X %02X %02X ",
2332 b1[u-3], b1[u-2], b1[u-1]);
2333 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2334 b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]);
2336 DISPLAY(" %02X %02X %02X ",
2337 b2[u-3], b2[u-2], b2[u-1]);
2338 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2339 b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]);
2343 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
2345 size_t const lengthMask = ((size_t)1 << logLength) - 1;
2346 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
2349 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
2351 U32 const logLength = FUZ_rand(seed) % maxLog;
2352 return FUZ_rLogLength(seed, logLength);
2355 /* Return value in range minVal <= v <= maxVal */
2356 static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
2358 U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal;
2359 return (U32)((FUZ_rand(seed) % mod) + minVal);
2362 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests)
2364 U32 const maxSrcLog = bigTests ? 24 : 22;
2365 static const U32 maxSampleLog = 19;
2366 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2367 BYTE* cNoiseBuffer[5];
2368 size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
2369 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2370 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2371 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2372 size_t const dstBufferSize = srcBufferSize;
2373 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2375 unsigned testNb = 0;
2376 U32 coreSeed = seed;
2377 ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
2378 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
2379 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2380 UTIL_time_t const startClock = UTIL_getTime();
2381 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2382 size_t dictSize = 0;
2384 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
2387 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2388 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2389 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2390 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2391 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2392 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2393 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2394 "Not enough memory, fuzzer tests cancelled");
2396 /* Create initial samples */
2397 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2398 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2399 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2400 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2401 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2402 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2403 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2405 /* catch up testNb */
2406 for (testNb=1; testNb < startTest; testNb++)
2407 FUZ_rand(&coreSeed);
2410 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2412 const BYTE* srcBuffer;
2413 size_t totalTestSize, totalGenSize, cSize;
2414 XXH64_state_t xxhState;
2416 U32 resetAllowed = 1;
2420 FUZ_rand(&coreSeed);
2421 lseed = coreSeed ^ prime32;
2422 if (nbTests >= testNb) {
2423 DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
2425 DISPLAYUPDATE(2, "\r%6u ", testNb);
2428 /* states full reset (deliberately not synchronized) */
2429 /* some issues can only happen when reusing states */
2430 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2431 ZSTD_freeCStream(zc);
2432 zc = ZSTD_createCStream();
2433 CHECK(zc==NULL, "ZSTD_createCStream : allocation error");
2436 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2437 ZSTD_freeDStream(zd);
2438 zd = ZSTD_createDStream();
2439 CHECK(zd==NULL, "ZSTD_createDStream : allocation error");
2440 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2443 /* srcBuffer selection [0-4] */
2444 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2445 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2449 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2450 buffNb = tnb[buffNb >> 3];
2452 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2453 buffNb = tnb[buffNb >> 3];
2455 srcBuffer = cNoiseBuffer[buffNb];
2458 /* compression init */
2459 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2460 && oldTestLog /* at least one test happened */ && resetAllowed) {
2461 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2462 maxTestSize = MIN(maxTestSize, srcBufferSize-16);
2463 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2464 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2465 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2468 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2469 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2470 U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
2472 (MAX(testLog, dictLog) / 3)))
2474 U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
2475 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2476 oldTestLog = testLog;
2477 /* random dictionary selection */
2478 dictSize = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2479 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2480 dict = srcBuffer + dictStart;
2482 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2483 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2484 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) );
2485 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) );
2486 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) );
2487 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) );
2488 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2489 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2492 /* multi-segments compression test */
2493 XXH64_reset(&xxhState, 0);
2494 { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2497 while(totalTestSize < maxTestSize) {
2498 /* compress random chunks into randomly sized dst buffers */
2499 { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2500 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2501 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2502 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2503 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
2504 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2505 outBuff.size = outBuff.pos + dstBuffSize;
2507 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
2509 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2510 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2511 totalTestSize += inBuff.pos;
2514 /* random flush operation, to mess around */
2515 if ((FUZ_rand(&lseed) & 15) == 0) {
2516 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2517 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2518 outBuff.size = outBuff.pos + adjustedDstSize;
2519 CHECK_Z( ZSTD_flushStream(zc, &outBuff) );
2522 /* final frame epilogue */
2523 { size_t remainingToFlush = (size_t)(-1);
2524 while (remainingToFlush) {
2525 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2526 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2527 outBuff.size = outBuff.pos + adjustedDstSize;
2528 remainingToFlush = ZSTD_endStream(zc, &outBuff);
2529 CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
2531 crcOrig = XXH64_digest(&xxhState);
2532 cSize = outBuff.pos;
2535 /* multi - fragments decompression test */
2536 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2537 CHECK_Z ( ZSTD_resetDStream(zd) );
2539 CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2541 { size_t decompressionResult = 1;
2542 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2543 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2544 for (totalGenSize = 0 ; decompressionResult ; ) {
2545 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2546 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2547 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2548 inBuff.size = inBuff.pos + readCSrcSize;
2549 outBuff.size = outBuff.pos + dstBuffSize;
2550 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2551 if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) {
2552 DISPLAY("checksum error : \n");
2553 findDiff(copyBuffer, dstBuffer, totalTestSize);
2555 CHECK( ZSTD_isError(decompressionResult), "decompression error : %s",
2556 ZSTD_getErrorName(decompressionResult) );
2558 CHECK (decompressionResult != 0, "frame not fully decoded");
2559 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)",
2560 (unsigned)outBuff.pos, (unsigned)totalTestSize);
2561 CHECK (inBuff.pos != cSize, "compressed data should be fully read")
2562 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
2563 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
2564 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
2567 /*===== noisy/erroneous src decompression test =====*/
2569 /* add some noise */
2570 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
2571 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
2572 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
2573 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
2574 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
2575 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
2576 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
2579 /* try decompression on noisy data */
2580 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
2581 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2582 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2583 while (outBuff.pos < dstBufferSize) {
2584 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2585 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2586 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
2587 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
2588 outBuff.size = outBuff.pos + adjustedDstSize;
2589 inBuff.size = inBuff.pos + adjustedCSrcSize;
2590 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2591 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
2592 /* No forward progress possible */
2593 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
2595 DISPLAY("\r%u fuzzer tests completed \n", testNb);
2598 ZSTD_freeCStream(zc);
2599 ZSTD_freeDStream(zd);
2600 ZSTD_freeDStream(zd_noise);
2601 free(cNoiseBuffer[0]);
2602 free(cNoiseBuffer[1]);
2603 free(cNoiseBuffer[2]);
2604 free(cNoiseBuffer[3]);
2605 free(cNoiseBuffer[4]);
2616 /** If useOpaqueAPI, sets param in cctxParams.
2617 * Otherwise, sets the param in zc. */
2618 static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
2619 ZSTD_cParameter param, unsigned value,
2623 return ZSTD_CCtxParams_setParameter(cctxParams, param, value);
2625 return ZSTD_CCtx_setParameter(zc, param, value);
2629 /* Tests for ZSTD_compress_generic() API */
2630 static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
2631 double compressibility, int bigTests)
2633 U32 const maxSrcLog = bigTests ? 24 : 22;
2634 static const U32 maxSampleLog = 19;
2635 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2636 BYTE* cNoiseBuffer[5];
2637 size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
2638 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2639 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2640 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2641 size_t const dstBufferSize = srcBufferSize;
2642 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2645 U32 coreSeed = seed;
2646 ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
2647 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
2648 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2649 UTIL_time_t const startClock = UTIL_getTime();
2650 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2651 size_t dictSize = 0;
2653 U32 windowLogMalus = 0; /* can survive between 2 loops */
2654 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
2655 U32 const nbThreadsMax = bigTests ? 4 : 2;
2656 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
2659 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2660 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2661 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2662 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2663 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2664 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2665 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2666 "Not enough memory, fuzzer tests cancelled");
2668 /* Create initial samples */
2669 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2670 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2671 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2672 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2673 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2674 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2675 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2677 /* catch up testNb */
2678 for (testNb=1; testNb < startTest; testNb++)
2679 FUZ_rand(&coreSeed);
2682 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2685 const BYTE* srcBuffer;
2686 size_t totalTestSize, totalGenSize, cSize;
2687 XXH64_state_t xxhState;
2689 U32 resetAllowed = 1;
2691 ZSTD_parameters savedParams;
2692 int isRefPrefix = 0;
2693 U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2696 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
2697 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
2698 FUZ_rand(&coreSeed);
2699 lseed = coreSeed ^ prime32;
2700 DISPLAYLEVEL(5, " *** Test %u *** \n", testNb);
2701 opaqueAPI = FUZ_rand(&lseed) & 1;
2703 /* states full reset (deliberately not synchronized) */
2704 /* some issues can only happen when reusing states */
2705 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2706 DISPLAYLEVEL(5, "Creating new context \n");
2708 zc = ZSTD_createCCtx();
2709 CHECK(zc == NULL, "ZSTD_createCCtx allocation error");
2712 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2713 ZSTD_freeDStream(zd);
2714 zd = ZSTD_createDStream();
2715 CHECK(zd == NULL, "ZSTD_createDStream allocation error");
2716 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2719 /* srcBuffer selection [0-4] */
2720 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2721 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2725 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2726 buffNb = tnb[buffNb >> 3];
2728 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2729 buffNb = tnb[buffNb >> 3];
2731 srcBuffer = cNoiseBuffer[buffNb];
2734 /* compression init */
2735 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
2736 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2737 && oldTestLog /* at least one test happened */
2739 /* just set a compression level */
2740 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2741 if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
2742 { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
2743 DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
2744 CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
2747 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2748 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2749 U32 const cLevelCandidate = (FUZ_rand(&lseed) %
2751 (MAX(testLog, dictLog) / 2))) +
2753 int const cLevel = MIN(cLevelCandidate, cLevelMax);
2754 DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel);
2755 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2756 DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize);
2757 oldTestLog = testLog;
2758 /* random dictionary selection */
2759 dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2760 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2761 dict = srcBuffer + dictStart;
2762 if (!dictSize) dict=NULL;
2764 pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2765 { ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
2766 const U32 windowLogMax = bigTests ? 24 : 20;
2767 const U32 searchLogMax = bigTests ? 15 : 13;
2769 DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
2771 /* mess with compression parameters */
2772 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
2773 cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
2774 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
2775 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
2776 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
2777 cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
2778 cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
2779 cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
2780 cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
2782 if (FUZ_rand(&lseed) & 1) {
2783 DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
2784 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
2785 assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
2786 windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
2788 if (FUZ_rand(&lseed) & 1) {
2789 DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
2790 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
2792 if (FUZ_rand(&lseed) & 1) {
2793 DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
2794 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
2796 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
2797 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
2798 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
2800 /* mess with long distance matching parameters */
2802 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) );
2803 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
2804 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
2805 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
2806 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
2807 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_srcSizeHint, FUZ_randomClampedLength(&lseed, ZSTD_SRCSIZEHINT_MIN, ZSTD_SRCSIZEHINT_MAX), opaqueAPI) );
2810 /* mess with frame parameters */
2811 if (FUZ_rand(&lseed) & 1) {
2812 int const checksumFlag = FUZ_rand(&lseed) & 1;
2813 DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
2814 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
2816 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2817 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2818 if (FUZ_rand(&lseed) & 1) {
2819 DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize);
2820 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2822 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2825 /* multi-threading parameters. Only adjust occasionally for small tests. */
2826 if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
2827 U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
2828 U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
2829 int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
2830 DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads);
2831 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
2832 if (nbThreads > 1) {
2833 U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
2834 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
2835 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
2838 /* Enable rsyncable mode 1 in 4 times. */
2840 int const rsyncable = (FUZ_rand(&lseed) % 4 == 0);
2841 DISPLAYLEVEL(5, "t%u: rsyncable : %d \n", testNb, rsyncable);
2842 setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, rsyncable, opaqueAPI);
2845 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
2846 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) );
2848 /* Apply parameters */
2850 DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb);
2851 CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
2854 if (FUZ_rand(&lseed) & 1) {
2855 if (FUZ_rand(&lseed) & 1) {
2856 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2858 CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
2862 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
2866 CHECK_Z(getCCtxParams(zc, &savedParams));
2868 /* multi-segments compression test */
2870 int const startSeed = lseed;
2871 XXH64_hash_t compressedCrcs[2];
2872 for (iter = 0; iter < 2; ++iter, lseed = startSeed) {
2873 ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2874 int const singlePass = (FUZ_rand(&lseed) & 3) == 0;
2877 XXH64_reset(&xxhState, 0);
2879 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2881 DISPLAYLEVEL(6, "t%u: Reloading prefix\n", testNb);
2882 /* Need to reload the prefix because it gets dropped after one compression */
2883 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
2886 /* Adjust number of workers occasionally - result must be deterministic independent of nbWorkers */
2887 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers));
2888 if (nbWorkers > 0 && (FUZ_rand(&lseed) & 7) == 0) {
2889 DISPLAYLEVEL(6, "t%u: Modify nbWorkers: %d -> %d \n", testNb, nbWorkers, nbWorkers + iter);
2890 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers + iter));
2894 ZSTD_inBuffer inBuff = { srcBuffer, maxTestSize, 0 };
2895 CHECK_Z(ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end));
2896 DISPLAYLEVEL(6, "t%u: Single pass compression: consumed %u bytes ; produced %u bytes \n",
2897 testNb, (unsigned)inBuff.pos, (unsigned)outBuff.pos);
2898 CHECK(inBuff.pos != inBuff.size, "Input not consumed!");
2899 crcOrig = XXH64(srcBuffer, maxTestSize, 0);
2900 totalTestSize = maxTestSize;
2903 for (totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
2904 /* compress random chunks into randomly sized dst buffers */
2905 size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2906 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2907 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2908 ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
2909 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2910 int forwardProgress;
2912 size_t const ipos = inBuff.pos;
2913 size_t const opos = outBuff.pos;
2915 if (outBuff.pos == outBuff.size) {
2916 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2917 size_t const dstBuffSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
2918 outBuff.size = outBuff.pos + dstBuffSize;
2920 CHECK_Z( ret = ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
2921 DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
2922 testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos);
2924 /* We've completed the flush */
2925 if (flush == ZSTD_e_flush && ret == 0)
2928 /* Ensure maximal forward progress for determinism */
2929 forwardProgress = (inBuff.pos != ipos) || (outBuff.pos != opos);
2930 } while (forwardProgress);
2931 assert(inBuff.pos == inBuff.size);
2933 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2934 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2935 totalTestSize += inBuff.pos;
2938 /* final frame epilogue */
2939 { size_t remainingToFlush = 1;
2940 while (remainingToFlush) {
2941 ZSTD_inBuffer inBuff = { NULL, 0, 0 };
2942 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2943 size_t const adjustedDstSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
2944 outBuff.size = outBuff.pos + adjustedDstSize;
2945 DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize);
2946 /* ZSTD_e_end guarantees maximal forward progress */
2947 remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
2948 DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos);
2949 CHECK( ZSTD_isError(remainingToFlush),
2950 "ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
2951 ZSTD_getErrorName(remainingToFlush) );
2953 crcOrig = XXH64_digest(&xxhState);
2955 cSize = outBuff.pos;
2956 compressedCrcs[iter] = XXH64(cBuffer, cSize, 0);
2957 DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize);
2959 CHECK(!(compressedCrcs[0] == compressedCrcs[1]), "Compression is not deterministic!");
2962 CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
2964 /* multi - fragments decompression test */
2965 if (FUZ_rand(&lseed) & 1) {
2966 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
2968 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2969 DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", (void const*)dict);
2970 CHECK_Z( ZSTD_resetDStream(zd) );
2973 DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
2974 CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2976 if (FUZ_rand(&lseed) & 1) {
2977 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
2979 { size_t decompressionResult = 1;
2980 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2981 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2982 for (totalGenSize = 0 ; decompressionResult ; ) {
2983 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2984 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2985 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2986 inBuff.size = inBuff.pos + readCSrcSize;
2987 outBuff.size = outBuff.pos + dstBuffSize;
2988 DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n",
2989 (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize);
2990 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2991 DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n",
2992 (unsigned)inBuff.pos, (unsigned)outBuff.pos);
2993 if (ZSTD_isError(decompressionResult)) {
2994 DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult));
2995 findDiff(copyBuffer, dstBuffer, totalTestSize);
2997 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
2998 CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize);
3000 CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize);
3001 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize);
3002 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
3003 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
3004 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
3007 /*===== noisy/erroneous src decompression test =====*/
3009 /* add some noise */
3010 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
3011 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
3012 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
3013 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
3014 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
3015 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
3016 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
3019 /* try decompression on noisy data */
3020 if (FUZ_rand(&lseed) & 1) {
3021 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_and_parameters));
3023 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_only));
3025 if (FUZ_rand(&lseed) & 1) {
3026 CHECK_Z(ZSTD_DCtx_setParameter(zd_noise, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
3028 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
3029 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
3030 while (outBuff.pos < dstBufferSize) {
3031 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3032 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
3033 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
3034 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
3035 outBuff.size = outBuff.pos + adjustedDstSize;
3036 inBuff.size = inBuff.pos + adjustedCSrcSize;
3037 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
3038 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
3039 /* Good so far, but no more progress possible */
3040 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
3042 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
3046 ZSTD_freeDStream(zd);
3047 ZSTD_freeDStream(zd_noise);
3048 ZSTD_freeCCtxParams(cctxParams);
3049 free(cNoiseBuffer[0]);
3050 free(cNoiseBuffer[1]);
3051 free(cNoiseBuffer[2]);
3052 free(cNoiseBuffer[3]);
3053 free(cNoiseBuffer[4]);
3064 /*-*******************************************************
3066 *********************************************************/
3067 static int FUZ_usage(const char* programName)
3069 DISPLAY( "Usage :\n");
3070 DISPLAY( " %s [args]\n", programName);
3072 DISPLAY( "Arguments :\n");
3073 DISPLAY( " -i# : Number of tests (default:%u)\n", nbTestsDefault);
3074 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
3075 DISPLAY( " -s# : Select seed (default:prompt user)\n");
3076 DISPLAY( " -t# : Select starting test number (default:0)\n");
3077 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
3078 DISPLAY( " -v : verbose\n");
3079 DISPLAY( " -p : pause at the end\n");
3080 DISPLAY( " -h : display help and exit\n");
3084 typedef enum { simple_api, advanced_api } e_api;
3086 int main(int argc, const char** argv)
3090 int nbTests = nbTestsDefault;
3092 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
3095 int bigTests = (sizeof(size_t) == 8);
3096 e_api selected_api = simple_api;
3097 const char* const programName = argv[0];
3100 /* Check command line */
3101 for(argNb=1; argNb<argc; argNb++) {
3102 const char* argument = argv[argNb];
3103 assert(argument != NULL);
3105 /* Parsing commands. Aggregated commands are allowed */
3106 if (argument[0]=='-') {
3108 if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
3109 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
3110 if (!strcmp(argument, "--big-tests")) { bigTests=1; continue; }
3113 while (*argument!=0) {
3117 return FUZ_usage(programName);
3129 case 'p': /* pause at the end */
3134 case 'i': /* limit tests by nb of iterations (default) */
3136 nbTests=0; g_clockTime=0;
3137 while ((*argument>='0') && (*argument<='9')) {
3139 nbTests += *argument - '0';
3144 case 'T': /* limit tests by time */
3146 nbTests=0; g_clockTime=0;
3147 while ((*argument>='0') && (*argument<='9')) {
3149 g_clockTime += *argument - '0';
3152 if (*argument=='m') { /* -T1m == -T60 */
3153 g_clockTime *=60, argument++;
3154 if (*argument=='n') argument++; /* -T1mn == -T60 */
3155 } else if (*argument=='s') argument++; /* -T10s == -T10 */
3156 g_clockTime *= SEC_TO_MICRO;
3159 case 's': /* manually select seed */
3163 while ((*argument>='0') && (*argument<='9')) {
3165 seed += *argument - '0';
3170 case 't': /* select starting test number */
3173 while ((*argument>='0') && (*argument<='9')) {
3175 testNb += *argument - '0';
3180 case 'P': /* compressibility % */
3183 while ((*argument>='0') && (*argument<='9')) {
3185 proba += *argument - '0';
3188 if (proba<0) proba=0;
3189 if (proba>100) proba=100;
3193 return FUZ_usage(programName);
3195 } } } /* for(argNb=1; argNb<argc; argNb++) */
3198 DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
3201 time_t const t = time(NULL);
3202 U32 const h = XXH32(&t, sizeof(t), 1);
3206 DISPLAY("Seed = %u\n", (unsigned)seed);
3207 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
3209 if (nbTests<=0) nbTests=1;
3212 result = basicUnitTests(0, ((double)proba) / 100, bigTests); /* constant seed for predictability */
3216 switch(selected_api)
3219 result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3222 result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3225 assert(0); /* impossible */
3231 DISPLAY("Press Enter \n");