2 * Copyright (c) Meta Platforms, Inc. and affiliates.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
12 /*-************************************
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
22 /*-************************************
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
28 #include <time.h> /* time_t, time(), to randomize seed */
29 #include <assert.h> /* assert */
30 #include "timefn.h" /* UTIL_time_t, UTIL_getTime */
32 #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
33 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
34 #include "zstd.h" /* ZSTD_compressBound */
35 #include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
36 #include "zdict.h" /* ZDICT_trainFromBuffer */
37 #include "datagen.h" /* RDG_genBuffer */
38 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
39 #include "xxhash.h" /* XXH64_* */
42 #include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
43 #include "external_matchfinder.h" /* zstreamSequenceProducer, EMF_testCase */
45 /*-************************************
47 **************************************/
52 static const int nbTestsDefault = 10000;
53 static const U32 g_cLevelMax_smallTests = 10;
54 #define COMPRESSIBLE_NOISE_LENGTH (10 MB)
55 #define FUZ_COMPRESSIBILITY_DEFAULT 50
56 static const U32 prime32 = 2654435761U;
59 /*-************************************
61 **************************************/
62 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
63 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \
64 DISPLAY(__VA_ARGS__); \
65 if (g_displayLevel>=4) fflush(stderr); }
66 static U32 g_displayLevel = 2;
68 static const U64 g_refreshRate = SEC_TO_MICRO / 6;
69 static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
71 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
72 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
73 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
74 if (g_displayLevel>=4) fflush(stderr); } }
76 static U64 g_clockTime = 0;
79 /*-*******************************************************
81 *********************************************************/
84 #define MIN(a,b) ((a)<(b)?(a):(b))
85 #define MAX(a,b) ((a)>(b)?(a):(b))
87 @return : a 27 bits random value, from a 32-bits `seed`.
88 `seed` is also modified */
89 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
90 static U32 FUZ_rand(U32* seedPtr)
92 static const U32 prime2 = 2246822519U;
93 U32 rand32 = *seedPtr;
96 rand32 = FUZ_rotl32(rand32, 13);
101 #define CHECK(cond, ...) { \
103 DISPLAY("Error => "); \
104 DISPLAY(__VA_ARGS__); \
105 DISPLAY(" (seed %u, test nb %u, line %u) \n", \
106 (unsigned)seed, testNb, __LINE__); \
107 goto _output_error; \
110 #define CHECK_Z(f) { \
111 size_t const err = f; \
112 CHECK(ZSTD_isError(err), "%s : %s ", \
113 #f, ZSTD_getErrorName(err)); \
116 #define CHECK_RET(ret, cond, ...) { \
118 DISPLAY("Error %llu => ", (unsigned long long)ret); \
119 DISPLAY(__VA_ARGS__); \
120 DISPLAY(" (line %u)\n", __LINE__); \
124 #define CHECK_RET_Z(f) { \
125 size_t const err = f; \
126 CHECK_RET(err, ZSTD_isError(err), "%s : %s ", \
127 #f, ZSTD_getErrorName(err)); \
131 /*======================================================
133 *======================================================*/
141 static const buffer_t kBuffNull = { NULL, 0 , 0 };
143 static void FUZ_freeDictionary(buffer_t dict)
148 static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
150 buffer_t dict = kBuffNull;
151 size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
152 size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
153 if (!blockSizes) return kBuffNull;
154 dict.start = malloc(requestedDictSize);
155 if (!dict.start) { free(blockSizes); return kBuffNull; }
157 for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
158 blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
160 { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
162 if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; }
163 dict.size = requestedDictSize;
164 dict.filled = dictSize;
169 /* Round trips data and updates xxh with the decompressed data produced */
170 static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
171 XXH64_state_t* xxh, void* data, size_t size,
172 ZSTD_EndDirective endOp)
174 static BYTE compressed[1024];
175 static BYTE uncompressed[1024];
177 ZSTD_inBuffer cin = {data, size, 0};
181 ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
182 ZSTD_inBuffer din = { compressed, 0, 0 };
183 ZSTD_outBuffer dout = { uncompressed, 0, 0 };
185 cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
186 if (ZSTD_isError(cret))
190 while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
194 dout.size = sizeof(uncompressed);
195 dret = ZSTD_decompressStream(dctx, &dout, &din);
196 if (ZSTD_isError(dret))
198 XXH64_update(xxh, dout.dst, dout.pos);
202 } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
206 /* Generates some data and round trips it */
207 static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
208 XXH64_state_t* xxh, SEQ_stream* seq,
209 SEQ_gen_type type, unsigned value)
211 static BYTE data[1024];
215 SEQ_outBuffer sout = {data, sizeof(data), 0};
217 gen = SEQ_gen(seq, type, value, &sout);
219 ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
220 if (ZSTD_isError(ret))
227 static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
230 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
231 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
232 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
233 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
234 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
235 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
236 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
237 savedParams->cParams.strategy = value;
239 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
240 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
241 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
242 savedParams->fParams.noDictIDFlag = !value;
246 static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
248 ZSTD_parameters params;
249 if (ZSTD_isError(getCCtxParams(zc, ¶ms))) return 10;
250 CHECK_RET(1, params.cParams.windowLog != savedParams.cParams.windowLog, "windowLog");
251 CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
252 CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
253 CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
254 CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
255 CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
257 CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
258 CHECK_RET(8, params.fParams.contentSizeFlag != savedParams.fParams.contentSizeFlag, "contentSizeFlag");
259 CHECK_RET(9, params.fParams.noDictIDFlag != savedParams.fParams.noDictIDFlag, "noDictIDFlag");
263 static int basicUnitTests(U32 seed, double compressibility, int bigTests)
265 size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
266 void* CNBuffer = malloc(CNBufferSize);
267 size_t const skippableFrameSize = 200 KB;
268 size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
269 void* compressedBuffer = malloc(compressedBufferSize);
270 size_t const decodedBufferSize = CNBufferSize;
271 void* decodedBuffer = malloc(decodedBufferSize);
275 U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */
276 ZSTD_CStream* zc = ZSTD_createCStream();
277 ZSTD_DStream* zd = ZSTD_createDStream();
278 ZSTD_CCtx* mtctx = ZSTD_createCCtx();
280 ZSTD_inBuffer inBuff, inBuff2;
281 ZSTD_outBuffer outBuff;
282 buffer_t dictionary = kBuffNull;
283 size_t const dictSize = 128 KB;
286 /* Create compressible test buffer */
287 if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd || !mtctx) {
288 DISPLAY("Not enough memory, aborting \n");
291 RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
293 CHECK_Z(ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2));
295 /* Create dictionary */
296 DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
297 dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB);
298 if (!dictionary.start) {
299 DISPLAY("Error creating dictionary, aborting \n");
302 dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
304 /* Basic compression test */
305 DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
306 CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
307 outBuff.dst = (char*)(compressedBuffer);
308 outBuff.size = compressedBufferSize;
310 inBuff.src = CNBuffer;
311 inBuff.size = CNBufferSize;
313 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
314 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
315 { size_t const r = ZSTD_endStream(zc, &outBuff);
316 if (r != 0) goto _output_error; } /* error, or some data not flushed */
317 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos);
319 /* generate skippable frame */
320 MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
321 MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
322 cSize = skippableFrameSize + 8;
324 /* Basic compression test using dict */
325 DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
326 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
327 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
328 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) );
329 outBuff.dst = (char*)(compressedBuffer)+cSize;
330 assert(compressedBufferSize > cSize);
331 outBuff.size = compressedBufferSize - cSize;
333 inBuff.src = CNBuffer;
334 inBuff.size = CNBufferSize;
336 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
337 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
338 { size_t const r = ZSTD_endStream(zc, &outBuff);
339 if (r != 0) goto _output_error; } /* error, or some data not flushed */
340 cSize += outBuff.pos;
341 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
342 (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
344 /* context size functions */
345 DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
346 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
347 size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams);
348 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */
349 if (ZSTD_isError(cstreamSize)) goto _output_error;
350 if (ZSTD_isError(cdictSize)) goto _output_error;
351 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize));
354 /* context size functions */
355 DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++);
356 { ZSTD_CCtx_params* const params = ZSTD_createCCtxParams();
357 size_t cstreamSize, cctxSize;
358 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) );
359 cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
360 CHECK_Z(cstreamSize);
361 cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
363 if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error;
364 ZSTD_freeCCtxParams(params);
365 DISPLAYLEVEL(3, "OK \n");
368 DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
369 { size_t const s = ZSTD_sizeof_CStream(zc);
370 if (ZSTD_isError(s)) goto _output_error;
371 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
374 /* Attempt bad compression parameters */
375 DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++);
377 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
378 params.cParams.minMatch = 2;
379 r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
380 if (!ZSTD_isError(r)) goto _output_error;
381 DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
384 /* skippable frame test */
385 DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
386 CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) );
387 inBuff.src = compressedBuffer;
390 outBuff.dst = decodedBuffer;
391 outBuff.size = CNBufferSize;
393 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
394 DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r);
395 if (r != 0) goto _output_error;
397 if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */
398 DISPLAYLEVEL(3, "OK \n");
400 /* Basic decompression test */
402 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
403 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
404 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */
405 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
406 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
407 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
408 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
409 DISPLAYLEVEL(3, "OK \n");
411 /* Reuse without init */
412 DISPLAYLEVEL(3, "test%3i : decompress again without init (reuse previous settings): ", testNb++);
414 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
415 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
416 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
417 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
418 DISPLAYLEVEL(3, "OK \n");
420 /* check regenerated data is byte exact */
421 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
423 for (i=0; i<CNBufferSize; i++) {
424 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
426 DISPLAYLEVEL(3, "OK \n");
428 /* check decompression fails early if first bytes are wrong */
429 DISPLAYLEVEL(3, "test%3i : early decompression error if first bytes are incorrect : ", testNb++);
430 { const char buf[3] = { 0 }; /* too short, not enough to start decoding header */
431 ZSTD_inBuffer inb = { buf, sizeof(buf), 0 };
432 size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inb);
433 if (!ZSTD_isError(remaining)) goto _output_error; /* should have errored out immediately (note: this does not test the exact error code) */
435 DISPLAYLEVEL(3, "OK \n");
437 /* context size functions */
438 DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
439 { ZSTD_frameHeader fhi;
440 const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
441 size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
442 if (gfhError!=0) goto _output_error;
443 DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize);
444 { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
445 /* uses ZSTD_initDStream_usingDict() */
446 + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
447 if (ZSTD_isError(s)) goto _output_error;
448 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
451 DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
452 { size_t const s = ZSTD_sizeof_DStream(zd);
453 if (ZSTD_isError(s)) goto _output_error;
454 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
457 /* Decompression by small increment */
458 DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
459 { /* skippable frame */
461 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
462 inBuff.src = compressedBuffer;
463 outBuff.dst = decodedBuffer;
466 while (r) { /* skippable frame */
467 size_t const inSize = (FUZ_rand(&coreSeed) & 15) + 1;
468 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + 1;
469 inBuff.size = inBuff.pos + inSize;
470 outBuff.size = outBuff.pos + outSize;
471 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
472 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream on skippable frame error : %s \n", ZSTD_getErrorName(r));
473 if (ZSTD_isError(r)) goto _output_error;
476 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
479 size_t const inSize = FUZ_rand(&coreSeed) & 15;
480 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + (!inSize); /* avoid having both sizes at 0 => would trigger a no_forward_progress error */
481 inBuff.size = inBuff.pos + inSize;
482 outBuff.size = outBuff.pos + outSize;
483 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
484 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(r));
485 if (ZSTD_isError(r)) goto _output_error;
488 if (outBuff.pos != CNBufferSize) DISPLAYLEVEL(4, "outBuff.pos != CNBufferSize : should have regenerated same amount ! \n");
489 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
490 if (inBuff.pos != cSize) DISPLAYLEVEL(4, "inBuff.pos != cSize : should have real all input ! \n");
491 if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
492 DISPLAYLEVEL(3, "OK \n");
494 /* check regenerated data is byte exact */
495 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
497 for (i=0; i<CNBufferSize; i++) {
498 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
500 DISPLAYLEVEL(3, "OK \n");
502 /* Decompression forward progress */
503 DISPLAYLEVEL(3, "test%3i : generate error when ZSTD_decompressStream() doesn't progress : ", testNb++);
504 { /* skippable frame */
507 int const maxDec = 100;
508 inBuff.src = compressedBuffer;
512 outBuff.dst = decodedBuffer;
514 outBuff.size = CNBufferSize-1; /* 1 byte missing */
516 for (decNb=0; decNb<maxDec; decNb++) {
517 if (r==0) ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
518 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
519 if (ZSTD_isError(r)) break;
521 if (!ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream should have triggered a no_forward_progress error \n");
522 if (!ZSTD_isError(r)) goto _output_error; /* should have triggered no_forward_progress error */
524 DISPLAYLEVEL(3, "OK \n");
526 DISPLAYLEVEL(3, "test%3i : NULL output and NULL input : ", testNb++);
533 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
534 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
535 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
536 outBuff.dst = (char*)(compressedBuffer);
537 outBuff.size = compressedBufferSize;
539 { size_t const r = ZSTD_endStream(zc, &outBuff);
540 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
542 inBuff.src = outBuff.dst;
543 inBuff.size = outBuff.pos;
548 CHECK_Z( ZSTD_initDStream(zd) );
549 { size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
550 if (ret != 0) goto _output_error;
552 DISPLAYLEVEL(3, "OK\n");
554 DISPLAYLEVEL(3, "test%3i : NULL output buffer with non-NULL input : ", testNb++);
556 const char* test = "aa";
563 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
564 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
565 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
566 outBuff.dst = (char*)(compressedBuffer);
567 outBuff.size = compressedBufferSize;
569 { size_t const r = ZSTD_endStream(zc, &outBuff);
570 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
572 inBuff.src = outBuff.dst;
573 inBuff.size = outBuff.pos;
578 CHECK_Z( ZSTD_initDStream(zd) );
579 CHECK_Z(ZSTD_decompressStream(zd, &outBuff, &inBuff));
582 DISPLAYLEVEL(3, "OK\n");
583 /* _srcSize compression test */
584 DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
585 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
586 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
587 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
588 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) );
589 outBuff.dst = (char*)(compressedBuffer);
590 outBuff.size = compressedBufferSize;
592 inBuff.src = CNBuffer;
593 inBuff.size = CNBufferSize;
595 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
596 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
597 { size_t const r = ZSTD_endStream(zc, &outBuff);
598 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
600 { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
601 CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
602 CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
604 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
606 /* wrong _srcSize compression test */
607 DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
608 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
609 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
610 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
611 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) );
612 outBuff.dst = (char*)(compressedBuffer);
613 outBuff.size = compressedBufferSize;
615 inBuff.src = CNBuffer;
616 inBuff.size = CNBufferSize;
618 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
619 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
620 { size_t const r = ZSTD_endStream(zc, &outBuff);
621 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
622 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
624 /* wrong _srcSize compression test */
625 DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
626 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
627 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
628 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
629 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) );
630 outBuff.dst = (char*)(compressedBuffer);
631 outBuff.size = compressedBufferSize;
633 inBuff.src = CNBuffer;
634 inBuff.size = CNBufferSize;
636 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
637 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
638 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
641 DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
642 { CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
643 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) );
644 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) );
645 outBuff.dst = (char*)compressedBuffer;
646 outBuff.size = compressedBufferSize;
648 inBuff.src = CNBuffer;
649 inBuff.size = CNBufferSize;
651 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
652 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
653 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
656 /* Compression state reuse scenario */
657 DISPLAYLEVEL(3, "test%3i : context reuse : ", testNb++);
658 ZSTD_freeCStream(zc);
659 zc = ZSTD_createCStream();
660 if (zc==NULL) goto _output_error; /* memory allocation issue */
662 { size_t const inSize = 513;
663 DISPLAYLEVEL(5, "use1 ");
664 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
665 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
666 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
667 inBuff.src = CNBuffer;
668 inBuff.size = inSize;
670 outBuff.dst = (char*)(compressedBuffer)+cSize;
671 outBuff.size = ZSTD_compressBound(inSize);
673 DISPLAYLEVEL(5, "compress1 ");
674 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
675 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
676 DISPLAYLEVEL(5, "end1 ");
677 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
680 { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
681 DISPLAYLEVEL(5, "use2 ");
682 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
683 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
684 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
685 inBuff.src = CNBuffer;
686 inBuff.size = inSize;
688 outBuff.dst = (char*)(compressedBuffer)+cSize;
689 outBuff.size = ZSTD_compressBound(inSize);
691 DISPLAYLEVEL(5, "compress2 ");
692 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
693 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
694 DISPLAYLEVEL(5, "end2 ");
695 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
697 DISPLAYLEVEL(3, "OK \n");
699 /* Decompression single pass with empty frame */
700 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, 1);
702 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass on empty frame : ", testNb++);
703 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
704 size_t const dctxSize = ZSTD_sizeof_DCtx(dctx);
705 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
707 outBuff.dst = decodedBuffer;
709 outBuff.size = CNBufferSize;
711 inBuff.src = compressedBuffer;
714 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
716 CHECK(r != 0, "Entire frame must be decompressed");
717 CHECK(outBuff.pos != 0, "Wrong size!");
718 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
720 CHECK(dctxSize != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
723 DISPLAYLEVEL(3, "OK \n");
725 DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2KB : ", testNb++);
727 ZSTD_DCtx* dctx = ZSTD_createDCtx();
728 size_t singlePassSize, streamingSize, streaming2KSize;
731 ZSTD_CCtx* cctx = ZSTD_createCCtx();
732 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
733 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18));
734 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
735 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048));
736 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize);
741 CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBufferSize, compressedBuffer, cSize));
742 singlePassSize = ZSTD_sizeof_DCtx(dctx);
743 CHECK_Z(singlePassSize);
745 inBuff.src = compressedBuffer;
748 outBuff.dst = decodedBuffer;
749 outBuff.size = decodedBufferSize;
751 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048));
755 size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
757 CHECK(r != 0, "Entire frame must be decompressed");
759 streaming2KSize = ZSTD_sizeof_DCtx(dctx);
760 CHECK_Z(streaming2KSize);
762 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters));
766 size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
768 CHECK(r != 0, "Entire frame must be decompressed");
770 streamingSize = ZSTD_sizeof_DCtx(dctx);
771 CHECK_Z(streamingSize);
773 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024));
776 CHECK(!ZSTD_isError(ZSTD_decompressStream(dctx, &outBuff, &inBuff)), "decompression must fail");
778 CHECK(streamingSize < singlePassSize + (1 << 18) + 3 * ZSTD_BLOCKSIZE_MAX, "Streaming doesn't use the right amount of memory");
779 CHECK(streamingSize != streaming2KSize + 3 * (ZSTD_BLOCKSIZE_MAX - 2048), "ZSTD_d_blockSizeMax didn't save the right amount of memory");
780 DISPLAYLEVEL(3, "| %zu | %zu | %zu | ", singlePassSize, streaming2KSize, streamingSize);
784 DISPLAYLEVEL(3, "OK \n");
786 /* Decompression with ZSTD_d_stableOutBuffer */
787 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1);
789 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
790 size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
792 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
794 outBuff.dst = decodedBuffer;
796 outBuff.size = CNBufferSize;
798 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass : ", testNb++);
799 inBuff.src = compressedBuffer;
802 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
804 CHECK(r != 0, "Entire frame must be decompressed");
805 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
806 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
808 CHECK(dctxSize0 != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
809 DISPLAYLEVEL(3, "OK \n");
811 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer : ", testNb++);
815 while (inBuff.pos < cSize) {
816 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
817 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
819 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
820 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
821 dctxSize1 = ZSTD_sizeof_DCtx(dctx);
822 CHECK(!(dctxSize0 < dctxSize1), "Input buffer allocated");
823 DISPLAYLEVEL(3, "OK \n");
825 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer too small : ", testNb++);
826 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
827 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
828 inBuff.src = compressedBuffer;
832 outBuff.size = CNBufferSize - 1;
833 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
834 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall, "Must error but got %s", ZSTD_getErrorName(r));
836 DISPLAYLEVEL(3, "OK \n");
838 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer modified : ", testNb++);
839 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
840 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
841 inBuff.src = compressedBuffer;
842 inBuff.size = cSize - 1;
845 outBuff.size = CNBufferSize;
846 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
849 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
850 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r));
852 DISPLAYLEVEL(3, "OK \n");
854 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++);
855 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
856 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0));
860 while (inBuff.pos < cSize) {
861 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
862 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
864 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
865 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
866 CHECK(!(dctxSize1 < ZSTD_sizeof_DCtx(dctx)), "Output buffer allocated");
867 DISPLAYLEVEL(3, "OK \n");
872 /* Compression with ZSTD_c_stable{In,Out}Buffer */
873 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
878 assert(cctx != NULL);
880 in.size = CNBufferSize;
881 out.dst = compressedBuffer;
882 out.size = compressedBufferSize;
883 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
884 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++);
885 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
886 CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
887 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
888 CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
889 /* @cctxSize2 : sizeof_CCtx when doing full streaming (no stable in/out) */
890 { ZSTD_CCtx* const cctx2 = ZSTD_createCCtx();
891 assert(cctx2 != NULL);
892 in.pos = out.pos = 0;
893 CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
894 CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
895 CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
896 ZSTD_freeCCtx(cctx2);
898 /* @cctxSize1 : sizeof_CCtx when doing single-shot compression (no streaming) */
899 { ZSTD_CCtx* const cctx1 = ZSTD_createCCtx();
900 ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
902 assert(cctx1 != NULL);
903 params.fParams.checksumFlag = 1;
904 cSize3 = ZSTD_compress_advanced(cctx1, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
906 CHECK(!(cSize == cSize3), "Must be same compressed size");
907 CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx1)), "Must be same CCtx size");
908 ZSTD_freeCCtx(cctx1);
910 CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
911 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
912 DISPLAYLEVEL(3, "OK \n");
914 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
915 { int stableInBuffer;
917 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
918 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
919 CHECK(!(stableInBuffer == 0), "Modified");
920 CHECK(!(stableOutBuffer == 0), "Modified");
921 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
922 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
923 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
924 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
925 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
926 CHECK(!(stableInBuffer == 1), "Modified");
927 CHECK(!(stableOutBuffer == 1), "Modified");
929 DISPLAYLEVEL(3, "OK \n");
931 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer : ", testNb++);
932 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
933 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
934 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
935 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
936 in.pos = out.pos = 0;
937 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
938 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
939 DISPLAYLEVEL(3, "OK \n");
941 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer allocated size : ", testNb++);
942 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
943 CHECK(!(cctxSize1 == cctxSize), "Must be the same size as single pass");
945 DISPLAYLEVEL(3, "OK \n");
947 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer only : ", testNb++);
948 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
949 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
950 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
951 in.pos = out.pos = 0;
952 out.size = cSize / 4;
954 size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
958 out.size = MIN(out.size + cSize / 4, compressedBufferSize);
960 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
961 DISPLAYLEVEL(3, "OK \n");
963 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++);
964 in.pos = out.pos = 0;
965 out.size = cSize / 4;
966 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
967 in.src = (char const*)in.src + in.pos;
970 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
971 CHECK(!ZSTD_isError(ret), "Must error");
972 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
974 DISPLAYLEVEL(3, "OK \n");
976 /* stableSrc + streaming */
977 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream, flushStream and endStream : ", testNb++);
978 CHECK_Z( ZSTD_initCStream(cctx, 1) );
979 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
980 { ZSTD_inBuffer inBuf;
981 ZSTD_outBuffer outBuf;
982 const size_t nonZeroStartPos = 18;
983 const size_t inputSize = 500;
984 inBuf.src = CNBuffer;
986 inBuf.pos = nonZeroStartPos;
987 outBuf.dst = (char*)(compressedBuffer)+cSize;
988 outBuf.size = ZSTD_compressBound(inputSize);
990 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
992 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
993 CHECK_Z( ZSTD_flushStream(cctx, &outBuf) );
994 inBuf.size = nonZeroStartPos + inputSize;
995 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
996 CHECK(ZSTD_endStream(cctx, &outBuf) != 0, "compression should be successful and fully flushed");
997 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
998 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
999 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
1001 CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
1002 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
1004 DISPLAYLEVEL(3, "OK \n");
1006 /* stableSrc + streaming */
1007 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream2, using different end directives : ", testNb++);
1008 CHECK_Z( ZSTD_initCStream(cctx, 1) );
1009 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
1010 { ZSTD_inBuffer inBuf;
1011 ZSTD_outBuffer outBuf;
1012 const size_t nonZeroStartPos = 18;
1013 const size_t inputSize = 500;
1014 inBuf.src = CNBuffer;
1016 inBuf.pos = nonZeroStartPos;
1017 outBuf.dst = (char*)(compressedBuffer)+cSize;
1018 outBuf.size = ZSTD_compressBound(inputSize);
1020 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
1022 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
1023 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_flush) );
1024 inBuf.size = nonZeroStartPos + inputSize;
1025 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
1026 CHECK( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_end) != 0, "compression should be successful and fully flushed");
1027 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
1028 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
1029 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
1031 CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
1032 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
1034 DISPLAYLEVEL(3, "OK \n");
1036 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer: context size : ", testNb++);
1037 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
1038 DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
1039 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
1040 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
1041 cctxSize1 = cctxSize;
1043 DISPLAYLEVEL(3, "OK \n");
1045 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer only : ", testNb++);
1046 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
1047 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1048 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
1050 in.pos = out.pos = 0;
1051 in.size = MIN(CNBufferSize, 10);
1052 out.size = compressedBufferSize;
1053 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1055 in.size = CNBufferSize - in.size;
1056 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
1057 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, out.pos));
1058 DISPLAYLEVEL(3, "OK \n");
1060 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer modify buffer : ", testNb++);
1061 in.pos = out.pos = 0;
1062 in.size = CNBufferSize;
1063 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1064 in.pos = out.pos = 0;
1065 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1066 CHECK(!ZSTD_isError(ret), "Must have errored");
1067 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
1069 DISPLAYLEVEL(3, "OK \n");
1071 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer: context size : ", testNb++);
1072 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
1073 DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
1074 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
1075 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
1077 DISPLAYLEVEL(3, "OK \n");
1079 ZSTD_freeCCtx(cctx);
1082 /* CDict scenario */
1083 DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
1084 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
1085 size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
1086 DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError);
1087 if (ZSTD_isError(initError)) goto _output_error;
1088 outBuff.dst = compressedBuffer;
1089 outBuff.size = compressedBufferSize;
1091 inBuff.src = CNBuffer;
1092 inBuff.size = CNBufferSize;
1094 DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
1095 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1096 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1097 { size_t const r = ZSTD_endStream(zc, &outBuff);
1098 DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r);
1099 if (r != 0) goto _output_error; /* error, or some data not flushed */
1101 cSize = outBuff.pos;
1102 ZSTD_freeCDict(cdict);
1103 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1106 DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
1107 { size_t const s = ZSTD_sizeof_CStream(zc);
1108 if (ZSTD_isError(s)) goto _output_error;
1109 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
1112 DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
1113 { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1114 if (dID != dictID) goto _output_error;
1115 DISPLAYLEVEL(4, "OK (%u) \n", dID);
1118 /* DDict scenario */
1119 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize);
1120 { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1121 size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
1122 if (ZSTD_isError(initError)) goto _output_error;
1123 outBuff.dst = decodedBuffer;
1124 outBuff.size = CNBufferSize;
1126 inBuff.src = compressedBuffer;
1127 inBuff.size = cSize;
1129 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1130 if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
1131 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
1132 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
1133 ZSTD_freeDDict(ddict);
1134 DISPLAYLEVEL(3, "OK \n");
1137 /* Memory restriction */
1138 DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
1139 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
1140 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */
1141 outBuff.dst = decodedBuffer;
1142 outBuff.size = CNBufferSize;
1144 inBuff.src = compressedBuffer;
1145 inBuff.size = cSize;
1147 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1148 if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
1149 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
1150 ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */
1152 DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
1153 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1154 int const maxLevel = 16; /* first level with zstd_opt */
1156 assert(maxLevel < ZSTD_maxCLevel());
1157 CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) );
1158 for (level = 1; level <= maxLevel; ++level) {
1159 ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level);
1160 size_t const maxSize = MIN(1 MB, CNBufferSize);
1162 for (size = 512; size <= maxSize; size <<= 1) {
1163 U64 const crcOrig = XXH64(CNBuffer, size, 0);
1164 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1165 ZSTD_parameters savedParams;
1166 getCCtxParams(cctx, &savedParams);
1167 outBuff.dst = compressedBuffer;
1168 outBuff.size = compressedBufferSize;
1170 inBuff.src = CNBuffer;
1173 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
1174 CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
1175 CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
1176 if (inBuff.pos != inBuff.size) goto _output_error;
1177 { ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
1178 ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
1179 CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
1180 if (decIn.pos != decIn.size) goto _output_error;
1181 if (decOut.pos != size) goto _output_error;
1182 { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
1183 if (crcDec != crcOrig) goto _output_error;
1185 ZSTD_freeCCtx(cctx);
1187 ZSTD_freeCDict(cdict);
1189 ZSTD_freeDCtx(dctx);
1191 DISPLAYLEVEL(3, "OK\n");
1193 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1194 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) );
1195 cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB));
1197 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++);
1199 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1200 /* We should fail to decompress without a dictionary. */
1201 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1202 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1203 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1204 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1205 if (!ZSTD_isError(ret)) goto _output_error;
1207 /* We should succeed to decompress with the dictionary. */
1208 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1209 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1210 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1211 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1212 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1213 if (in.pos != in.size) goto _output_error;
1215 /* The dictionary should persist across calls. */
1216 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1217 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1218 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1219 if (in.pos != in.size) goto _output_error;
1221 /* The dictionary should not be cleared by ZSTD_reset_session_only. */
1222 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
1223 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1224 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1225 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1226 if (in.pos != in.size) goto _output_error;
1228 /* When we reset the context the dictionary is cleared. */
1229 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1230 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1231 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1232 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1233 if (!ZSTD_isError(ret)) goto _output_error;
1235 ZSTD_freeDCtx(dctx);
1237 DISPLAYLEVEL(3, "OK \n");
1239 DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
1241 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1242 /* We should succeed to decompress with the dictionary. */
1243 ZSTD_resetDStream(dctx);
1244 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1245 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1246 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1247 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1248 if (in.pos != in.size) goto _output_error;
1250 /* The dictionary should not be cleared by ZSTD_resetDStream(). */
1251 ZSTD_resetDStream(dctx);
1252 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1253 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1254 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1255 if (in.pos != in.size) goto _output_error;
1257 /* The dictionary should be cleared by ZSTD_initDStream(). */
1258 CHECK_Z( ZSTD_initDStream(dctx) );
1259 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1260 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1261 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1262 if (!ZSTD_isError(ret)) goto _output_error;
1264 ZSTD_freeDCtx(dctx);
1266 DISPLAYLEVEL(3, "OK \n");
1268 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++);
1270 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1271 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1272 /* We should succeed to decompress with the ddict. */
1273 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1274 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
1275 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1276 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1277 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1278 if (in.pos != in.size) goto _output_error;
1280 /* The ddict should persist across calls. */
1281 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1282 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1283 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1284 if (in.pos != in.size) goto _output_error;
1286 /* When we reset the context the ddict is cleared. */
1287 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1288 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1289 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1290 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1291 if (!ZSTD_isError(ret)) goto _output_error;
1293 ZSTD_freeDCtx(dctx);
1294 ZSTD_freeDDict(ddict);
1296 DISPLAYLEVEL(3, "OK \n");
1298 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
1300 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1301 /* We should succeed to decompress with the prefix. */
1302 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1303 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) );
1304 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1305 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1306 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1307 if (in.pos != in.size) goto _output_error;
1309 /* The prefix should be cleared after the first compression. */
1310 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1311 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1312 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1313 if (!ZSTD_isError(ret)) goto _output_error;
1315 ZSTD_freeDCtx(dctx);
1317 DISPLAYLEVEL(3, "OK \n");
1319 DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++);
1321 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1322 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1324 /* We should succeed to decompress with the dictionary. */
1325 CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) );
1326 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1327 /* The dictionary should persist across calls. */
1328 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1329 /* We should succeed to decompress with the ddict. */
1330 CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) );
1331 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1332 /* The ddict should persist across calls. */
1333 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1334 /* When we reset the context the ddict is cleared. */
1335 CHECK_Z( ZSTD_initDStream(dctx) );
1336 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize);
1337 if (!ZSTD_isError(ret)) goto _output_error;
1338 ZSTD_freeDCtx(dctx);
1339 ZSTD_freeDDict(ddict);
1341 DISPLAYLEVEL(3, "OK \n");
1343 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
1344 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
1345 ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
1346 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
1347 size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
1348 if (ZSTD_isError(initError)) goto _output_error;
1349 outBuff.dst = compressedBuffer;
1350 outBuff.size = compressedBufferSize;
1352 inBuff.src = CNBuffer;
1353 inBuff.size = CNBufferSize;
1355 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1356 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1357 { size_t const r = ZSTD_endStream(zc, &outBuff);
1358 if (r != 0) goto _output_error; } /* error, or some data not flushed */
1359 cSize = outBuff.pos;
1360 ZSTD_freeCDict(cdict);
1361 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1364 DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
1365 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1366 if (did != 0) goto _output_error;
1368 DISPLAYLEVEL(3, "OK (not detected) \n");
1370 DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
1371 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1372 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1373 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1376 DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
1377 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) );
1378 outBuff.dst = compressedBuffer;
1379 outBuff.size = compressedBufferSize;
1381 inBuff.src = CNBuffer;
1382 inBuff.size = CNBufferSize;
1384 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1385 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1386 cSize = outBuff.pos;
1387 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1389 DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++);
1390 CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) );
1391 outBuff.dst = decodedBuffer;
1392 outBuff.size = CNBufferSize;
1394 inBuff.src = compressedBuffer;
1395 inBuff.size = cSize;
1397 CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
1398 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1399 if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
1400 DISPLAYLEVEL(3, "OK \n");
1402 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
1403 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1404 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1405 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1408 DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
1409 outBuff.dst = compressedBuffer;
1410 outBuff.size = compressedBufferSize;
1412 inBuff.src = CNBuffer;
1413 inBuff.size = CNBufferSize;
1415 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1416 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1417 cSize = outBuff.pos;
1418 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1420 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
1421 CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) );
1422 DISPLAYLEVEL(3, "OK \n");
1425 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
1426 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1427 params.fParams.contentSizeFlag = 1;
1428 CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
1429 } /* cstream advanced shall write content size = 0 */
1430 outBuff.dst = compressedBuffer;
1431 outBuff.size = compressedBufferSize;
1433 inBuff.src = CNBuffer;
1436 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1437 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1438 cSize = outBuff.pos;
1439 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1440 DISPLAYLEVEL(3, "OK \n");
1442 DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++);
1443 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1444 params.fParams.contentSizeFlag = 1;
1445 CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
1446 } /* cstream advanced shall write content size = 0 */
1447 inBuff.src = CNBuffer;
1450 outBuff.dst = compressedBuffer;
1451 outBuff.size = compressedBufferSize;
1453 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1454 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1455 cSize = outBuff.pos;
1456 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1458 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1459 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1460 outBuff.dst = compressedBuffer;
1461 outBuff.size = compressedBufferSize;
1463 inBuff.src = CNBuffer;
1466 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1467 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1468 cSize = outBuff.pos;
1469 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
1470 DISPLAYLEVEL(3, "OK \n");
1472 /* Basic multithreading compression test */
1473 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
1475 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1476 CHECK(jobSize != 0, "job size non-zero");
1477 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1478 CHECK(jobSize != 0, "job size non-zero");
1480 outBuff.dst = compressedBuffer;
1481 outBuff.size = compressedBufferSize;
1483 inBuff.src = CNBuffer;
1484 inBuff.size = CNBufferSize;
1486 { size_t const compressResult = ZSTD_compressStream2(mtctx, &outBuff, &inBuff, ZSTD_e_end);
1487 if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */
1489 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1490 { size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos);
1491 if (compressedSize != outBuff.pos) goto _output_error; /* must be a full valid frame */
1493 DISPLAYLEVEL(3, "OK \n");
1495 /* Complex multithreading + dictionary test */
1496 { U32 const nbWorkers = 2;
1497 size_t const jobSize = 4 * 1 MB;
1498 size_t const srcSize = jobSize * nbWorkers; /* we want each job to have predictable size */
1499 size_t const segLength = 2 KB;
1500 size_t const offset = 600 KB; /* must be larger than window defined in cdict */
1501 size_t const start = jobSize + (offset-1);
1502 const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
1503 BYTE* const dst = (BYTE*)CNBuffer + start - offset;
1504 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize);
1505 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
1506 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
1507 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
1508 assert(start > offset);
1509 assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
1510 memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
1511 outBuff.dst = compressedBuffer;
1512 outBuff.size = compressedBufferSize;
1514 inBuff.src = CNBuffer;
1515 inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
1518 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
1519 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1520 DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
1521 CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
1522 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1523 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
1524 ZSTD_freeCDict(cdict);
1526 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1527 cSize = outBuff.pos;
1528 DISPLAYLEVEL(3, "OK \n");
1530 DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
1531 { ZSTD_DStream* const dstream = ZSTD_createDCtx();
1532 ZSTD_frameHeader zfh;
1533 ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
1534 DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize);
1535 outBuff.dst = decodedBuffer;
1536 outBuff.size = CNBufferSize;
1538 inBuff.src = compressedBuffer;
1540 CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
1541 inBuff.size = 1; /* avoid shortcut to single-pass mode */
1542 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1543 inBuff.size = cSize;
1544 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1545 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1546 ZSTD_freeDStream(dstream);
1548 DISPLAYLEVEL(3, "OK \n");
1550 DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
1551 { unsigned const kMaxWindowLog = 24;
1553 ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
1556 SEQ_stream seq = SEQ_initStream(0x87654321);
1560 XXH64_reset(&xxh, 0);
1561 cParams.windowLog = kMaxWindowLog;
1562 cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1563 ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1565 if (!cdict || !ddict) goto _output_error;
1567 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1568 ZSTD_resetDStream(zd);
1569 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1570 CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
1571 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
1572 /* Test all values < 300 */
1573 for (value = 0; value < 300; ++value) {
1574 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1575 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1578 /* Test values 2^8 to 2^17 */
1579 for (value = (1 << 8); value < (1 << 17); value <<= 1) {
1580 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1581 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1582 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
1585 /* Test offset values up to the max window log */
1586 for (value = 8; value <= kMaxWindowLog; ++value) {
1587 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
1590 CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
1591 CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
1593 ZSTD_freeCDict(cdict);
1594 ZSTD_freeDDict(ddict);
1596 DISPLAYLEVEL(3, "OK \n");
1598 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
1600 CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
1601 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1602 CHECK(level != 11, "Compression level does not match");
1603 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1604 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1605 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1606 CHECK(level != 11, "Compression level does not match");
1608 DISPLAYLEVEL(3, "OK \n");
1610 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++);
1611 { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0);
1612 CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN));
1613 CHECK(badParameters(zc, params), "Compression parameters do not match");
1614 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1615 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1616 CHECK(badParameters(zc, params), "Compression parameters do not match");
1618 DISPLAYLEVEL(3, "OK \n");
1620 DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++);
1621 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1622 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX));
1624 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint));
1625 CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match");
1627 CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
1628 DISPLAYLEVEL(3, "OK \n");
1630 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy compress with hashLog = 29 and searchLog = 4 : ", testNb++);
1632 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
1633 ZSTD_inBuffer in = { CNBuffer, CNBufferSize, 0 };
1634 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1635 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_strategy, ZSTD_lazy));
1636 /* Force enable the row based match finder */
1637 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
1638 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_searchLog, 4));
1639 /* Set windowLog to 29 so the hashLog doesn't get sized down */
1640 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, 29));
1641 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_hashLog, 29));
1642 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1643 /* Compress with continue first so the hashLog doesn't get sized down */
1644 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_continue));
1645 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_end));
1647 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
1649 DISPLAYLEVEL(3, "OK \n");
1651 DISPLAYLEVEL(3, "test%3i : Test offset == windowSize : ", testNb++);
1654 int const kMaxWindowLog = bigTests ? 29 : 26;
1655 size_t const kNbSequences = 10000;
1656 size_t const kMaxSrcSize = (1u << kMaxWindowLog) + 10 * kNbSequences;
1657 char* src = calloc(kMaxSrcSize, 1);
1658 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
1659 for (windowLog = ZSTD_WINDOWLOG_MIN; windowLog <= kMaxWindowLog; ++windowLog) {
1660 size_t const srcSize = ((size_t)1 << windowLog) + 10 * (kNbSequences - 1);
1662 sequences[0].offset = 32;
1663 sequences[0].litLength = 32;
1664 sequences[0].matchLength = (1u << windowLog) - 32;
1665 sequences[0].rep = 0;
1668 for (i = 1; i < kNbSequences; ++i) {
1669 sequences[i].offset = (1u << windowLog) - (FUZ_rand(&seed) % 8);
1670 sequences[i].litLength = FUZ_rand(&seed) & 7;
1671 sequences[i].matchLength = 10 - sequences[i].litLength;
1672 sequences[i].rep = 0;
1676 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1677 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1678 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_minMatch, 3));
1679 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, 1));
1680 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, windowLog));
1681 assert(srcSize <= kMaxSrcSize);
1682 cSize = ZSTD_compressSequences(zc, compressedBuffer, compressedBufferSize, sequences, kNbSequences, src, srcSize);
1684 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
1685 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, windowLog))
1687 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1688 size_t decompressedBytes = 0;
1690 ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1691 size_t const ret = ZSTD_decompressStream(zd, &out, &in);
1693 CHECK(decompressedBytes + out.pos > srcSize, "Output too large");
1694 CHECK(memcmp(out.dst, src + decompressedBytes, out.pos), "Corrupted");
1695 decompressedBytes += out.pos;
1700 CHECK(decompressedBytes != srcSize, "Output wrong size");
1706 DISPLAYLEVEL(3, "OK \n");
1708 /* Overlen overwriting window data bug */
1709 DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
1710 { /* This test has a window size of 1024 bytes and consists of 3 blocks:
1711 1. 'a' repeated 517 times
1712 2. 'b' repeated 516 times
1713 3. a compressed block with no literals and 3 sequence commands:
1714 litlength = 0, offset = 24, match length = 24
1715 litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
1716 litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
1718 const char* testCase =
1719 "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
1720 "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
1721 "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
1722 ZSTD_DStream* const zds = ZSTD_createDStream();
1723 if (zds==NULL) goto _output_error;
1725 CHECK_Z( ZSTD_initDStream(zds) );
1726 inBuff.src = testCase;
1729 outBuff.dst = decodedBuffer;
1730 outBuff.size = CNBufferSize;
1733 while (inBuff.pos < inBuff.size) {
1734 CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) );
1737 ZSTD_freeDStream(zds);
1739 DISPLAYLEVEL(3, "OK \n");
1741 /* Small Sequence Section bug */
1742 DISPLAYLEVEL(3, "test%3i : decompress blocks with small sequences section : ", testNb++);
1743 { /* This test consists of 3 blocks. Each block has one sequence.
1744 The sequence has literal length of 10, match length of 10 and offset of 10.
1745 The sequence value and compression mode for the blocks are following:
1746 The order of values are ll, ml, of.
1747 - First block : (10, 7, 13) (rle, rle, rle)
1748 - size of sequences section: 6 bytes (1 byte for nbSeq, 1 byte for encoding mode, 3 bytes for rle, 1 byte bitstream)
1749 - Second block : (10, 7, 1) (repeat, repeat, rle)
1750 - size of sequences section: 4 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 bytes for rle, 1 byte bitstream)
1751 - Third block : (10, 7, 1) (repeat, repeat, repeat)
1752 - size of sequences section: 3 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 byte bitstream) */
1754 unsigned char compressed[] = {
1755 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x3c, 0x35, 0x01, 0x00, 0xf0, 0x85, 0x08,
1756 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1757 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac,
1758 0x69, 0x94, 0x89, 0x1c, 0x03, 0x44, 0x0a, 0x07, 0x00, 0xb4, 0x04, 0x80,
1761 unsigned int compressedSize = 51;
1762 unsigned char decompressed[] = {
1763 0x85, 0x08, 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x85, 0x08,
1764 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1765 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0x4c, 0x6b, 0xa9, 0x8b, 0xbc, 0xc5,
1766 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94,
1767 0x89, 0x1c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, 0x89, 0x1c
1769 unsigned int decompressedSize = 60;
1771 ZSTD_DStream* const zds = ZSTD_createDStream();
1772 if (zds==NULL) goto _output_error;
1774 CHECK_Z( ZSTD_initDStream(zds) );
1775 inBuff.src = compressed;
1776 inBuff.size = compressedSize;
1778 outBuff.dst = decodedBuffer;
1779 outBuff.size = CNBufferSize;
1782 CHECK(ZSTD_decompressStream(zds, &outBuff, &inBuff) != 0,
1783 "Decompress did not reach the end of frame");
1784 CHECK(inBuff.pos != inBuff.size, "Decompress did not fully consume input");
1785 CHECK(outBuff.pos != decompressedSize, "Decompressed size does not match");
1786 CHECK(memcmp(outBuff.dst, decompressed, decompressedSize) != 0,
1787 "Decompressed data does not match");
1789 ZSTD_freeDStream(zds);
1791 DISPLAYLEVEL(3, "OK \n");
1793 DISPLAYLEVEL(3, "test%3i : raw block can be streamed: ", testNb++);
1794 { size_t const inputSize = 10000;
1795 size_t const compCapacity = ZSTD_compressBound(inputSize);
1796 BYTE* const input = (BYTE*)malloc(inputSize);
1797 BYTE* const comp = (BYTE*)malloc(compCapacity);
1798 BYTE* const decomp = (BYTE*)malloc(inputSize);
1800 CHECK(input == NULL || comp == NULL || decomp == NULL, "failed to alloc buffers");
1802 RDG_genBuffer(input, inputSize, 0.0, 0.0, seed);
1803 { size_t const compSize = ZSTD_compress(comp, compCapacity, input, inputSize, -(int)inputSize);
1804 ZSTD_inBuffer in = { comp, 0, 0 };
1805 ZSTD_outBuffer out = { decomp, 0, 0 };
1807 CHECK_Z( ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters) );
1808 while (in.size < compSize) {
1809 in.size = MIN(in.size + 100, compSize);
1810 while (in.pos < in.size) {
1811 size_t const outPos = out.pos;
1812 if (out.pos == out.size) {
1813 out.size = MIN(out.size + 10, inputSize);
1815 CHECK_Z( ZSTD_decompressStream(zd, &out, &in) );
1816 CHECK(!(out.pos > outPos), "We are not streaming (no output generated)");
1819 CHECK(in.pos != compSize, "Not all input consumed!");
1820 CHECK(out.pos != inputSize, "Not all output produced!");
1822 CHECK(memcmp(input, decomp, inputSize), "round trip failed!");
1828 DISPLAYLEVEL(3, "OK \n");
1830 DISPLAYLEVEL(3, "test%3i : dictionary + uncompressible block + reusing tables checks offset table validity: ", testNb++);
1831 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1832 dictionary.start, dictionary.filled,
1833 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1834 ZSTD_getCParams(3, 0, dictionary.filled),
1836 const size_t inbufsize = 2 * 128 * 1024; /* 2 blocks */
1837 const size_t outbufsize = ZSTD_compressBound(inbufsize);
1838 size_t inbufpos = 0;
1839 size_t cursegmentlen;
1840 BYTE *inbuf = (BYTE *)malloc(inbufsize);
1841 BYTE *outbuf = (BYTE *)malloc(outbufsize);
1842 BYTE *checkbuf = (BYTE *)malloc(inbufsize);
1845 CHECK(cdict == NULL, "failed to alloc cdict");
1846 CHECK(inbuf == NULL, "failed to alloc input buffer");
1848 /* first block is uncompressible */
1849 cursegmentlen = 128 * 1024;
1850 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0., 0., seed);
1851 inbufpos += cursegmentlen;
1853 /* second block is compressible */
1854 cursegmentlen = 128 * 1024 - 256;
1855 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0.05, 0., seed);
1856 inbufpos += cursegmentlen;
1858 /* and includes a very long backref */
1859 cursegmentlen = 128;
1860 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 256, cursegmentlen);
1861 inbufpos += cursegmentlen;
1863 /* and includes a very long backref */
1864 cursegmentlen = 128;
1865 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 128, cursegmentlen);
1866 inbufpos += cursegmentlen;
1868 ret = ZSTD_compress_usingCDict(zc, outbuf, outbufsize, inbuf, inbufpos, cdict);
1871 ret = ZSTD_decompress_usingDict(zd, checkbuf, inbufsize, outbuf, ret, dictionary.start, dictionary.filled);
1874 CHECK(memcmp(inbuf, checkbuf, inbufpos), "start and finish buffers don't match");
1876 ZSTD_freeCDict(cdict);
1881 DISPLAYLEVEL(3, "OK \n");
1883 DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++);
1884 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1885 dictionary.start, dictionary.filled,
1886 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1887 ZSTD_getCParams(3, 0, dictionary.filled),
1889 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1890 int remainingInput = 256 * 1024;
1893 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1894 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1895 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1896 /* Write a bunch of 6 byte blocks */
1897 while (remainingInput > 0) {
1898 char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
1899 const size_t kSmallBlockSize = sizeof(testBuffer);
1900 ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
1902 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
1903 CHECK(in.pos != in.size, "input not fully consumed");
1904 remainingInput -= kSmallBlockSize;
1906 /* Write several very long offset matches into the dictionary */
1907 for (offset = 1024; offset >= 0; offset -= 128) {
1908 ZSTD_inBuffer in = {(BYTE*)dictionary.start + offset, 128, 0};
1909 ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
1910 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
1911 CHECK(in.pos != in.size, "input not fully consumed");
1913 /* Ensure decompression works */
1914 CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled));
1916 ZSTD_freeCDict(cdict);
1918 DISPLAYLEVEL(3, "OK \n");
1920 DISPLAYLEVEL(3, "test%3i : Block-Level External Sequence Producer API: ", testNb++);
1922 size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
1923 BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
1924 size_t const checkBufSize = CNBufferSize;
1925 BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
1927 EMF_testCase sequenceProducerState;
1929 CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
1931 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1933 /* Reference external matchfinder outside the test loop to
1934 * check that the reference is preserved across compressions */
1935 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1937 for (enableFallback = 0; enableFallback <= 1; enableFallback++) {
1939 size_t const numTestCases = 9;
1941 EMF_testCase const testCases[] = {
1948 EMF_INVALID_MATCHLEN,
1950 EMF_INVALID_LAST_LITS
1953 ZSTD_ErrorCode const errorCodes[] = {
1954 ZSTD_error_no_error,
1955 ZSTD_error_no_error,
1956 ZSTD_error_sequenceProducer_failed,
1957 ZSTD_error_sequenceProducer_failed,
1958 ZSTD_error_sequenceProducer_failed,
1959 ZSTD_error_externalSequences_invalid,
1960 ZSTD_error_externalSequences_invalid,
1961 ZSTD_error_externalSequences_invalid,
1962 ZSTD_error_externalSequences_invalid
1965 for (testCaseId = 0; testCaseId < numTestCases; testCaseId++) {
1968 int const compressionShouldSucceed = (
1969 (errorCodes[testCaseId] == ZSTD_error_no_error) ||
1970 (enableFallback && errorCodes[testCaseId] == ZSTD_error_sequenceProducer_failed)
1973 int const testWithSequenceValidation = (
1974 testCases[testCaseId] == EMF_INVALID_OFFSET
1977 sequenceProducerState = testCases[testCaseId];
1979 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1980 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, testWithSequenceValidation));
1981 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
1982 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
1984 if (compressionShouldSucceed) {
1985 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
1986 CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
1987 CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
1989 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
1991 ZSTD_getErrorCode(res) != errorCodes[testCaseId],
1992 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
1997 /* Test compression with external matchfinder + empty src buffer */
2000 sequenceProducerState = EMF_ZERO_SEQS;
2001 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
2002 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
2003 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
2004 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
2005 CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
2009 /* Test that reset clears the external matchfinder */
2010 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
2011 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
2012 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
2013 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
2015 /* Test that registering mFinder == NULL clears the external matchfinder */
2016 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
2017 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2018 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
2019 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
2020 ZSTD_registerSequenceProducer(zc, NULL, NULL); /* clear the external matchfinder */
2021 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
2023 /* Test that external matchfinder doesn't interact with older APIs */
2024 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
2025 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2026 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
2027 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
2028 CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
2030 /* Test that compression returns the correct error with LDM */
2031 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
2034 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2035 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
2036 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2037 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
2039 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
2040 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
2044 #ifdef ZSTD_MULTITHREAD
2045 /* Test that compression returns the correct error with nbWorkers > 0 */
2046 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
2049 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2050 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
2051 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2052 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
2054 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
2055 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
2063 DISPLAYLEVEL(3, "OK \n");
2066 /* Test maxBlockSize cctx param functionality */
2067 DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
2069 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2071 /* Quick test to make sure maxBlockSize bounds are enforced */
2072 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
2073 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
2075 /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
2077 size_t srcSize = 2 << 10;
2078 void* const src = CNBuffer;
2079 size_t dstSize = ZSTD_compressBound(srcSize);
2080 void* const dst1 = compressedBuffer;
2081 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2082 size_t size1, size2;
2083 void* const checkBuf = malloc(srcSize);
2084 memset(src, 'x', srcSize);
2086 /* maxBlockSize = 1KB */
2087 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2088 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2090 if (ZSTD_isError(size1)) goto _output_error;
2091 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2092 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2094 /* maxBlockSize = 3KB */
2095 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2096 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2098 if (ZSTD_isError(size2)) goto _output_error;
2099 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2100 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2102 assert(size1 - size2 == 4); /* We add another RLE block with header + character */
2103 assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
2105 /* maxBlockSize = 1KB, windowLog = 10 */
2106 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2107 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2108 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2110 if (ZSTD_isError(size1)) goto _output_error;
2111 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2112 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2114 /* maxBlockSize = 3KB, windowLog = 10 */
2115 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2116 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2117 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2119 if (ZSTD_isError(size2)) goto _output_error;
2120 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2121 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2123 assert(size1 == size2);
2124 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2129 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2131 /* Test maxBlockSize = 0 is valid */
2132 { size_t srcSize = 256 << 10;
2133 void* const src = CNBuffer;
2134 size_t dstSize = ZSTD_compressBound(srcSize);
2135 void* const dst1 = compressedBuffer;
2136 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2137 size_t size1, size2;
2138 void* const checkBuf = malloc(srcSize);
2140 /* maxBlockSize = 0 */
2141 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
2142 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2144 if (ZSTD_isError(size1)) goto _output_error;
2145 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2146 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2148 /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
2149 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
2150 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2152 if (ZSTD_isError(size2)) goto _output_error;
2153 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2154 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2156 assert(size1 == size2);
2157 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2160 ZSTD_freeCCtx(cctx);
2162 DISPLAYLEVEL(3, "OK \n");
2164 /* Test Sequence Validation */
2165 DISPLAYLEVEL(3, "test%3i : Testing sequence validation: ", testNb++);
2167 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2169 /* Test minMatch >= 4, matchLength < 4 */
2171 size_t srcSize = 11;
2172 void* const src = CNBuffer;
2173 size_t dstSize = ZSTD_compressBound(srcSize);
2174 void* const dst = compressedBuffer;
2175 size_t const kNbSequences = 4;
2176 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2178 memset(src, 'x', srcSize);
2180 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2181 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2182 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2183 sequences[3] = (ZSTD_Sequence) {0, 1, 0, 0};
2185 /* Test with sequence validation */
2186 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2187 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2188 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2190 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2191 sequences, kNbSequences,
2194 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2195 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2197 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2199 /* Test without sequence validation */
2200 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2201 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2202 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2204 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2205 sequences, kNbSequences,
2208 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2209 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2214 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2217 /* Test with no block delim */
2220 void* const src = CNBuffer;
2221 size_t dstSize = ZSTD_compressBound(srcSize);
2222 void* const dst = compressedBuffer;
2223 size_t const kNbSequences = 1;
2224 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2225 void* const checkBuf = malloc(srcSize);
2227 memset(src, 'x', srcSize);
2229 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2231 /* Test with sequence validation */
2232 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 3));
2233 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters));
2234 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2236 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2237 sequences, kNbSequences,
2240 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2241 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst, cSize));
2242 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2248 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2250 { /* Test case with two additional sequences */
2251 size_t srcSize = 19;
2252 void* const src = CNBuffer;
2253 size_t dstSize = ZSTD_compressBound(srcSize);
2254 void* const dst = compressedBuffer;
2255 size_t const kNbSequences = 7;
2256 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2258 memset(src, 'x', srcSize);
2260 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2261 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2262 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2263 sequences[3] = (ZSTD_Sequence) {1, 0, 3, 0};
2264 sequences[4] = (ZSTD_Sequence) {1, 0, 3, 0};
2265 sequences[5] = (ZSTD_Sequence) {1, 0, 3, 0};
2266 sequences[6] = (ZSTD_Sequence) {0, 0, 0, 0};
2268 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2269 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2270 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2272 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2273 sequences, kNbSequences,
2276 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2277 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2279 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2281 /* Test without sequence validation */
2282 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2283 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2284 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2286 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2287 sequences, kNbSequences,
2290 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2291 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2295 ZSTD_freeCCtx(cctx);
2297 DISPLAYLEVEL(3, "OK \n");
2300 DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
2302 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2303 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2305 /* Test large offset, small window size*/
2307 size_t srcSize = 21;
2308 void* const src = CNBuffer;
2309 size_t dstSize = ZSTD_compressBound(srcSize);
2310 void* const dst = compressedBuffer;
2311 size_t const kNbSequences = 4;
2312 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2313 void* const checkBuf = malloc(srcSize);
2314 const size_t largeDictSize = 1 << 25;
2315 ZSTD_CDict* cdict = NULL;
2316 ZSTD_DDict* ddict = NULL;
2318 /* Generate large dictionary */
2319 void* dictBuffer = calloc(largeDictSize, 1);
2320 ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, largeDictSize);
2321 cParams.minMatch = ZSTD_MINMATCH_MIN;
2322 cParams.hashLog = ZSTD_HASHLOG_MIN;
2323 cParams.chainLog = ZSTD_CHAINLOG_MIN;
2325 cdict = ZSTD_createCDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
2326 ddict = ZSTD_createDDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
2328 ZSTD_CCtx_refCDict(cctx, cdict);
2329 ZSTD_DCtx_refDDict(dctx, ddict);
2331 sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0};
2332 sequences[1] = (ZSTD_Sequence) {1 << 25, 0, 3, 0};
2333 sequences[2] = (ZSTD_Sequence) {1 << 25, 0, 9, 0};
2334 sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0};
2336 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2337 sequences, kNbSequences,
2340 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2343 size_t dSize = ZSTD_decompressDCtx(dctx, checkBuf, srcSize, dst, cSize);
2344 CHECK(ZSTD_isError(dSize), "Should not throw an error");
2345 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2351 ZSTD_freeCDict(cdict);
2352 ZSTD_freeDDict(ddict);
2354 ZSTD_freeCCtx(cctx);
2355 ZSTD_freeDCtx(dctx);
2357 DISPLAYLEVEL(3, "OK \n");
2359 DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx: ", testNb++);
2361 size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
2362 BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
2363 size_t const checkBufSize = CNBufferSize;
2364 BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
2365 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2366 ZSTD_CCtx* staticCCtx;
2368 EMF_testCase seqProdState;
2370 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_validateSequences, 1));
2371 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableSeqProducerFallback, 0));
2372 ZSTD_CCtxParams_registerSequenceProducer(params, &seqProdState, zstreamSequenceProducer);
2375 size_t const cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2376 cctxBuf = malloc(cctxSize);
2377 staticCCtx = ZSTD_initStaticCCtx(cctxBuf, cctxSize);
2378 ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params);
2381 // Check that compression with external sequence producer succeeds when expected
2382 seqProdState = EMF_LOTS_OF_SEQS;
2385 size_t const cResult = ZSTD_compress2(staticCCtx, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2386 CHECK(ZSTD_isError(cResult), "EMF: Compression error: %s", ZSTD_getErrorName(cResult));
2387 dResult = ZSTD_decompress(checkBuf, checkBufSize, dstBuf, cResult);
2388 CHECK(ZSTD_isError(dResult), "EMF: Decompression error: %s", ZSTD_getErrorName(dResult));
2389 CHECK(dResult != CNBufferSize, "EMF: Corruption!");
2390 CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
2393 // Check that compression with external sequence producer fails when expected
2394 seqProdState = EMF_BIG_ERROR;
2396 size_t const cResult = ZSTD_compress2(staticCCtx, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2397 CHECK(!ZSTD_isError(cResult), "EMF: Should have raised an error!");
2399 ZSTD_getErrorCode(cResult) != ZSTD_error_sequenceProducer_failed,
2400 "EMF: Wrong error code: %s", ZSTD_getErrorName(cResult)
2407 ZSTD_freeCCtxParams(params);
2409 DISPLAYLEVEL(3, "OK \n");
2411 DISPLAYLEVEL(3, "test%3i : Decoder should reject invalid frame header on legacy frames: ", testNb++);
2413 const unsigned char compressed[] = { 0x26,0xb5,0x2f,0xfd,0x50,0x91,0xfd,0xd8,0xb5 };
2414 const size_t compressedSize = 9;
2415 size_t const dSize = ZSTD_decompress(NULL, 0, compressed, compressedSize);
2416 CHECK(!ZSTD_isError(dSize), "must reject when legacy frame header is invalid");
2418 DISPLAYLEVEL(3, "OK \n");
2420 DISPLAYLEVEL(3, "test%3i : Test single-shot fallback for magicless mode: ", testNb++);
2423 size_t const srcSize = COMPRESSIBLE_NOISE_LENGTH;
2424 void* src = malloc(srcSize);
2425 size_t const dstSize = ZSTD_compressBound(srcSize);
2426 void* dst = malloc(dstSize);
2427 size_t const valSize = srcSize;
2428 void* val = malloc(valSize);
2429 ZSTD_inBuffer inBuf = { dst, dstSize, 0 };
2430 ZSTD_outBuffer outBuf = { val, valSize, 0 };
2431 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2432 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2433 CHECK(!src || !dst || !val || !dctx || !cctx, "memory allocation failure");
2435 // Write test data for decompression to dst
2436 RDG_genBuffer(src, srcSize, compressibility, 0.0, 0xdeadbeef);
2437 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless));
2438 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
2440 // Run decompression
2441 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless));
2442 CHECK_Z(ZSTD_decompressStream(dctx, &outBuf, &inBuf));
2445 CHECK(outBuf.pos != srcSize, "decompressed size must match");
2446 CHECK(memcmp(src, val, srcSize) != 0, "decompressed data must match");
2449 free(src); free(dst); free(val);
2450 ZSTD_freeCCtx(cctx);
2451 ZSTD_freeDCtx(dctx);
2453 DISPLAYLEVEL(3, "OK \n");
2456 FUZ_freeDictionary(dictionary);
2457 ZSTD_freeCStream(zc);
2458 ZSTD_freeDStream(zd);
2459 ZSTD_freeCCtx(mtctx);
2461 free(compressedBuffer);
2462 free(decodedBuffer);
2467 DISPLAY("Error detected in Unit tests ! \n");
2472 /* ====== Fuzzer tests ====== */
2474 static size_t findDiff(const void* buf1, const void* buf2, size_t max)
2476 const BYTE* b1 = (const BYTE*)buf1;
2477 const BYTE* b2 = (const BYTE*)buf2;
2479 for (u=0; u<max; u++) {
2480 if (b1[u] != b2[u]) break;
2483 DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max);
2486 DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max);
2488 DISPLAY(" %02X %02X %02X ",
2489 b1[u-3], b1[u-2], b1[u-1]);
2490 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2491 b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]);
2493 DISPLAY(" %02X %02X %02X ",
2494 b2[u-3], b2[u-2], b2[u-1]);
2495 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2496 b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]);
2500 static size_t FUZ_rLogLength(U32* seed, U32 logLength)
2502 size_t const lengthMask = ((size_t)1 << logLength) - 1;
2503 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
2506 static size_t FUZ_randomLength(U32* seed, U32 maxLog)
2508 U32 const logLength = FUZ_rand(seed) % maxLog;
2509 return FUZ_rLogLength(seed, logLength);
2512 /* Return value in range minVal <= v <= maxVal */
2513 static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
2515 U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal;
2516 return (U32)((FUZ_rand(seed) % mod) + minVal);
2519 static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests)
2521 U32 const maxSrcLog = bigTests ? 24 : 22;
2522 static const U32 maxSampleLog = 19;
2523 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2524 BYTE* cNoiseBuffer[5];
2525 size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
2526 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2527 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2528 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2529 size_t const dstBufferSize = srcBufferSize;
2530 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2532 unsigned testNb = 0;
2533 U32 coreSeed = seed;
2534 ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
2535 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
2536 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2537 UTIL_time_t const startClock = UTIL_getTime();
2538 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2539 size_t dictSize = 0;
2541 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
2544 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2545 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2546 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2547 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2548 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2549 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2550 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2551 "Not enough memory, fuzzer tests cancelled");
2553 /* Create initial samples */
2554 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2555 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2556 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2557 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2558 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2559 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2560 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2562 /* catch up testNb */
2563 for (testNb=1; testNb < startTest; testNb++)
2564 FUZ_rand(&coreSeed);
2567 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2569 const BYTE* srcBuffer;
2570 size_t totalTestSize, totalGenSize, cSize;
2571 XXH64_state_t xxhState;
2573 U32 resetAllowed = 1;
2577 FUZ_rand(&coreSeed);
2578 lseed = coreSeed ^ prime32;
2579 if (nbTests >= testNb) {
2580 DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
2582 DISPLAYUPDATE(2, "\r%6u ", testNb);
2585 /* states full reset (deliberately not synchronized) */
2586 /* some issues can only happen when reusing states */
2587 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2588 ZSTD_freeCStream(zc);
2589 zc = ZSTD_createCStream();
2590 CHECK(zc==NULL, "ZSTD_createCStream : allocation error");
2593 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2594 ZSTD_freeDStream(zd);
2595 zd = ZSTD_createDStream();
2596 CHECK(zd==NULL, "ZSTD_createDStream : allocation error");
2597 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2600 /* srcBuffer selection [0-4] */
2601 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2602 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2606 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2607 buffNb = tnb[buffNb >> 3];
2609 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2610 buffNb = tnb[buffNb >> 3];
2612 srcBuffer = cNoiseBuffer[buffNb];
2615 /* compression init */
2616 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2617 && oldTestLog /* at least one test happened */ && resetAllowed) {
2618 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2619 maxTestSize = MIN(maxTestSize, srcBufferSize-16);
2620 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2621 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2622 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2625 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2626 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2627 U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
2629 (MAX(testLog, dictLog) / 3)))
2631 U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
2632 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2633 oldTestLog = testLog;
2634 /* random dictionary selection */
2635 dictSize = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2636 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2637 dict = srcBuffer + dictStart;
2639 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2640 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2641 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) );
2642 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) );
2643 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) );
2644 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) );
2645 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2646 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2649 /* multi-segments compression test */
2650 XXH64_reset(&xxhState, 0);
2651 { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2654 while(totalTestSize < maxTestSize) {
2655 /* compress random chunks into randomly sized dst buffers */
2656 { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2657 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2658 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2659 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2660 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
2661 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2662 outBuff.size = outBuff.pos + dstBuffSize;
2664 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
2666 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2667 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2668 totalTestSize += inBuff.pos;
2671 /* random flush operation, to mess around */
2672 if ((FUZ_rand(&lseed) & 15) == 0) {
2673 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2674 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2675 outBuff.size = outBuff.pos + adjustedDstSize;
2676 CHECK_Z( ZSTD_flushStream(zc, &outBuff) );
2679 /* final frame epilogue */
2680 { size_t remainingToFlush = (size_t)(-1);
2681 while (remainingToFlush) {
2682 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2683 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2684 outBuff.size = outBuff.pos + adjustedDstSize;
2685 remainingToFlush = ZSTD_endStream(zc, &outBuff);
2686 CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
2688 crcOrig = XXH64_digest(&xxhState);
2689 cSize = outBuff.pos;
2692 /* multi - fragments decompression test */
2693 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2694 CHECK_Z ( ZSTD_resetDStream(zd) );
2696 CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2698 { size_t decompressionResult = 1;
2699 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2700 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2701 for (totalGenSize = 0 ; decompressionResult ; ) {
2702 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2703 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2704 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2705 inBuff.size = inBuff.pos + readCSrcSize;
2706 outBuff.size = outBuff.pos + dstBuffSize;
2707 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2708 if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) {
2709 DISPLAY("checksum error : \n");
2710 findDiff(copyBuffer, dstBuffer, totalTestSize);
2712 CHECK( ZSTD_isError(decompressionResult), "decompression error : %s",
2713 ZSTD_getErrorName(decompressionResult) );
2715 CHECK (decompressionResult != 0, "frame not fully decoded");
2716 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)",
2717 (unsigned)outBuff.pos, (unsigned)totalTestSize);
2718 CHECK (inBuff.pos != cSize, "compressed data should be fully read")
2719 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
2720 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
2721 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
2724 /*===== noisy/erroneous src decompression test =====*/
2726 /* add some noise */
2727 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
2728 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
2729 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
2730 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
2731 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
2732 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
2733 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
2736 /* try decompression on noisy data */
2737 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
2738 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2739 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2740 while (outBuff.pos < dstBufferSize) {
2741 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2742 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2743 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
2744 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
2745 outBuff.size = outBuff.pos + adjustedDstSize;
2746 inBuff.size = inBuff.pos + adjustedCSrcSize;
2747 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2748 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
2749 /* No forward progress possible */
2750 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
2752 DISPLAY("\r%u fuzzer tests completed \n", testNb);
2755 ZSTD_freeCStream(zc);
2756 ZSTD_freeDStream(zd);
2757 ZSTD_freeDStream(zd_noise);
2758 free(cNoiseBuffer[0]);
2759 free(cNoiseBuffer[1]);
2760 free(cNoiseBuffer[2]);
2761 free(cNoiseBuffer[3]);
2762 free(cNoiseBuffer[4]);
2773 /** If useOpaqueAPI, sets param in cctxParams.
2774 * Otherwise, sets the param in zc. */
2775 static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
2776 ZSTD_cParameter param, unsigned value,
2780 return ZSTD_CCtxParams_setParameter(cctxParams, param, value);
2782 return ZSTD_CCtx_setParameter(zc, param, value);
2786 /* Tests for ZSTD_compress_generic() API */
2787 static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
2788 double compressibility, int bigTests)
2790 U32 const maxSrcLog = bigTests ? 24 : 22;
2791 static const U32 maxSampleLog = 19;
2792 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2793 BYTE* cNoiseBuffer[5];
2794 size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
2795 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2796 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2797 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2798 size_t const dstBufferSize = srcBufferSize;
2799 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2802 U32 coreSeed = seed;
2803 ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
2804 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
2805 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2806 UTIL_time_t const startClock = UTIL_getTime();
2807 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2808 size_t dictSize = 0;
2810 U32 windowLogMalus = 0; /* can survive between 2 loops */
2811 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
2812 U32 const nbThreadsMax = bigTests ? 4 : 2;
2813 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
2816 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2817 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2818 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2819 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2820 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2821 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2822 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2823 "Not enough memory, fuzzer tests cancelled");
2825 /* Create initial samples */
2826 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2827 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2828 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2829 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2830 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2831 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2832 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2834 /* catch up testNb */
2835 for (testNb=1; testNb < startTest; testNb++)
2836 FUZ_rand(&coreSeed);
2839 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2842 const BYTE* srcBuffer;
2843 size_t totalTestSize, totalGenSize, cSize;
2844 XXH64_state_t xxhState;
2846 U32 resetAllowed = 1;
2848 ZSTD_parameters savedParams;
2849 int isRefPrefix = 0;
2850 U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2853 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
2854 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
2855 FUZ_rand(&coreSeed);
2856 lseed = coreSeed ^ prime32;
2857 DISPLAYLEVEL(5, " *** Test %u *** \n", testNb);
2858 opaqueAPI = FUZ_rand(&lseed) & 1;
2860 /* states full reset (deliberately not synchronized) */
2861 /* some issues can only happen when reusing states */
2862 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2863 DISPLAYLEVEL(5, "Creating new context \n");
2865 zc = ZSTD_createCCtx();
2866 CHECK(zc == NULL, "ZSTD_createCCtx allocation error");
2869 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2870 ZSTD_freeDStream(zd);
2871 zd = ZSTD_createDStream();
2872 CHECK(zd == NULL, "ZSTD_createDStream allocation error");
2873 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2876 /* srcBuffer selection [0-4] */
2877 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2878 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2882 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2883 buffNb = tnb[buffNb >> 3];
2885 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2886 buffNb = tnb[buffNb >> 3];
2888 srcBuffer = cNoiseBuffer[buffNb];
2891 /* compression init */
2892 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
2893 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2894 && oldTestLog /* at least one test happened */
2896 /* just set a compression level */
2897 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2898 if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
2899 { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
2900 DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
2901 CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
2904 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2905 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2906 U32 const cLevelCandidate = (FUZ_rand(&lseed) %
2908 (MAX(testLog, dictLog) / 2))) +
2910 int const cLevel = MIN(cLevelCandidate, cLevelMax);
2911 DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel);
2912 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2913 DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize);
2914 oldTestLog = testLog;
2915 /* random dictionary selection */
2916 dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2917 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2918 dict = srcBuffer + dictStart;
2919 if (!dictSize) dict=NULL;
2921 pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2922 { ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
2923 const U32 windowLogMax = bigTests ? 24 : 20;
2924 const U32 searchLogMax = bigTests ? 15 : 13;
2926 DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
2928 /* mess with compression parameters */
2929 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
2930 cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
2931 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
2932 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
2933 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
2934 cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
2935 cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
2936 cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
2937 cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
2939 if (FUZ_rand(&lseed) & 1) {
2940 DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
2941 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
2942 assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
2943 windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
2945 if (FUZ_rand(&lseed) & 1) {
2946 DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
2947 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
2949 if (FUZ_rand(&lseed) & 1) {
2950 DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
2951 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
2953 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
2954 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
2955 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
2957 /* mess with long distance matching parameters */
2959 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) );
2960 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
2961 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
2962 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
2963 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
2964 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_srcSizeHint, FUZ_randomClampedLength(&lseed, ZSTD_SRCSIZEHINT_MIN, ZSTD_SRCSIZEHINT_MAX), opaqueAPI) );
2967 /* mess with frame parameters */
2968 if (FUZ_rand(&lseed) & 1) {
2969 int const checksumFlag = FUZ_rand(&lseed) & 1;
2970 DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
2971 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
2973 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2974 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2975 if (FUZ_rand(&lseed) & 1) {
2976 DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize);
2977 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2979 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2982 /* multi-threading parameters. Only adjust occasionally for small tests. */
2983 if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
2984 U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
2985 U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
2986 int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
2987 DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads);
2988 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
2989 if (nbThreads > 1) {
2990 U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
2991 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
2992 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
2995 /* Enable rsyncable mode 1 in 4 times. */
2997 int const rsyncable = (FUZ_rand(&lseed) % 4 == 0);
2998 DISPLAYLEVEL(5, "t%u: rsyncable : %d \n", testNb, rsyncable);
2999 setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, rsyncable, opaqueAPI);
3002 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
3003 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) );
3005 /* Set max block size parameters */
3006 if (FUZ_rand(&lseed) & 1) {
3007 int maxBlockSize = (int)(FUZ_rand(&lseed) % ZSTD_BLOCKSIZE_MAX);
3008 maxBlockSize = MAX(1024, maxBlockSize);
3009 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_maxBlockSize, maxBlockSize, opaqueAPI) );
3012 /* Apply parameters */
3014 DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb);
3015 CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
3018 if (FUZ_rand(&lseed) & 1) {
3019 if (FUZ_rand(&lseed) & 1) {
3020 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
3022 CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
3026 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
3030 CHECK_Z(getCCtxParams(zc, &savedParams));
3032 /* multi-segments compression test */
3034 int const startSeed = lseed;
3035 XXH64_hash_t compressedCrcs[2];
3036 for (iter = 0; iter < 2; ++iter, lseed = startSeed) {
3037 ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
3038 int const singlePass = (FUZ_rand(&lseed) & 3) == 0;
3041 XXH64_reset(&xxhState, 0);
3043 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
3045 DISPLAYLEVEL(6, "t%u: Reloading prefix\n", testNb);
3046 /* Need to reload the prefix because it gets dropped after one compression */
3047 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
3050 /* Adjust number of workers occasionally - result must be deterministic independent of nbWorkers */
3051 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers));
3052 if (nbWorkers > 0 && (FUZ_rand(&lseed) & 7) == 0) {
3053 DISPLAYLEVEL(6, "t%u: Modify nbWorkers: %d -> %d \n", testNb, nbWorkers, nbWorkers + iter);
3054 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers + iter));
3058 ZSTD_inBuffer inBuff = { srcBuffer, maxTestSize, 0 };
3059 CHECK_Z(ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end));
3060 DISPLAYLEVEL(6, "t%u: Single pass compression: consumed %u bytes ; produced %u bytes \n",
3061 testNb, (unsigned)inBuff.pos, (unsigned)outBuff.pos);
3062 CHECK(inBuff.pos != inBuff.size, "Input not consumed!");
3063 crcOrig = XXH64(srcBuffer, maxTestSize, 0);
3064 totalTestSize = maxTestSize;
3067 for (totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
3068 /* compress random chunks into randomly sized dst buffers */
3069 size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3070 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
3071 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
3072 ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
3073 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
3074 int forwardProgress;
3076 size_t const ipos = inBuff.pos;
3077 size_t const opos = outBuff.pos;
3079 if (outBuff.pos == outBuff.size) {
3080 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
3081 size_t const dstBuffSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
3082 outBuff.size = outBuff.pos + dstBuffSize;
3084 CHECK_Z( ret = ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
3085 DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
3086 testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos);
3088 /* We've completed the flush */
3089 if (flush == ZSTD_e_flush && ret == 0)
3092 /* Ensure maximal forward progress for determinism */
3093 forwardProgress = (inBuff.pos != ipos) || (outBuff.pos != opos);
3094 } while (forwardProgress);
3095 assert(inBuff.pos == inBuff.size);
3097 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
3098 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
3099 totalTestSize += inBuff.pos;
3102 /* final frame epilogue */
3103 { size_t remainingToFlush = 1;
3104 while (remainingToFlush) {
3105 ZSTD_inBuffer inBuff = { NULL, 0, 0 };
3106 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
3107 size_t const adjustedDstSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
3108 outBuff.size = outBuff.pos + adjustedDstSize;
3109 DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize);
3110 /* ZSTD_e_end guarantees maximal forward progress */
3111 remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
3112 DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos);
3113 CHECK( ZSTD_isError(remainingToFlush),
3114 "ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
3115 ZSTD_getErrorName(remainingToFlush) );
3117 crcOrig = XXH64_digest(&xxhState);
3119 cSize = outBuff.pos;
3120 compressedCrcs[iter] = XXH64(cBuffer, cSize, 0);
3121 DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize);
3123 CHECK(!(compressedCrcs[0] == compressedCrcs[1]), "Compression is not deterministic!");
3126 CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
3128 /* multi - fragments decompression test */
3129 if (FUZ_rand(&lseed) & 1) {
3130 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
3132 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
3133 DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", (void const*)dict);
3134 CHECK_Z( ZSTD_resetDStream(zd) );
3137 DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
3138 CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
3140 if (FUZ_rand(&lseed) & 1) {
3141 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
3143 if (FUZ_rand(&lseed) & 1) {
3145 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_maxBlockSize, &maxBlockSize));
3146 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_maxBlockSize, maxBlockSize));
3148 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_maxBlockSize, 0));
3150 { size_t decompressionResult = 1;
3151 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
3152 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
3153 for (totalGenSize = 0 ; decompressionResult ; ) {
3154 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3155 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
3156 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
3157 inBuff.size = inBuff.pos + readCSrcSize;
3158 outBuff.size = outBuff.pos + dstBuffSize;
3159 DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n",
3160 (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize);
3161 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
3162 DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n",
3163 (unsigned)inBuff.pos, (unsigned)outBuff.pos);
3164 if (ZSTD_isError(decompressionResult)) {
3165 DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult));
3166 findDiff(copyBuffer, dstBuffer, totalTestSize);
3168 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
3169 CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize);
3171 CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize);
3172 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize);
3173 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
3174 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
3175 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
3178 /*===== noisy/erroneous src decompression test =====*/
3180 /* add some noise */
3181 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
3182 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
3183 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
3184 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
3185 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
3186 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
3187 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
3190 /* try decompression on noisy data */
3191 if (FUZ_rand(&lseed) & 1) {
3192 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_and_parameters));
3194 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_only));
3196 if (FUZ_rand(&lseed) & 1) {
3197 CHECK_Z(ZSTD_DCtx_setParameter(zd_noise, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
3199 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
3200 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
3201 while (outBuff.pos < dstBufferSize) {
3202 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3203 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
3204 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
3205 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
3206 outBuff.size = outBuff.pos + adjustedDstSize;
3207 inBuff.size = inBuff.pos + adjustedCSrcSize;
3208 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
3209 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
3210 /* Good so far, but no more progress possible */
3211 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
3213 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
3217 ZSTD_freeDStream(zd);
3218 ZSTD_freeDStream(zd_noise);
3219 ZSTD_freeCCtxParams(cctxParams);
3220 free(cNoiseBuffer[0]);
3221 free(cNoiseBuffer[1]);
3222 free(cNoiseBuffer[2]);
3223 free(cNoiseBuffer[3]);
3224 free(cNoiseBuffer[4]);
3235 /*-*******************************************************
3237 *********************************************************/
3238 static int FUZ_usage(const char* programName)
3240 DISPLAY( "Usage :\n");
3241 DISPLAY( " %s [args]\n", programName);
3243 DISPLAY( "Arguments :\n");
3244 DISPLAY( " -i# : Number of tests (default:%u)\n", nbTestsDefault);
3245 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
3246 DISPLAY( " -s# : Select seed (default:prompt user)\n");
3247 DISPLAY( " -t# : Select starting test number (default:0)\n");
3248 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
3249 DISPLAY( " -v : verbose\n");
3250 DISPLAY( " -p : pause at the end\n");
3251 DISPLAY( " -h : display help and exit\n");
3255 typedef enum { simple_api, advanced_api } e_api;
3257 int main(int argc, const char** argv)
3261 int nbTests = nbTestsDefault;
3263 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
3266 int bigTests = (sizeof(size_t) == 8);
3267 e_api selected_api = simple_api;
3268 const char* const programName = argv[0];
3271 /* Check command line */
3272 for(argNb=1; argNb<argc; argNb++) {
3273 const char* argument = argv[argNb];
3274 assert(argument != NULL);
3276 /* Parsing commands. Aggregated commands are allowed */
3277 if (argument[0]=='-') {
3279 if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
3280 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
3281 if (!strcmp(argument, "--big-tests")) { bigTests=1; continue; }
3284 while (*argument!=0) {
3288 return FUZ_usage(programName);
3300 case 'p': /* pause at the end */
3305 case 'i': /* limit tests by nb of iterations (default) */
3307 nbTests=0; g_clockTime=0;
3308 while ((*argument>='0') && (*argument<='9')) {
3310 nbTests += *argument - '0';
3315 case 'T': /* limit tests by time */
3317 nbTests=0; g_clockTime=0;
3318 while ((*argument>='0') && (*argument<='9')) {
3320 g_clockTime += *argument - '0';
3323 if (*argument=='m') { /* -T1m == -T60 */
3324 g_clockTime *=60, argument++;
3325 if (*argument=='n') argument++; /* -T1mn == -T60 */
3326 } else if (*argument=='s') argument++; /* -T10s == -T10 */
3327 g_clockTime *= SEC_TO_MICRO;
3330 case 's': /* manually select seed */
3334 while ((*argument>='0') && (*argument<='9')) {
3336 seed += *argument - '0';
3341 case 't': /* select starting test number */
3344 while ((*argument>='0') && (*argument<='9')) {
3346 testNb += *argument - '0';
3351 case 'P': /* compressibility % */
3354 while ((*argument>='0') && (*argument<='9')) {
3356 proba += *argument - '0';
3359 if (proba<0) proba=0;
3360 if (proba>100) proba=100;
3364 return FUZ_usage(programName);
3366 } } } /* for(argNb=1; argNb<argc; argNb++) */
3369 DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
3372 time_t const t = time(NULL);
3373 U32 const h = XXH32(&t, sizeof(t), 1);
3377 DISPLAY("Seed = %u\n", (unsigned)seed);
3378 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
3380 if (nbTests<=0) nbTests=1;
3383 result = basicUnitTests(0, ((double)proba) / 100, bigTests); /* constant seed for predictability */
3387 switch(selected_api)
3390 result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3393 result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3396 assert(0); /* impossible */
3402 DISPLAY("Press Enter \n");