notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
git subrepo pull (merge) --force deps/libchdr
[pcsx_rearmed.git] / deps / libchdr / deps / zstd-1.5.5 / tests / zstreamtest.c
CommitLineData
648db22b 1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12/*-************************************
13 * Compiler specific
14 **************************************/
15#ifdef _MSC_VER /* Visual Studio */
16# define _CRT_SECURE_NO_WARNINGS /* fgets */
17# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
19#endif
20
21
22/*-************************************
23 * Includes
24 **************************************/
25#include <stdlib.h> /* free */
26#include <stdio.h> /* fgets, sscanf */
27#include <string.h> /* strcmp */
28#include <time.h> /* time_t, time(), to randomize seed */
29#include <assert.h> /* assert */
30#include "timefn.h" /* UTIL_time_t, UTIL_getTime */
31#include "mem.h"
32#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
33#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
34#include "zstd.h" /* ZSTD_compressBound */
35#include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
36#include "zdict.h" /* ZDICT_trainFromBuffer */
37#include "datagen.h" /* RDG_genBuffer */
38#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
39#include "xxhash.h" /* XXH64_* */
40#include "seqgen.h"
41#include "util.h"
42#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
43#include "external_matchfinder.h" /* zstreamSequenceProducer, EMF_testCase */
44
45/*-************************************
46 * Constants
47 **************************************/
48#define KB *(1U<<10)
49#define MB *(1U<<20)
50#define GB *(1U<<30)
51
52static const int nbTestsDefault = 10000;
53static const U32 g_cLevelMax_smallTests = 10;
54#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
55#define FUZ_COMPRESSIBILITY_DEFAULT 50
56static const U32 prime32 = 2654435761U;
57
58
59/*-************************************
60 * Display Macros
61 **************************************/
62#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
63#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \
64 DISPLAY(__VA_ARGS__); \
65 if (g_displayLevel>=4) fflush(stderr); }
66static U32 g_displayLevel = 2;
67
68static const U64 g_refreshRate = SEC_TO_MICRO / 6;
69static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
70
71#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
72 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
73 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
74 if (g_displayLevel>=4) fflush(stderr); } }
75
76static U64 g_clockTime = 0;
77
78
79/*-*******************************************************
80 * Check macros
81 *********************************************************/
82#undef MIN
83#undef MAX
84#define MIN(a,b) ((a)<(b)?(a):(b))
85#define MAX(a,b) ((a)>(b)?(a):(b))
86/*! FUZ_rand() :
87 @return : a 27 bits random value, from a 32-bits `seed`.
88 `seed` is also modified */
89#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
90static U32 FUZ_rand(U32* seedPtr)
91{
92 static const U32 prime2 = 2246822519U;
93 U32 rand32 = *seedPtr;
94 rand32 *= prime32;
95 rand32 += prime2;
96 rand32 = FUZ_rotl32(rand32, 13);
97 *seedPtr = rand32;
98 return rand32 >> 5;
99}
100
101#define CHECK(cond, ...) { \
102 if (cond) { \
103 DISPLAY("Error => "); \
104 DISPLAY(__VA_ARGS__); \
105 DISPLAY(" (seed %u, test nb %u, line %u) \n", \
106 (unsigned)seed, testNb, __LINE__); \
107 goto _output_error; \
108} }
109
110#define CHECK_Z(f) { \
111 size_t const err = f; \
112 CHECK(ZSTD_isError(err), "%s : %s ", \
113 #f, ZSTD_getErrorName(err)); \
114}
115
116#define CHECK_RET(ret, cond, ...) { \
117 if (cond) { \
118 DISPLAY("Error %llu => ", (unsigned long long)ret); \
119 DISPLAY(__VA_ARGS__); \
120 DISPLAY(" (line %u)\n", __LINE__); \
121 return ret; \
122} }
123
124#define CHECK_RET_Z(f) { \
125 size_t const err = f; \
126 CHECK_RET(err, ZSTD_isError(err), "%s : %s ", \
127 #f, ZSTD_getErrorName(err)); \
128}
129
130
131/*======================================================
132 * Basic Unit tests
133 *======================================================*/
134
135typedef struct {
136 void* start;
137 size_t size;
138 size_t filled;
139} buffer_t;
140
141static const buffer_t kBuffNull = { NULL, 0 , 0 };
142
143static void FUZ_freeDictionary(buffer_t dict)
144{
145 free(dict.start);
146}
147
148static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
149{
150 buffer_t dict = kBuffNull;
151 size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
152 size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
153 if (!blockSizes) return kBuffNull;
154 dict.start = malloc(requestedDictSize);
155 if (!dict.start) { free(blockSizes); return kBuffNull; }
156 { size_t nb;
157 for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
158 blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
159 }
160 { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
161 free(blockSizes);
162 if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; }
163 dict.size = requestedDictSize;
164 dict.filled = dictSize;
165 return dict;
166 }
167}
168
169/* Round trips data and updates xxh with the decompressed data produced */
170static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
171 XXH64_state_t* xxh, void* data, size_t size,
172 ZSTD_EndDirective endOp)
173{
174 static BYTE compressed[1024];
175 static BYTE uncompressed[1024];
176
177 ZSTD_inBuffer cin = {data, size, 0};
178 size_t cret;
179
180 do {
181 ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
182 ZSTD_inBuffer din = { compressed, 0, 0 };
183 ZSTD_outBuffer dout = { uncompressed, 0, 0 };
184
185 cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
186 if (ZSTD_isError(cret))
187 return cret;
188
189 din.size = cout.pos;
190 while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
191 size_t dret;
192
193 dout.pos = 0;
194 dout.size = sizeof(uncompressed);
195 dret = ZSTD_decompressStream(dctx, &dout, &din);
196 if (ZSTD_isError(dret))
197 return dret;
198 XXH64_update(xxh, dout.dst, dout.pos);
199 if (dret == 0)
200 break;
201 }
202 } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
203 return 0;
204}
205
206/* Generates some data and round trips it */
207static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
208 XXH64_state_t* xxh, SEQ_stream* seq,
209 SEQ_gen_type type, unsigned value)
210{
211 static BYTE data[1024];
212 size_t gen;
213
214 do {
215 SEQ_outBuffer sout = {data, sizeof(data), 0};
216 size_t ret;
217 gen = SEQ_gen(seq, type, value, &sout);
218
219 ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
220 if (ZSTD_isError(ret))
221 return ret;
222 } while (gen != 0);
223
224 return 0;
225}
226
227static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
228{
229 int value;
230 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
231 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
232 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
233 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
234 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
235 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
236 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
237 savedParams->cParams.strategy = value;
238
239 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
240 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
241 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
242 savedParams->fParams.noDictIDFlag = !value;
243 return 0;
244}
245
246static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
247{
248 ZSTD_parameters params;
249 if (ZSTD_isError(getCCtxParams(zc, &params))) return 10;
250 CHECK_RET(1, params.cParams.windowLog != savedParams.cParams.windowLog, "windowLog");
251 CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
252 CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
253 CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
254 CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
255 CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
256
257 CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
258 CHECK_RET(8, params.fParams.contentSizeFlag != savedParams.fParams.contentSizeFlag, "contentSizeFlag");
259 CHECK_RET(9, params.fParams.noDictIDFlag != savedParams.fParams.noDictIDFlag, "noDictIDFlag");
260 return 0;
261}
262
263static int basicUnitTests(U32 seed, double compressibility, int bigTests)
264{
265 size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
266 void* CNBuffer = malloc(CNBufferSize);
267 size_t const skippableFrameSize = 200 KB;
268 size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
269 void* compressedBuffer = malloc(compressedBufferSize);
270 size_t const decodedBufferSize = CNBufferSize;
271 void* decodedBuffer = malloc(decodedBufferSize);
272 size_t cSize;
273 int testResult = 0;
274 int testNb = 1;
275 U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */
276 ZSTD_CStream* zc = ZSTD_createCStream();
277 ZSTD_DStream* zd = ZSTD_createDStream();
278 ZSTD_CCtx* mtctx = ZSTD_createCCtx();
279
280 ZSTD_inBuffer inBuff, inBuff2;
281 ZSTD_outBuffer outBuff;
282 buffer_t dictionary = kBuffNull;
283 size_t const dictSize = 128 KB;
284 unsigned dictID = 0;
285
286 /* Create compressible test buffer */
287 if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd || !mtctx) {
288 DISPLAY("Not enough memory, aborting \n");
289 goto _output_error;
290 }
291 RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
292
293 CHECK_Z(ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2));
294
295 /* Create dictionary */
296 DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
297 dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB);
298 if (!dictionary.start) {
299 DISPLAY("Error creating dictionary, aborting \n");
300 goto _output_error;
301 }
302 dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
303
304 /* Basic compression test */
305 DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
306 CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
307 outBuff.dst = (char*)(compressedBuffer);
308 outBuff.size = compressedBufferSize;
309 outBuff.pos = 0;
310 inBuff.src = CNBuffer;
311 inBuff.size = CNBufferSize;
312 inBuff.pos = 0;
313 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
314 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
315 { size_t const r = ZSTD_endStream(zc, &outBuff);
316 if (r != 0) goto _output_error; } /* error, or some data not flushed */
317 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos);
318
319 /* generate skippable frame */
320 MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
321 MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
322 cSize = skippableFrameSize + 8;
323
324 /* Basic compression test using dict */
325 DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
326 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
327 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
328 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) );
329 outBuff.dst = (char*)(compressedBuffer)+cSize;
330 assert(compressedBufferSize > cSize);
331 outBuff.size = compressedBufferSize - cSize;
332 outBuff.pos = 0;
333 inBuff.src = CNBuffer;
334 inBuff.size = CNBufferSize;
335 inBuff.pos = 0;
336 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
337 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
338 { size_t const r = ZSTD_endStream(zc, &outBuff);
339 if (r != 0) goto _output_error; } /* error, or some data not flushed */
340 cSize += outBuff.pos;
341 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
342 (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
343
344 /* context size functions */
345 DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
346 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
347 size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams);
348 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */
349 if (ZSTD_isError(cstreamSize)) goto _output_error;
350 if (ZSTD_isError(cdictSize)) goto _output_error;
351 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize));
352 }
353
354 /* context size functions */
355 DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++);
356 { ZSTD_CCtx_params* const params = ZSTD_createCCtxParams();
357 size_t cstreamSize, cctxSize;
358 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) );
359 cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
360 CHECK_Z(cstreamSize);
361 cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
362 CHECK_Z(cctxSize);
363 if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error;
364 ZSTD_freeCCtxParams(params);
365 DISPLAYLEVEL(3, "OK \n");
366 }
367
368 DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
369 { size_t const s = ZSTD_sizeof_CStream(zc);
370 if (ZSTD_isError(s)) goto _output_error;
371 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
372 }
373
374 /* Attempt bad compression parameters */
375 DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++);
376 { size_t r;
377 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
378 params.cParams.minMatch = 2;
379 r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
380 if (!ZSTD_isError(r)) goto _output_error;
381 DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
382 }
383
384 /* skippable frame test */
385 DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
386 CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) );
387 inBuff.src = compressedBuffer;
388 inBuff.size = cSize;
389 inBuff.pos = 0;
390 outBuff.dst = decodedBuffer;
391 outBuff.size = CNBufferSize;
392 outBuff.pos = 0;
393 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
394 DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r);
395 if (r != 0) goto _output_error;
396 }
397 if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */
398 DISPLAYLEVEL(3, "OK \n");
399
400 /* Basic decompression test */
401 inBuff2 = inBuff;
402 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
403 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
404 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */
405 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
406 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
407 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
408 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
409 DISPLAYLEVEL(3, "OK \n");
410
411 /* Re-use without init */
412 DISPLAYLEVEL(3, "test%3i : decompress again without init (re-use previous settings): ", testNb++);
413 outBuff.pos = 0;
414 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
415 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
416 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
417 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
418 DISPLAYLEVEL(3, "OK \n");
419
420 /* check regenerated data is byte exact */
421 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
422 { size_t i;
423 for (i=0; i<CNBufferSize; i++) {
424 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
425 } }
426 DISPLAYLEVEL(3, "OK \n");
427
428 /* check decompression fails early if first bytes are wrong */
429 DISPLAYLEVEL(3, "test%3i : early decompression error if first bytes are incorrect : ", testNb++);
430 { const char buf[3] = { 0 }; /* too short, not enough to start decoding header */
431 ZSTD_inBuffer inb = { buf, sizeof(buf), 0 };
432 size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inb);
433 if (!ZSTD_isError(remaining)) goto _output_error; /* should have errored out immediately (note: this does not test the exact error code) */
434 }
435 DISPLAYLEVEL(3, "OK \n");
436
437 /* context size functions */
438 DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
439 { ZSTD_frameHeader fhi;
440 const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
441 size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
442 if (gfhError!=0) goto _output_error;
443 DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize);
444 { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
445 /* uses ZSTD_initDStream_usingDict() */
446 + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
447 if (ZSTD_isError(s)) goto _output_error;
448 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
449 } }
450
451 DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
452 { size_t const s = ZSTD_sizeof_DStream(zd);
453 if (ZSTD_isError(s)) goto _output_error;
454 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
455 }
456
457 /* Decompression by small increment */
458 DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
459 { /* skippable frame */
460 size_t r = 1;
461 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
462 inBuff.src = compressedBuffer;
463 outBuff.dst = decodedBuffer;
464 inBuff.pos = 0;
465 outBuff.pos = 0;
466 while (r) { /* skippable frame */
467 size_t const inSize = (FUZ_rand(&coreSeed) & 15) + 1;
468 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + 1;
469 inBuff.size = inBuff.pos + inSize;
470 outBuff.size = outBuff.pos + outSize;
471 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
472 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream on skippable frame error : %s \n", ZSTD_getErrorName(r));
473 if (ZSTD_isError(r)) goto _output_error;
474 }
475 /* normal frame */
476 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
477 r=1;
478 while (r) {
479 size_t const inSize = FUZ_rand(&coreSeed) & 15;
480 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + (!inSize); /* avoid having both sizes at 0 => would trigger a no_forward_progress error */
481 inBuff.size = inBuff.pos + inSize;
482 outBuff.size = outBuff.pos + outSize;
483 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
484 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(r));
485 if (ZSTD_isError(r)) goto _output_error;
486 }
487 }
488 if (outBuff.pos != CNBufferSize) DISPLAYLEVEL(4, "outBuff.pos != CNBufferSize : should have regenerated same amount ! \n");
489 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
490 if (inBuff.pos != cSize) DISPLAYLEVEL(4, "inBuff.pos != cSize : should have real all input ! \n");
491 if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
492 DISPLAYLEVEL(3, "OK \n");
493
494 /* check regenerated data is byte exact */
495 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
496 { size_t i;
497 for (i=0; i<CNBufferSize; i++) {
498 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
499 } }
500 DISPLAYLEVEL(3, "OK \n");
501
502 /* Decompression forward progress */
503 DISPLAYLEVEL(3, "test%3i : generate error when ZSTD_decompressStream() doesn't progress : ", testNb++);
504 { /* skippable frame */
505 size_t r = 0;
506 int decNb = 0;
507 int const maxDec = 100;
508 inBuff.src = compressedBuffer;
509 inBuff.size = cSize;
510 inBuff.pos = 0;
511
512 outBuff.dst = decodedBuffer;
513 outBuff.pos = 0;
514 outBuff.size = CNBufferSize-1; /* 1 byte missing */
515
516 for (decNb=0; decNb<maxDec; decNb++) {
517 if (r==0) ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
518 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
519 if (ZSTD_isError(r)) break;
520 }
521 if (!ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream should have triggered a no_forward_progress error \n");
522 if (!ZSTD_isError(r)) goto _output_error; /* should have triggered no_forward_progress error */
523 }
524 DISPLAYLEVEL(3, "OK \n");
525
526 DISPLAYLEVEL(3, "test%3i : NULL output and NULL input : ", testNb++);
527 inBuff.src = NULL;
528 inBuff.size = 0;
529 inBuff.pos = 0;
530 outBuff.dst = NULL;
531 outBuff.size = 0;
532 outBuff.pos = 0;
533 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
534 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
535 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
536 outBuff.dst = (char*)(compressedBuffer);
537 outBuff.size = compressedBufferSize;
538 outBuff.pos = 0;
539 { size_t const r = ZSTD_endStream(zc, &outBuff);
540 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
541 }
542 inBuff.src = outBuff.dst;
543 inBuff.size = outBuff.pos;
544 inBuff.pos = 0;
545 outBuff.dst = NULL;
546 outBuff.size = 0;
547 outBuff.pos = 0;
548 CHECK_Z( ZSTD_initDStream(zd) );
549 { size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
550 if (ret != 0) goto _output_error;
551 }
552 DISPLAYLEVEL(3, "OK\n");
553
554 DISPLAYLEVEL(3, "test%3i : NULL output buffer with non-NULL input : ", testNb++);
555 {
556 const char* test = "aa";
557 inBuff.src = test;
558 inBuff.size = 2;
559 inBuff.pos = 0;
560 outBuff.dst = NULL;
561 outBuff.size = 0;
562 outBuff.pos = 0;
563 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
564 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
565 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
566 outBuff.dst = (char*)(compressedBuffer);
567 outBuff.size = compressedBufferSize;
568 outBuff.pos = 0;
569 { size_t const r = ZSTD_endStream(zc, &outBuff);
570 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
571 }
572 inBuff.src = outBuff.dst;
573 inBuff.size = outBuff.pos;
574 inBuff.pos = 0;
575 outBuff.dst = NULL;
576 outBuff.size = 0;
577 outBuff.pos = 0;
578 CHECK_Z( ZSTD_initDStream(zd) );
579 CHECK_Z(ZSTD_decompressStream(zd, &outBuff, &inBuff));
580 }
581
582 DISPLAYLEVEL(3, "OK\n");
583 /* _srcSize compression test */
584 DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
585 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
586 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
587 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
588 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) );
589 outBuff.dst = (char*)(compressedBuffer);
590 outBuff.size = compressedBufferSize;
591 outBuff.pos = 0;
592 inBuff.src = CNBuffer;
593 inBuff.size = CNBufferSize;
594 inBuff.pos = 0;
595 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
596 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
597 { size_t const r = ZSTD_endStream(zc, &outBuff);
598 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
599 }
600 { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
601 CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
602 CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
603 }
604 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
605
606 /* wrong _srcSize compression test */
607 DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
608 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
609 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
610 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
611 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) );
612 outBuff.dst = (char*)(compressedBuffer);
613 outBuff.size = compressedBufferSize;
614 outBuff.pos = 0;
615 inBuff.src = CNBuffer;
616 inBuff.size = CNBufferSize;
617 inBuff.pos = 0;
618 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
619 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
620 { size_t const r = ZSTD_endStream(zc, &outBuff);
621 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
622 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
623
624 /* wrong _srcSize compression test */
625 DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
626 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
627 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
628 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
629 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) );
630 outBuff.dst = (char*)(compressedBuffer);
631 outBuff.size = compressedBufferSize;
632 outBuff.pos = 0;
633 inBuff.src = CNBuffer;
634 inBuff.size = CNBufferSize;
635 inBuff.pos = 0;
636 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
637 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
638 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
639 }
640
641 DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
642 { CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
643 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) );
644 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) );
645 outBuff.dst = (char*)compressedBuffer;
646 outBuff.size = compressedBufferSize;
647 outBuff.pos = 0;
648 inBuff.src = CNBuffer;
649 inBuff.size = CNBufferSize;
650 inBuff.pos = 0;
651 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
652 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
653 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
654 } }
655
656 /* Compression state re-use scenario */
657 DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++);
658 ZSTD_freeCStream(zc);
659 zc = ZSTD_createCStream();
660 if (zc==NULL) goto _output_error; /* memory allocation issue */
661 /* use 1 */
662 { size_t const inSize = 513;
663 DISPLAYLEVEL(5, "use1 ");
664 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
665 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
666 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
667 inBuff.src = CNBuffer;
668 inBuff.size = inSize;
669 inBuff.pos = 0;
670 outBuff.dst = (char*)(compressedBuffer)+cSize;
671 outBuff.size = ZSTD_compressBound(inSize);
672 outBuff.pos = 0;
673 DISPLAYLEVEL(5, "compress1 ");
674 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
675 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
676 DISPLAYLEVEL(5, "end1 ");
677 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
678 }
679 /* use 2 */
680 { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
681 DISPLAYLEVEL(5, "use2 ");
682 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
683 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
684 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
685 inBuff.src = CNBuffer;
686 inBuff.size = inSize;
687 inBuff.pos = 0;
688 outBuff.dst = (char*)(compressedBuffer)+cSize;
689 outBuff.size = ZSTD_compressBound(inSize);
690 outBuff.pos = 0;
691 DISPLAYLEVEL(5, "compress2 ");
692 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
693 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
694 DISPLAYLEVEL(5, "end2 ");
695 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
696 }
697 DISPLAYLEVEL(3, "OK \n");
698
699 /* Decompression single pass with empty frame */
700 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, 1);
701 CHECK_Z(cSize);
702 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass on empty frame : ", testNb++);
703 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
704 size_t const dctxSize = ZSTD_sizeof_DCtx(dctx);
705 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
706
707 outBuff.dst = decodedBuffer;
708 outBuff.pos = 0;
709 outBuff.size = CNBufferSize;
710
711 inBuff.src = compressedBuffer;
712 inBuff.size = cSize;
713 inBuff.pos = 0;
714 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
715 CHECK_Z(r);
716 CHECK(r != 0, "Entire frame must be decompressed");
717 CHECK(outBuff.pos != 0, "Wrong size!");
718 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
719 }
720 CHECK(dctxSize != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
721 ZSTD_freeDCtx(dctx);
722 }
723 DISPLAYLEVEL(3, "OK \n");
724
725 /* Decompression with ZSTD_d_stableOutBuffer */
726 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1);
727 CHECK_Z(cSize);
728 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
729 size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
730 size_t dctxSize1;
731 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
732
733 outBuff.dst = decodedBuffer;
734 outBuff.pos = 0;
735 outBuff.size = CNBufferSize;
736
737 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass : ", testNb++);
738 inBuff.src = compressedBuffer;
739 inBuff.size = cSize;
740 inBuff.pos = 0;
741 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
742 CHECK_Z(r);
743 CHECK(r != 0, "Entire frame must be decompressed");
744 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
745 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
746 }
747 CHECK(dctxSize0 != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
748 DISPLAYLEVEL(3, "OK \n");
749
750 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer : ", testNb++);
751 outBuff.pos = 0;
752 inBuff.pos = 0;
753 inBuff.size = 0;
754 while (inBuff.pos < cSize) {
755 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
756 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
757 }
758 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
759 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
760 dctxSize1 = ZSTD_sizeof_DCtx(dctx);
761 CHECK(!(dctxSize0 < dctxSize1), "Input buffer allocated");
762 DISPLAYLEVEL(3, "OK \n");
763
764 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer too small : ", testNb++);
765 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
766 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
767 inBuff.src = compressedBuffer;
768 inBuff.size = cSize;
769 inBuff.pos = 0;
770 outBuff.pos = 0;
771 outBuff.size = CNBufferSize - 1;
772 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
773 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall, "Must error but got %s", ZSTD_getErrorName(r));
774 }
775 DISPLAYLEVEL(3, "OK \n");
776
777 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer modified : ", testNb++);
778 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
779 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
780 inBuff.src = compressedBuffer;
781 inBuff.size = cSize - 1;
782 inBuff.pos = 0;
783 outBuff.pos = 0;
784 outBuff.size = CNBufferSize;
785 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
786 ++inBuff.size;
787 outBuff.pos = 0;
788 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
789 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r));
790 }
791 DISPLAYLEVEL(3, "OK \n");
792
793 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++);
794 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
795 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0));
796 outBuff.pos = 0;
797 inBuff.pos = 0;
798 inBuff.size = 0;
799 while (inBuff.pos < cSize) {
800 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
801 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
802 }
803 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
804 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
805 CHECK(!(dctxSize1 < ZSTD_sizeof_DCtx(dctx)), "Output buffer allocated");
806 DISPLAYLEVEL(3, "OK \n");
807
808 ZSTD_freeDCtx(dctx);
809 }
810
811 /* Compression with ZSTD_c_stable{In,Out}Buffer */
812 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
813 ZSTD_inBuffer in;
814 ZSTD_outBuffer out;
815 size_t cctxSize1;
816 size_t cctxSize2;
817 assert(cctx != NULL);
818 in.src = CNBuffer;
819 in.size = CNBufferSize;
820 out.dst = compressedBuffer;
821 out.size = compressedBufferSize;
822 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
823 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++);
824 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
825 CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
826 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
827 CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
828 /* @cctxSize2 : sizeof_CCtx when doing full streaming (no stable in/out) */
829 { ZSTD_CCtx* const cctx2 = ZSTD_createCCtx();
830 assert(cctx2 != NULL);
831 in.pos = out.pos = 0;
832 CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
833 CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
834 CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
835 ZSTD_freeCCtx(cctx2);
836 }
837 /* @cctxSize1 : sizeof_CCtx when doing single-shot compression (no streaming) */
838 { ZSTD_CCtx* const cctx1 = ZSTD_createCCtx();
839 ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
840 size_t cSize3;
841 assert(cctx1 != NULL);
842 params.fParams.checksumFlag = 1;
843 cSize3 = ZSTD_compress_advanced(cctx1, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
844 CHECK_Z(cSize3);
845 CHECK(!(cSize == cSize3), "Must be same compressed size");
846 CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx1)), "Must be same CCtx size");
847 ZSTD_freeCCtx(cctx1);
848 }
849 CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
850 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
851 DISPLAYLEVEL(3, "OK \n");
852
853 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
854 { int stableInBuffer;
855 int stableOutBuffer;
856 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
857 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
858 CHECK(!(stableInBuffer == 0), "Modified");
859 CHECK(!(stableOutBuffer == 0), "Modified");
860 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
861 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
862 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
863 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
864 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
865 CHECK(!(stableInBuffer == 1), "Modified");
866 CHECK(!(stableOutBuffer == 1), "Modified");
867 }
868 DISPLAYLEVEL(3, "OK \n");
869
870 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer : ", testNb++);
871 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
872 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
873 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
874 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
875 in.pos = out.pos = 0;
876 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
877 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
878 DISPLAYLEVEL(3, "OK \n");
879
880 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer allocated size : ", testNb++);
881 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
882 CHECK(!(cctxSize1 == cctxSize), "Must be the same size as single pass");
883 }
884 DISPLAYLEVEL(3, "OK \n");
885
886 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer only : ", testNb++);
887 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
888 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
889 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
890 in.pos = out.pos = 0;
891 out.size = cSize / 4;
892 for (;;) {
893 size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
894 CHECK_Z(ret);
895 if (ret == 0)
896 break;
897 out.size = MIN(out.size + cSize / 4, compressedBufferSize);
898 }
899 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
900 DISPLAYLEVEL(3, "OK \n");
901
902 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++);
903 in.pos = out.pos = 0;
904 out.size = cSize / 4;
905 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
906 in.src = (char const*)in.src + in.pos;
907 in.size -= in.pos;
908 in.pos = 0;
909 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
910 CHECK(!ZSTD_isError(ret), "Must error");
911 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
912 }
913 DISPLAYLEVEL(3, "OK \n");
914
915 /* stableSrc + streaming */
916 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream, flushStream and endStream : ", testNb++);
917 CHECK_Z( ZSTD_initCStream(cctx, 1) );
918 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
919 { ZSTD_inBuffer inBuf;
920 ZSTD_outBuffer outBuf;
921 const size_t nonZeroStartPos = 18;
922 const size_t inputSize = 500;
923 inBuf.src = CNBuffer;
924 inBuf.size = 100;
925 inBuf.pos = nonZeroStartPos;
926 outBuf.dst = (char*)(compressedBuffer)+cSize;
927 outBuf.size = ZSTD_compressBound(inputSize);
928 outBuf.pos = 0;
929 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
930 inBuf.size = 200;
931 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
932 CHECK_Z( ZSTD_flushStream(cctx, &outBuf) );
933 inBuf.size = nonZeroStartPos + inputSize;
934 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
935 CHECK(ZSTD_endStream(cctx, &outBuf) != 0, "compression should be successful and fully flushed");
936 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
937 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
938 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
939 CHECK_Z(decSize);
940 CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
941 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
942 } }
943 DISPLAYLEVEL(3, "OK \n");
944
945 /* stableSrc + streaming */
946 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream2, using different end directives : ", testNb++);
947 CHECK_Z( ZSTD_initCStream(cctx, 1) );
948 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
949 { ZSTD_inBuffer inBuf;
950 ZSTD_outBuffer outBuf;
951 const size_t nonZeroStartPos = 18;
952 const size_t inputSize = 500;
953 inBuf.src = CNBuffer;
954 inBuf.size = 100;
955 inBuf.pos = nonZeroStartPos;
956 outBuf.dst = (char*)(compressedBuffer)+cSize;
957 outBuf.size = ZSTD_compressBound(inputSize);
958 outBuf.pos = 0;
959 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
960 inBuf.size = 200;
961 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
962 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_flush) );
963 inBuf.size = nonZeroStartPos + inputSize;
964 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
965 CHECK( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_end) != 0, "compression should be successful and fully flushed");
966 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
967 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
968 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
969 CHECK_Z(decSize);
970 CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
971 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
972 } }
973 DISPLAYLEVEL(3, "OK \n");
974
975 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer: context size : ", testNb++);
976 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
977 DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
978 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
979 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
980 cctxSize1 = cctxSize;
981 }
982 DISPLAYLEVEL(3, "OK \n");
983
984 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer only : ", testNb++);
985 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
986 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
987 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
988 in.src = CNBuffer;
989 in.pos = out.pos = 0;
990 in.size = MIN(CNBufferSize, 10);
991 out.size = compressedBufferSize;
992 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
993 in.pos = 0;
994 in.size = CNBufferSize - in.size;
995 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
996 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, out.pos));
997 DISPLAYLEVEL(3, "OK \n");
998
999 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer modify buffer : ", testNb++);
1000 in.pos = out.pos = 0;
1001 in.size = CNBufferSize;
1002 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1003 in.pos = out.pos = 0;
1004 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1005 CHECK(!ZSTD_isError(ret), "Must have errored");
1006 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
1007 }
1008 DISPLAYLEVEL(3, "OK \n");
1009
1010 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer: context size : ", testNb++);
1011 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
1012 DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
1013 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
1014 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
1015 }
1016 DISPLAYLEVEL(3, "OK \n");
1017
1018 ZSTD_freeCCtx(cctx);
1019 }
1020
1021 /* CDict scenario */
1022 DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
1023 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
1024 size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
1025 DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError);
1026 if (ZSTD_isError(initError)) goto _output_error;
1027 outBuff.dst = compressedBuffer;
1028 outBuff.size = compressedBufferSize;
1029 outBuff.pos = 0;
1030 inBuff.src = CNBuffer;
1031 inBuff.size = CNBufferSize;
1032 inBuff.pos = 0;
1033 DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
1034 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1035 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1036 { size_t const r = ZSTD_endStream(zc, &outBuff);
1037 DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r);
1038 if (r != 0) goto _output_error; /* error, or some data not flushed */
1039 }
1040 cSize = outBuff.pos;
1041 ZSTD_freeCDict(cdict);
1042 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1043 }
1044
1045 DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
1046 { size_t const s = ZSTD_sizeof_CStream(zc);
1047 if (ZSTD_isError(s)) goto _output_error;
1048 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
1049 }
1050
1051 DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
1052 { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1053 if (dID != dictID) goto _output_error;
1054 DISPLAYLEVEL(4, "OK (%u) \n", dID);
1055 }
1056
1057 /* DDict scenario */
1058 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize);
1059 { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1060 size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
1061 if (ZSTD_isError(initError)) goto _output_error;
1062 outBuff.dst = decodedBuffer;
1063 outBuff.size = CNBufferSize;
1064 outBuff.pos = 0;
1065 inBuff.src = compressedBuffer;
1066 inBuff.size = cSize;
1067 inBuff.pos = 0;
1068 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1069 if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
1070 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
1071 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
1072 ZSTD_freeDDict(ddict);
1073 DISPLAYLEVEL(3, "OK \n");
1074 }
1075
1076 /* Memory restriction */
1077 DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
1078 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
1079 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */
1080 outBuff.dst = decodedBuffer;
1081 outBuff.size = CNBufferSize;
1082 outBuff.pos = 0;
1083 inBuff.src = compressedBuffer;
1084 inBuff.size = cSize;
1085 inBuff.pos = 0;
1086 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1087 if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
1088 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
1089 ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */
1090
1091 DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
1092 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1093 int const maxLevel = 16; /* first level with zstd_opt */
1094 int level;
1095 assert(maxLevel < ZSTD_maxCLevel());
1096 CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) );
1097 for (level = 1; level <= maxLevel; ++level) {
1098 ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level);
1099 size_t const maxSize = MIN(1 MB, CNBufferSize);
1100 size_t size;
1101 for (size = 512; size <= maxSize; size <<= 1) {
1102 U64 const crcOrig = XXH64(CNBuffer, size, 0);
1103 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1104 ZSTD_parameters savedParams;
1105 getCCtxParams(cctx, &savedParams);
1106 outBuff.dst = compressedBuffer;
1107 outBuff.size = compressedBufferSize;
1108 outBuff.pos = 0;
1109 inBuff.src = CNBuffer;
1110 inBuff.size = size;
1111 inBuff.pos = 0;
1112 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
1113 CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
1114 CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
1115 if (inBuff.pos != inBuff.size) goto _output_error;
1116 { ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
1117 ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
1118 CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
1119 if (decIn.pos != decIn.size) goto _output_error;
1120 if (decOut.pos != size) goto _output_error;
1121 { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
1122 if (crcDec != crcOrig) goto _output_error;
1123 } }
1124 ZSTD_freeCCtx(cctx);
1125 }
1126 ZSTD_freeCDict(cdict);
1127 }
1128 ZSTD_freeDCtx(dctx);
1129 }
1130 DISPLAYLEVEL(3, "OK\n");
1131
1132 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1133 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) );
1134 cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB));
1135 CHECK_Z(cSize);
1136 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++);
1137 {
1138 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1139 /* We should fail to decompress without a dictionary. */
1140 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1141 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1142 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1143 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1144 if (!ZSTD_isError(ret)) goto _output_error;
1145 }
1146 /* We should succeed to decompress with the dictionary. */
1147 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1148 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1149 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1150 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1151 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1152 if (in.pos != in.size) goto _output_error;
1153 }
1154 /* The dictionary should persist across calls. */
1155 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1156 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1157 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1158 if (in.pos != in.size) goto _output_error;
1159 }
1160 /* The dictionary should not be cleared by ZSTD_reset_session_only. */
1161 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
1162 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1163 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1164 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1165 if (in.pos != in.size) goto _output_error;
1166 }
1167 /* When we reset the context the dictionary is cleared. */
1168 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1169 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1170 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1171 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1172 if (!ZSTD_isError(ret)) goto _output_error;
1173 }
1174 ZSTD_freeDCtx(dctx);
1175 }
1176 DISPLAYLEVEL(3, "OK \n");
1177
1178 DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
1179 {
1180 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1181 /* We should succeed to decompress with the dictionary. */
1182 ZSTD_resetDStream(dctx);
1183 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1184 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1185 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1186 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1187 if (in.pos != in.size) goto _output_error;
1188 }
1189 /* The dictionary should not be cleared by ZSTD_resetDStream(). */
1190 ZSTD_resetDStream(dctx);
1191 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1192 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1193 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1194 if (in.pos != in.size) goto _output_error;
1195 }
1196 /* The dictionary should be cleared by ZSTD_initDStream(). */
1197 CHECK_Z( ZSTD_initDStream(dctx) );
1198 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1199 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1200 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1201 if (!ZSTD_isError(ret)) goto _output_error;
1202 }
1203 ZSTD_freeDCtx(dctx);
1204 }
1205 DISPLAYLEVEL(3, "OK \n");
1206
1207 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++);
1208 {
1209 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1210 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1211 /* We should succeed to decompress with the ddict. */
1212 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1213 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
1214 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1215 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1216 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1217 if (in.pos != in.size) goto _output_error;
1218 }
1219 /* The ddict should persist across calls. */
1220 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1221 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1222 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1223 if (in.pos != in.size) goto _output_error;
1224 }
1225 /* When we reset the context the ddict is cleared. */
1226 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1227 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1228 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1229 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1230 if (!ZSTD_isError(ret)) goto _output_error;
1231 }
1232 ZSTD_freeDCtx(dctx);
1233 ZSTD_freeDDict(ddict);
1234 }
1235 DISPLAYLEVEL(3, "OK \n");
1236
1237 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
1238 {
1239 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1240 /* We should succeed to decompress with the prefix. */
1241 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1242 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) );
1243 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1244 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1245 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1246 if (in.pos != in.size) goto _output_error;
1247 }
1248 /* The prefix should be cleared after the first compression. */
1249 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1250 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1251 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1252 if (!ZSTD_isError(ret)) goto _output_error;
1253 }
1254 ZSTD_freeDCtx(dctx);
1255 }
1256 DISPLAYLEVEL(3, "OK \n");
1257
1258 DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++);
1259 {
1260 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1261 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1262 size_t ret;
1263 /* We should succeed to decompress with the dictionary. */
1264 CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) );
1265 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1266 /* The dictionary should persist across calls. */
1267 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1268 /* We should succeed to decompress with the ddict. */
1269 CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) );
1270 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1271 /* The ddict should persist across calls. */
1272 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1273 /* When we reset the context the ddict is cleared. */
1274 CHECK_Z( ZSTD_initDStream(dctx) );
1275 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize);
1276 if (!ZSTD_isError(ret)) goto _output_error;
1277 ZSTD_freeDCtx(dctx);
1278 ZSTD_freeDDict(ddict);
1279 }
1280 DISPLAYLEVEL(3, "OK \n");
1281
1282 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
1283 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
1284 ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
1285 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
1286 size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
1287 if (ZSTD_isError(initError)) goto _output_error;
1288 outBuff.dst = compressedBuffer;
1289 outBuff.size = compressedBufferSize;
1290 outBuff.pos = 0;
1291 inBuff.src = CNBuffer;
1292 inBuff.size = CNBufferSize;
1293 inBuff.pos = 0;
1294 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1295 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1296 { size_t const r = ZSTD_endStream(zc, &outBuff);
1297 if (r != 0) goto _output_error; } /* error, or some data not flushed */
1298 cSize = outBuff.pos;
1299 ZSTD_freeCDict(cdict);
1300 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1301 }
1302
1303 DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
1304 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1305 if (did != 0) goto _output_error;
1306 }
1307 DISPLAYLEVEL(3, "OK (not detected) \n");
1308
1309 DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
1310 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1311 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1312 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1313 }
1314
1315 DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
1316 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) );
1317 outBuff.dst = compressedBuffer;
1318 outBuff.size = compressedBufferSize;
1319 outBuff.pos = 0;
1320 inBuff.src = CNBuffer;
1321 inBuff.size = CNBufferSize;
1322 inBuff.pos = 0;
1323 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1324 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1325 cSize = outBuff.pos;
1326 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1327
1328 DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++);
1329 CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) );
1330 outBuff.dst = decodedBuffer;
1331 outBuff.size = CNBufferSize;
1332 outBuff.pos = 0;
1333 inBuff.src = compressedBuffer;
1334 inBuff.size = cSize;
1335 inBuff.pos = 0;
1336 CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
1337 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1338 if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
1339 DISPLAYLEVEL(3, "OK \n");
1340
1341 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
1342 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1343 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1344 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1345 }
1346
1347 DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
1348 outBuff.dst = compressedBuffer;
1349 outBuff.size = compressedBufferSize;
1350 outBuff.pos = 0;
1351 inBuff.src = CNBuffer;
1352 inBuff.size = CNBufferSize;
1353 inBuff.pos = 0;
1354 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1355 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1356 cSize = outBuff.pos;
1357 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1358
1359 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
1360 CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) );
1361 DISPLAYLEVEL(3, "OK \n");
1362
1363 /* Empty srcSize */
1364 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
1365 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1366 params.fParams.contentSizeFlag = 1;
1367 CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
1368 } /* cstream advanced shall write content size = 0 */
1369 outBuff.dst = compressedBuffer;
1370 outBuff.size = compressedBufferSize;
1371 outBuff.pos = 0;
1372 inBuff.src = CNBuffer;
1373 inBuff.size = 0;
1374 inBuff.pos = 0;
1375 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1376 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1377 cSize = outBuff.pos;
1378 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1379 DISPLAYLEVEL(3, "OK \n");
1380
1381 DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++);
1382 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1383 params.fParams.contentSizeFlag = 1;
1384 CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
1385 } /* cstream advanced shall write content size = 0 */
1386 inBuff.src = CNBuffer;
1387 inBuff.size = 0;
1388 inBuff.pos = 0;
1389 outBuff.dst = compressedBuffer;
1390 outBuff.size = compressedBufferSize;
1391 outBuff.pos = 0;
1392 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1393 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1394 cSize = outBuff.pos;
1395 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1396
1397 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1398 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1399 outBuff.dst = compressedBuffer;
1400 outBuff.size = compressedBufferSize;
1401 outBuff.pos = 0;
1402 inBuff.src = CNBuffer;
1403 inBuff.size = 0;
1404 inBuff.pos = 0;
1405 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1406 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1407 cSize = outBuff.pos;
1408 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
1409 DISPLAYLEVEL(3, "OK \n");
1410
1411 /* Basic multithreading compression test */
1412 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
1413 { int jobSize;
1414 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1415 CHECK(jobSize != 0, "job size non-zero");
1416 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1417 CHECK(jobSize != 0, "job size non-zero");
1418 }
1419 outBuff.dst = compressedBuffer;
1420 outBuff.size = compressedBufferSize;
1421 outBuff.pos = 0;
1422 inBuff.src = CNBuffer;
1423 inBuff.size = CNBufferSize;
1424 inBuff.pos = 0;
1425 { size_t const compressResult = ZSTD_compressStream2(mtctx, &outBuff, &inBuff, ZSTD_e_end);
1426 if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */
1427 }
1428 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1429 { size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos);
1430 if (compressedSize != outBuff.pos) goto _output_error; /* must be a full valid frame */
1431 }
1432 DISPLAYLEVEL(3, "OK \n");
1433
1434 /* Complex multithreading + dictionary test */
1435 { U32 const nbWorkers = 2;
1436 size_t const jobSize = 4 * 1 MB;
1437 size_t const srcSize = jobSize * nbWorkers; /* we want each job to have predictable size */
1438 size_t const segLength = 2 KB;
1439 size_t const offset = 600 KB; /* must be larger than window defined in cdict */
1440 size_t const start = jobSize + (offset-1);
1441 const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
1442 BYTE* const dst = (BYTE*)CNBuffer + start - offset;
1443 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize);
1444 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
1445 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
1446 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
1447 assert(start > offset);
1448 assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
1449 memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
1450 outBuff.dst = compressedBuffer;
1451 outBuff.size = compressedBufferSize;
1452 outBuff.pos = 0;
1453 inBuff.src = CNBuffer;
1454 inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
1455 inBuff.pos = 0;
1456 }
1457 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
1458 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1459 DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
1460 CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
1461 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1462 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
1463 ZSTD_freeCDict(cdict);
1464 }
1465 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1466 cSize = outBuff.pos;
1467 DISPLAYLEVEL(3, "OK \n");
1468
1469 DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
1470 { ZSTD_DStream* const dstream = ZSTD_createDCtx();
1471 ZSTD_frameHeader zfh;
1472 ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
1473 DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize);
1474 outBuff.dst = decodedBuffer;
1475 outBuff.size = CNBufferSize;
1476 outBuff.pos = 0;
1477 inBuff.src = compressedBuffer;
1478 inBuff.pos = 0;
1479 CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
1480 inBuff.size = 1; /* avoid shortcut to single-pass mode */
1481 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1482 inBuff.size = cSize;
1483 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1484 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1485 ZSTD_freeDStream(dstream);
1486 }
1487 DISPLAYLEVEL(3, "OK \n");
1488
1489 DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
1490 { unsigned const kMaxWindowLog = 24;
1491 unsigned value;
1492 ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
1493 ZSTD_CDict* cdict;
1494 ZSTD_DDict* ddict;
1495 SEQ_stream seq = SEQ_initStream(0x87654321);
1496 SEQ_gen_type type;
1497 XXH64_state_t xxh;
1498
1499 XXH64_reset(&xxh, 0);
1500 cParams.windowLog = kMaxWindowLog;
1501 cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1502 ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1503
1504 if (!cdict || !ddict) goto _output_error;
1505
1506 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1507 ZSTD_resetDStream(zd);
1508 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1509 CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
1510 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
1511 /* Test all values < 300 */
1512 for (value = 0; value < 300; ++value) {
1513 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1514 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1515 }
1516 }
1517 /* Test values 2^8 to 2^17 */
1518 for (value = (1 << 8); value < (1 << 17); value <<= 1) {
1519 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1520 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1521 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
1522 }
1523 }
1524 /* Test offset values up to the max window log */
1525 for (value = 8; value <= kMaxWindowLog; ++value) {
1526 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
1527 }
1528
1529 CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
1530 CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
1531
1532 ZSTD_freeCDict(cdict);
1533 ZSTD_freeDDict(ddict);
1534 }
1535 DISPLAYLEVEL(3, "OK \n");
1536
1537 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
1538 { int level;
1539 CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
1540 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1541 CHECK(level != 11, "Compression level does not match");
1542 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1543 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1544 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1545 CHECK(level != 11, "Compression level does not match");
1546 }
1547 DISPLAYLEVEL(3, "OK \n");
1548
1549 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++);
1550 { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0);
1551 CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN));
1552 CHECK(badParameters(zc, params), "Compression parameters do not match");
1553 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1554 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1555 CHECK(badParameters(zc, params), "Compression parameters do not match");
1556 }
1557 DISPLAYLEVEL(3, "OK \n");
1558
1559 DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++);
1560 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1561 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX));
1562 { int srcSizeHint;
1563 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint));
1564 CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match");
1565 }
1566 CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
1567 DISPLAYLEVEL(3, "OK \n");
1568
1569 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy compress with hashLog = 29 and searchLog = 4 : ", testNb++);
1570 if (MEM_64bits()) {
1571 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
1572 ZSTD_inBuffer in = { CNBuffer, CNBufferSize, 0 };
1573 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1574 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_strategy, ZSTD_lazy));
1575 /* Force enable the row based match finder */
1576 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
1577 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_searchLog, 4));
1578 /* Set windowLog to 29 so the hashLog doesn't get sized down */
1579 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, 29));
1580 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_hashLog, 29));
1581 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1582 /* Compress with continue first so the hashLog doesn't get sized down */
1583 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_continue));
1584 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_end));
1585 cSize = out.pos;
1586 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
1587 }
1588 DISPLAYLEVEL(3, "OK \n");
1589
1590 DISPLAYLEVEL(3, "test%3i : Test offset == windowSize : ", testNb++);
1591 {
1592 int windowLog;
1593 int const kMaxWindowLog = bigTests ? 29 : 26;
1594 size_t const kNbSequences = 10000;
1595 size_t const kMaxSrcSize = (1u << kMaxWindowLog) + 10 * kNbSequences;
1596 char* src = calloc(kMaxSrcSize, 1);
1597 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
1598 for (windowLog = ZSTD_WINDOWLOG_MIN; windowLog <= kMaxWindowLog; ++windowLog) {
1599 size_t const srcSize = ((size_t)1 << windowLog) + 10 * (kNbSequences - 1);
1600
1601 sequences[0].offset = 32;
1602 sequences[0].litLength = 32;
1603 sequences[0].matchLength = (1u << windowLog) - 32;
1604 sequences[0].rep = 0;
1605 {
1606 size_t i;
1607 for (i = 1; i < kNbSequences; ++i) {
1608 sequences[i].offset = (1u << windowLog) - (FUZ_rand(&seed) % 8);
1609 sequences[i].litLength = FUZ_rand(&seed) & 7;
1610 sequences[i].matchLength = 10 - sequences[i].litLength;
1611 sequences[i].rep = 0;
1612 }
1613 }
1614
1615 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1616 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1617 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_minMatch, 3));
1618 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, 1));
1619 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, windowLog));
1620 assert(srcSize <= kMaxSrcSize);
1621 cSize = ZSTD_compressSequences(zc, compressedBuffer, compressedBufferSize, sequences, kNbSequences, src, srcSize);
1622 CHECK_Z(cSize);
1623 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
1624 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, windowLog))
1625 {
1626 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1627 size_t decompressedBytes = 0;
1628 for (;;) {
1629 ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1630 size_t const ret = ZSTD_decompressStream(zd, &out, &in);
1631 CHECK_Z(ret);
1632 CHECK(decompressedBytes + out.pos > srcSize, "Output too large");
1633 CHECK(memcmp(out.dst, src + decompressedBytes, out.pos), "Corrupted");
1634 decompressedBytes += out.pos;
1635 if (ret == 0) {
1636 break;
1637 }
1638 }
1639 CHECK(decompressedBytes != srcSize, "Output wrong size");
1640 }
1641 }
1642 free(sequences);
1643 free(src);
1644 }
1645 DISPLAYLEVEL(3, "OK \n");
1646
1647 /* Overlen overwriting window data bug */
1648 DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
1649 { /* This test has a window size of 1024 bytes and consists of 3 blocks:
1650 1. 'a' repeated 517 times
1651 2. 'b' repeated 516 times
1652 3. a compressed block with no literals and 3 sequence commands:
1653 litlength = 0, offset = 24, match length = 24
1654 litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
1655 litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
1656
1657 const char* testCase =
1658 "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
1659 "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
1660 "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
1661 ZSTD_DStream* const zds = ZSTD_createDStream();
1662 if (zds==NULL) goto _output_error;
1663
1664 CHECK_Z( ZSTD_initDStream(zds) );
1665 inBuff.src = testCase;
1666 inBuff.size = 47;
1667 inBuff.pos = 0;
1668 outBuff.dst = decodedBuffer;
1669 outBuff.size = CNBufferSize;
1670 outBuff.pos = 0;
1671
1672 while (inBuff.pos < inBuff.size) {
1673 CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) );
1674 }
1675
1676 ZSTD_freeDStream(zds);
1677 }
1678 DISPLAYLEVEL(3, "OK \n");
1679
1680 /* Small Sequence Section bug */
1681 DISPLAYLEVEL(3, "test%3i : decompress blocks with small sequences section : ", testNb++);
1682 { /* This test consists of 3 blocks. Each block has one sequence.
1683 The sequence has literal length of 10, match length of 10 and offset of 10.
1684 The sequence value and compression mode for the blocks are following:
1685 The order of values are ll, ml, of.
1686 - First block : (10, 7, 13) (rle, rle, rle)
1687 - size of sequences section: 6 bytes (1 byte for nbSeq, 1 byte for encoding mode, 3 bytes for rle, 1 byte bitstream)
1688 - Second block : (10, 7, 1) (repeat, repeat, rle)
1689 - size of sequences section: 4 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 bytes for rle, 1 byte bitstream)
1690 - Third block : (10, 7, 1) (repeat, repeat, repeat)
1691 - size of sequences section: 3 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 byte bitstream) */
1692
1693 unsigned char compressed[] = {
1694 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x3c, 0x35, 0x01, 0x00, 0xf0, 0x85, 0x08,
1695 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1696 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac,
1697 0x69, 0x94, 0x89, 0x1c, 0x03, 0x44, 0x0a, 0x07, 0x00, 0xb4, 0x04, 0x80,
1698 0x40, 0x0a, 0xa4
1699 };
1700 unsigned int compressedSize = 51;
1701 unsigned char decompressed[] = {
1702 0x85, 0x08, 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x85, 0x08,
1703 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1704 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0x4c, 0x6b, 0xa9, 0x8b, 0xbc, 0xc5,
1705 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94,
1706 0x89, 0x1c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, 0x89, 0x1c
1707 };
1708 unsigned int decompressedSize = 60;
1709
1710 ZSTD_DStream* const zds = ZSTD_createDStream();
1711 if (zds==NULL) goto _output_error;
1712
1713 CHECK_Z( ZSTD_initDStream(zds) );
1714 inBuff.src = compressed;
1715 inBuff.size = compressedSize;
1716 inBuff.pos = 0;
1717 outBuff.dst = decodedBuffer;
1718 outBuff.size = CNBufferSize;
1719 outBuff.pos = 0;
1720
1721 CHECK(ZSTD_decompressStream(zds, &outBuff, &inBuff) != 0,
1722 "Decompress did not reach the end of frame");
1723 CHECK(inBuff.pos != inBuff.size, "Decompress did not fully consume input");
1724 CHECK(outBuff.pos != decompressedSize, "Decompressed size does not match");
1725 CHECK(memcmp(outBuff.dst, decompressed, decompressedSize) != 0,
1726 "Decompressed data does not match");
1727
1728 ZSTD_freeDStream(zds);
1729 }
1730 DISPLAYLEVEL(3, "OK \n");
1731
1732 DISPLAYLEVEL(3, "test%3i : raw block can be streamed: ", testNb++);
1733 { size_t const inputSize = 10000;
1734 size_t const compCapacity = ZSTD_compressBound(inputSize);
1735 BYTE* const input = (BYTE*)malloc(inputSize);
1736 BYTE* const comp = (BYTE*)malloc(compCapacity);
1737 BYTE* const decomp = (BYTE*)malloc(inputSize);
1738
1739 CHECK(input == NULL || comp == NULL || decomp == NULL, "failed to alloc buffers");
1740
1741 RDG_genBuffer(input, inputSize, 0.0, 0.0, seed);
1742 { size_t const compSize = ZSTD_compress(comp, compCapacity, input, inputSize, -(int)inputSize);
1743 ZSTD_inBuffer in = { comp, 0, 0 };
1744 ZSTD_outBuffer out = { decomp, 0, 0 };
1745 CHECK_Z(compSize);
1746 CHECK_Z( ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters) );
1747 while (in.size < compSize) {
1748 in.size = MIN(in.size + 100, compSize);
1749 while (in.pos < in.size) {
1750 size_t const outPos = out.pos;
1751 if (out.pos == out.size) {
1752 out.size = MIN(out.size + 10, inputSize);
1753 }
1754 CHECK_Z( ZSTD_decompressStream(zd, &out, &in) );
1755 CHECK(!(out.pos > outPos), "We are not streaming (no output generated)");
1756 }
1757 }
1758 CHECK(in.pos != compSize, "Not all input consumed!");
1759 CHECK(out.pos != inputSize, "Not all output produced!");
1760 }
1761 CHECK(memcmp(input, decomp, inputSize), "round trip failed!");
1762
1763 free(input);
1764 free(comp);
1765 free(decomp);
1766 }
1767 DISPLAYLEVEL(3, "OK \n");
1768
1769 DISPLAYLEVEL(3, "test%3i : dictionary + uncompressible block + reusing tables checks offset table validity: ", testNb++);
1770 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1771 dictionary.start, dictionary.filled,
1772 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1773 ZSTD_getCParams(3, 0, dictionary.filled),
1774 ZSTD_defaultCMem);
1775 const size_t inbufsize = 2 * 128 * 1024; /* 2 blocks */
1776 const size_t outbufsize = ZSTD_compressBound(inbufsize);
1777 size_t inbufpos = 0;
1778 size_t cursegmentlen;
1779 BYTE *inbuf = (BYTE *)malloc(inbufsize);
1780 BYTE *outbuf = (BYTE *)malloc(outbufsize);
1781 BYTE *checkbuf = (BYTE *)malloc(inbufsize);
1782 size_t ret;
1783
1784 CHECK(cdict == NULL, "failed to alloc cdict");
1785 CHECK(inbuf == NULL, "failed to alloc input buffer");
1786
1787 /* first block is uncompressible */
1788 cursegmentlen = 128 * 1024;
1789 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0., 0., seed);
1790 inbufpos += cursegmentlen;
1791
1792 /* second block is compressible */
1793 cursegmentlen = 128 * 1024 - 256;
1794 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0.05, 0., seed);
1795 inbufpos += cursegmentlen;
1796
1797 /* and includes a very long backref */
1798 cursegmentlen = 128;
1799 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 256, cursegmentlen);
1800 inbufpos += cursegmentlen;
1801
1802 /* and includes a very long backref */
1803 cursegmentlen = 128;
1804 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 128, cursegmentlen);
1805 inbufpos += cursegmentlen;
1806
1807 ret = ZSTD_compress_usingCDict(zc, outbuf, outbufsize, inbuf, inbufpos, cdict);
1808 CHECK_Z(ret);
1809
1810 ret = ZSTD_decompress_usingDict(zd, checkbuf, inbufsize, outbuf, ret, dictionary.start, dictionary.filled);
1811 CHECK_Z(ret);
1812
1813 CHECK(memcmp(inbuf, checkbuf, inbufpos), "start and finish buffers don't match");
1814
1815 ZSTD_freeCDict(cdict);
1816 free(inbuf);
1817 free(outbuf);
1818 free(checkbuf);
1819 }
1820 DISPLAYLEVEL(3, "OK \n");
1821
1822 DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++);
1823 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1824 dictionary.start, dictionary.filled,
1825 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1826 ZSTD_getCParams(3, 0, dictionary.filled),
1827 ZSTD_defaultCMem);
1828 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1829 int remainingInput = 256 * 1024;
1830 int offset;
1831
1832 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1833 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1834 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1835 /* Write a bunch of 6 byte blocks */
1836 while (remainingInput > 0) {
1837 char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
1838 const size_t kSmallBlockSize = sizeof(testBuffer);
1839 ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
1840
1841 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
1842 CHECK(in.pos != in.size, "input not fully consumed");
1843 remainingInput -= kSmallBlockSize;
1844 }
1845 /* Write several very long offset matches into the dictionary */
1846 for (offset = 1024; offset >= 0; offset -= 128) {
1847 ZSTD_inBuffer in = {(BYTE*)dictionary.start + offset, 128, 0};
1848 ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
1849 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
1850 CHECK(in.pos != in.size, "input not fully consumed");
1851 }
1852 /* Ensure decompression works */
1853 CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled));
1854
1855 ZSTD_freeCDict(cdict);
1856 }
1857 DISPLAYLEVEL(3, "OK \n");
1858
1859 DISPLAYLEVEL(3, "test%3i : Block-Level External Sequence Producer API: ", testNb++);
1860 {
1861 size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
1862 BYTE* const dstBuf = (BYTE*)malloc(ZSTD_compressBound(dstBufSize));
1863 size_t const checkBufSize = CNBufferSize;
1864 BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
1865 int enableFallback;
1866 EMF_testCase sequenceProducerState;
1867
1868 CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
1869
1870 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1871
1872 /* Reference external matchfinder outside the test loop to
1873 * check that the reference is preserved across compressions */
1874 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1875
1876 for (enableFallback = 0; enableFallback <= 1; enableFallback++) {
1877 size_t testCaseId;
1878 size_t const numTestCases = 9;
1879
1880 EMF_testCase const testCases[] = {
1881 EMF_ONE_BIG_SEQ,
1882 EMF_LOTS_OF_SEQS,
1883 EMF_ZERO_SEQS,
1884 EMF_BIG_ERROR,
1885 EMF_SMALL_ERROR,
1886 EMF_INVALID_OFFSET,
1887 EMF_INVALID_MATCHLEN,
1888 EMF_INVALID_LITLEN,
1889 EMF_INVALID_LAST_LITS
1890 };
1891
1892 ZSTD_ErrorCode const errorCodes[] = {
1893 ZSTD_error_no_error,
1894 ZSTD_error_no_error,
1895 ZSTD_error_sequenceProducer_failed,
1896 ZSTD_error_sequenceProducer_failed,
1897 ZSTD_error_sequenceProducer_failed,
1898 ZSTD_error_externalSequences_invalid,
1899 ZSTD_error_externalSequences_invalid,
1900 ZSTD_error_externalSequences_invalid,
1901 ZSTD_error_externalSequences_invalid
1902 };
1903
1904 for (testCaseId = 0; testCaseId < numTestCases; testCaseId++) {
1905 size_t res;
1906
1907 int const compressionShouldSucceed = (
1908 (errorCodes[testCaseId] == ZSTD_error_no_error) ||
1909 (enableFallback && errorCodes[testCaseId] == ZSTD_error_sequenceProducer_failed)
1910 );
1911
1912 int const testWithSequenceValidation = (
1913 testCases[testCaseId] == EMF_INVALID_OFFSET
1914 );
1915
1916 sequenceProducerState = testCases[testCaseId];
1917
1918 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1919 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, testWithSequenceValidation));
1920 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
1921 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
1922
1923 if (compressionShouldSucceed) {
1924 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
1925 CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
1926 CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
1927 } else {
1928 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
1929 CHECK(
1930 ZSTD_getErrorCode(res) != errorCodes[testCaseId],
1931 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
1932 );
1933 }
1934 }
1935
1936 /* Test compression with external matchfinder + empty src buffer */
1937 {
1938 size_t res;
1939 sequenceProducerState = EMF_ZERO_SEQS;
1940 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1941 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
1942 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
1943 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
1944 CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
1945 }
1946 }
1947
1948 /* Test that reset clears the external matchfinder */
1949 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1950 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
1951 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
1952 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
1953
1954 /* Test that registering mFinder == NULL clears the external matchfinder */
1955 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1956 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1957 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
1958 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
1959 ZSTD_registerSequenceProducer(zc, NULL, NULL); /* clear the external matchfinder */
1960 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
1961
1962 /* Test that external matchfinder doesn't interact with older APIs */
1963 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1964 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1965 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
1966 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
1967 CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
1968
1969 /* Test that compression returns the correct error with LDM */
1970 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1971 {
1972 size_t res;
1973 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1974 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1975 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
1976 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
1977 CHECK(
1978 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
1979 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
1980 );
1981 }
1982
1983#ifdef ZSTD_MULTITHREAD
1984 /* Test that compression returns the correct error with nbWorkers > 0 */
1985 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1986 {
1987 size_t res;
1988 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1989 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
1990 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
1991 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
1992 CHECK(
1993 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
1994 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
1995 );
1996 }
1997#endif
1998
1999 free(dstBuf);
2000 free(checkBuf);
2001 }
2002 DISPLAYLEVEL(3, "OK \n");
2003
2004
2005 /* Test maxBlockSize cctx param functionality */
2006 DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
2007 {
2008 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2009
2010 /* Quick test to make sure maxBlockSize bounds are enforced */
2011 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
2012 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
2013
2014 /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
2015 {
2016 size_t srcSize = 2 << 10;
2017 void* const src = CNBuffer;
2018 size_t dstSize = ZSTD_compressBound(srcSize);
2019 void* const dst1 = compressedBuffer;
2020 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2021 size_t size1, size2;
2022 void* const checkBuf = malloc(srcSize);
2023 memset(src, 'x', srcSize);
2024
2025 /* maxBlockSize = 1KB */
2026 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2027 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2028
2029 if (ZSTD_isError(size1)) goto _output_error;
2030 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2031 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2032
2033 /* maxBlockSize = 3KB */
2034 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2035 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2036
2037 if (ZSTD_isError(size2)) goto _output_error;
2038 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2039 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2040
2041 assert(size1 - size2 == 4); /* We add another RLE block with header + character */
2042 assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
2043
2044 /* maxBlockSize = 1KB, windowLog = 10 */
2045 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2046 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2047 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2048
2049 if (ZSTD_isError(size1)) goto _output_error;
2050 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2051 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2052
2053 /* maxBlockSize = 3KB, windowLog = 10 */
2054 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2055 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2056 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2057
2058 if (ZSTD_isError(size2)) goto _output_error;
2059 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2060 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2061
2062 assert(size1 == size2);
2063 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2064
2065 free(checkBuf);
2066 }
2067
2068 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2069
2070 /* Test maxBlockSize = 0 is valid */
2071 { size_t srcSize = 256 << 10;
2072 void* const src = CNBuffer;
2073 size_t dstSize = ZSTD_compressBound(srcSize);
2074 void* const dst1 = compressedBuffer;
2075 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2076 size_t size1, size2;
2077 void* const checkBuf = malloc(srcSize);
2078
2079 /* maxBlockSize = 0 */
2080 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
2081 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2082
2083 if (ZSTD_isError(size1)) goto _output_error;
2084 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2085 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2086
2087 /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
2088 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
2089 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2090
2091 if (ZSTD_isError(size2)) goto _output_error;
2092 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2093 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2094
2095 assert(size1 == size2);
2096 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2097 free(checkBuf);
2098 }
2099 ZSTD_freeCCtx(cctx);
2100 }
2101 DISPLAYLEVEL(3, "OK \n");
2102
2103 /* Test Sequence Validation */
2104 DISPLAYLEVEL(3, "test%3i : Testing sequence validation: ", testNb++);
2105 {
2106 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2107
2108 /* Test minMatch >= 4, matchLength < 4 */
2109 {
2110 size_t srcSize = 11;
2111 void* const src = CNBuffer;
2112 size_t dstSize = ZSTD_compressBound(srcSize);
2113 void* const dst = compressedBuffer;
2114 size_t const kNbSequences = 4;
2115 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2116
2117 memset(src, 'x', srcSize);
2118
2119 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2120 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2121 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2122 sequences[3] = (ZSTD_Sequence) {0, 1, 0, 0};
2123
2124 /* Test with sequence validation */
2125 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2126 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2127 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2128
2129 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2130 sequences, kNbSequences,
2131 src, srcSize);
2132
2133 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2134 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2135
2136 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2137
2138 /* Test without sequence validation */
2139 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2140 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2141 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2142
2143 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2144 sequences, kNbSequences,
2145 src, srcSize);
2146
2147 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2148 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2149
2150 free(sequences);
2151 }
2152
2153 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2154
2155
2156 /* Test with no block delim */
2157 {
2158 size_t srcSize = 4;
2159 void* const src = CNBuffer;
2160 size_t dstSize = ZSTD_compressBound(srcSize);
2161 void* const dst = compressedBuffer;
2162 size_t const kNbSequences = 1;
2163 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2164 void* const checkBuf = malloc(srcSize);
2165
2166 memset(src, 'x', srcSize);
2167
2168 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2169
2170 /* Test with sequence validation */
2171 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 3));
2172 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters));
2173 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2174
2175 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2176 sequences, kNbSequences,
2177 src, srcSize);
2178
2179 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2180 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst, cSize));
2181 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2182
2183 free(sequences);
2184 free(checkBuf);
2185 }
2186
2187 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2188
2189 { /* Test case with two additional sequences */
2190 size_t srcSize = 19;
2191 void* const src = CNBuffer;
2192 size_t dstSize = ZSTD_compressBound(srcSize);
2193 void* const dst = compressedBuffer;
2194 size_t const kNbSequences = 7;
2195 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2196
2197 memset(src, 'x', srcSize);
2198
2199 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2200 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2201 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2202 sequences[3] = (ZSTD_Sequence) {1, 0, 3, 0};
2203 sequences[4] = (ZSTD_Sequence) {1, 0, 3, 0};
2204 sequences[5] = (ZSTD_Sequence) {1, 0, 3, 0};
2205 sequences[6] = (ZSTD_Sequence) {0, 0, 0, 0};
2206
2207 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2208 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2209 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2210
2211 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2212 sequences, kNbSequences,
2213 src, srcSize);
2214
2215 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2216 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2217
2218 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2219
2220 /* Test without sequence validation */
2221 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2222 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2223 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2224
2225 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2226 sequences, kNbSequences,
2227 src, srcSize);
2228
2229 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2230 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2231
2232 free(sequences);
2233 }
2234 ZSTD_freeCCtx(cctx);
2235 }
2236 DISPLAYLEVEL(3, "OK \n");
2237
2238
2239 DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
2240 {
2241 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2242 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2243
2244 /* Test large offset, small window size*/
2245 {
2246 size_t srcSize = 21;
2247 void* const src = CNBuffer;
2248 size_t dstSize = ZSTD_compressBound(srcSize);
2249 void* const dst = compressedBuffer;
2250 size_t const kNbSequences = 4;
2251 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2252 void* const checkBuf = malloc(srcSize);
2253 const size_t largeDictSize = 1 << 25;
2254 ZSTD_CDict* cdict = NULL;
2255 ZSTD_DDict* ddict = NULL;
2256
2257 /* Generate large dictionary */
2258 void* dictBuffer = calloc(largeDictSize, 1);
2259 ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, largeDictSize);
2260 cParams.minMatch = ZSTD_MINMATCH_MIN;
2261 cParams.hashLog = ZSTD_HASHLOG_MIN;
2262 cParams.chainLog = ZSTD_CHAINLOG_MIN;
2263
2264 cdict = ZSTD_createCDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
2265 ddict = ZSTD_createDDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
2266
2267 ZSTD_CCtx_refCDict(cctx, cdict);
2268 ZSTD_DCtx_refDDict(dctx, ddict);
2269
2270 sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0};
2271 sequences[1] = (ZSTD_Sequence) {1 << 25, 0, 3, 0};
2272 sequences[2] = (ZSTD_Sequence) {1 << 25, 0, 9, 0};
2273 sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0};
2274
2275 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2276 sequences, kNbSequences,
2277 src, srcSize);
2278
2279 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2280
2281 {
2282 size_t dSize = ZSTD_decompressDCtx(dctx, checkBuf, srcSize, dst, cSize);
2283 CHECK(ZSTD_isError(dSize), "Should not throw an error");
2284 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2285 }
2286
2287 free(sequences);
2288 free(checkBuf);
2289 free(dictBuffer);
2290 ZSTD_freeCDict(cdict);
2291 ZSTD_freeDDict(ddict);
2292 }
2293 ZSTD_freeCCtx(cctx);
2294 ZSTD_freeDCtx(dctx);
2295 }
2296 DISPLAYLEVEL(3, "OK \n");
2297
2298_end:
2299 FUZ_freeDictionary(dictionary);
2300 ZSTD_freeCStream(zc);
2301 ZSTD_freeDStream(zd);
2302 ZSTD_freeCCtx(mtctx);
2303 free(CNBuffer);
2304 free(compressedBuffer);
2305 free(decodedBuffer);
2306 return testResult;
2307
2308_output_error:
2309 testResult = 1;
2310 DISPLAY("Error detected in Unit tests ! \n");
2311 goto _end;
2312}
2313
2314
2315/* ====== Fuzzer tests ====== */
2316
2317static size_t findDiff(const void* buf1, const void* buf2, size_t max)
2318{
2319 const BYTE* b1 = (const BYTE*)buf1;
2320 const BYTE* b2 = (const BYTE*)buf2;
2321 size_t u;
2322 for (u=0; u<max; u++) {
2323 if (b1[u] != b2[u]) break;
2324 }
2325 if (u==max) {
2326 DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max);
2327 return u;
2328 }
2329 DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max);
2330 if (u>=3)
2331 DISPLAY(" %02X %02X %02X ",
2332 b1[u-3], b1[u-2], b1[u-1]);
2333 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2334 b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]);
2335 if (u>=3)
2336 DISPLAY(" %02X %02X %02X ",
2337 b2[u-3], b2[u-2], b2[u-1]);
2338 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2339 b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]);
2340 return u;
2341}
2342
2343static size_t FUZ_rLogLength(U32* seed, U32 logLength)
2344{
2345 size_t const lengthMask = ((size_t)1 << logLength) - 1;
2346 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
2347}
2348
2349static size_t FUZ_randomLength(U32* seed, U32 maxLog)
2350{
2351 U32 const logLength = FUZ_rand(seed) % maxLog;
2352 return FUZ_rLogLength(seed, logLength);
2353}
2354
2355/* Return value in range minVal <= v <= maxVal */
2356static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
2357{
2358 U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal;
2359 return (U32)((FUZ_rand(seed) % mod) + minVal);
2360}
2361
2362static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests)
2363{
2364 U32 const maxSrcLog = bigTests ? 24 : 22;
2365 static const U32 maxSampleLog = 19;
2366 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2367 BYTE* cNoiseBuffer[5];
2368 size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
2369 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2370 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2371 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2372 size_t const dstBufferSize = srcBufferSize;
2373 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2374 U32 result = 0;
2375 unsigned testNb = 0;
2376 U32 coreSeed = seed;
2377 ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
2378 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
2379 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2380 UTIL_time_t const startClock = UTIL_getTime();
2381 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2382 size_t dictSize = 0;
2383 U32 oldTestLog = 0;
2384 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
2385
2386 /* allocations */
2387 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2388 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2389 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2390 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2391 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2392 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2393 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2394 "Not enough memory, fuzzer tests cancelled");
2395
2396 /* Create initial samples */
2397 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2398 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2399 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2400 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2401 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2402 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2403 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2404
2405 /* catch up testNb */
2406 for (testNb=1; testNb < startTest; testNb++)
2407 FUZ_rand(&coreSeed);
2408
2409 /* test loop */
2410 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2411 U32 lseed;
2412 const BYTE* srcBuffer;
2413 size_t totalTestSize, totalGenSize, cSize;
2414 XXH64_state_t xxhState;
2415 U64 crcOrig;
2416 U32 resetAllowed = 1;
2417 size_t maxTestSize;
2418
2419 /* init */
2420 FUZ_rand(&coreSeed);
2421 lseed = coreSeed ^ prime32;
2422 if (nbTests >= testNb) {
2423 DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
2424 } else {
2425 DISPLAYUPDATE(2, "\r%6u ", testNb);
2426 }
2427
2428 /* states full reset (deliberately not synchronized) */
2429 /* some issues can only happen when reusing states */
2430 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2431 ZSTD_freeCStream(zc);
2432 zc = ZSTD_createCStream();
2433 CHECK(zc==NULL, "ZSTD_createCStream : allocation error");
2434 resetAllowed=0;
2435 }
2436 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2437 ZSTD_freeDStream(zd);
2438 zd = ZSTD_createDStream();
2439 CHECK(zd==NULL, "ZSTD_createDStream : allocation error");
2440 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2441 }
2442
2443 /* srcBuffer selection [0-4] */
2444 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2445 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2446 else {
2447 buffNb >>= 3;
2448 if (buffNb & 7) {
2449 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2450 buffNb = tnb[buffNb >> 3];
2451 } else {
2452 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2453 buffNb = tnb[buffNb >> 3];
2454 } }
2455 srcBuffer = cNoiseBuffer[buffNb];
2456 }
2457
2458 /* compression init */
2459 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2460 && oldTestLog /* at least one test happened */ && resetAllowed) {
2461 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2462 maxTestSize = MIN(maxTestSize, srcBufferSize-16);
2463 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2464 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2465 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2466 }
2467 } else {
2468 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2469 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2470 U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
2471 (ZSTD_maxCLevel() -
2472 (MAX(testLog, dictLog) / 3)))
2473 + 1;
2474 U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
2475 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2476 oldTestLog = testLog;
2477 /* random dictionary selection */
2478 dictSize = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2479 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2480 dict = srcBuffer + dictStart;
2481 }
2482 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2483 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2484 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) );
2485 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) );
2486 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) );
2487 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) );
2488 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2489 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2490 } }
2491
2492 /* multi-segments compression test */
2493 XXH64_reset(&xxhState, 0);
2494 { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2495 cSize=0;
2496 totalTestSize=0;
2497 while(totalTestSize < maxTestSize) {
2498 /* compress random chunks into randomly sized dst buffers */
2499 { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2500 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2501 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2502 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2503 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
2504 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2505 outBuff.size = outBuff.pos + dstBuffSize;
2506
2507 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
2508
2509 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2510 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2511 totalTestSize += inBuff.pos;
2512 }
2513
2514 /* random flush operation, to mess around */
2515 if ((FUZ_rand(&lseed) & 15) == 0) {
2516 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2517 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2518 outBuff.size = outBuff.pos + adjustedDstSize;
2519 CHECK_Z( ZSTD_flushStream(zc, &outBuff) );
2520 } }
2521
2522 /* final frame epilogue */
2523 { size_t remainingToFlush = (size_t)(-1);
2524 while (remainingToFlush) {
2525 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2526 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2527 outBuff.size = outBuff.pos + adjustedDstSize;
2528 remainingToFlush = ZSTD_endStream(zc, &outBuff);
2529 CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
2530 } }
2531 crcOrig = XXH64_digest(&xxhState);
2532 cSize = outBuff.pos;
2533 }
2534
2535 /* multi - fragments decompression test */
2536 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2537 CHECK_Z ( ZSTD_resetDStream(zd) );
2538 } else {
2539 CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2540 }
2541 { size_t decompressionResult = 1;
2542 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2543 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2544 for (totalGenSize = 0 ; decompressionResult ; ) {
2545 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2546 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2547 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2548 inBuff.size = inBuff.pos + readCSrcSize;
2549 outBuff.size = outBuff.pos + dstBuffSize;
2550 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2551 if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) {
2552 DISPLAY("checksum error : \n");
2553 findDiff(copyBuffer, dstBuffer, totalTestSize);
2554 }
2555 CHECK( ZSTD_isError(decompressionResult), "decompression error : %s",
2556 ZSTD_getErrorName(decompressionResult) );
2557 }
2558 CHECK (decompressionResult != 0, "frame not fully decoded");
2559 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)",
2560 (unsigned)outBuff.pos, (unsigned)totalTestSize);
2561 CHECK (inBuff.pos != cSize, "compressed data should be fully read")
2562 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
2563 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
2564 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
2565 } }
2566
2567 /*===== noisy/erroneous src decompression test =====*/
2568
2569 /* add some noise */
2570 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
2571 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
2572 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
2573 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
2574 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
2575 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
2576 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
2577 } }
2578
2579 /* try decompression on noisy data */
2580 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
2581 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2582 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2583 while (outBuff.pos < dstBufferSize) {
2584 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2585 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2586 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
2587 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
2588 outBuff.size = outBuff.pos + adjustedDstSize;
2589 inBuff.size = inBuff.pos + adjustedCSrcSize;
2590 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2591 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
2592 /* No forward progress possible */
2593 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
2594 } } } }
2595 DISPLAY("\r%u fuzzer tests completed \n", testNb);
2596
2597_cleanup:
2598 ZSTD_freeCStream(zc);
2599 ZSTD_freeDStream(zd);
2600 ZSTD_freeDStream(zd_noise);
2601 free(cNoiseBuffer[0]);
2602 free(cNoiseBuffer[1]);
2603 free(cNoiseBuffer[2]);
2604 free(cNoiseBuffer[3]);
2605 free(cNoiseBuffer[4]);
2606 free(copyBuffer);
2607 free(cBuffer);
2608 free(dstBuffer);
2609 return result;
2610
2611_output_error:
2612 result = 1;
2613 goto _cleanup;
2614}
2615
2616/** If useOpaqueAPI, sets param in cctxParams.
2617 * Otherwise, sets the param in zc. */
2618static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
2619 ZSTD_cParameter param, unsigned value,
2620 int useOpaqueAPI)
2621{
2622 if (useOpaqueAPI) {
2623 return ZSTD_CCtxParams_setParameter(cctxParams, param, value);
2624 } else {
2625 return ZSTD_CCtx_setParameter(zc, param, value);
2626 }
2627}
2628
2629/* Tests for ZSTD_compress_generic() API */
2630static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
2631 double compressibility, int bigTests)
2632{
2633 U32 const maxSrcLog = bigTests ? 24 : 22;
2634 static const U32 maxSampleLog = 19;
2635 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2636 BYTE* cNoiseBuffer[5];
2637 size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
2638 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2639 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2640 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2641 size_t const dstBufferSize = srcBufferSize;
2642 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2643 U32 result = 0;
2644 int testNb = 0;
2645 U32 coreSeed = seed;
2646 ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
2647 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
2648 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2649 UTIL_time_t const startClock = UTIL_getTime();
2650 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2651 size_t dictSize = 0;
2652 U32 oldTestLog = 0;
2653 U32 windowLogMalus = 0; /* can survive between 2 loops */
2654 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
2655 U32 const nbThreadsMax = bigTests ? 4 : 2;
2656 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
2657
2658 /* allocations */
2659 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2660 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2661 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2662 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2663 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2664 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2665 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2666 "Not enough memory, fuzzer tests cancelled");
2667
2668 /* Create initial samples */
2669 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2670 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2671 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2672 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2673 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2674 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2675 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2676
2677 /* catch up testNb */
2678 for (testNb=1; testNb < startTest; testNb++)
2679 FUZ_rand(&coreSeed);
2680
2681 /* test loop */
2682 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2683 U32 lseed;
2684 int opaqueAPI;
2685 const BYTE* srcBuffer;
2686 size_t totalTestSize, totalGenSize, cSize;
2687 XXH64_state_t xxhState;
2688 U64 crcOrig;
2689 U32 resetAllowed = 1;
2690 size_t maxTestSize;
2691 ZSTD_parameters savedParams;
2692 int isRefPrefix = 0;
2693 U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2694
2695 /* init */
2696 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
2697 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
2698 FUZ_rand(&coreSeed);
2699 lseed = coreSeed ^ prime32;
2700 DISPLAYLEVEL(5, " *** Test %u *** \n", testNb);
2701 opaqueAPI = FUZ_rand(&lseed) & 1;
2702
2703 /* states full reset (deliberately not synchronized) */
2704 /* some issues can only happen when reusing states */
2705 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2706 DISPLAYLEVEL(5, "Creating new context \n");
2707 ZSTD_freeCCtx(zc);
2708 zc = ZSTD_createCCtx();
2709 CHECK(zc == NULL, "ZSTD_createCCtx allocation error");
2710 resetAllowed = 0;
2711 }
2712 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2713 ZSTD_freeDStream(zd);
2714 zd = ZSTD_createDStream();
2715 CHECK(zd == NULL, "ZSTD_createDStream allocation error");
2716 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2717 }
2718
2719 /* srcBuffer selection [0-4] */
2720 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2721 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2722 else {
2723 buffNb >>= 3;
2724 if (buffNb & 7) {
2725 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2726 buffNb = tnb[buffNb >> 3];
2727 } else {
2728 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2729 buffNb = tnb[buffNb >> 3];
2730 } }
2731 srcBuffer = cNoiseBuffer[buffNb];
2732 }
2733
2734 /* compression init */
2735 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
2736 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2737 && oldTestLog /* at least one test happened */
2738 && resetAllowed) {
2739 /* just set a compression level */
2740 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2741 if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
2742 { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
2743 DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
2744 CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
2745 }
2746 } else {
2747 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2748 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2749 U32 const cLevelCandidate = (FUZ_rand(&lseed) %
2750 (ZSTD_maxCLevel() -
2751 (MAX(testLog, dictLog) / 2))) +
2752 1;
2753 int const cLevel = MIN(cLevelCandidate, cLevelMax);
2754 DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel);
2755 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2756 DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize);
2757 oldTestLog = testLog;
2758 /* random dictionary selection */
2759 dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2760 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2761 dict = srcBuffer + dictStart;
2762 if (!dictSize) dict=NULL;
2763 }
2764 pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2765 { ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
2766 const U32 windowLogMax = bigTests ? 24 : 20;
2767 const U32 searchLogMax = bigTests ? 15 : 13;
2768 if (dictSize)
2769 DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
2770
2771 /* mess with compression parameters */
2772 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
2773 cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
2774 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
2775 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
2776 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
2777 cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
2778 cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
2779 cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
2780 cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
2781
2782 if (FUZ_rand(&lseed) & 1) {
2783 DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
2784 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
2785 assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
2786 windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
2787 }
2788 if (FUZ_rand(&lseed) & 1) {
2789 DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
2790 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
2791 }
2792 if (FUZ_rand(&lseed) & 1) {
2793 DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
2794 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
2795 }
2796 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
2797 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
2798 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
2799
2800 /* mess with long distance matching parameters */
2801 if (bigTests) {
2802 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) );
2803 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
2804 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
2805 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
2806 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
2807 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_srcSizeHint, FUZ_randomClampedLength(&lseed, ZSTD_SRCSIZEHINT_MIN, ZSTD_SRCSIZEHINT_MAX), opaqueAPI) );
2808 }
2809
2810 /* mess with frame parameters */
2811 if (FUZ_rand(&lseed) & 1) {
2812 int const checksumFlag = FUZ_rand(&lseed) & 1;
2813 DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
2814 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
2815 }
2816 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2817 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2818 if (FUZ_rand(&lseed) & 1) {
2819 DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize);
2820 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2821 } else {
2822 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2823 }
2824
2825 /* multi-threading parameters. Only adjust occasionally for small tests. */
2826 if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
2827 U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
2828 U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
2829 int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
2830 DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads);
2831 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
2832 if (nbThreads > 1) {
2833 U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
2834 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
2835 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
2836 }
2837 }
2838 /* Enable rsyncable mode 1 in 4 times. */
2839 {
2840 int const rsyncable = (FUZ_rand(&lseed) % 4 == 0);
2841 DISPLAYLEVEL(5, "t%u: rsyncable : %d \n", testNb, rsyncable);
2842 setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, rsyncable, opaqueAPI);
2843 }
2844
2845 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
2846 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) );
2847
2848 /* Apply parameters */
2849 if (opaqueAPI) {
2850 DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb);
2851 CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
2852 }
2853
2854 if (FUZ_rand(&lseed) & 1) {
2855 if (FUZ_rand(&lseed) & 1) {
2856 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2857 } else {
2858 CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
2859 }
2860 } else {
2861 isRefPrefix = 1;
2862 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
2863 }
2864 } }
2865
2866 CHECK_Z(getCCtxParams(zc, &savedParams));
2867
2868 /* multi-segments compression test */
2869 { int iter;
2870 int const startSeed = lseed;
2871 XXH64_hash_t compressedCrcs[2];
2872 for (iter = 0; iter < 2; ++iter, lseed = startSeed) {
2873 ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2874 int const singlePass = (FUZ_rand(&lseed) & 3) == 0;
2875 int nbWorkers;
2876
2877 XXH64_reset(&xxhState, 0);
2878
2879 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2880 if (isRefPrefix) {
2881 DISPLAYLEVEL(6, "t%u: Reloading prefix\n", testNb);
2882 /* Need to reload the prefix because it gets dropped after one compression */
2883 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
2884 }
2885
2886 /* Adjust number of workers occasionally - result must be deterministic independent of nbWorkers */
2887 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers));
2888 if (nbWorkers > 0 && (FUZ_rand(&lseed) & 7) == 0) {
2889 DISPLAYLEVEL(6, "t%u: Modify nbWorkers: %d -> %d \n", testNb, nbWorkers, nbWorkers + iter);
2890 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers + iter));
2891 }
2892
2893 if (singlePass) {
2894 ZSTD_inBuffer inBuff = { srcBuffer, maxTestSize, 0 };
2895 CHECK_Z(ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end));
2896 DISPLAYLEVEL(6, "t%u: Single pass compression: consumed %u bytes ; produced %u bytes \n",
2897 testNb, (unsigned)inBuff.pos, (unsigned)outBuff.pos);
2898 CHECK(inBuff.pos != inBuff.size, "Input not consumed!");
2899 crcOrig = XXH64(srcBuffer, maxTestSize, 0);
2900 totalTestSize = maxTestSize;
2901 } else {
2902 outBuff.size = 0;
2903 for (totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
2904 /* compress random chunks into randomly sized dst buffers */
2905 size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2906 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2907 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2908 ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
2909 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2910 int forwardProgress;
2911 do {
2912 size_t const ipos = inBuff.pos;
2913 size_t const opos = outBuff.pos;
2914 size_t ret;
2915 if (outBuff.pos == outBuff.size) {
2916 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2917 size_t const dstBuffSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
2918 outBuff.size = outBuff.pos + dstBuffSize;
2919 }
2920 CHECK_Z( ret = ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
2921 DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
2922 testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos);
2923
2924 /* We've completed the flush */
2925 if (flush == ZSTD_e_flush && ret == 0)
2926 break;
2927
2928 /* Ensure maximal forward progress for determinism */
2929 forwardProgress = (inBuff.pos != ipos) || (outBuff.pos != opos);
2930 } while (forwardProgress);
2931 assert(inBuff.pos == inBuff.size);
2932
2933 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2934 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2935 totalTestSize += inBuff.pos;
2936 }
2937
2938 /* final frame epilogue */
2939 { size_t remainingToFlush = 1;
2940 while (remainingToFlush) {
2941 ZSTD_inBuffer inBuff = { NULL, 0, 0 };
2942 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
2943 size_t const adjustedDstSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
2944 outBuff.size = outBuff.pos + adjustedDstSize;
2945 DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize);
2946 /* ZSTD_e_end guarantees maximal forward progress */
2947 remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
2948 DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos);
2949 CHECK( ZSTD_isError(remainingToFlush),
2950 "ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
2951 ZSTD_getErrorName(remainingToFlush) );
2952 } }
2953 crcOrig = XXH64_digest(&xxhState);
2954 }
2955 cSize = outBuff.pos;
2956 compressedCrcs[iter] = XXH64(cBuffer, cSize, 0);
2957 DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize);
2958 }
2959 CHECK(!(compressedCrcs[0] == compressedCrcs[1]), "Compression is not deterministic!");
2960 }
2961
2962 CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
2963
2964 /* multi - fragments decompression test */
2965 if (FUZ_rand(&lseed) & 1) {
2966 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
2967 }
2968 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2969 DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", (void const*)dict);
2970 CHECK_Z( ZSTD_resetDStream(zd) );
2971 } else {
2972 if (dictSize)
2973 DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
2974 CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2975 }
2976 if (FUZ_rand(&lseed) & 1) {
2977 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
2978 }
2979 { size_t decompressionResult = 1;
2980 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2981 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2982 for (totalGenSize = 0 ; decompressionResult ; ) {
2983 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2984 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2985 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2986 inBuff.size = inBuff.pos + readCSrcSize;
2987 outBuff.size = outBuff.pos + dstBuffSize;
2988 DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n",
2989 (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize);
2990 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2991 DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n",
2992 (unsigned)inBuff.pos, (unsigned)outBuff.pos);
2993 if (ZSTD_isError(decompressionResult)) {
2994 DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult));
2995 findDiff(copyBuffer, dstBuffer, totalTestSize);
2996 }
2997 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
2998 CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize);
2999 }
3000 CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize);
3001 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize);
3002 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
3003 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
3004 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
3005 } }
3006
3007 /*===== noisy/erroneous src decompression test =====*/
3008
3009 /* add some noise */
3010 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
3011 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
3012 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
3013 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
3014 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
3015 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
3016 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
3017 } }
3018
3019 /* try decompression on noisy data */
3020 if (FUZ_rand(&lseed) & 1) {
3021 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_and_parameters));
3022 } else {
3023 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_only));
3024 }
3025 if (FUZ_rand(&lseed) & 1) {
3026 CHECK_Z(ZSTD_DCtx_setParameter(zd_noise, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
3027 }
3028 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
3029 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
3030 while (outBuff.pos < dstBufferSize) {
3031 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3032 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
3033 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
3034 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
3035 outBuff.size = outBuff.pos + adjustedDstSize;
3036 inBuff.size = inBuff.pos + adjustedCSrcSize;
3037 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
3038 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
3039 /* Good so far, but no more progress possible */
3040 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
3041 } } } }
3042 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
3043
3044_cleanup:
3045 ZSTD_freeCCtx(zc);
3046 ZSTD_freeDStream(zd);
3047 ZSTD_freeDStream(zd_noise);
3048 ZSTD_freeCCtxParams(cctxParams);
3049 free(cNoiseBuffer[0]);
3050 free(cNoiseBuffer[1]);
3051 free(cNoiseBuffer[2]);
3052 free(cNoiseBuffer[3]);
3053 free(cNoiseBuffer[4]);
3054 free(copyBuffer);
3055 free(cBuffer);
3056 free(dstBuffer);
3057 return result;
3058
3059_output_error:
3060 result = 1;
3061 goto _cleanup;
3062}
3063
3064/*-*******************************************************
3065* Command line
3066*********************************************************/
3067static int FUZ_usage(const char* programName)
3068{
3069 DISPLAY( "Usage :\n");
3070 DISPLAY( " %s [args]\n", programName);
3071 DISPLAY( "\n");
3072 DISPLAY( "Arguments :\n");
3073 DISPLAY( " -i# : Number of tests (default:%u)\n", nbTestsDefault);
3074 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
3075 DISPLAY( " -s# : Select seed (default:prompt user)\n");
3076 DISPLAY( " -t# : Select starting test number (default:0)\n");
3077 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
3078 DISPLAY( " -v : verbose\n");
3079 DISPLAY( " -p : pause at the end\n");
3080 DISPLAY( " -h : display help and exit\n");
3081 return 0;
3082}
3083
3084typedef enum { simple_api, advanced_api } e_api;
3085
3086int main(int argc, const char** argv)
3087{
3088 U32 seed = 0;
3089 int seedset = 0;
3090 int nbTests = nbTestsDefault;
3091 int testNb = 0;
3092 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
3093 int result = 0;
3094 int mainPause = 0;
3095 int bigTests = (sizeof(size_t) == 8);
3096 e_api selected_api = simple_api;
3097 const char* const programName = argv[0];
3098 int argNb;
3099
3100 /* Check command line */
3101 for(argNb=1; argNb<argc; argNb++) {
3102 const char* argument = argv[argNb];
3103 assert(argument != NULL);
3104
3105 /* Parsing commands. Aggregated commands are allowed */
3106 if (argument[0]=='-') {
3107
3108 if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
3109 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
3110 if (!strcmp(argument, "--big-tests")) { bigTests=1; continue; }
3111
3112 argument++;
3113 while (*argument!=0) {
3114 switch(*argument)
3115 {
3116 case 'h':
3117 return FUZ_usage(programName);
3118
3119 case 'v':
3120 argument++;
3121 g_displayLevel++;
3122 break;
3123
3124 case 'q':
3125 argument++;
3126 g_displayLevel--;
3127 break;
3128
3129 case 'p': /* pause at the end */
3130 argument++;
3131 mainPause = 1;
3132 break;
3133
3134 case 'i': /* limit tests by nb of iterations (default) */
3135 argument++;
3136 nbTests=0; g_clockTime=0;
3137 while ((*argument>='0') && (*argument<='9')) {
3138 nbTests *= 10;
3139 nbTests += *argument - '0';
3140 argument++;
3141 }
3142 break;
3143
3144 case 'T': /* limit tests by time */
3145 argument++;
3146 nbTests=0; g_clockTime=0;
3147 while ((*argument>='0') && (*argument<='9')) {
3148 g_clockTime *= 10;
3149 g_clockTime += *argument - '0';
3150 argument++;
3151 }
3152 if (*argument=='m') { /* -T1m == -T60 */
3153 g_clockTime *=60, argument++;
3154 if (*argument=='n') argument++; /* -T1mn == -T60 */
3155 } else if (*argument=='s') argument++; /* -T10s == -T10 */
3156 g_clockTime *= SEC_TO_MICRO;
3157 break;
3158
3159 case 's': /* manually select seed */
3160 argument++;
3161 seedset=1;
3162 seed=0;
3163 while ((*argument>='0') && (*argument<='9')) {
3164 seed *= 10;
3165 seed += *argument - '0';
3166 argument++;
3167 }
3168 break;
3169
3170 case 't': /* select starting test number */
3171 argument++;
3172 testNb=0;
3173 while ((*argument>='0') && (*argument<='9')) {
3174 testNb *= 10;
3175 testNb += *argument - '0';
3176 argument++;
3177 }
3178 break;
3179
3180 case 'P': /* compressibility % */
3181 argument++;
3182 proba=0;
3183 while ((*argument>='0') && (*argument<='9')) {
3184 proba *= 10;
3185 proba += *argument - '0';
3186 argument++;
3187 }
3188 if (proba<0) proba=0;
3189 if (proba>100) proba=100;
3190 break;
3191
3192 default:
3193 return FUZ_usage(programName);
3194 }
3195 } } } /* for(argNb=1; argNb<argc; argNb++) */
3196
3197 /* Get Seed */
3198 DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
3199
3200 if (!seedset) {
3201 time_t const t = time(NULL);
3202 U32 const h = XXH32(&t, sizeof(t), 1);
3203 seed = h % 10000;
3204 }
3205
3206 DISPLAY("Seed = %u\n", (unsigned)seed);
3207 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
3208
3209 if (nbTests<=0) nbTests=1;
3210
3211 if (testNb==0) {
3212 result = basicUnitTests(0, ((double)proba) / 100, bigTests); /* constant seed for predictability */
3213 }
3214
3215 if (!result) {
3216 switch(selected_api)
3217 {
3218 case simple_api :
3219 result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3220 break;
3221 case advanced_api :
3222 result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3223 break;
3224 default :
3225 assert(0); /* impossible */
3226 }
3227 }
3228
3229 if (mainPause) {
3230 int unused;
3231 DISPLAY("Press Enter \n");
3232 unused = getchar();
3233 (void)unused;
3234 }
3235 return result;
3236}
ARM optimized PCSX fork