notaz.gp2x.de / pcsx_rearmed.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
gpu_neon: revive the old tests
[pcsx_rearmed.git] / deps / libchdr / deps / zstd-1.5.6 / tests / zstreamtest.c
CommitLineData
648db22b 1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12/*-************************************
13 * Compiler specific
14 **************************************/
15#ifdef _MSC_VER /* Visual Studio */
16# define _CRT_SECURE_NO_WARNINGS /* fgets */
17# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */
19#endif
20
21
22/*-************************************
23 * Includes
24 **************************************/
25#include <stdlib.h> /* free */
26#include <stdio.h> /* fgets, sscanf */
27#include <string.h> /* strcmp */
28#include <time.h> /* time_t, time(), to randomize seed */
29#include <assert.h> /* assert */
30#include "timefn.h" /* UTIL_time_t, UTIL_getTime */
31#include "mem.h"
32#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
33#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */
34#include "zstd.h" /* ZSTD_compressBound */
35#include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */
36#include "zdict.h" /* ZDICT_trainFromBuffer */
37#include "datagen.h" /* RDG_genBuffer */
38#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
39#include "xxhash.h" /* XXH64_* */
40#include "seqgen.h"
41#include "util.h"
42#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */
43#include "external_matchfinder.h" /* zstreamSequenceProducer, EMF_testCase */
44
45/*-************************************
46 * Constants
47 **************************************/
48#define KB *(1U<<10)
49#define MB *(1U<<20)
50#define GB *(1U<<30)
51
52static const int nbTestsDefault = 10000;
53static const U32 g_cLevelMax_smallTests = 10;
54#define COMPRESSIBLE_NOISE_LENGTH (10 MB)
55#define FUZ_COMPRESSIBILITY_DEFAULT 50
56static const U32 prime32 = 2654435761U;
57
58
59/*-************************************
60 * Display Macros
61 **************************************/
62#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
63#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \
64 DISPLAY(__VA_ARGS__); \
65 if (g_displayLevel>=4) fflush(stderr); }
66static U32 g_displayLevel = 2;
67
68static const U64 g_refreshRate = SEC_TO_MICRO / 6;
69static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
70
71#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
72 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
73 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
74 if (g_displayLevel>=4) fflush(stderr); } }
75
76static U64 g_clockTime = 0;
77
78
79/*-*******************************************************
80 * Check macros
81 *********************************************************/
82#undef MIN
83#undef MAX
84#define MIN(a,b) ((a)<(b)?(a):(b))
85#define MAX(a,b) ((a)>(b)?(a):(b))
86/*! FUZ_rand() :
87 @return : a 27 bits random value, from a 32-bits `seed`.
88 `seed` is also modified */
89#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
90static U32 FUZ_rand(U32* seedPtr)
91{
92 static const U32 prime2 = 2246822519U;
93 U32 rand32 = *seedPtr;
94 rand32 *= prime32;
95 rand32 += prime2;
96 rand32 = FUZ_rotl32(rand32, 13);
97 *seedPtr = rand32;
98 return rand32 >> 5;
99}
100
101#define CHECK(cond, ...) { \
102 if (cond) { \
103 DISPLAY("Error => "); \
104 DISPLAY(__VA_ARGS__); \
105 DISPLAY(" (seed %u, test nb %u, line %u) \n", \
106 (unsigned)seed, testNb, __LINE__); \
107 goto _output_error; \
108} }
109
110#define CHECK_Z(f) { \
111 size_t const err = f; \
112 CHECK(ZSTD_isError(err), "%s : %s ", \
113 #f, ZSTD_getErrorName(err)); \
114}
115
116#define CHECK_RET(ret, cond, ...) { \
117 if (cond) { \
118 DISPLAY("Error %llu => ", (unsigned long long)ret); \
119 DISPLAY(__VA_ARGS__); \
120 DISPLAY(" (line %u)\n", __LINE__); \
121 return ret; \
122} }
123
124#define CHECK_RET_Z(f) { \
125 size_t const err = f; \
126 CHECK_RET(err, ZSTD_isError(err), "%s : %s ", \
127 #f, ZSTD_getErrorName(err)); \
128}
129
130
131/*======================================================
132 * Basic Unit tests
133 *======================================================*/
134
135typedef struct {
136 void* start;
137 size_t size;
138 size_t filled;
139} buffer_t;
140
141static const buffer_t kBuffNull = { NULL, 0 , 0 };
142
143static void FUZ_freeDictionary(buffer_t dict)
144{
145 free(dict.start);
146}
147
148static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize)
149{
150 buffer_t dict = kBuffNull;
151 size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize;
152 size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t));
153 if (!blockSizes) return kBuffNull;
154 dict.start = malloc(requestedDictSize);
155 if (!dict.start) { free(blockSizes); return kBuffNull; }
156 { size_t nb;
157 for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize;
158 blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1));
159 }
160 { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks);
161 free(blockSizes);
162 if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; }
163 dict.size = requestedDictSize;
164 dict.filled = dictSize;
165 return dict;
166 }
167}
168
169/* Round trips data and updates xxh with the decompressed data produced */
170static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
171 XXH64_state_t* xxh, void* data, size_t size,
172 ZSTD_EndDirective endOp)
173{
174 static BYTE compressed[1024];
175 static BYTE uncompressed[1024];
176
177 ZSTD_inBuffer cin = {data, size, 0};
178 size_t cret;
179
180 do {
181 ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 };
182 ZSTD_inBuffer din = { compressed, 0, 0 };
183 ZSTD_outBuffer dout = { uncompressed, 0, 0 };
184
185 cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp);
186 if (ZSTD_isError(cret))
187 return cret;
188
189 din.size = cout.pos;
190 while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) {
191 size_t dret;
192
193 dout.pos = 0;
194 dout.size = sizeof(uncompressed);
195 dret = ZSTD_decompressStream(dctx, &dout, &din);
196 if (ZSTD_isError(dret))
197 return dret;
198 XXH64_update(xxh, dout.dst, dout.pos);
199 if (dret == 0)
200 break;
201 }
202 } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0));
203 return 0;
204}
205
206/* Generates some data and round trips it */
207static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx,
208 XXH64_state_t* xxh, SEQ_stream* seq,
209 SEQ_gen_type type, unsigned value)
210{
211 static BYTE data[1024];
212 size_t gen;
213
214 do {
215 SEQ_outBuffer sout = {data, sizeof(data), 0};
216 size_t ret;
217 gen = SEQ_gen(seq, type, value, &sout);
218
219 ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue);
220 if (ZSTD_isError(ret))
221 return ret;
222 } while (gen != 0);
223
224 return 0;
225}
226
227static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams)
228{
229 int value;
230 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog));
231 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog));
232 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog));
233 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog));
234 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch));
235 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength));
236 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value));
237 savedParams->cParams.strategy = value;
238
239 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag));
240 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag));
241 CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value));
242 savedParams->fParams.noDictIDFlag = !value;
243 return 0;
244}
245
246static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams)
247{
248 ZSTD_parameters params;
249 if (ZSTD_isError(getCCtxParams(zc, &params))) return 10;
250 CHECK_RET(1, params.cParams.windowLog != savedParams.cParams.windowLog, "windowLog");
251 CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog");
252 CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog");
253 CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog");
254 CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch");
255 CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength");
256
257 CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag");
258 CHECK_RET(8, params.fParams.contentSizeFlag != savedParams.fParams.contentSizeFlag, "contentSizeFlag");
259 CHECK_RET(9, params.fParams.noDictIDFlag != savedParams.fParams.noDictIDFlag, "noDictIDFlag");
260 return 0;
261}
262
263static int basicUnitTests(U32 seed, double compressibility, int bigTests)
264{
265 size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH;
266 void* CNBuffer = malloc(CNBufferSize);
267 size_t const skippableFrameSize = 200 KB;
268 size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH);
269 void* compressedBuffer = malloc(compressedBufferSize);
270 size_t const decodedBufferSize = CNBufferSize;
271 void* decodedBuffer = malloc(decodedBufferSize);
272 size_t cSize;
273 int testResult = 0;
274 int testNb = 1;
275 U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */
276 ZSTD_CStream* zc = ZSTD_createCStream();
277 ZSTD_DStream* zd = ZSTD_createDStream();
278 ZSTD_CCtx* mtctx = ZSTD_createCCtx();
279
280 ZSTD_inBuffer inBuff, inBuff2;
281 ZSTD_outBuffer outBuff;
282 buffer_t dictionary = kBuffNull;
283 size_t const dictSize = 128 KB;
284 unsigned dictID = 0;
285
286 /* Create compressible test buffer */
287 if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd || !mtctx) {
288 DISPLAY("Not enough memory, aborting \n");
289 goto _output_error;
290 }
291 RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed);
292
293 CHECK_Z(ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2));
294
295 /* Create dictionary */
296 DISPLAYLEVEL(3, "creating dictionary for unit tests \n");
297 dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB);
298 if (!dictionary.start) {
299 DISPLAY("Error creating dictionary, aborting \n");
300 goto _output_error;
301 }
302 dictID = ZDICT_getDictID(dictionary.start, dictionary.filled);
303
304 /* Basic compression test */
305 DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
306 CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) );
307 outBuff.dst = (char*)(compressedBuffer);
308 outBuff.size = compressedBufferSize;
309 outBuff.pos = 0;
310 inBuff.src = CNBuffer;
311 inBuff.size = CNBufferSize;
312 inBuff.pos = 0;
313 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
314 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
315 { size_t const r = ZSTD_endStream(zc, &outBuff);
316 if (r != 0) goto _output_error; } /* error, or some data not flushed */
317 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos);
318
319 /* generate skippable frame */
320 MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START);
321 MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize);
322 cSize = skippableFrameSize + 8;
323
324 /* Basic compression test using dict */
325 DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
326 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
327 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
328 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, CNBuffer, dictSize) );
329 outBuff.dst = (char*)(compressedBuffer)+cSize;
330 assert(compressedBufferSize > cSize);
331 outBuff.size = compressedBufferSize - cSize;
332 outBuff.pos = 0;
333 inBuff.src = CNBuffer;
334 inBuff.size = CNBufferSize;
335 inBuff.pos = 0;
336 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
337 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
338 { size_t const r = ZSTD_endStream(zc, &outBuff);
339 if (r != 0) goto _output_error; } /* error, or some data not flushed */
340 cSize += outBuff.pos;
341 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
342 (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
343
344 /* context size functions */
345 DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++);
346 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize);
347 size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams);
348 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */
349 if (ZSTD_isError(cstreamSize)) goto _output_error;
350 if (ZSTD_isError(cdictSize)) goto _output_error;
351 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize));
352 }
353
354 /* context size functions */
355 DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++);
356 { ZSTD_CCtx_params* const params = ZSTD_createCCtxParams();
357 size_t cstreamSize, cctxSize;
358 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) );
359 cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params);
360 CHECK_Z(cstreamSize);
361 cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
362 CHECK_Z(cctxSize);
363 if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error;
364 ZSTD_freeCCtxParams(params);
365 DISPLAYLEVEL(3, "OK \n");
366 }
367
368 DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++);
369 { size_t const s = ZSTD_sizeof_CStream(zc);
370 if (ZSTD_isError(s)) goto _output_error;
371 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
372 }
373
374 /* Attempt bad compression parameters */
375 DISPLAYLEVEL(3, "test%3i : use bad compression parameters with ZSTD_initCStream_advanced : ", testNb++);
376 { size_t r;
377 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
378 params.cParams.minMatch = 2;
379 r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0);
380 if (!ZSTD_isError(r)) goto _output_error;
381 DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r));
382 }
383
384 /* skippable frame test */
385 DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++);
386 CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) );
387 inBuff.src = compressedBuffer;
388 inBuff.size = cSize;
389 inBuff.pos = 0;
390 outBuff.dst = decodedBuffer;
391 outBuff.size = CNBufferSize;
392 outBuff.pos = 0;
393 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
394 DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r);
395 if (r != 0) goto _output_error;
396 }
397 if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */
398 DISPLAYLEVEL(3, "OK \n");
399
400 /* Basic decompression test */
401 inBuff2 = inBuff;
402 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
403 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
404 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */
405 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff);
406 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
407 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
408 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
409 DISPLAYLEVEL(3, "OK \n");
410
f535537f 411 /* Reuse without init */
412 DISPLAYLEVEL(3, "test%3i : decompress again without init (reuse previous settings): ", testNb++);
648db22b 413 outBuff.pos = 0;
414 { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2);
415 if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
416 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
417 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
418 DISPLAYLEVEL(3, "OK \n");
419
420 /* check regenerated data is byte exact */
421 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
422 { size_t i;
423 for (i=0; i<CNBufferSize; i++) {
424 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
425 } }
426 DISPLAYLEVEL(3, "OK \n");
427
428 /* check decompression fails early if first bytes are wrong */
429 DISPLAYLEVEL(3, "test%3i : early decompression error if first bytes are incorrect : ", testNb++);
430 { const char buf[3] = { 0 }; /* too short, not enough to start decoding header */
431 ZSTD_inBuffer inb = { buf, sizeof(buf), 0 };
432 size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inb);
433 if (!ZSTD_isError(remaining)) goto _output_error; /* should have errored out immediately (note: this does not test the exact error code) */
434 }
435 DISPLAYLEVEL(3, "OK \n");
436
437 /* context size functions */
438 DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++);
439 { ZSTD_frameHeader fhi;
440 const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8);
441 size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize);
442 if (gfhError!=0) goto _output_error;
443 DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize);
444 { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize)
445 /* uses ZSTD_initDStream_usingDict() */
446 + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
447 if (ZSTD_isError(s)) goto _output_error;
448 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
449 } }
450
451 DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++);
452 { size_t const s = ZSTD_sizeof_DStream(zd);
453 if (ZSTD_isError(s)) goto _output_error;
454 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
455 }
456
457 /* Decompression by small increment */
458 DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++);
459 { /* skippable frame */
460 size_t r = 1;
461 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
462 inBuff.src = compressedBuffer;
463 outBuff.dst = decodedBuffer;
464 inBuff.pos = 0;
465 outBuff.pos = 0;
466 while (r) { /* skippable frame */
467 size_t const inSize = (FUZ_rand(&coreSeed) & 15) + 1;
468 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + 1;
469 inBuff.size = inBuff.pos + inSize;
470 outBuff.size = outBuff.pos + outSize;
471 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
472 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream on skippable frame error : %s \n", ZSTD_getErrorName(r));
473 if (ZSTD_isError(r)) goto _output_error;
474 }
475 /* normal frame */
476 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
477 r=1;
478 while (r) {
479 size_t const inSize = FUZ_rand(&coreSeed) & 15;
480 size_t const outSize = (FUZ_rand(&coreSeed) & 15) + (!inSize); /* avoid having both sizes at 0 => would trigger a no_forward_progress error */
481 inBuff.size = inBuff.pos + inSize;
482 outBuff.size = outBuff.pos + outSize;
483 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
484 if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(r));
485 if (ZSTD_isError(r)) goto _output_error;
486 }
487 }
488 if (outBuff.pos != CNBufferSize) DISPLAYLEVEL(4, "outBuff.pos != CNBufferSize : should have regenerated same amount ! \n");
489 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
490 if (inBuff.pos != cSize) DISPLAYLEVEL(4, "inBuff.pos != cSize : should have real all input ! \n");
491 if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */
492 DISPLAYLEVEL(3, "OK \n");
493
494 /* check regenerated data is byte exact */
495 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
496 { size_t i;
497 for (i=0; i<CNBufferSize; i++) {
498 if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error;
499 } }
500 DISPLAYLEVEL(3, "OK \n");
501
502 /* Decompression forward progress */
503 DISPLAYLEVEL(3, "test%3i : generate error when ZSTD_decompressStream() doesn't progress : ", testNb++);
504 { /* skippable frame */
505 size_t r = 0;
506 int decNb = 0;
507 int const maxDec = 100;
508 inBuff.src = compressedBuffer;
509 inBuff.size = cSize;
510 inBuff.pos = 0;
511
512 outBuff.dst = decodedBuffer;
513 outBuff.pos = 0;
514 outBuff.size = CNBufferSize-1; /* 1 byte missing */
515
516 for (decNb=0; decNb<maxDec; decNb++) {
517 if (r==0) ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
518 r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
519 if (ZSTD_isError(r)) break;
520 }
521 if (!ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream should have triggered a no_forward_progress error \n");
522 if (!ZSTD_isError(r)) goto _output_error; /* should have triggered no_forward_progress error */
523 }
524 DISPLAYLEVEL(3, "OK \n");
525
526 DISPLAYLEVEL(3, "test%3i : NULL output and NULL input : ", testNb++);
527 inBuff.src = NULL;
528 inBuff.size = 0;
529 inBuff.pos = 0;
530 outBuff.dst = NULL;
531 outBuff.size = 0;
532 outBuff.pos = 0;
533 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
534 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
535 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
536 outBuff.dst = (char*)(compressedBuffer);
537 outBuff.size = compressedBufferSize;
538 outBuff.pos = 0;
539 { size_t const r = ZSTD_endStream(zc, &outBuff);
540 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
541 }
542 inBuff.src = outBuff.dst;
543 inBuff.size = outBuff.pos;
544 inBuff.pos = 0;
545 outBuff.dst = NULL;
546 outBuff.size = 0;
547 outBuff.pos = 0;
548 CHECK_Z( ZSTD_initDStream(zd) );
549 { size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
550 if (ret != 0) goto _output_error;
551 }
552 DISPLAYLEVEL(3, "OK\n");
553
554 DISPLAYLEVEL(3, "test%3i : NULL output buffer with non-NULL input : ", testNb++);
555 {
556 const char* test = "aa";
557 inBuff.src = test;
558 inBuff.size = 2;
559 inBuff.pos = 0;
560 outBuff.dst = NULL;
561 outBuff.size = 0;
562 outBuff.pos = 0;
563 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
564 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
565 CHECK_Z( ZSTD_endStream(zc, &outBuff) );
566 outBuff.dst = (char*)(compressedBuffer);
567 outBuff.size = compressedBufferSize;
568 outBuff.pos = 0;
569 { size_t const r = ZSTD_endStream(zc, &outBuff);
570 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
571 }
572 inBuff.src = outBuff.dst;
573 inBuff.size = outBuff.pos;
574 inBuff.pos = 0;
575 outBuff.dst = NULL;
576 outBuff.size = 0;
577 outBuff.pos = 0;
578 CHECK_Z( ZSTD_initDStream(zd) );
579 CHECK_Z(ZSTD_decompressStream(zd, &outBuff, &inBuff));
580 }
581
582 DISPLAYLEVEL(3, "OK\n");
583 /* _srcSize compression test */
584 DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
585 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
586 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
587 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
588 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize) );
589 outBuff.dst = (char*)(compressedBuffer);
590 outBuff.size = compressedBufferSize;
591 outBuff.pos = 0;
592 inBuff.src = CNBuffer;
593 inBuff.size = CNBufferSize;
594 inBuff.pos = 0;
595 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
596 CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
597 { size_t const r = ZSTD_endStream(zc, &outBuff);
598 CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
599 }
600 { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos);
601 CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!");
602 CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize);
603 }
604 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100);
605
606 /* wrong _srcSize compression test */
607 DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
608 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
609 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
610 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
611 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize+1) );
612 outBuff.dst = (char*)(compressedBuffer);
613 outBuff.size = compressedBufferSize;
614 outBuff.pos = 0;
615 inBuff.src = CNBuffer;
616 inBuff.size = CNBufferSize;
617 inBuff.pos = 0;
618 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
619 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
620 { size_t const r = ZSTD_endStream(zc, &outBuff);
621 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
622 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); }
623
624 /* wrong _srcSize compression test */
625 DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
626 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
627 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) );
628 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 1) );
629 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize-1) );
630 outBuff.dst = (char*)(compressedBuffer);
631 outBuff.size = compressedBufferSize;
632 outBuff.pos = 0;
633 inBuff.src = CNBuffer;
634 inBuff.size = CNBufferSize;
635 inBuff.pos = 0;
636 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
637 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
638 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
639 }
640
641 DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1);
642 { CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
643 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, 0) );
644 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, CNBufferSize - MIN(CNBufferSize, 200 KB)) );
645 outBuff.dst = (char*)compressedBuffer;
646 outBuff.size = compressedBufferSize;
647 outBuff.pos = 0;
648 inBuff.src = CNBuffer;
649 inBuff.size = CNBufferSize;
650 inBuff.pos = 0;
651 { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff);
652 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */
653 DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r));
654 } }
655
f535537f 656 /* Compression state reuse scenario */
657 DISPLAYLEVEL(3, "test%3i : context reuse : ", testNb++);
648db22b 658 ZSTD_freeCStream(zc);
659 zc = ZSTD_createCStream();
660 if (zc==NULL) goto _output_error; /* memory allocation issue */
661 /* use 1 */
662 { size_t const inSize = 513;
663 DISPLAYLEVEL(5, "use1 ");
664 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
665 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
666 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
667 inBuff.src = CNBuffer;
668 inBuff.size = inSize;
669 inBuff.pos = 0;
670 outBuff.dst = (char*)(compressedBuffer)+cSize;
671 outBuff.size = ZSTD_compressBound(inSize);
672 outBuff.pos = 0;
673 DISPLAYLEVEL(5, "compress1 ");
674 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
675 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
676 DISPLAYLEVEL(5, "end1 ");
677 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
678 }
679 /* use 2 */
680 { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */
681 DISPLAYLEVEL(5, "use2 ");
682 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
683 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 19) );
684 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, inSize) );
685 inBuff.src = CNBuffer;
686 inBuff.size = inSize;
687 inBuff.pos = 0;
688 outBuff.dst = (char*)(compressedBuffer)+cSize;
689 outBuff.size = ZSTD_compressBound(inSize);
690 outBuff.pos = 0;
691 DISPLAYLEVEL(5, "compress2 ");
692 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
693 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
694 DISPLAYLEVEL(5, "end2 ");
695 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; /* error, or some data not flushed */
696 }
697 DISPLAYLEVEL(3, "OK \n");
698
699 /* Decompression single pass with empty frame */
700 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, 1);
701 CHECK_Z(cSize);
702 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass on empty frame : ", testNb++);
703 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
704 size_t const dctxSize = ZSTD_sizeof_DCtx(dctx);
705 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
706
707 outBuff.dst = decodedBuffer;
708 outBuff.pos = 0;
709 outBuff.size = CNBufferSize;
710
711 inBuff.src = compressedBuffer;
712 inBuff.size = cSize;
713 inBuff.pos = 0;
714 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
715 CHECK_Z(r);
716 CHECK(r != 0, "Entire frame must be decompressed");
717 CHECK(outBuff.pos != 0, "Wrong size!");
718 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
719 }
720 CHECK(dctxSize != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
721 ZSTD_freeDCtx(dctx);
722 }
723 DISPLAYLEVEL(3, "OK \n");
724
f535537f 725 DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2KB : ", testNb++);
726 {
727 ZSTD_DCtx* dctx = ZSTD_createDCtx();
728 size_t singlePassSize, streamingSize, streaming2KSize;
729
730 {
731 ZSTD_CCtx* cctx = ZSTD_createCCtx();
732 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
733 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18));
734 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
735 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048));
736 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize);
737 CHECK_Z(cSize);
738 ZSTD_freeCCtx(cctx);
739 }
740
741 CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBufferSize, compressedBuffer, cSize));
742 singlePassSize = ZSTD_sizeof_DCtx(dctx);
743 CHECK_Z(singlePassSize);
744
745 inBuff.src = compressedBuffer;
746 inBuff.size = cSize;
747
748 outBuff.dst = decodedBuffer;
749 outBuff.size = decodedBufferSize;
750
751 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048));
752 inBuff.pos = 0;
753 outBuff.pos = 0;
754 {
755 size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
756 CHECK_Z(r);
757 CHECK(r != 0, "Entire frame must be decompressed");
758 }
759 streaming2KSize = ZSTD_sizeof_DCtx(dctx);
760 CHECK_Z(streaming2KSize);
761
762 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters));
763 inBuff.pos = 0;
764 outBuff.pos = 0;
765 {
766 size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
767 CHECK_Z(r);
768 CHECK(r != 0, "Entire frame must be decompressed");
769 }
770 streamingSize = ZSTD_sizeof_DCtx(dctx);
771 CHECK_Z(streamingSize);
772
773 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024));
774 inBuff.pos = 0;
775 outBuff.pos = 0;
776 CHECK(!ZSTD_isError(ZSTD_decompressStream(dctx, &outBuff, &inBuff)), "decompression must fail");
777
778 CHECK(streamingSize < singlePassSize + (1 << 18) + 3 * ZSTD_BLOCKSIZE_MAX, "Streaming doesn't use the right amount of memory");
779 CHECK(streamingSize != streaming2KSize + 3 * (ZSTD_BLOCKSIZE_MAX - 2048), "ZSTD_d_blockSizeMax didn't save the right amount of memory");
780 DISPLAYLEVEL(3, "| %zu | %zu | %zu | ", singlePassSize, streaming2KSize, streamingSize);
781
782 ZSTD_freeDCtx(dctx);
783 }
784 DISPLAYLEVEL(3, "OK \n");
785
648db22b 786 /* Decompression with ZSTD_d_stableOutBuffer */
787 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1);
788 CHECK_Z(cSize);
789 { ZSTD_DCtx* dctx = ZSTD_createDCtx();
790 size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx);
791 size_t dctxSize1;
792 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
793
794 outBuff.dst = decodedBuffer;
795 outBuff.pos = 0;
796 outBuff.size = CNBufferSize;
797
798 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass : ", testNb++);
799 inBuff.src = compressedBuffer;
800 inBuff.size = cSize;
801 inBuff.pos = 0;
802 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
803 CHECK_Z(r);
804 CHECK(r != 0, "Entire frame must be decompressed");
805 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
806 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
807 }
808 CHECK(dctxSize0 != ZSTD_sizeof_DCtx(dctx), "No buffers allocated");
809 DISPLAYLEVEL(3, "OK \n");
810
811 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer : ", testNb++);
812 outBuff.pos = 0;
813 inBuff.pos = 0;
814 inBuff.size = 0;
815 while (inBuff.pos < cSize) {
816 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
817 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
818 }
819 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
820 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
821 dctxSize1 = ZSTD_sizeof_DCtx(dctx);
822 CHECK(!(dctxSize0 < dctxSize1), "Input buffer allocated");
823 DISPLAYLEVEL(3, "OK \n");
824
825 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer too small : ", testNb++);
826 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
827 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
828 inBuff.src = compressedBuffer;
829 inBuff.size = cSize;
830 inBuff.pos = 0;
831 outBuff.pos = 0;
832 outBuff.size = CNBufferSize - 1;
833 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
834 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall, "Must error but got %s", ZSTD_getErrorName(r));
835 }
836 DISPLAYLEVEL(3, "OK \n");
837
838 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer modified : ", testNb++);
839 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
840 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1));
841 inBuff.src = compressedBuffer;
842 inBuff.size = cSize - 1;
843 inBuff.pos = 0;
844 outBuff.pos = 0;
845 outBuff.size = CNBufferSize;
846 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
847 ++inBuff.size;
848 outBuff.pos = 0;
849 { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff);
850 CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r));
851 }
852 DISPLAYLEVEL(3, "OK \n");
853
854 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++);
855 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
856 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0));
857 outBuff.pos = 0;
858 inBuff.pos = 0;
859 inBuff.size = 0;
860 while (inBuff.pos < cSize) {
861 inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15));
862 CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff));
863 }
864 CHECK(outBuff.pos != CNBufferSize, "Wrong size!");
865 CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!");
866 CHECK(!(dctxSize1 < ZSTD_sizeof_DCtx(dctx)), "Output buffer allocated");
867 DISPLAYLEVEL(3, "OK \n");
868
869 ZSTD_freeDCtx(dctx);
870 }
871
872 /* Compression with ZSTD_c_stable{In,Out}Buffer */
873 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
874 ZSTD_inBuffer in;
875 ZSTD_outBuffer out;
876 size_t cctxSize1;
877 size_t cctxSize2;
878 assert(cctx != NULL);
879 in.src = CNBuffer;
880 in.size = CNBufferSize;
881 out.dst = compressedBuffer;
882 out.size = compressedBufferSize;
883 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
884 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() uses stable input and output : ", testNb++);
885 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
886 CHECK(!(cSize < ZSTD_compressBound(CNBufferSize)), "cSize too large for test");
887 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, cSize + 4, CNBuffer, CNBufferSize));
888 CHECK_Z(cctxSize1 = ZSTD_sizeof_CCtx(cctx));
889 /* @cctxSize2 : sizeof_CCtx when doing full streaming (no stable in/out) */
890 { ZSTD_CCtx* const cctx2 = ZSTD_createCCtx();
891 assert(cctx2 != NULL);
892 in.pos = out.pos = 0;
893 CHECK_Z(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_continue));
894 CHECK(!(ZSTD_compressStream2(cctx2, &out, &in, ZSTD_e_end) == 0), "Not finished");
895 CHECK_Z(cctxSize2 = ZSTD_sizeof_CCtx(cctx2));
896 ZSTD_freeCCtx(cctx2);
897 }
898 /* @cctxSize1 : sizeof_CCtx when doing single-shot compression (no streaming) */
899 { ZSTD_CCtx* const cctx1 = ZSTD_createCCtx();
900 ZSTD_parameters params = ZSTD_getParams(0, CNBufferSize, 0);
901 size_t cSize3;
902 assert(cctx1 != NULL);
903 params.fParams.checksumFlag = 1;
904 cSize3 = ZSTD_compress_advanced(cctx1, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, NULL, 0, params);
905 CHECK_Z(cSize3);
906 CHECK(!(cSize == cSize3), "Must be same compressed size");
907 CHECK(!(cctxSize1 == ZSTD_sizeof_CCtx(cctx1)), "Must be same CCtx size");
908 ZSTD_freeCCtx(cctx1);
909 }
910 CHECK(!(cctxSize1 < cctxSize2), "Stable buffers means less allocated size");
911 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
912 DISPLAYLEVEL(3, "OK \n");
913
914 DISPLAYLEVEL(3, "test%3i : ZSTD_compress2() doesn't modify user parameters : ", testNb++);
915 { int stableInBuffer;
916 int stableOutBuffer;
917 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
918 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
919 CHECK(!(stableInBuffer == 0), "Modified");
920 CHECK(!(stableOutBuffer == 0), "Modified");
921 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
922 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
923 CHECK_Z(cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize));
924 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableInBuffer, &stableInBuffer));
925 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_stableOutBuffer, &stableOutBuffer));
926 CHECK(!(stableInBuffer == 1), "Modified");
927 CHECK(!(stableOutBuffer == 1), "Modified");
928 }
929 DISPLAYLEVEL(3, "OK \n");
930
931 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer : ", testNb++);
932 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
933 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
934 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
935 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
936 in.pos = out.pos = 0;
937 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
938 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
939 DISPLAYLEVEL(3, "OK \n");
940
941 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer and ZSTD_c_stableOutBuffer allocated size : ", testNb++);
942 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
943 CHECK(!(cctxSize1 == cctxSize), "Must be the same size as single pass");
944 }
945 DISPLAYLEVEL(3, "OK \n");
946
947 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer only : ", testNb++);
948 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
949 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
950 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1));
951 in.pos = out.pos = 0;
952 out.size = cSize / 4;
953 for (;;) {
954 size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
955 CHECK_Z(ret);
956 if (ret == 0)
957 break;
958 out.size = MIN(out.size + cSize / 4, compressedBufferSize);
959 }
960 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
961 DISPLAYLEVEL(3, "OK \n");
962
963 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableInBuffer modify buffer : ", testNb++);
964 in.pos = out.pos = 0;
965 out.size = cSize / 4;
966 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
967 in.src = (char const*)in.src + in.pos;
968 in.size -= in.pos;
969 in.pos = 0;
970 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
971 CHECK(!ZSTD_isError(ret), "Must error");
972 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
973 }
974 DISPLAYLEVEL(3, "OK \n");
975
976 /* stableSrc + streaming */
977 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream, flushStream and endStream : ", testNb++);
978 CHECK_Z( ZSTD_initCStream(cctx, 1) );
979 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
980 { ZSTD_inBuffer inBuf;
981 ZSTD_outBuffer outBuf;
982 const size_t nonZeroStartPos = 18;
983 const size_t inputSize = 500;
984 inBuf.src = CNBuffer;
985 inBuf.size = 100;
986 inBuf.pos = nonZeroStartPos;
987 outBuf.dst = (char*)(compressedBuffer)+cSize;
988 outBuf.size = ZSTD_compressBound(inputSize);
989 outBuf.pos = 0;
990 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
991 inBuf.size = 200;
992 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
993 CHECK_Z( ZSTD_flushStream(cctx, &outBuf) );
994 inBuf.size = nonZeroStartPos + inputSize;
995 CHECK_Z( ZSTD_compressStream(cctx, &outBuf, &inBuf) );
996 CHECK(ZSTD_endStream(cctx, &outBuf) != 0, "compression should be successful and fully flushed");
997 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
998 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
999 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
1000 CHECK_Z(decSize);
1001 CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
1002 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
1003 } }
1004 DISPLAYLEVEL(3, "OK \n");
1005
1006 /* stableSrc + streaming */
1007 DISPLAYLEVEL(3, "test%3i : ZSTD_c_stableInBuffer compatibility with compressStream2, using different end directives : ", testNb++);
1008 CHECK_Z( ZSTD_initCStream(cctx, 1) );
1009 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableInBuffer, 1) );
1010 { ZSTD_inBuffer inBuf;
1011 ZSTD_outBuffer outBuf;
1012 const size_t nonZeroStartPos = 18;
1013 const size_t inputSize = 500;
1014 inBuf.src = CNBuffer;
1015 inBuf.size = 100;
1016 inBuf.pos = nonZeroStartPos;
1017 outBuf.dst = (char*)(compressedBuffer)+cSize;
1018 outBuf.size = ZSTD_compressBound(inputSize);
1019 outBuf.pos = 0;
1020 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
1021 inBuf.size = 200;
1022 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
1023 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_flush) );
1024 inBuf.size = nonZeroStartPos + inputSize;
1025 CHECK_Z( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_continue) );
1026 CHECK( ZSTD_compressStream2(cctx, &outBuf, &inBuf, ZSTD_e_end) != 0, "compression should be successful and fully flushed");
1027 { const void* const realSrcStart = (const char*)inBuf.src + nonZeroStartPos;
1028 void* const verifBuf = (char*)outBuf.dst + outBuf.pos;
1029 const size_t decSize = ZSTD_decompress(verifBuf, inputSize, outBuf.dst, outBuf.pos);
1030 CHECK_Z(decSize);
1031 CHECK(decSize != inputSize, "regenerated %zu bytes, instead of %zu", decSize, inputSize);
1032 CHECK(memcmp(realSrcStart, verifBuf, inputSize) != 0, "regenerated data different from original");
1033 } }
1034 DISPLAYLEVEL(3, "OK \n");
1035
1036 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableInBuffer: context size : ", testNb++);
1037 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
1038 DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
1039 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass");
1040 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
1041 cctxSize1 = cctxSize;
1042 }
1043 DISPLAYLEVEL(3, "OK \n");
1044
1045 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer only : ", testNb++);
1046 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
1047 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1048 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_stableOutBuffer, 1));
1049 in.src = CNBuffer;
1050 in.pos = out.pos = 0;
1051 in.size = MIN(CNBufferSize, 10);
1052 out.size = compressedBufferSize;
1053 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1054 in.pos = 0;
1055 in.size = CNBufferSize - in.size;
1056 CHECK(!(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) == 0), "Not finished");
1057 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, out.pos));
1058 DISPLAYLEVEL(3, "OK \n");
1059
1060 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() ZSTD_c_stableOutBuffer modify buffer : ", testNb++);
1061 in.pos = out.pos = 0;
1062 in.size = CNBufferSize;
1063 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1064 in.pos = out.pos = 0;
1065 { size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1066 CHECK(!ZSTD_isError(ret), "Must have errored");
1067 CHECK(!(ZSTD_getErrorCode(ret) == ZSTD_error_stabilityCondition_notRespected), "Must be this error");
1068 }
1069 DISPLAYLEVEL(3, "OK \n");
1070
1071 DISPLAYLEVEL(3, "test%3i : ZSTD_compressStream2() with ZSTD_c_stableOutBuffer: context size : ", testNb++);
1072 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
1073 DISPLAYLEVEL(4, "cctxSize1=%zu; cctxSize=%zu; cctxSize2=%zu : ", cctxSize1, cctxSize, cctxSize2);
1074 CHECK(!(cctxSize1 < cctxSize), "Must be bigger than single-pass and stableInBuffer");
1075 CHECK(!(cctxSize < cctxSize2), "Must be smaller than streaming");
1076 }
1077 DISPLAYLEVEL(3, "OK \n");
1078
1079 ZSTD_freeCCtx(cctx);
1080 }
1081
1082 /* CDict scenario */
1083 DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++);
1084 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ );
1085 size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict);
1086 DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError);
1087 if (ZSTD_isError(initError)) goto _output_error;
1088 outBuff.dst = compressedBuffer;
1089 outBuff.size = compressedBufferSize;
1090 outBuff.pos = 0;
1091 inBuff.src = CNBuffer;
1092 inBuff.size = CNBufferSize;
1093 inBuff.pos = 0;
1094 DISPLAYLEVEL(5, "- starting ZSTD_compressStream ");
1095 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1096 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1097 { size_t const r = ZSTD_endStream(zc, &outBuff);
1098 DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r);
1099 if (r != 0) goto _output_error; /* error, or some data not flushed */
1100 }
1101 cSize = outBuff.pos;
1102 ZSTD_freeCDict(cdict);
1103 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1104 }
1105
1106 DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++);
1107 { size_t const s = ZSTD_sizeof_CStream(zc);
1108 if (ZSTD_isError(s)) goto _output_error;
1109 DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s);
1110 }
1111
1112 DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++);
1113 { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1114 if (dID != dictID) goto _output_error;
1115 DISPLAYLEVEL(4, "OK (%u) \n", dID);
1116 }
1117
1118 /* DDict scenario */
1119 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize);
1120 { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1121 size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict);
1122 if (ZSTD_isError(initError)) goto _output_error;
1123 outBuff.dst = decodedBuffer;
1124 outBuff.size = CNBufferSize;
1125 outBuff.pos = 0;
1126 inBuff.src = compressedBuffer;
1127 inBuff.size = cSize;
1128 inBuff.pos = 0;
1129 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1130 if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */
1131 if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */
1132 if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */
1133 ZSTD_freeDDict(ddict);
1134 DISPLAYLEVEL(3, "OK \n");
1135 }
1136
1137 /* Memory restriction */
1138 DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++);
1139 ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize);
1140 CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */
1141 outBuff.dst = decodedBuffer;
1142 outBuff.size = CNBufferSize;
1143 outBuff.pos = 0;
1144 inBuff.src = compressedBuffer;
1145 inBuff.size = cSize;
1146 inBuff.pos = 0;
1147 { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff);
1148 if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */
1149 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); }
1150 ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */
1151
1152 DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++);
1153 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1154 int const maxLevel = 16; /* first level with zstd_opt */
1155 int level;
1156 assert(maxLevel < ZSTD_maxCLevel());
1157 CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) );
1158 for (level = 1; level <= maxLevel; ++level) {
1159 ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level);
1160 size_t const maxSize = MIN(1 MB, CNBufferSize);
1161 size_t size;
1162 for (size = 512; size <= maxSize; size <<= 1) {
1163 U64 const crcOrig = XXH64(CNBuffer, size, 0);
1164 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1165 ZSTD_parameters savedParams;
1166 getCCtxParams(cctx, &savedParams);
1167 outBuff.dst = compressedBuffer;
1168 outBuff.size = compressedBufferSize;
1169 outBuff.pos = 0;
1170 inBuff.src = CNBuffer;
1171 inBuff.size = size;
1172 inBuff.pos = 0;
1173 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
1174 CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end));
1175 CHECK(badParameters(cctx, savedParams), "Bad CCtx params");
1176 if (inBuff.pos != inBuff.size) goto _output_error;
1177 { ZSTD_outBuffer decOut = {decodedBuffer, size, 0};
1178 ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0};
1179 CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) );
1180 if (decIn.pos != decIn.size) goto _output_error;
1181 if (decOut.pos != size) goto _output_error;
1182 { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0);
1183 if (crcDec != crcOrig) goto _output_error;
1184 } }
1185 ZSTD_freeCCtx(cctx);
1186 }
1187 ZSTD_freeCDict(cdict);
1188 }
1189 ZSTD_freeDCtx(dctx);
1190 }
1191 DISPLAYLEVEL(3, "OK\n");
1192
1193 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1194 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) );
1195 cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB));
1196 CHECK_Z(cSize);
1197 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++);
1198 {
1199 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1200 /* We should fail to decompress without a dictionary. */
1201 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1202 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1203 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1204 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1205 if (!ZSTD_isError(ret)) goto _output_error;
1206 }
1207 /* We should succeed to decompress with the dictionary. */
1208 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1209 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1210 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1211 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1212 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1213 if (in.pos != in.size) goto _output_error;
1214 }
1215 /* The dictionary should persist across calls. */
1216 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1217 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1218 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1219 if (in.pos != in.size) goto _output_error;
1220 }
1221 /* The dictionary should not be cleared by ZSTD_reset_session_only. */
1222 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only);
1223 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1224 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1225 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1226 if (in.pos != in.size) goto _output_error;
1227 }
1228 /* When we reset the context the dictionary is cleared. */
1229 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1230 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1231 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1232 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1233 if (!ZSTD_isError(ret)) goto _output_error;
1234 }
1235 ZSTD_freeDCtx(dctx);
1236 }
1237 DISPLAYLEVEL(3, "OK \n");
1238
1239 DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++);
1240 {
1241 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1242 /* We should succeed to decompress with the dictionary. */
1243 ZSTD_resetDStream(dctx);
1244 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) );
1245 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1246 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1247 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1248 if (in.pos != in.size) goto _output_error;
1249 }
1250 /* The dictionary should not be cleared by ZSTD_resetDStream(). */
1251 ZSTD_resetDStream(dctx);
1252 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1253 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1254 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1255 if (in.pos != in.size) goto _output_error;
1256 }
1257 /* The dictionary should be cleared by ZSTD_initDStream(). */
1258 CHECK_Z( ZSTD_initDStream(dctx) );
1259 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1260 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1261 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1262 if (!ZSTD_isError(ret)) goto _output_error;
1263 }
1264 ZSTD_freeDCtx(dctx);
1265 }
1266 DISPLAYLEVEL(3, "OK \n");
1267
1268 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++);
1269 {
1270 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1271 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1272 /* We should succeed to decompress with the ddict. */
1273 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1274 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
1275 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1276 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1277 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1278 if (in.pos != in.size) goto _output_error;
1279 }
1280 /* The ddict should persist across calls. */
1281 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1282 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1283 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1284 if (in.pos != in.size) goto _output_error;
1285 }
1286 /* When we reset the context the ddict is cleared. */
1287 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1288 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1289 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1290 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1291 if (!ZSTD_isError(ret)) goto _output_error;
1292 }
1293 ZSTD_freeDCtx(dctx);
1294 ZSTD_freeDDict(ddict);
1295 }
1296 DISPLAYLEVEL(3, "OK \n");
1297
1298 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
1299 {
1300 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1301 /* We should succeed to decompress with the prefix. */
1302 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
1303 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) );
1304 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1305 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1306 if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error;
1307 if (in.pos != in.size) goto _output_error;
1308 }
1309 /* The prefix should be cleared after the first compression. */
1310 { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1311 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1312 size_t const ret = ZSTD_decompressStream(dctx, &out, &in);
1313 if (!ZSTD_isError(ret)) goto _output_error;
1314 }
1315 ZSTD_freeDCtx(dctx);
1316 }
1317 DISPLAYLEVEL(3, "OK \n");
1318
1319 DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++);
1320 {
1321 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1322 ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1323 size_t ret;
1324 /* We should succeed to decompress with the dictionary. */
1325 CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) );
1326 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1327 /* The dictionary should persist across calls. */
1328 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1329 /* We should succeed to decompress with the ddict. */
1330 CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) );
1331 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1332 /* The ddict should persist across calls. */
1333 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) );
1334 /* When we reset the context the ddict is cleared. */
1335 CHECK_Z( ZSTD_initDStream(dctx) );
1336 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize);
1337 if (!ZSTD_isError(ret)) goto _output_error;
1338 ZSTD_freeDCtx(dctx);
1339 ZSTD_freeDDict(ddict);
1340 }
1341 DISPLAYLEVEL(3, "OK \n");
1342
1343 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++);
1344 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled);
1345 ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */};
1346 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
1347 size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize);
1348 if (ZSTD_isError(initError)) goto _output_error;
1349 outBuff.dst = compressedBuffer;
1350 outBuff.size = compressedBufferSize;
1351 outBuff.pos = 0;
1352 inBuff.src = CNBuffer;
1353 inBuff.size = CNBufferSize;
1354 inBuff.pos = 0;
1355 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1356 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1357 { size_t const r = ZSTD_endStream(zc, &outBuff);
1358 if (r != 0) goto _output_error; } /* error, or some data not flushed */
1359 cSize = outBuff.pos;
1360 ZSTD_freeCDict(cdict);
1361 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1362 }
1363
1364 DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++);
1365 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
1366 if (did != 0) goto _output_error;
1367 }
1368 DISPLAYLEVEL(3, "OK (not detected) \n");
1369
1370 DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++);
1371 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1372 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1373 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1374 }
1375
1376 DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++);
1377 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) );
1378 outBuff.dst = compressedBuffer;
1379 outBuff.size = compressedBufferSize;
1380 outBuff.pos = 0;
1381 inBuff.src = CNBuffer;
1382 inBuff.size = CNBufferSize;
1383 inBuff.pos = 0;
1384 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1385 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1386 cSize = outBuff.pos;
1387 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1388
1389 DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++);
1390 CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) );
1391 outBuff.dst = decodedBuffer;
1392 outBuff.size = CNBufferSize;
1393 outBuff.pos = 0;
1394 inBuff.src = compressedBuffer;
1395 inBuff.size = cSize;
1396 inBuff.pos = 0;
1397 CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) );
1398 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1399 if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */
1400 DISPLAYLEVEL(3, "OK \n");
1401
1402 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++);
1403 { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize);
1404 if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */
1405 DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r));
1406 }
1407
1408 DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++);
1409 outBuff.dst = compressedBuffer;
1410 outBuff.size = compressedBufferSize;
1411 outBuff.pos = 0;
1412 inBuff.src = CNBuffer;
1413 inBuff.size = CNBufferSize;
1414 inBuff.pos = 0;
1415 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1416 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1417 cSize = outBuff.pos;
1418 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100);
1419
1420 DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++);
1421 CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) );
1422 DISPLAYLEVEL(3, "OK \n");
1423
1424 /* Empty srcSize */
1425 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++);
1426 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1427 params.fParams.contentSizeFlag = 1;
1428 CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) );
1429 } /* cstream advanced shall write content size = 0 */
1430 outBuff.dst = compressedBuffer;
1431 outBuff.size = compressedBufferSize;
1432 outBuff.pos = 0;
1433 inBuff.src = CNBuffer;
1434 inBuff.size = 0;
1435 inBuff.pos = 0;
1436 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1437 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1438 cSize = outBuff.pos;
1439 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1440 DISPLAYLEVEL(3, "OK \n");
1441
1442 DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly with ZSTD_initCStream_advanced : ", testNb++);
1443 { ZSTD_parameters params = ZSTD_getParams(5, 0, 0);
1444 params.fParams.contentSizeFlag = 1;
1445 CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) );
1446 } /* cstream advanced shall write content size = 0 */
1447 inBuff.src = CNBuffer;
1448 inBuff.size = 0;
1449 inBuff.pos = 0;
1450 outBuff.dst = compressedBuffer;
1451 outBuff.size = compressedBufferSize;
1452 outBuff.pos = 0;
1453 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1454 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1455 cSize = outBuff.pos;
1456 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error;
1457
1458 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1459 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1460 outBuff.dst = compressedBuffer;
1461 outBuff.size = compressedBufferSize;
1462 outBuff.pos = 0;
1463 inBuff.src = CNBuffer;
1464 inBuff.size = 0;
1465 inBuff.pos = 0;
1466 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
1467 if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error;
1468 cSize = outBuff.pos;
1469 if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
1470 DISPLAYLEVEL(3, "OK \n");
1471
1472 /* Basic multithreading compression test */
1473 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);
1474 { int jobSize;
1475 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1476 CHECK(jobSize != 0, "job size non-zero");
1477 CHECK_Z( ZSTD_CCtx_getParameter(mtctx, ZSTD_c_jobSize, &jobSize));
1478 CHECK(jobSize != 0, "job size non-zero");
1479 }
1480 outBuff.dst = compressedBuffer;
1481 outBuff.size = compressedBufferSize;
1482 outBuff.pos = 0;
1483 inBuff.src = CNBuffer;
1484 inBuff.size = CNBufferSize;
1485 inBuff.pos = 0;
1486 { size_t const compressResult = ZSTD_compressStream2(mtctx, &outBuff, &inBuff, ZSTD_e_end);
1487 if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */
1488 }
1489 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1490 { size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos);
1491 if (compressedSize != outBuff.pos) goto _output_error; /* must be a full valid frame */
1492 }
1493 DISPLAYLEVEL(3, "OK \n");
1494
1495 /* Complex multithreading + dictionary test */
1496 { U32 const nbWorkers = 2;
1497 size_t const jobSize = 4 * 1 MB;
1498 size_t const srcSize = jobSize * nbWorkers; /* we want each job to have predictable size */
1499 size_t const segLength = 2 KB;
1500 size_t const offset = 600 KB; /* must be larger than window defined in cdict */
1501 size_t const start = jobSize + (offset-1);
1502 const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start;
1503 BYTE* const dst = (BYTE*)CNBuffer + start - offset;
1504 DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize);
1505 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) );
1506 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) );
1507 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) );
1508 assert(start > offset);
1509 assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH);
1510 memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */
1511 outBuff.dst = compressedBuffer;
1512 outBuff.size = compressedBufferSize;
1513 outBuff.pos = 0;
1514 inBuff.src = CNBuffer;
1515 inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH);
1516 inBuff.pos = 0;
1517 }
1518 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */
1519 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1520 DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog);
1521 CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) );
1522 CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) );
1523 CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */
1524 ZSTD_freeCDict(cdict);
1525 }
1526 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1527 cSize = outBuff.pos;
1528 DISPLAYLEVEL(3, "OK \n");
1529
1530 DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++);
1531 { ZSTD_DStream* const dstream = ZSTD_createDCtx();
1532 ZSTD_frameHeader zfh;
1533 ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize);
1534 DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize);
1535 outBuff.dst = decodedBuffer;
1536 outBuff.size = CNBufferSize;
1537 outBuff.pos = 0;
1538 inBuff.src = compressedBuffer;
1539 inBuff.pos = 0;
1540 CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) );
1541 inBuff.size = 1; /* avoid shortcut to single-pass mode */
1542 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1543 inBuff.size = cSize;
1544 CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) );
1545 if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */
1546 ZSTD_freeDStream(dstream);
1547 }
1548 DISPLAYLEVEL(3, "OK \n");
1549
1550 DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++);
1551 { unsigned const kMaxWindowLog = 24;
1552 unsigned value;
1553 ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024);
1554 ZSTD_CDict* cdict;
1555 ZSTD_DDict* ddict;
1556 SEQ_stream seq = SEQ_initStream(0x87654321);
1557 SEQ_gen_type type;
1558 XXH64_state_t xxh;
1559
1560 XXH64_reset(&xxh, 0);
1561 cParams.windowLog = kMaxWindowLog;
1562 cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
1563 ddict = ZSTD_createDDict(dictionary.start, dictionary.filled);
1564
1565 if (!cdict || !ddict) goto _output_error;
1566
1567 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1568 ZSTD_resetDStream(zd);
1569 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1570 CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict));
1571 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog));
1572 /* Test all values < 300 */
1573 for (value = 0; value < 300; ++value) {
1574 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1575 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1576 }
1577 }
1578 /* Test values 2^8 to 2^17 */
1579 for (value = (1 << 8); value < (1 << 17); value <<= 1) {
1580 for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) {
1581 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value));
1582 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2)));
1583 }
1584 }
1585 /* Test offset values up to the max window log */
1586 for (value = 8; value <= kMaxWindowLog; ++value) {
1587 CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1));
1588 }
1589
1590 CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end));
1591 CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match");
1592
1593 ZSTD_freeCDict(cdict);
1594 ZSTD_freeDDict(ddict);
1595 }
1596 DISPLAYLEVEL(3, "OK \n");
1597
1598 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++);
1599 { int level;
1600 CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN));
1601 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1602 CHECK(level != 11, "Compression level does not match");
1603 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1604 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1605 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level));
1606 CHECK(level != 11, "Compression level does not match");
1607 }
1608 DISPLAYLEVEL(3, "OK \n");
1609
1610 DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++);
1611 { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0);
1612 CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN));
1613 CHECK(badParameters(zc, params), "Compression parameters do not match");
1614 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
1615 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, ZSTD_CONTENTSIZE_UNKNOWN) );
1616 CHECK(badParameters(zc, params), "Compression parameters do not match");
1617 }
1618 DISPLAYLEVEL(3, "OK \n");
1619
1620 DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++);
1621 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
1622 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX));
1623 { int srcSizeHint;
1624 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint));
1625 CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match");
1626 }
1627 CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error");
1628 DISPLAYLEVEL(3, "OK \n");
1629
1630 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy compress with hashLog = 29 and searchLog = 4 : ", testNb++);
1631 if (MEM_64bits()) {
1632 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
1633 ZSTD_inBuffer in = { CNBuffer, CNBufferSize, 0 };
1634 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1635 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_strategy, ZSTD_lazy));
1636 /* Force enable the row based match finder */
1637 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
1638 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_searchLog, 4));
1639 /* Set windowLog to 29 so the hashLog doesn't get sized down */
1640 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, 29));
1641 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_hashLog, 29));
1642 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1643 /* Compress with continue first so the hashLog doesn't get sized down */
1644 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_continue));
1645 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_end));
1646 cSize = out.pos;
1647 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize));
1648 }
1649 DISPLAYLEVEL(3, "OK \n");
1650
1651 DISPLAYLEVEL(3, "test%3i : Test offset == windowSize : ", testNb++);
1652 {
1653 int windowLog;
1654 int const kMaxWindowLog = bigTests ? 29 : 26;
1655 size_t const kNbSequences = 10000;
1656 size_t const kMaxSrcSize = (1u << kMaxWindowLog) + 10 * kNbSequences;
1657 char* src = calloc(kMaxSrcSize, 1);
1658 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
1659 for (windowLog = ZSTD_WINDOWLOG_MIN; windowLog <= kMaxWindowLog; ++windowLog) {
1660 size_t const srcSize = ((size_t)1 << windowLog) + 10 * (kNbSequences - 1);
1661
1662 sequences[0].offset = 32;
1663 sequences[0].litLength = 32;
1664 sequences[0].matchLength = (1u << windowLog) - 32;
1665 sequences[0].rep = 0;
1666 {
1667 size_t i;
1668 for (i = 1; i < kNbSequences; ++i) {
1669 sequences[i].offset = (1u << windowLog) - (FUZ_rand(&seed) % 8);
1670 sequences[i].litLength = FUZ_rand(&seed) & 7;
1671 sequences[i].matchLength = 10 - sequences[i].litLength;
1672 sequences[i].rep = 0;
1673 }
1674 }
1675
1676 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1677 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1678 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_minMatch, 3));
1679 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, 1));
1680 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, windowLog));
1681 assert(srcSize <= kMaxSrcSize);
1682 cSize = ZSTD_compressSequences(zc, compressedBuffer, compressedBufferSize, sequences, kNbSequences, src, srcSize);
1683 CHECK_Z(cSize);
1684 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
1685 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, windowLog))
1686 {
1687 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1688 size_t decompressedBytes = 0;
1689 for (;;) {
1690 ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0};
1691 size_t const ret = ZSTD_decompressStream(zd, &out, &in);
1692 CHECK_Z(ret);
1693 CHECK(decompressedBytes + out.pos > srcSize, "Output too large");
1694 CHECK(memcmp(out.dst, src + decompressedBytes, out.pos), "Corrupted");
1695 decompressedBytes += out.pos;
1696 if (ret == 0) {
1697 break;
1698 }
1699 }
1700 CHECK(decompressedBytes != srcSize, "Output wrong size");
1701 }
1702 }
1703 free(sequences);
1704 free(src);
1705 }
1706 DISPLAYLEVEL(3, "OK \n");
1707
1708 /* Overlen overwriting window data bug */
1709 DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++);
1710 { /* This test has a window size of 1024 bytes and consists of 3 blocks:
1711 1. 'a' repeated 517 times
1712 2. 'b' repeated 516 times
1713 3. a compressed block with no literals and 3 sequence commands:
1714 litlength = 0, offset = 24, match length = 24
1715 litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3)
1716 litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */
1717
1718 const char* testCase =
1719 "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A"
1720 "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00"
1721 "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0";
1722 ZSTD_DStream* const zds = ZSTD_createDStream();
1723 if (zds==NULL) goto _output_error;
1724
1725 CHECK_Z( ZSTD_initDStream(zds) );
1726 inBuff.src = testCase;
1727 inBuff.size = 47;
1728 inBuff.pos = 0;
1729 outBuff.dst = decodedBuffer;
1730 outBuff.size = CNBufferSize;
1731 outBuff.pos = 0;
1732
1733 while (inBuff.pos < inBuff.size) {
1734 CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) );
1735 }
1736
1737 ZSTD_freeDStream(zds);
1738 }
1739 DISPLAYLEVEL(3, "OK \n");
1740
1741 /* Small Sequence Section bug */
1742 DISPLAYLEVEL(3, "test%3i : decompress blocks with small sequences section : ", testNb++);
1743 { /* This test consists of 3 blocks. Each block has one sequence.
1744 The sequence has literal length of 10, match length of 10 and offset of 10.
1745 The sequence value and compression mode for the blocks are following:
1746 The order of values are ll, ml, of.
1747 - First block : (10, 7, 13) (rle, rle, rle)
1748 - size of sequences section: 6 bytes (1 byte for nbSeq, 1 byte for encoding mode, 3 bytes for rle, 1 byte bitstream)
1749 - Second block : (10, 7, 1) (repeat, repeat, rle)
1750 - size of sequences section: 4 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 bytes for rle, 1 byte bitstream)
1751 - Third block : (10, 7, 1) (repeat, repeat, repeat)
1752 - size of sequences section: 3 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 byte bitstream) */
1753
1754 unsigned char compressed[] = {
1755 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x3c, 0x35, 0x01, 0x00, 0xf0, 0x85, 0x08,
1756 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1757 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac,
1758 0x69, 0x94, 0x89, 0x1c, 0x03, 0x44, 0x0a, 0x07, 0x00, 0xb4, 0x04, 0x80,
1759 0x40, 0x0a, 0xa4
1760 };
1761 unsigned int compressedSize = 51;
1762 unsigned char decompressed[] = {
1763 0x85, 0x08, 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x85, 0x08,
1764 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b,
1765 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0x4c, 0x6b, 0xa9, 0x8b, 0xbc, 0xc5,
1766 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94,
1767 0x89, 0x1c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, 0x89, 0x1c
1768 };
1769 unsigned int decompressedSize = 60;
1770
1771 ZSTD_DStream* const zds = ZSTD_createDStream();
1772 if (zds==NULL) goto _output_error;
1773
1774 CHECK_Z( ZSTD_initDStream(zds) );
1775 inBuff.src = compressed;
1776 inBuff.size = compressedSize;
1777 inBuff.pos = 0;
1778 outBuff.dst = decodedBuffer;
1779 outBuff.size = CNBufferSize;
1780 outBuff.pos = 0;
1781
1782 CHECK(ZSTD_decompressStream(zds, &outBuff, &inBuff) != 0,
1783 "Decompress did not reach the end of frame");
1784 CHECK(inBuff.pos != inBuff.size, "Decompress did not fully consume input");
1785 CHECK(outBuff.pos != decompressedSize, "Decompressed size does not match");
1786 CHECK(memcmp(outBuff.dst, decompressed, decompressedSize) != 0,
1787 "Decompressed data does not match");
1788
1789 ZSTD_freeDStream(zds);
1790 }
1791 DISPLAYLEVEL(3, "OK \n");
1792
1793 DISPLAYLEVEL(3, "test%3i : raw block can be streamed: ", testNb++);
1794 { size_t const inputSize = 10000;
1795 size_t const compCapacity = ZSTD_compressBound(inputSize);
1796 BYTE* const input = (BYTE*)malloc(inputSize);
1797 BYTE* const comp = (BYTE*)malloc(compCapacity);
1798 BYTE* const decomp = (BYTE*)malloc(inputSize);
1799
1800 CHECK(input == NULL || comp == NULL || decomp == NULL, "failed to alloc buffers");
1801
1802 RDG_genBuffer(input, inputSize, 0.0, 0.0, seed);
1803 { size_t const compSize = ZSTD_compress(comp, compCapacity, input, inputSize, -(int)inputSize);
1804 ZSTD_inBuffer in = { comp, 0, 0 };
1805 ZSTD_outBuffer out = { decomp, 0, 0 };
1806 CHECK_Z(compSize);
1807 CHECK_Z( ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters) );
1808 while (in.size < compSize) {
1809 in.size = MIN(in.size + 100, compSize);
1810 while (in.pos < in.size) {
1811 size_t const outPos = out.pos;
1812 if (out.pos == out.size) {
1813 out.size = MIN(out.size + 10, inputSize);
1814 }
1815 CHECK_Z( ZSTD_decompressStream(zd, &out, &in) );
1816 CHECK(!(out.pos > outPos), "We are not streaming (no output generated)");
1817 }
1818 }
1819 CHECK(in.pos != compSize, "Not all input consumed!");
1820 CHECK(out.pos != inputSize, "Not all output produced!");
1821 }
1822 CHECK(memcmp(input, decomp, inputSize), "round trip failed!");
1823
1824 free(input);
1825 free(comp);
1826 free(decomp);
1827 }
1828 DISPLAYLEVEL(3, "OK \n");
1829
1830 DISPLAYLEVEL(3, "test%3i : dictionary + uncompressible block + reusing tables checks offset table validity: ", testNb++);
1831 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1832 dictionary.start, dictionary.filled,
1833 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1834 ZSTD_getCParams(3, 0, dictionary.filled),
1835 ZSTD_defaultCMem);
1836 const size_t inbufsize = 2 * 128 * 1024; /* 2 blocks */
1837 const size_t outbufsize = ZSTD_compressBound(inbufsize);
1838 size_t inbufpos = 0;
1839 size_t cursegmentlen;
1840 BYTE *inbuf = (BYTE *)malloc(inbufsize);
1841 BYTE *outbuf = (BYTE *)malloc(outbufsize);
1842 BYTE *checkbuf = (BYTE *)malloc(inbufsize);
1843 size_t ret;
1844
1845 CHECK(cdict == NULL, "failed to alloc cdict");
1846 CHECK(inbuf == NULL, "failed to alloc input buffer");
1847
1848 /* first block is uncompressible */
1849 cursegmentlen = 128 * 1024;
1850 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0., 0., seed);
1851 inbufpos += cursegmentlen;
1852
1853 /* second block is compressible */
1854 cursegmentlen = 128 * 1024 - 256;
1855 RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0.05, 0., seed);
1856 inbufpos += cursegmentlen;
1857
1858 /* and includes a very long backref */
1859 cursegmentlen = 128;
1860 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 256, cursegmentlen);
1861 inbufpos += cursegmentlen;
1862
1863 /* and includes a very long backref */
1864 cursegmentlen = 128;
1865 memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 128, cursegmentlen);
1866 inbufpos += cursegmentlen;
1867
1868 ret = ZSTD_compress_usingCDict(zc, outbuf, outbufsize, inbuf, inbufpos, cdict);
1869 CHECK_Z(ret);
1870
1871 ret = ZSTD_decompress_usingDict(zd, checkbuf, inbufsize, outbuf, ret, dictionary.start, dictionary.filled);
1872 CHECK_Z(ret);
1873
1874 CHECK(memcmp(inbuf, checkbuf, inbufpos), "start and finish buffers don't match");
1875
1876 ZSTD_freeCDict(cdict);
1877 free(inbuf);
1878 free(outbuf);
1879 free(checkbuf);
1880 }
1881 DISPLAYLEVEL(3, "OK \n");
1882
1883 DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++);
1884 { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(
1885 dictionary.start, dictionary.filled,
1886 ZSTD_dlm_byRef, ZSTD_dct_fullDict,
1887 ZSTD_getCParams(3, 0, dictionary.filled),
1888 ZSTD_defaultCMem);
1889 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1890 int remainingInput = 256 * 1024;
1891 int offset;
1892
1893 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1894 CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict));
1895 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1));
1896 /* Write a bunch of 6 byte blocks */
1897 while (remainingInput > 0) {
1898 char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA";
1899 const size_t kSmallBlockSize = sizeof(testBuffer);
1900 ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0};
1901
1902 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush));
1903 CHECK(in.pos != in.size, "input not fully consumed");
1904 remainingInput -= kSmallBlockSize;
1905 }
1906 /* Write several very long offset matches into the dictionary */
1907 for (offset = 1024; offset >= 0; offset -= 128) {
1908 ZSTD_inBuffer in = {(BYTE*)dictionary.start + offset, 128, 0};
1909 ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end;
1910 CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush));
1911 CHECK(in.pos != in.size, "input not fully consumed");
1912 }
1913 /* Ensure decompression works */
1914 CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled));
1915
1916 ZSTD_freeCDict(cdict);
1917 }
1918 DISPLAYLEVEL(3, "OK \n");
1919
1920 DISPLAYLEVEL(3, "test%3i : Block-Level External Sequence Producer API: ", testNb++);
1921 {
1922 size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
f535537f 1923 BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
648db22b 1924 size_t const checkBufSize = CNBufferSize;
1925 BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
1926 int enableFallback;
1927 EMF_testCase sequenceProducerState;
1928
1929 CHECK(dstBuf == NULL || checkBuf == NULL, "allocation failed");
1930
1931 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
1932
1933 /* Reference external matchfinder outside the test loop to
1934 * check that the reference is preserved across compressions */
1935 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
1936
1937 for (enableFallback = 0; enableFallback <= 1; enableFallback++) {
1938 size_t testCaseId;
1939 size_t const numTestCases = 9;
1940
1941 EMF_testCase const testCases[] = {
1942 EMF_ONE_BIG_SEQ,
1943 EMF_LOTS_OF_SEQS,
1944 EMF_ZERO_SEQS,
1945 EMF_BIG_ERROR,
1946 EMF_SMALL_ERROR,
1947 EMF_INVALID_OFFSET,
1948 EMF_INVALID_MATCHLEN,
1949 EMF_INVALID_LITLEN,
1950 EMF_INVALID_LAST_LITS
1951 };
1952
1953 ZSTD_ErrorCode const errorCodes[] = {
1954 ZSTD_error_no_error,
1955 ZSTD_error_no_error,
1956 ZSTD_error_sequenceProducer_failed,
1957 ZSTD_error_sequenceProducer_failed,
1958 ZSTD_error_sequenceProducer_failed,
1959 ZSTD_error_externalSequences_invalid,
1960 ZSTD_error_externalSequences_invalid,
1961 ZSTD_error_externalSequences_invalid,
1962 ZSTD_error_externalSequences_invalid
1963 };
1964
1965 for (testCaseId = 0; testCaseId < numTestCases; testCaseId++) {
1966 size_t res;
1967
1968 int const compressionShouldSucceed = (
1969 (errorCodes[testCaseId] == ZSTD_error_no_error) ||
1970 (enableFallback && errorCodes[testCaseId] == ZSTD_error_sequenceProducer_failed)
1971 );
1972
1973 int const testWithSequenceValidation = (
1974 testCases[testCaseId] == EMF_INVALID_OFFSET
1975 );
1976
1977 sequenceProducerState = testCases[testCaseId];
1978
1979 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
1980 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_validateSequences, testWithSequenceValidation));
1981 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
1982 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
1983
1984 if (compressionShouldSucceed) {
1985 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
1986 CHECK_Z(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res));
1987 CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
1988 } else {
1989 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
1990 CHECK(
1991 ZSTD_getErrorCode(res) != errorCodes[testCaseId],
1992 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
1993 );
1994 }
1995 }
1996
1997 /* Test compression with external matchfinder + empty src buffer */
1998 {
1999 size_t res;
2000 sequenceProducerState = EMF_ZERO_SEQS;
2001 ZSTD_CCtx_reset(zc, ZSTD_reset_session_only);
2002 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, enableFallback));
2003 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, 0);
2004 CHECK(ZSTD_isError(res), "EMF: Compression error: %s", ZSTD_getErrorName(res));
2005 CHECK(ZSTD_decompress(checkBuf, checkBufSize, dstBuf, res) != 0, "EMF: Empty src round trip failed!");
2006 }
2007 }
2008
2009 /* Test that reset clears the external matchfinder */
2010 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
2011 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
2012 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
2013 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
2014
2015 /* Test that registering mFinder == NULL clears the external matchfinder */
2016 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
2017 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2018 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder wasn't cleared */
2019 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
2020 ZSTD_registerSequenceProducer(zc, NULL, NULL); /* clear the external matchfinder */
2021 CHECK_Z(ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize));
2022
2023 /* Test that external matchfinder doesn't interact with older APIs */
2024 ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters);
2025 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2026 sequenceProducerState = EMF_BIG_ERROR; /* ensure zstd will fail if the matchfinder is used */
2027 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableSeqProducerFallback, 0));
2028 CHECK_Z(ZSTD_compressCCtx(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize, 3));
2029
2030 /* Test that compression returns the correct error with LDM */
2031 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
2032 {
2033 size_t res;
2034 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2035 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
2036 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2037 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
2038 CHECK(
2039 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
2040 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
2041 );
2042 }
2043
2044#ifdef ZSTD_MULTITHREAD
2045 /* Test that compression returns the correct error with nbWorkers > 0 */
2046 CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters));
2047 {
2048 size_t res;
2049 ZSTD_registerSequenceProducer(zc, &sequenceProducerState, zstreamSequenceProducer);
2050 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, 1));
2051 res = ZSTD_compress2(zc, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2052 CHECK(!ZSTD_isError(res), "EMF: Should have raised an error!");
2053 CHECK(
2054 ZSTD_getErrorCode(res) != ZSTD_error_parameter_combination_unsupported,
2055 "EMF: Wrong error code: %s", ZSTD_getErrorName(res)
2056 );
2057 }
2058#endif
2059
2060 free(dstBuf);
2061 free(checkBuf);
2062 }
2063 DISPLAYLEVEL(3, "OK \n");
2064
2065
2066 /* Test maxBlockSize cctx param functionality */
2067 DISPLAYLEVEL(3, "test%3i : Testing maxBlockSize PR#3418: ", testNb++);
2068 {
2069 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2070
2071 /* Quick test to make sure maxBlockSize bounds are enforced */
2072 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN - 1)));
2073 assert(ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX + 1)));
2074
2075 /* Test maxBlockSize < windowSize and windowSize < maxBlockSize*/
2076 {
2077 size_t srcSize = 2 << 10;
2078 void* const src = CNBuffer;
2079 size_t dstSize = ZSTD_compressBound(srcSize);
2080 void* const dst1 = compressedBuffer;
2081 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2082 size_t size1, size2;
2083 void* const checkBuf = malloc(srcSize);
2084 memset(src, 'x', srcSize);
2085
2086 /* maxBlockSize = 1KB */
2087 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2088 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2089
2090 if (ZSTD_isError(size1)) goto _output_error;
2091 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2092 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2093
2094 /* maxBlockSize = 3KB */
2095 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2096 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2097
2098 if (ZSTD_isError(size2)) goto _output_error;
2099 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2100 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2101
2102 assert(size1 - size2 == 4); /* We add another RLE block with header + character */
2103 assert(memcmp(dst1, dst2, size2) != 0); /* Compressed output should not be equal */
2104
2105 /* maxBlockSize = 1KB, windowLog = 10 */
2106 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 1u << 10));
2107 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2108 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2109
2110 if (ZSTD_isError(size1)) goto _output_error;
2111 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2112 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2113
2114 /* maxBlockSize = 3KB, windowLog = 10 */
2115 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 3u << 10));
2116 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 10));
2117 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2118
2119 if (ZSTD_isError(size2)) goto _output_error;
2120 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2121 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2122
2123 assert(size1 == size2);
2124 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2125
2126 free(checkBuf);
2127 }
2128
2129 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2130
2131 /* Test maxBlockSize = 0 is valid */
2132 { size_t srcSize = 256 << 10;
2133 void* const src = CNBuffer;
2134 size_t dstSize = ZSTD_compressBound(srcSize);
2135 void* const dst1 = compressedBuffer;
2136 void* const dst2 = (BYTE*)compressedBuffer + dstSize;
2137 size_t size1, size2;
2138 void* const checkBuf = malloc(srcSize);
2139
2140 /* maxBlockSize = 0 */
2141 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 0));
2142 size1 = ZSTD_compress2(cctx, dst1, dstSize, src, srcSize);
2143
2144 if (ZSTD_isError(size1)) goto _output_error;
2145 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst1, size1));
2146 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2147
2148 /* maxBlockSize = ZSTD_BLOCKSIZE_MAX */
2149 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX));
2150 size2 = ZSTD_compress2(cctx, dst2, dstSize, src, srcSize);
2151
2152 if (ZSTD_isError(size2)) goto _output_error;
2153 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst2, size2));
2154 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2155
2156 assert(size1 == size2);
2157 assert(memcmp(dst1, dst2, size1) == 0); /* Compressed output should be equal */
2158 free(checkBuf);
2159 }
2160 ZSTD_freeCCtx(cctx);
2161 }
2162 DISPLAYLEVEL(3, "OK \n");
2163
2164 /* Test Sequence Validation */
2165 DISPLAYLEVEL(3, "test%3i : Testing sequence validation: ", testNb++);
2166 {
2167 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2168
2169 /* Test minMatch >= 4, matchLength < 4 */
2170 {
2171 size_t srcSize = 11;
2172 void* const src = CNBuffer;
2173 size_t dstSize = ZSTD_compressBound(srcSize);
2174 void* const dst = compressedBuffer;
2175 size_t const kNbSequences = 4;
2176 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2177
2178 memset(src, 'x', srcSize);
2179
2180 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2181 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2182 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2183 sequences[3] = (ZSTD_Sequence) {0, 1, 0, 0};
2184
2185 /* Test with sequence validation */
2186 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2187 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2188 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2189
2190 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2191 sequences, kNbSequences,
2192 src, srcSize);
2193
2194 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2195 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2196
2197 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2198
2199 /* Test without sequence validation */
2200 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2201 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2202 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2203
2204 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2205 sequences, kNbSequences,
2206 src, srcSize);
2207
2208 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2209 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2210
2211 free(sequences);
2212 }
2213
2214 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2215
2216
2217 /* Test with no block delim */
2218 {
2219 size_t srcSize = 4;
2220 void* const src = CNBuffer;
2221 size_t dstSize = ZSTD_compressBound(srcSize);
2222 void* const dst = compressedBuffer;
2223 size_t const kNbSequences = 1;
2224 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2225 void* const checkBuf = malloc(srcSize);
2226
2227 memset(src, 'x', srcSize);
2228
2229 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2230
2231 /* Test with sequence validation */
2232 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 3));
2233 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters));
2234 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2235
2236 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2237 sequences, kNbSequences,
2238 src, srcSize);
2239
2240 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2241 CHECK_Z(ZSTD_decompress(checkBuf, srcSize, dst, cSize));
2242 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2243
2244 free(sequences);
2245 free(checkBuf);
2246 }
2247
2248 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2249
2250 { /* Test case with two additional sequences */
2251 size_t srcSize = 19;
2252 void* const src = CNBuffer;
2253 size_t dstSize = ZSTD_compressBound(srcSize);
2254 void* const dst = compressedBuffer;
2255 size_t const kNbSequences = 7;
2256 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2257
2258 memset(src, 'x', srcSize);
2259
2260 sequences[0] = (ZSTD_Sequence) {1, 1, 3, 0};
2261 sequences[1] = (ZSTD_Sequence) {1, 0, 3, 0};
2262 sequences[2] = (ZSTD_Sequence) {1, 0, 3, 0};
2263 sequences[3] = (ZSTD_Sequence) {1, 0, 3, 0};
2264 sequences[4] = (ZSTD_Sequence) {1, 0, 3, 0};
2265 sequences[5] = (ZSTD_Sequence) {1, 0, 3, 0};
2266 sequences[6] = (ZSTD_Sequence) {0, 0, 0, 0};
2267
2268 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2269 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2270 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 1));
2271
2272 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2273 sequences, kNbSequences,
2274 src, srcSize);
2275
2276 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2277 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2278
2279 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2280
2281 /* Test without sequence validation */
2282 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 5));
2283 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters));
2284 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_validateSequences, 0));
2285
2286 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2287 sequences, kNbSequences,
2288 src, srcSize);
2289
2290 CHECK(!ZSTD_isError(cSize), "Should throw an error"); /* maxNbSeq is too small and an assert will fail */
2291 CHECK(ZSTD_getErrorCode(cSize) != ZSTD_error_externalSequences_invalid, "Wrong error code: %s", ZSTD_getErrorName(cSize)); /* fails sequence validation */
2292
2293 free(sequences);
2294 }
2295 ZSTD_freeCCtx(cctx);
2296 }
2297 DISPLAYLEVEL(3, "OK \n");
2298
2299
2300 DISPLAYLEVEL(3, "test%3i : Testing large offset with small window size: ", testNb++);
2301 {
2302 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2303 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2304
2305 /* Test large offset, small window size*/
2306 {
2307 size_t srcSize = 21;
2308 void* const src = CNBuffer;
2309 size_t dstSize = ZSTD_compressBound(srcSize);
2310 void* const dst = compressedBuffer;
2311 size_t const kNbSequences = 4;
2312 ZSTD_Sequence* sequences = malloc(sizeof(ZSTD_Sequence) * kNbSequences);
2313 void* const checkBuf = malloc(srcSize);
2314 const size_t largeDictSize = 1 << 25;
2315 ZSTD_CDict* cdict = NULL;
2316 ZSTD_DDict* ddict = NULL;
2317
2318 /* Generate large dictionary */
2319 void* dictBuffer = calloc(largeDictSize, 1);
2320 ZSTD_compressionParameters cParams = ZSTD_getCParams(1, srcSize, largeDictSize);
2321 cParams.minMatch = ZSTD_MINMATCH_MIN;
2322 cParams.hashLog = ZSTD_HASHLOG_MIN;
2323 cParams.chainLog = ZSTD_CHAINLOG_MIN;
2324
2325 cdict = ZSTD_createCDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, cParams, ZSTD_defaultCMem);
2326 ddict = ZSTD_createDDict_advanced(dictBuffer, largeDictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent, ZSTD_defaultCMem);
2327
2328 ZSTD_CCtx_refCDict(cctx, cdict);
2329 ZSTD_DCtx_refDDict(dctx, ddict);
2330
2331 sequences[0] = (ZSTD_Sequence) {3, 3, 3, 0};
2332 sequences[1] = (ZSTD_Sequence) {1 << 25, 0, 3, 0};
2333 sequences[2] = (ZSTD_Sequence) {1 << 25, 0, 9, 0};
2334 sequences[3] = (ZSTD_Sequence) {3, 0, 3, 0};
2335
2336 cSize = ZSTD_compressSequences(cctx, dst, dstSize,
2337 sequences, kNbSequences,
2338 src, srcSize);
2339
2340 CHECK(ZSTD_isError(cSize), "Should not throw an error");
2341
2342 {
2343 size_t dSize = ZSTD_decompressDCtx(dctx, checkBuf, srcSize, dst, cSize);
2344 CHECK(ZSTD_isError(dSize), "Should not throw an error");
2345 CHECK(memcmp(src, checkBuf, srcSize) != 0, "Corruption!");
2346 }
2347
2348 free(sequences);
2349 free(checkBuf);
2350 free(dictBuffer);
2351 ZSTD_freeCDict(cdict);
2352 ZSTD_freeDDict(ddict);
2353 }
2354 ZSTD_freeCCtx(cctx);
2355 ZSTD_freeDCtx(dctx);
2356 }
2357 DISPLAYLEVEL(3, "OK \n");
2358
f535537f 2359 DISPLAYLEVEL(3, "test%3i : Testing external sequence producer with static CCtx: ", testNb++);
2360 {
2361 size_t const dstBufSize = ZSTD_compressBound(CNBufferSize);
2362 BYTE* const dstBuf = (BYTE*)malloc(dstBufSize);
2363 size_t const checkBufSize = CNBufferSize;
2364 BYTE* const checkBuf = (BYTE*)malloc(checkBufSize);
2365 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2366 ZSTD_CCtx* staticCCtx;
2367 void* cctxBuf;
2368 EMF_testCase seqProdState;
2369
2370 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_validateSequences, 1));
2371 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_enableSeqProducerFallback, 0));
2372 ZSTD_CCtxParams_registerSequenceProducer(params, &seqProdState, zstreamSequenceProducer);
2373
2374 {
2375 size_t const cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2376 cctxBuf = malloc(cctxSize);
2377 staticCCtx = ZSTD_initStaticCCtx(cctxBuf, cctxSize);
2378 ZSTD_CCtx_setParametersUsingCCtxParams(staticCCtx, params);
2379 }
2380
2381 // Check that compression with external sequence producer succeeds when expected
2382 seqProdState = EMF_LOTS_OF_SEQS;
2383 {
2384 size_t dResult;
2385 size_t const cResult = ZSTD_compress2(staticCCtx, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2386 CHECK(ZSTD_isError(cResult), "EMF: Compression error: %s", ZSTD_getErrorName(cResult));
2387 dResult = ZSTD_decompress(checkBuf, checkBufSize, dstBuf, cResult);
2388 CHECK(ZSTD_isError(dResult), "EMF: Decompression error: %s", ZSTD_getErrorName(dResult));
2389 CHECK(dResult != CNBufferSize, "EMF: Corruption!");
2390 CHECK(memcmp(CNBuffer, checkBuf, CNBufferSize) != 0, "EMF: Corruption!");
2391 }
2392
2393 // Check that compression with external sequence producer fails when expected
2394 seqProdState = EMF_BIG_ERROR;
2395 {
2396 size_t const cResult = ZSTD_compress2(staticCCtx, dstBuf, dstBufSize, CNBuffer, CNBufferSize);
2397 CHECK(!ZSTD_isError(cResult), "EMF: Should have raised an error!");
2398 CHECK(
2399 ZSTD_getErrorCode(cResult) != ZSTD_error_sequenceProducer_failed,
2400 "EMF: Wrong error code: %s", ZSTD_getErrorName(cResult)
2401 );
2402 }
2403
2404 free(dstBuf);
2405 free(checkBuf);
2406 free(cctxBuf);
2407 ZSTD_freeCCtxParams(params);
2408 }
2409 DISPLAYLEVEL(3, "OK \n");
2410
2411 DISPLAYLEVEL(3, "test%3i : Decoder should reject invalid frame header on legacy frames: ", testNb++);
2412 {
2413 const unsigned char compressed[] = { 0x26,0xb5,0x2f,0xfd,0x50,0x91,0xfd,0xd8,0xb5 };
2414 const size_t compressedSize = 9;
2415 size_t const dSize = ZSTD_decompress(NULL, 0, compressed, compressedSize);
2416 CHECK(!ZSTD_isError(dSize), "must reject when legacy frame header is invalid");
2417 }
2418 DISPLAYLEVEL(3, "OK \n");
2419
2420 DISPLAYLEVEL(3, "test%3i : Test single-shot fallback for magicless mode: ", testNb++);
2421 {
2422 // Aquire resources
2423 size_t const srcSize = COMPRESSIBLE_NOISE_LENGTH;
2424 void* src = malloc(srcSize);
2425 size_t const dstSize = ZSTD_compressBound(srcSize);
2426 void* dst = malloc(dstSize);
2427 size_t const valSize = srcSize;
2428 void* val = malloc(valSize);
2429 ZSTD_inBuffer inBuf = { dst, dstSize, 0 };
2430 ZSTD_outBuffer outBuf = { val, valSize, 0 };
2431 ZSTD_CCtx* cctx = ZSTD_createCCtx();
2432 ZSTD_DCtx* dctx = ZSTD_createDCtx();
2433 CHECK(!src || !dst || !val || !dctx || !cctx, "memory allocation failure");
2434
2435 // Write test data for decompression to dst
2436 RDG_genBuffer(src, srcSize, compressibility, 0.0, 0xdeadbeef);
2437 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless));
2438 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
2439
2440 // Run decompression
2441 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless));
2442 CHECK_Z(ZSTD_decompressStream(dctx, &outBuf, &inBuf));
2443
2444 // Validate
2445 CHECK(outBuf.pos != srcSize, "decompressed size must match");
2446 CHECK(memcmp(src, val, srcSize) != 0, "decompressed data must match");
2447
2448 // Cleanup
2449 free(src); free(dst); free(val);
2450 ZSTD_freeCCtx(cctx);
2451 ZSTD_freeDCtx(dctx);
2452 }
2453 DISPLAYLEVEL(3, "OK \n");
2454
648db22b 2455_end:
2456 FUZ_freeDictionary(dictionary);
2457 ZSTD_freeCStream(zc);
2458 ZSTD_freeDStream(zd);
2459 ZSTD_freeCCtx(mtctx);
2460 free(CNBuffer);
2461 free(compressedBuffer);
2462 free(decodedBuffer);
2463 return testResult;
2464
2465_output_error:
2466 testResult = 1;
2467 DISPLAY("Error detected in Unit tests ! \n");
2468 goto _end;
2469}
2470
2471
2472/* ====== Fuzzer tests ====== */
2473
2474static size_t findDiff(const void* buf1, const void* buf2, size_t max)
2475{
2476 const BYTE* b1 = (const BYTE*)buf1;
2477 const BYTE* b2 = (const BYTE*)buf2;
2478 size_t u;
2479 for (u=0; u<max; u++) {
2480 if (b1[u] != b2[u]) break;
2481 }
2482 if (u==max) {
2483 DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max);
2484 return u;
2485 }
2486 DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max);
2487 if (u>=3)
2488 DISPLAY(" %02X %02X %02X ",
2489 b1[u-3], b1[u-2], b1[u-1]);
2490 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2491 b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]);
2492 if (u>=3)
2493 DISPLAY(" %02X %02X %02X ",
2494 b2[u-3], b2[u-2], b2[u-1]);
2495 DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n",
2496 b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]);
2497 return u;
2498}
2499
2500static size_t FUZ_rLogLength(U32* seed, U32 logLength)
2501{
2502 size_t const lengthMask = ((size_t)1 << logLength) - 1;
2503 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
2504}
2505
2506static size_t FUZ_randomLength(U32* seed, U32 maxLog)
2507{
2508 U32 const logLength = FUZ_rand(seed) % maxLog;
2509 return FUZ_rLogLength(seed, logLength);
2510}
2511
2512/* Return value in range minVal <= v <= maxVal */
2513static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal)
2514{
2515 U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal;
2516 return (U32)((FUZ_rand(seed) % mod) + minVal);
2517}
2518
2519static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests)
2520{
2521 U32 const maxSrcLog = bigTests ? 24 : 22;
2522 static const U32 maxSampleLog = 19;
2523 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2524 BYTE* cNoiseBuffer[5];
2525 size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog);
2526 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2527 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2528 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2529 size_t const dstBufferSize = srcBufferSize;
2530 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2531 U32 result = 0;
2532 unsigned testNb = 0;
2533 U32 coreSeed = seed;
2534 ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */
2535 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */
2536 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2537 UTIL_time_t const startClock = UTIL_getTime();
2538 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2539 size_t dictSize = 0;
2540 U32 oldTestLog = 0;
2541 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests;
2542
2543 /* allocations */
2544 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2545 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2546 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2547 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2548 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2549 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2550 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2551 "Not enough memory, fuzzer tests cancelled");
2552
2553 /* Create initial samples */
2554 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2555 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2556 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2557 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2558 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2559 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2560 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2561
2562 /* catch up testNb */
2563 for (testNb=1; testNb < startTest; testNb++)
2564 FUZ_rand(&coreSeed);
2565
2566 /* test loop */
2567 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2568 U32 lseed;
2569 const BYTE* srcBuffer;
2570 size_t totalTestSize, totalGenSize, cSize;
2571 XXH64_state_t xxhState;
2572 U64 crcOrig;
2573 U32 resetAllowed = 1;
2574 size_t maxTestSize;
2575
2576 /* init */
2577 FUZ_rand(&coreSeed);
2578 lseed = coreSeed ^ prime32;
2579 if (nbTests >= testNb) {
2580 DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests);
2581 } else {
2582 DISPLAYUPDATE(2, "\r%6u ", testNb);
2583 }
2584
2585 /* states full reset (deliberately not synchronized) */
2586 /* some issues can only happen when reusing states */
2587 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2588 ZSTD_freeCStream(zc);
2589 zc = ZSTD_createCStream();
2590 CHECK(zc==NULL, "ZSTD_createCStream : allocation error");
2591 resetAllowed=0;
2592 }
2593 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2594 ZSTD_freeDStream(zd);
2595 zd = ZSTD_createDStream();
2596 CHECK(zd==NULL, "ZSTD_createDStream : allocation error");
2597 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2598 }
2599
2600 /* srcBuffer selection [0-4] */
2601 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2602 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2603 else {
2604 buffNb >>= 3;
2605 if (buffNb & 7) {
2606 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2607 buffNb = tnb[buffNb >> 3];
2608 } else {
2609 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2610 buffNb = tnb[buffNb >> 3];
2611 } }
2612 srcBuffer = cNoiseBuffer[buffNb];
2613 }
2614
2615 /* compression init */
2616 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2617 && oldTestLog /* at least one test happened */ && resetAllowed) {
2618 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2619 maxTestSize = MIN(maxTestSize, srcBufferSize-16);
2620 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2621 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2622 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2623 }
2624 } else {
2625 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2626 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2627 U32 const cLevelCandidate = ( FUZ_rand(&lseed) %
2628 (ZSTD_maxCLevel() -
2629 (MAX(testLog, dictLog) / 3)))
2630 + 1;
2631 U32 const cLevel = MIN(cLevelCandidate, cLevelMax);
2632 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2633 oldTestLog = testLog;
2634 /* random dictionary selection */
2635 dictSize = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2636 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2637 dict = srcBuffer + dictStart;
2638 }
2639 { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2640 CHECK_Z( ZSTD_CCtx_reset(zc, ZSTD_reset_session_only) );
2641 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, cLevel) );
2642 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, FUZ_rand(&lseed) & 1) );
2643 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1) );
2644 CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1) );
2645 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2646 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
2647 } }
2648
2649 /* multi-segments compression test */
2650 XXH64_reset(&xxhState, 0);
2651 { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
2652 cSize=0;
2653 totalTestSize=0;
2654 while(totalTestSize < maxTestSize) {
2655 /* compress random chunks into randomly sized dst buffers */
2656 { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2657 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
2658 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
2659 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2660 size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize);
2661 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
2662 outBuff.size = outBuff.pos + dstBuffSize;
2663
2664 CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
2665
2666 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
2667 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
2668 totalTestSize += inBuff.pos;
2669 }
2670
2671 /* random flush operation, to mess around */
2672 if ((FUZ_rand(&lseed) & 15) == 0) {
2673 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2674 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2675 outBuff.size = outBuff.pos + adjustedDstSize;
2676 CHECK_Z( ZSTD_flushStream(zc, &outBuff) );
2677 } }
2678
2679 /* final frame epilogue */
2680 { size_t remainingToFlush = (size_t)(-1);
2681 while (remainingToFlush) {
2682 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2683 size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize);
2684 outBuff.size = outBuff.pos + adjustedDstSize;
2685 remainingToFlush = ZSTD_endStream(zc, &outBuff);
2686 CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush));
2687 } }
2688 crcOrig = XXH64_digest(&xxhState);
2689 cSize = outBuff.pos;
2690 }
2691
2692 /* multi - fragments decompression test */
2693 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
2694 CHECK_Z ( ZSTD_resetDStream(zd) );
2695 } else {
2696 CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
2697 }
2698 { size_t decompressionResult = 1;
2699 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2700 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2701 for (totalGenSize = 0 ; decompressionResult ; ) {
2702 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2703 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2704 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
2705 inBuff.size = inBuff.pos + readCSrcSize;
2706 outBuff.size = outBuff.pos + dstBuffSize;
2707 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2708 if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) {
2709 DISPLAY("checksum error : \n");
2710 findDiff(copyBuffer, dstBuffer, totalTestSize);
2711 }
2712 CHECK( ZSTD_isError(decompressionResult), "decompression error : %s",
2713 ZSTD_getErrorName(decompressionResult) );
2714 }
2715 CHECK (decompressionResult != 0, "frame not fully decoded");
2716 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)",
2717 (unsigned)outBuff.pos, (unsigned)totalTestSize);
2718 CHECK (inBuff.pos != cSize, "compressed data should be fully read")
2719 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
2720 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
2721 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
2722 } }
2723
2724 /*===== noisy/erroneous src decompression test =====*/
2725
2726 /* add some noise */
2727 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
2728 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
2729 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
2730 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
2731 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
2732 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
2733 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
2734 } }
2735
2736 /* try decompression on noisy data */
2737 CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */
2738 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
2739 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
2740 while (outBuff.pos < dstBufferSize) {
2741 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
2742 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
2743 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
2744 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
2745 outBuff.size = outBuff.pos + adjustedDstSize;
2746 inBuff.size = inBuff.pos + adjustedCSrcSize;
2747 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
2748 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
2749 /* No forward progress possible */
2750 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
2751 } } } }
2752 DISPLAY("\r%u fuzzer tests completed \n", testNb);
2753
2754_cleanup:
2755 ZSTD_freeCStream(zc);
2756 ZSTD_freeDStream(zd);
2757 ZSTD_freeDStream(zd_noise);
2758 free(cNoiseBuffer[0]);
2759 free(cNoiseBuffer[1]);
2760 free(cNoiseBuffer[2]);
2761 free(cNoiseBuffer[3]);
2762 free(cNoiseBuffer[4]);
2763 free(copyBuffer);
2764 free(cBuffer);
2765 free(dstBuffer);
2766 return result;
2767
2768_output_error:
2769 result = 1;
2770 goto _cleanup;
2771}
2772
2773/** If useOpaqueAPI, sets param in cctxParams.
2774 * Otherwise, sets the param in zc. */
2775static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams,
2776 ZSTD_cParameter param, unsigned value,
2777 int useOpaqueAPI)
2778{
2779 if (useOpaqueAPI) {
2780 return ZSTD_CCtxParams_setParameter(cctxParams, param, value);
2781 } else {
2782 return ZSTD_CCtx_setParameter(zc, param, value);
2783 }
2784}
2785
2786/* Tests for ZSTD_compress_generic() API */
2787static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest,
2788 double compressibility, int bigTests)
2789{
2790 U32 const maxSrcLog = bigTests ? 24 : 22;
2791 static const U32 maxSampleLog = 19;
2792 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
2793 BYTE* cNoiseBuffer[5];
2794 size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog);
2795 BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize);
2796 size_t const cBufferSize = ZSTD_compressBound(srcBufferSize);
2797 BYTE* const cBuffer = (BYTE*)malloc (cBufferSize);
2798 size_t const dstBufferSize = srcBufferSize;
2799 BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize);
2800 U32 result = 0;
2801 int testNb = 0;
2802 U32 coreSeed = seed;
2803 ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */
2804 ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */
2805 ZSTD_DStream* const zd_noise = ZSTD_createDStream();
2806 UTIL_time_t const startClock = UTIL_getTime();
2807 const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */
2808 size_t dictSize = 0;
2809 U32 oldTestLog = 0;
2810 U32 windowLogMalus = 0; /* can survive between 2 loops */
2811 U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests;
2812 U32 const nbThreadsMax = bigTests ? 4 : 2;
2813 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
2814
2815 /* allocations */
2816 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
2817 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
2818 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
2819 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
2820 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
2821 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] ||
2822 !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise ,
2823 "Not enough memory, fuzzer tests cancelled");
2824
2825 /* Create initial samples */
2826 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
2827 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
2828 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
2829 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
2830 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
2831 memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */
2832 CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */
2833
2834 /* catch up testNb */
2835 for (testNb=1; testNb < startTest; testNb++)
2836 FUZ_rand(&coreSeed);
2837
2838 /* test loop */
2839 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) {
2840 U32 lseed;
2841 int opaqueAPI;
2842 const BYTE* srcBuffer;
2843 size_t totalTestSize, totalGenSize, cSize;
2844 XXH64_state_t xxhState;
2845 U64 crcOrig;
2846 U32 resetAllowed = 1;
2847 size_t maxTestSize;
2848 ZSTD_parameters savedParams;
2849 int isRefPrefix = 0;
2850 U64 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2851
2852 /* init */
2853 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
2854 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
2855 FUZ_rand(&coreSeed);
2856 lseed = coreSeed ^ prime32;
2857 DISPLAYLEVEL(5, " *** Test %u *** \n", testNb);
2858 opaqueAPI = FUZ_rand(&lseed) & 1;
2859
2860 /* states full reset (deliberately not synchronized) */
2861 /* some issues can only happen when reusing states */
2862 if ((FUZ_rand(&lseed) & 0xFF) == 131) {
2863 DISPLAYLEVEL(5, "Creating new context \n");
2864 ZSTD_freeCCtx(zc);
2865 zc = ZSTD_createCCtx();
2866 CHECK(zc == NULL, "ZSTD_createCCtx allocation error");
2867 resetAllowed = 0;
2868 }
2869 if ((FUZ_rand(&lseed) & 0xFF) == 132) {
2870 ZSTD_freeDStream(zd);
2871 zd = ZSTD_createDStream();
2872 CHECK(zd == NULL, "ZSTD_createDStream allocation error");
2873 ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */
2874 }
2875
2876 /* srcBuffer selection [0-4] */
2877 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
2878 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
2879 else {
2880 buffNb >>= 3;
2881 if (buffNb & 7) {
2882 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
2883 buffNb = tnb[buffNb >> 3];
2884 } else {
2885 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
2886 buffNb = tnb[buffNb >> 3];
2887 } }
2888 srcBuffer = cNoiseBuffer[buffNb];
2889 }
2890
2891 /* compression init */
2892 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/
2893 if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */
2894 && oldTestLog /* at least one test happened */
2895 && resetAllowed) {
2896 /* just set a compression level */
2897 maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2);
2898 if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1;
2899 { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1;
2900 DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel);
2901 CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) );
2902 }
2903 } else {
2904 U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
2905 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
2906 U32 const cLevelCandidate = (FUZ_rand(&lseed) %
2907 (ZSTD_maxCLevel() -
2908 (MAX(testLog, dictLog) / 2))) +
2909 1;
2910 int const cLevel = MIN(cLevelCandidate, cLevelMax);
2911 DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel);
2912 maxTestSize = FUZ_rLogLength(&lseed, testLog);
2913 DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize);
2914 oldTestLog = testLog;
2915 /* random dictionary selection */
2916 dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0;
2917 { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize);
2918 dict = srcBuffer + dictStart;
2919 if (!dictSize) dict=NULL;
2920 }
2921 pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize;
2922 { ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize);
2923 const U32 windowLogMax = bigTests ? 24 : 20;
2924 const U32 searchLogMax = bigTests ? 15 : 13;
2925 if (dictSize)
2926 DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize);
2927
2928 /* mess with compression parameters */
2929 cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1;
2930 cParams.windowLog = MIN(windowLogMax, cParams.windowLog);
2931 cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1;
2932 cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1;
2933 cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1;
2934 cParams.searchLog = MIN(searchLogMax, cParams.searchLog);
2935 cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1;
2936 cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128)));
2937 cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize);
2938
2939 if (FUZ_rand(&lseed) & 1) {
2940 DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog);
2941 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) );
2942 assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */
2943 windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5;
2944 }
2945 if (FUZ_rand(&lseed) & 1) {
2946 DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog);
2947 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) );
2948 }
2949 if (FUZ_rand(&lseed) & 1) {
2950 DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog);
2951 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) );
2952 }
2953 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) );
2954 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) );
2955 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) );
2956
2957 /* mess with long distance matching parameters */
2958 if (bigTests) {
2959 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_randomClampedLength(&lseed, ZSTD_ps_auto, ZSTD_ps_disable), opaqueAPI) );
2960 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) );
2961 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) );
2962 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) );
2963 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) );
2964 if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_srcSizeHint, FUZ_randomClampedLength(&lseed, ZSTD_SRCSIZEHINT_MIN, ZSTD_SRCSIZEHINT_MAX), opaqueAPI) );
2965 }
2966
2967 /* mess with frame parameters */
2968 if (FUZ_rand(&lseed) & 1) {
2969 int const checksumFlag = FUZ_rand(&lseed) & 1;
2970 DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag);
2971 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) );
2972 }
2973 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2974 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) );
2975 if (FUZ_rand(&lseed) & 1) {
2976 DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize);
2977 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
2978 } else {
2979 pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN;
2980 }
2981
2982 /* multi-threading parameters. Only adjust occasionally for small tests. */
2983 if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) {
2984 U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1;
2985 U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1;
2986 int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax);
2987 DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads);
2988 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) );
2989 if (nbThreads > 1) {
2990 U32 const jobLog = FUZ_rand(&lseed) % (testLog+1);
2991 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) );
2992 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) );
2993 }
2994 }
2995 /* Enable rsyncable mode 1 in 4 times. */
2996 {
2997 int const rsyncable = (FUZ_rand(&lseed) % 4 == 0);
2998 DISPLAYLEVEL(5, "t%u: rsyncable : %d \n", testNb, rsyncable);
2999 setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, rsyncable, opaqueAPI);
3000 }
3001
3002 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) );
3003 if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_deterministicRefPrefix, FUZ_rand(&lseed) & 1, opaqueAPI) );
3004
f535537f 3005 /* Set max block size parameters */
3006 if (FUZ_rand(&lseed) & 1) {
3007 int maxBlockSize = (int)(FUZ_rand(&lseed) % ZSTD_BLOCKSIZE_MAX);
3008 maxBlockSize = MAX(1024, maxBlockSize);
3009 CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_maxBlockSize, maxBlockSize, opaqueAPI) );
3010 }
3011
648db22b 3012 /* Apply parameters */
3013 if (opaqueAPI) {
3014 DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb);
3015 CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) );
3016 }
3017
3018 if (FUZ_rand(&lseed) & 1) {
3019 if (FUZ_rand(&lseed) & 1) {
3020 CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) );
3021 } else {
3022 CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) );
3023 }
3024 } else {
3025 isRefPrefix = 1;
3026 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
3027 }
3028 } }
3029
3030 CHECK_Z(getCCtxParams(zc, &savedParams));
3031
3032 /* multi-segments compression test */
3033 { int iter;
3034 int const startSeed = lseed;
3035 XXH64_hash_t compressedCrcs[2];
3036 for (iter = 0; iter < 2; ++iter, lseed = startSeed) {
3037 ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ;
3038 int const singlePass = (FUZ_rand(&lseed) & 3) == 0;
3039 int nbWorkers;
3040
3041 XXH64_reset(&xxhState, 0);
3042
3043 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) );
3044 if (isRefPrefix) {
3045 DISPLAYLEVEL(6, "t%u: Reloading prefix\n", testNb);
3046 /* Need to reload the prefix because it gets dropped after one compression */
3047 CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) );
3048 }
3049
3050 /* Adjust number of workers occasionally - result must be deterministic independent of nbWorkers */
3051 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_nbWorkers, &nbWorkers));
3052 if (nbWorkers > 0 && (FUZ_rand(&lseed) & 7) == 0) {
3053 DISPLAYLEVEL(6, "t%u: Modify nbWorkers: %d -> %d \n", testNb, nbWorkers, nbWorkers + iter);
3054 CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers + iter));
3055 }
3056
3057 if (singlePass) {
3058 ZSTD_inBuffer inBuff = { srcBuffer, maxTestSize, 0 };
3059 CHECK_Z(ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end));
3060 DISPLAYLEVEL(6, "t%u: Single pass compression: consumed %u bytes ; produced %u bytes \n",
3061 testNb, (unsigned)inBuff.pos, (unsigned)outBuff.pos);
3062 CHECK(inBuff.pos != inBuff.size, "Input not consumed!");
3063 crcOrig = XXH64(srcBuffer, maxTestSize, 0);
3064 totalTestSize = maxTestSize;
3065 } else {
3066 outBuff.size = 0;
3067 for (totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) {
3068 /* compress random chunks into randomly sized dst buffers */
3069 size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3070 size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize);
3071 size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize);
3072 ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush;
3073 ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 };
3074 int forwardProgress;
3075 do {
3076 size_t const ipos = inBuff.pos;
3077 size_t const opos = outBuff.pos;
3078 size_t ret;
3079 if (outBuff.pos == outBuff.size) {
3080 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
3081 size_t const dstBuffSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
3082 outBuff.size = outBuff.pos + dstBuffSize;
3083 }
3084 CHECK_Z( ret = ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) );
3085 DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n",
3086 testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos);
3087
3088 /* We've completed the flush */
3089 if (flush == ZSTD_e_flush && ret == 0)
3090 break;
3091
3092 /* Ensure maximal forward progress for determinism */
3093 forwardProgress = (inBuff.pos != ipos) || (outBuff.pos != opos);
3094 } while (forwardProgress);
3095 assert(inBuff.pos == inBuff.size);
3096
3097 XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos);
3098 memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos);
3099 totalTestSize += inBuff.pos;
3100 }
3101
3102 /* final frame epilogue */
3103 { size_t remainingToFlush = 1;
3104 while (remainingToFlush) {
3105 ZSTD_inBuffer inBuff = { NULL, 0, 0 };
3106 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1);
3107 size_t const adjustedDstSize = MIN(cBufferSize - outBuff.pos, randomDstSize);
3108 outBuff.size = outBuff.pos + adjustedDstSize;
3109 DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize);
3110 /* ZSTD_e_end guarantees maximal forward progress */
3111 remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end);
3112 DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos);
3113 CHECK( ZSTD_isError(remainingToFlush),
3114 "ZSTD_compressStream2 w/ ZSTD_e_end error : %s",
3115 ZSTD_getErrorName(remainingToFlush) );
3116 } }
3117 crcOrig = XXH64_digest(&xxhState);
3118 }
3119 cSize = outBuff.pos;
3120 compressedCrcs[iter] = XXH64(cBuffer, cSize, 0);
3121 DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize);
3122 }
3123 CHECK(!(compressedCrcs[0] == compressedCrcs[1]), "Compression is not deterministic!");
3124 }
3125
3126 CHECK(badParameters(zc, savedParams), "CCtx params are wrong");
3127
3128 /* multi - fragments decompression test */
3129 if (FUZ_rand(&lseed) & 1) {
3130 CHECK_Z(ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters));
3131 }
3132 if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) {
3133 DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", (void const*)dict);
3134 CHECK_Z( ZSTD_resetDStream(zd) );
3135 } else {
3136 if (dictSize)
3137 DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize);
3138 CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) );
3139 }
3140 if (FUZ_rand(&lseed) & 1) {
3141 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
3142 }
f535537f 3143 if (FUZ_rand(&lseed) & 1) {
3144 int maxBlockSize;
3145 CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_maxBlockSize, &maxBlockSize));
3146 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_maxBlockSize, maxBlockSize));
3147 } else {
3148 CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_maxBlockSize, 0));
3149 }
648db22b 3150 { size_t decompressionResult = 1;
3151 ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
3152 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
3153 for (totalGenSize = 0 ; decompressionResult ; ) {
3154 size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3155 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
3156 size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize);
3157 inBuff.size = inBuff.pos + readCSrcSize;
3158 outBuff.size = outBuff.pos + dstBuffSize;
3159 DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n",
3160 (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize);
3161 decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff);
3162 DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n",
3163 (unsigned)inBuff.pos, (unsigned)outBuff.pos);
3164 if (ZSTD_isError(decompressionResult)) {
3165 DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult));
3166 findDiff(copyBuffer, dstBuffer, totalTestSize);
3167 }
3168 CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult));
3169 CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize);
3170 }
3171 CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize);
3172 CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize);
3173 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
3174 if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize);
3175 CHECK (crcDest!=crcOrig, "decompressed data corrupted");
3176 } }
3177
3178 /*===== noisy/erroneous src decompression test =====*/
3179
3180 /* add some noise */
3181 { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2;
3182 U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) {
3183 size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog);
3184 size_t const noiseSize = MIN((cSize/3) , randomNoiseSize);
3185 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize);
3186 size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize);
3187 memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize);
3188 } }
3189
3190 /* try decompression on noisy data */
3191 if (FUZ_rand(&lseed) & 1) {
3192 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_and_parameters));
3193 } else {
3194 CHECK_Z(ZSTD_DCtx_reset(zd_noise, ZSTD_reset_session_only));
3195 }
3196 if (FUZ_rand(&lseed) & 1) {
3197 CHECK_Z(ZSTD_DCtx_setParameter(zd_noise, ZSTD_d_disableHuffmanAssembly, FUZ_rand(&lseed) & 1));
3198 }
3199 { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 };
3200 ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 };
3201 while (outBuff.pos < dstBufferSize) {
3202 size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog);
3203 size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog);
3204 size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize);
3205 size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize);
3206 outBuff.size = outBuff.pos + adjustedDstSize;
3207 inBuff.size = inBuff.pos + adjustedCSrcSize;
3208 { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff);
3209 if (ZSTD_isError(decompressError)) break; /* error correctly detected */
3210 /* Good so far, but no more progress possible */
3211 if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break;
3212 } } } }
3213 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
3214
3215_cleanup:
3216 ZSTD_freeCCtx(zc);
3217 ZSTD_freeDStream(zd);
3218 ZSTD_freeDStream(zd_noise);
3219 ZSTD_freeCCtxParams(cctxParams);
3220 free(cNoiseBuffer[0]);
3221 free(cNoiseBuffer[1]);
3222 free(cNoiseBuffer[2]);
3223 free(cNoiseBuffer[3]);
3224 free(cNoiseBuffer[4]);
3225 free(copyBuffer);
3226 free(cBuffer);
3227 free(dstBuffer);
3228 return result;
3229
3230_output_error:
3231 result = 1;
3232 goto _cleanup;
3233}
3234
3235/*-*******************************************************
3236* Command line
3237*********************************************************/
3238static int FUZ_usage(const char* programName)
3239{
3240 DISPLAY( "Usage :\n");
3241 DISPLAY( " %s [args]\n", programName);
3242 DISPLAY( "\n");
3243 DISPLAY( "Arguments :\n");
3244 DISPLAY( " -i# : Number of tests (default:%u)\n", nbTestsDefault);
3245 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
3246 DISPLAY( " -s# : Select seed (default:prompt user)\n");
3247 DISPLAY( " -t# : Select starting test number (default:0)\n");
3248 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT);
3249 DISPLAY( " -v : verbose\n");
3250 DISPLAY( " -p : pause at the end\n");
3251 DISPLAY( " -h : display help and exit\n");
3252 return 0;
3253}
3254
3255typedef enum { simple_api, advanced_api } e_api;
3256
3257int main(int argc, const char** argv)
3258{
3259 U32 seed = 0;
3260 int seedset = 0;
3261 int nbTests = nbTestsDefault;
3262 int testNb = 0;
3263 int proba = FUZ_COMPRESSIBILITY_DEFAULT;
3264 int result = 0;
3265 int mainPause = 0;
3266 int bigTests = (sizeof(size_t) == 8);
3267 e_api selected_api = simple_api;
3268 const char* const programName = argv[0];
3269 int argNb;
3270
3271 /* Check command line */
3272 for(argNb=1; argNb<argc; argNb++) {
3273 const char* argument = argv[argNb];
3274 assert(argument != NULL);
3275
3276 /* Parsing commands. Aggregated commands are allowed */
3277 if (argument[0]=='-') {
3278
3279 if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; }
3280 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
3281 if (!strcmp(argument, "--big-tests")) { bigTests=1; continue; }
3282
3283 argument++;
3284 while (*argument!=0) {
3285 switch(*argument)
3286 {
3287 case 'h':
3288 return FUZ_usage(programName);
3289
3290 case 'v':
3291 argument++;
3292 g_displayLevel++;
3293 break;
3294
3295 case 'q':
3296 argument++;
3297 g_displayLevel--;
3298 break;
3299
3300 case 'p': /* pause at the end */
3301 argument++;
3302 mainPause = 1;
3303 break;
3304
3305 case 'i': /* limit tests by nb of iterations (default) */
3306 argument++;
3307 nbTests=0; g_clockTime=0;
3308 while ((*argument>='0') && (*argument<='9')) {
3309 nbTests *= 10;
3310 nbTests += *argument - '0';
3311 argument++;
3312 }
3313 break;
3314
3315 case 'T': /* limit tests by time */
3316 argument++;
3317 nbTests=0; g_clockTime=0;
3318 while ((*argument>='0') && (*argument<='9')) {
3319 g_clockTime *= 10;
3320 g_clockTime += *argument - '0';
3321 argument++;
3322 }
3323 if (*argument=='m') { /* -T1m == -T60 */
3324 g_clockTime *=60, argument++;
3325 if (*argument=='n') argument++; /* -T1mn == -T60 */
3326 } else if (*argument=='s') argument++; /* -T10s == -T10 */
3327 g_clockTime *= SEC_TO_MICRO;
3328 break;
3329
3330 case 's': /* manually select seed */
3331 argument++;
3332 seedset=1;
3333 seed=0;
3334 while ((*argument>='0') && (*argument<='9')) {
3335 seed *= 10;
3336 seed += *argument - '0';
3337 argument++;
3338 }
3339 break;
3340
3341 case 't': /* select starting test number */
3342 argument++;
3343 testNb=0;
3344 while ((*argument>='0') && (*argument<='9')) {
3345 testNb *= 10;
3346 testNb += *argument - '0';
3347 argument++;
3348 }
3349 break;
3350
3351 case 'P': /* compressibility % */
3352 argument++;
3353 proba=0;
3354 while ((*argument>='0') && (*argument<='9')) {
3355 proba *= 10;
3356 proba += *argument - '0';
3357 argument++;
3358 }
3359 if (proba<0) proba=0;
3360 if (proba>100) proba=100;
3361 break;
3362
3363 default:
3364 return FUZ_usage(programName);
3365 }
3366 } } } /* for(argNb=1; argNb<argc; argNb++) */
3367
3368 /* Get Seed */
3369 DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
3370
3371 if (!seedset) {
3372 time_t const t = time(NULL);
3373 U32 const h = XXH32(&t, sizeof(t), 1);
3374 seed = h % 10000;
3375 }
3376
3377 DISPLAY("Seed = %u\n", (unsigned)seed);
3378 if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba);
3379
3380 if (nbTests<=0) nbTests=1;
3381
3382 if (testNb==0) {
3383 result = basicUnitTests(0, ((double)proba) / 100, bigTests); /* constant seed for predictability */
3384 }
3385
3386 if (!result) {
3387 switch(selected_api)
3388 {
3389 case simple_api :
3390 result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3391 break;
3392 case advanced_api :
3393 result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests);
3394 break;
3395 default :
3396 assert(0); /* impossible */
3397 }
3398 }
3399
3400 if (mainPause) {
3401 int unused;
3402 DISPLAY("Press Enter \n");
3403 unused = getchar();
3404 (void)unused;
3405 }
3406 return result;
3407}
ARM optimized PCSX fork