git subrepo pull (merge) --force deps/libchdr
[pcsx_rearmed.git] / deps / libchdr / deps / zstd-1.5.5 / tests / fuzzer.c
CommitLineData
648db22b 1/*
2 * Copyright (c) Meta Platforms, Inc. and affiliates.
3 * All rights reserved.
4 *
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
9 */
10
11
12/*-************************************
13* Compiler specific
14**************************************/
15#ifdef _MSC_VER /* Visual Studio */
16# define _CRT_SECURE_NO_WARNINGS /* fgets */
17# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
19#endif
20
21
22/*-************************************
23* Includes
24**************************************/
25#include <stdlib.h> /* free */
26#include <stdio.h> /* fgets, sscanf */
27#include <string.h> /* strcmp */
28#include <time.h> /* time(), time_t */
29#undef NDEBUG /* always enable assert() */
30#include <assert.h>
31#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
32#include "debug.h" /* DEBUG_STATIC_ASSERT */
33#include "fse.h"
34#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */
35#include "zstd.h" /* ZSTD_VERSION_STRING */
36#include "zstd_errors.h" /* ZSTD_getErrorCode */
37#define ZDICT_STATIC_LINKING_ONLY
38#include "zdict.h" /* ZDICT_trainFromBuffer */
39#include "mem.h"
40#include "datagen.h" /* RDG_genBuffer */
41#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
42#include "xxhash.h" /* XXH64 */
43#include "util.h"
44#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
45/* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */
46#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */
47#include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */
48
49
50/*-************************************
51* Constants
52**************************************/
53#define GB *(1U<<30)
54
55static const int FUZ_compressibility_default = 50;
56static const int nbTestsDefault = 30000;
57
58
59/*-************************************
60* Display Macros
61**************************************/
62#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
63#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
64static U32 g_displayLevel = 2;
65
66static const U64 g_refreshRate = SEC_TO_MICRO / 6;
67static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER;
68
69#define DISPLAYUPDATE(l, ...) \
70 if (g_displayLevel>=l) { \
71 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
72 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
73 if (g_displayLevel>=4) fflush(stderr); } \
74 }
75
76
77/*-*******************************************************
78* Compile time test
79*********************************************************/
80#undef MIN
81#undef MAX
82/* Declaring the function, to avoid -Wmissing-prototype */
83void FUZ_bug976(void);
84void FUZ_bug976(void)
85{ /* these constants shall not depend on MIN() macro */
86 DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31);
87 DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31);
88}
89
90
91/*-*******************************************************
92* Internal functions
93*********************************************************/
94#define MIN(a,b) ((a)<(b)?(a):(b))
95#define MAX(a,b) ((a)>(b)?(a):(b))
96
97#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
98static U32 FUZ_rand(U32* src)
99{
100 static const U32 prime1 = 2654435761U;
101 static const U32 prime2 = 2246822519U;
102 U32 rand32 = *src;
103 rand32 *= prime1;
104 rand32 += prime2;
105 rand32 = FUZ_rotl32(rand32, 13);
106 *src = rand32;
107 return rand32 >> 5;
108}
109
110static U32 FUZ_highbit32(U32 v32)
111{
112 unsigned nbBits = 0;
113 if (v32==0) return 0;
114 while (v32) v32 >>= 1, nbBits++;
115 return nbBits;
116}
117
118
119/*=============================================
120* Test macros
121=============================================*/
122#define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } }
123
124#define CHECK_Z(f) { \
125 size_t const err = f; \
126 if (ZSTD_isError(err)) { \
127 DISPLAY("Error => %s : %s ", \
128 #f, ZSTD_getErrorName(err)); \
129 exit(1); \
130} }
131
132#define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); }
133#define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn)
134#define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; }
135
136#define CHECK_OP(op, lhs, rhs) { \
137 if (!((lhs) op (rhs))) { \
138 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
139 exit(1); \
140 } \
141}
142#define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
143#define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
144
145
146/*=============================================
147* Memory Tests
148=============================================*/
149#if defined(__APPLE__) && defined(__MACH__)
150
151#include <malloc/malloc.h> /* malloc_size */
152
153typedef struct {
154 unsigned long long totalMalloc;
155 size_t currentMalloc;
156 size_t peakMalloc;
157 unsigned nbMalloc;
158 unsigned nbFree;
159} mallocCounter_t;
160
161static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 };
162
163static void* FUZ_mallocDebug(void* counter, size_t size)
164{
165 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
166 void* const ptr = malloc(size);
167 if (ptr==NULL) return NULL;
168 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
169 (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */
170 mcPtr->totalMalloc += size;
171 mcPtr->currentMalloc += size;
172 if (mcPtr->currentMalloc > mcPtr->peakMalloc)
173 mcPtr->peakMalloc = mcPtr->currentMalloc;
174 mcPtr->nbMalloc += 1;
175 return ptr;
176}
177
178static void FUZ_freeDebug(void* counter, void* address)
179{
180 mallocCounter_t* const mcPtr = (mallocCounter_t*)counter;
181 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10));
182 mcPtr->nbFree += 1;
183 mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */
184 free(address);
185}
186
187static void FUZ_displayMallocStats(mallocCounter_t count)
188{
189 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n",
190 (unsigned)(count.peakMalloc >> 10),
191 count.nbMalloc,
192 (unsigned)(count.totalMalloc >> 10));
193}
194
195static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part,
196 void* inBuffer, size_t inSize, void* outBuffer, size_t outSize)
197{
198 /* test only played in verbose mode, as they are long */
199 if (g_displayLevel<3) return 0;
200
201 /* Create compressible noise */
202 if (!inBuffer || !outBuffer) {
203 DISPLAY("Not enough memory, aborting\n");
204 exit(1);
205 }
206 RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed);
207
208 /* simple compression tests */
209 if (part <= 1)
210 { int compressionLevel;
211 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
212 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
213 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
214 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
215 CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) );
216 ZSTD_freeCCtx(cctx);
217 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel);
218 FUZ_displayMallocStats(malcount);
219 } }
220
221 /* streaming compression tests */
222 if (part <= 2)
223 { int compressionLevel;
224 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
225 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
226 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
227 ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem);
228 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
229 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
230 CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) );
231 CHECK_Z( ZSTD_compressStream(cstream, &out, &in) );
232 CHECK_Z( ZSTD_endStream(cstream, &out) );
233 ZSTD_freeCStream(cstream);
234 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel);
235 FUZ_displayMallocStats(malcount);
236 } }
237
238 /* advanced MT API test */
239 if (part <= 3)
240 { int nbThreads;
241 for (nbThreads=1; nbThreads<=4; nbThreads++) {
242 int compressionLevel;
243 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
244 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
245 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
246 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
247 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
248 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
249 CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) );
250 ZSTD_freeCCtx(cctx);
251 DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ",
252 nbThreads, compressionLevel);
253 FUZ_displayMallocStats(malcount);
254 } } }
255
256 /* advanced MT streaming API test */
257 if (part <= 4)
258 { int nbThreads;
259 for (nbThreads=1; nbThreads<=4; nbThreads++) {
260 int compressionLevel;
261 for (compressionLevel=1; compressionLevel<=6; compressionLevel++) {
262 mallocCounter_t malcount = INIT_MALLOC_COUNTER;
263 ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount };
264 ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem);
265 ZSTD_outBuffer out = { outBuffer, outSize, 0 };
266 ZSTD_inBuffer in = { inBuffer, inSize, 0 };
267 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
268 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) );
269 CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) );
270 while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {}
271 ZSTD_freeCCtx(cctx);
272 DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ",
273 nbThreads, compressionLevel);
274 FUZ_displayMallocStats(malcount);
275 } } }
276
277 return 0;
278}
279
280static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
281{
282 size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */
283 size_t const outSize = ZSTD_compressBound(inSize);
284 void* const inBuffer = malloc(inSize);
285 void* const outBuffer = malloc(outSize);
286 int result;
287
288 /* Create compressible noise */
289 if (!inBuffer || !outBuffer) {
290 DISPLAY("Not enough memory, aborting \n");
291 exit(1);
292 }
293
294 result = FUZ_mallocTests_internal(seed, compressibility, part,
295 inBuffer, inSize, outBuffer, outSize);
296
297 free(inBuffer);
298 free(outBuffer);
299 return result;
300}
301
302#else
303
304static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part)
305{
306 (void)seed; (void)compressibility; (void)part;
307 return 0;
308}
309
310#endif
311
312static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize,
313 BYTE* src, size_t size, ZSTD_sequenceFormat_e format)
314{
315 size_t i;
316 size_t j;
317 for(i = 0; i < seqsSize; ++i) {
318 assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size);
319 assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size);
320 if (format == ZSTD_sf_noBlockDelimiters) {
321 assert(seqs[i].matchLength != 0 || seqs[i].offset != 0);
322 }
323
324 memcpy(dst, src, seqs[i].litLength);
325 dst += seqs[i].litLength;
326 src += seqs[i].litLength;
327 size -= seqs[i].litLength;
328
329 if (seqs[i].offset != 0) {
330 for (j = 0; j < seqs[i].matchLength; ++j)
331 dst[j] = dst[j - seqs[i].offset];
332 dst += seqs[i].matchLength;
333 src += seqs[i].matchLength;
334 size -= seqs[i].matchLength;
335 }
336 }
337 if (format == ZSTD_sf_noBlockDelimiters) {
338 memcpy(dst, src, size);
339 }
340}
341
342#ifdef ZSTD_MULTITHREAD
343
344typedef struct {
345 ZSTD_CCtx* cctx;
346 ZSTD_threadPool* pool;
347 void* CNBuffer;
348 size_t CNBuffSize;
349 void* compressedBuffer;
350 size_t compressedBufferSize;
351 void* decodedBuffer;
352 int err;
353} threadPoolTests_compressionJob_payload;
354
355static void* threadPoolTests_compressionJob(void* payload) {
356 threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload;
357 size_t cSize;
358 if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1;
359 cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize);
360 if (ZSTD_isError(cSize)) args->err = 1;
361 if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1;
362 return payload;
363}
364
365static int threadPoolTests(void) {
366 int testResult = 0;
367 size_t err;
368
369 size_t const CNBuffSize = 5 MB;
370 void* const CNBuffer = malloc(CNBuffSize);
371 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
372 void* const compressedBuffer = malloc(compressedBufferSize);
373 void* const decodedBuffer = malloc(CNBuffSize);
374
375 size_t const kPoolNumThreads = 8;
376
377 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0);
378
379 DISPLAYLEVEL(3, "thread pool test : threadPool re-use roundtrips: ");
380 {
381 ZSTD_CCtx* cctx = ZSTD_createCCtx();
382 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
383
384 size_t nbThreads = 1;
385 for (; nbThreads <= kPoolNumThreads; ++nbThreads) {
386 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
387 ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads);
388 err = ZSTD_CCtx_refThreadPool(cctx, pool);
389 if (ZSTD_isError(err)) {
390 DISPLAYLEVEL(3, "refThreadPool error!\n");
391 ZSTD_freeCCtx(cctx);
392 goto _output_error;
393 }
394 err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
395 if (ZSTD_isError(err)) {
396 DISPLAYLEVEL(3, "Compression error!\n");
397 ZSTD_freeCCtx(cctx);
398 goto _output_error;
399 }
400 err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err);
401 if (ZSTD_isError(err)) {
402 DISPLAYLEVEL(3, "Decompression error!\n");
403 ZSTD_freeCCtx(cctx);
404 goto _output_error;
405 }
406 }
407
408 ZSTD_freeCCtx(cctx);
409 ZSTD_freeThreadPool(pool);
410 }
411 DISPLAYLEVEL(3, "OK \n");
412
413 DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: ");
414 {
415 void* const decodedBuffer2 = malloc(CNBuffSize);
416 void* const compressedBuffer2 = malloc(compressedBufferSize);
417 ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads);
418 ZSTD_CCtx* cctx1 = ZSTD_createCCtx();
419 ZSTD_CCtx* cctx2 = ZSTD_createCCtx();
420
421 ZSTD_pthread_t t1;
422 ZSTD_pthread_t t2;
423 threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize,
424 compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */};
425 threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize,
426 compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */};
427
428 ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2);
429 ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2);
430 ZSTD_CCtx_refThreadPool(cctx1, pool);
431 ZSTD_CCtx_refThreadPool(cctx2, pool);
432
433 ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1);
434 ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2);
435 ZSTD_pthread_join(t1);
436 ZSTD_pthread_join(t2);
437
438 assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize));
439 free(decodedBuffer2);
440 free(compressedBuffer2);
441
442 ZSTD_freeThreadPool(pool);
443 ZSTD_freeCCtx(cctx1);
444 ZSTD_freeCCtx(cctx2);
445
446 if (p1.err || p2.err) goto _output_error;
447 }
448 DISPLAYLEVEL(3, "OK \n");
449
450_end:
451 free(CNBuffer);
452 free(compressedBuffer);
453 free(decodedBuffer);
454 return testResult;
455
456_output_error:
457 testResult = 1;
458 DISPLAY("Error detected in Unit tests ! \n");
459 goto _end;
460}
461#endif /* ZSTD_MULTITHREAD */
462
463/*=============================================
464* Unit tests
465=============================================*/
466
467static void test_compressBound(unsigned tnb)
468{
469 DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb);
470
471 /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND
472 * for a large range of known valid values */
473 DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4);
474 { int s;
475 for (s=0; s<30; s++) {
476 size_t const w = (size_t)1 << s;
477 CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w));
478 } }
479
480 /* Ensure error if srcSize too big */
481 { size_t const w = ZSTD_MAX_INPUT_SIZE + 1;
482 CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */
483 CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0);
484 }
485
486 DISPLAYLEVEL(3, "OK \n");
487}
488
489static void test_decompressBound(unsigned tnb)
490{
491 DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb);
492
493 /* Simple compression, with size : should provide size; */
494 { const char example[] = "abcd";
495 char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))];
496 size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0);
497 CHECK_Z(cSize);
498 CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example));
499 }
500
501 /* Simple small compression without size : should provide 1 block size */
502 { char cBuffer[ZSTD_COMPRESSBOUND(0)];
503 ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 };
504 ZSTD_inBuffer in = { NULL, 0, 0 };
505 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
506 assert(cctx);
507 CHECK_Z( ZSTD_initCStream(cctx, 0) );
508 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
509 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
510 CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX );
511 ZSTD_freeCCtx(cctx);
512 }
513
514 /* Attempt to overflow 32-bit intermediate multiplication result
515 * This requires dBound >= 4 GB, aka 2^32.
516 * This requires 2^32 / 2^17 = 2^15 blocks
517 * => create 2^15 blocks (can be empty, or just 1 byte). */
518 { const char input[] = "a";
519 size_t const nbBlocks = (1 << 15) + 1;
520 size_t blockNb;
521 size_t const outCapacity = 1 << 18; /* large margin */
522 char* const outBuffer = malloc (outCapacity);
523 ZSTD_outBuffer out = { outBuffer, outCapacity, 0 };
524 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
525 assert(cctx);
526 assert(outBuffer);
527 CHECK_Z( ZSTD_initCStream(cctx, 0) );
528 for (blockNb=0; blockNb<nbBlocks; blockNb++) {
529 ZSTD_inBuffer in = { input, sizeof(input), 0 };
530 CHECK_Z( ZSTD_compressStream(cctx, &out, &in) );
531 CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 );
532 }
533 CHECK_EQ( ZSTD_endStream(cctx, &out), 0 );
534 CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000LLU /* 4 GB */ );
535 ZSTD_freeCCtx(cctx);
536 free(outBuffer);
537 }
538
539 DISPLAYLEVEL(3, "OK \n");
540}
541
542static void test_setCParams(unsigned tnb)
543{
544 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
545 ZSTD_compressionParameters cparams;
546 assert(cctx);
547
548 DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb);
549
550 /* valid cparams */
551 cparams = ZSTD_getCParams(1, 0, 0);
552 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
553
554 /* invalid cparams (must fail) */
555 cparams.windowLog = 99;
556 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
557
558 free(cctx);
559 DISPLAYLEVEL(3, "OK \n");
560}
561
562static int basicUnitTests(U32 const seed, double compressibility)
563{
564 size_t const CNBuffSize = 5 MB;
565 void* const CNBuffer = malloc(CNBuffSize);
566 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
567 void* const compressedBuffer = malloc(compressedBufferSize);
568 void* const decodedBuffer = malloc(CNBuffSize);
569 int testResult = 0;
570 unsigned testNb=0;
571 size_t cSize;
572
573 /* Create compressible noise */
574 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
575 DISPLAY("Not enough memory, aborting\n");
576 testResult = 1;
577 goto _end;
578 }
579 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
580
581 /* Basic tests */
582 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++);
583 { const char* errorString = ZSTD_getErrorName(0);
584 DISPLAYLEVEL(3, "OK : %s \n", errorString);
585 }
586
587 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++);
588 { const char* errorString = ZSTD_getErrorName(499);
589 DISPLAYLEVEL(3, "OK : %s \n", errorString);
590 }
591
592 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++);
593 { int const mcl = ZSTD_minCLevel();
594 DISPLAYLEVEL(3, "%i (OK) \n", mcl);
595 }
596
597 DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++);
598 { int const defaultCLevel = ZSTD_defaultCLevel();
599 if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error;
600 DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel);
601 }
602
603 DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++);
604 { unsigned const vn = ZSTD_versionNumber();
605 DISPLAYLEVEL(3, "%u (OK) \n", vn);
606 }
607
608 test_compressBound(testNb++);
609
610 test_decompressBound(testNb++);
611
612 test_setCParams(testNb++);
613
614 DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++);
615 {
616 ZSTD_compressionParameters params;
617 memset(&params, 0, sizeof(params));
618 params.windowLog = 10;
619 params.hashLog = 19;
620 params.chainLog = 19;
621 params = ZSTD_adjustCParams(params, 1000, 100000);
622 if (params.hashLog != 18) goto _output_error;
623 if (params.chainLog != 17) goto _output_error;
624 }
625 DISPLAYLEVEL(3, "OK \n");
626
627 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize);
628 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
629 if (cctx==NULL) goto _output_error;
630 CHECK_VAR(cSize, ZSTD_compressCCtx(cctx,
631 compressedBuffer, compressedBufferSize,
632 CNBuffer, CNBuffSize, 1) );
633 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
634
635 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++);
636 { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx);
637 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize);
638 }
639 ZSTD_freeCCtx(cctx);
640 }
641
642 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++);
643 {
644 char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
645 size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8);
646 if (!ZSTD_isError(size)) goto _output_error;
647 }
648 DISPLAYLEVEL(3, "OK \n");
649
650 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++);
651 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
652 if (rSize != CNBuffSize) goto _output_error;
653 }
654 DISPLAYLEVEL(3, "OK \n");
655
656 DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++);
657 { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize);
658 if (rSize != CNBuffSize) goto _output_error;
659 }
660 DISPLAYLEVEL(3, "OK \n");
661
662 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++);
663 { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize);
664 if (rSize != CNBuffSize) goto _output_error;
665 }
666 DISPLAYLEVEL(3, "OK \n");
667
668 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++);
669 {
670 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize);
671 if (bound != CNBuffSize) goto _output_error;
672 }
673 DISPLAYLEVEL(3, "OK \n");
674
675 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++);
676 {
677 unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1);
678 if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error;
679 }
680 DISPLAYLEVEL(3, "OK \n");
681
682 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
683 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
684 if (r != CNBuffSize) goto _output_error; }
685 DISPLAYLEVEL(3, "OK \n");
686
687 DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize);
688 {
689 ZSTD_DCtx* dctx = ZSTD_createDCtx();
690 size_t r;
691 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1));
692 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
693 if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
694 ZSTD_freeDCtx(dctx);
695 }
696 DISPLAYLEVEL(3, "OK \n");
697
698 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
699 { size_t u;
700 for (u=0; u<CNBuffSize; u++) {
701 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
702 } }
703 DISPLAYLEVEL(3, "OK \n");
704
705 DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++);
706 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
707 ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 };
708 ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 };
709 if (cctx==NULL) goto _output_error;
710 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */
711 CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */
712 ZSTD_freeCCtx(cctx);
713 }
714 DISPLAYLEVEL(3, "OK \n");
715
716 DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++);
717 {
718 ZSTD_parameters params = ZSTD_getParams(3, 0, 0);
719 assert(!ZSTD_checkCParams(params.cParams));
720 }
721 DISPLAYLEVEL(3, "OK \n");
722
723 DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++);
724 {
725 ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem);
726 assert(dctx != NULL);
727 assert(ZSTD_sizeof_DCtx(dctx) != 0);
728 ZSTD_freeDCtx(dctx);
729 }
730 DISPLAYLEVEL(3, "OK \n");
731
732 DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++);
733 {
734 /* %p takes a void*. In ISO C, it's illegal to cast a function pointer
735 * to a data pointer. (Although in POSIX you're required to be allowed
736 * to do it...) So we have to fall back to our trusty friend memcpy. */
737 unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) =
738 ZSTD_getDictID_fromDDict;
739 ZSTD_DStream* (* const funcptr_createDStream)(
740 ZSTD_customMem customMem) = ZSTD_createDStream_advanced;
741 void (* const funcptr_copyDCtx)(
742 ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx;
743 ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) =
744 ZSTD_nextInputType;
745 const void *voidptr_getDictID;
746 const void *voidptr_createDStream;
747 const void *voidptr_copyDCtx;
748 const void *voidptr_nextInputType;
749 DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID));
750 memcpy(
751 (void*)&voidptr_getDictID,
752 (const void*)&funcptr_getDictID,
753 sizeof(void*));
754 memcpy(
755 (void*)&voidptr_createDStream,
756 (const void*)&funcptr_createDStream,
757 sizeof(void*));
758 memcpy(
759 (void*)&voidptr_copyDCtx,
760 (const void*)&funcptr_copyDCtx,
761 sizeof(void*));
762 memcpy(
763 (void*)&voidptr_nextInputType,
764 (const void*)&funcptr_nextInputType,
765 sizeof(void*));
766 DISPLAYLEVEL(3, "%p ", voidptr_getDictID);
767 DISPLAYLEVEL(3, "%p ", voidptr_createDStream);
768 DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx);
769 DISPLAYLEVEL(3, "%p ", voidptr_nextInputType);
770 }
771 DISPLAYLEVEL(3, ": OK \n");
772
773 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++);
774 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
775 { size_t const r = ZSTD_decompress_usingDict(dctx,
776 decodedBuffer, CNBuffSize,
777 compressedBuffer, cSize,
778 NULL, 0);
779 if (r != CNBuffSize) goto _output_error;
780 }
781 ZSTD_freeDCtx(dctx);
782 }
783 DISPLAYLEVEL(3, "OK \n");
784
785 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++);
786 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
787 { size_t const r = ZSTD_decompress_usingDDict(dctx,
788 decodedBuffer, CNBuffSize,
789 compressedBuffer, cSize,
790 NULL);
791 if (r != CNBuffSize) goto _output_error;
792 }
793 ZSTD_freeDCtx(dctx);
794 }
795 DISPLAYLEVEL(3, "OK \n");
796
797 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++);
798 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
799 if (!ZSTD_isError(r)) goto _output_error;
800 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; }
801 DISPLAYLEVEL(3, "OK \n");
802
803 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++);
804 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1);
805 if (!ZSTD_isError(r)) goto _output_error;
806 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
807 DISPLAYLEVEL(3, "OK \n");
808
809 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++);
810 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize);
811 if (!ZSTD_isError(r)) goto _output_error;
812 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
813 DISPLAYLEVEL(3, "OK \n");
814
815 DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++);
816 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize);
817 if (!ZSTD_isError(r)) goto _output_error;
818 if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; }
819 DISPLAYLEVEL(3, "OK \n");
820
821 DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++);
822 { /* create compressed buffer with checksumming enabled */
823 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
824 if (!cctx) {
825 DISPLAY("Not enough memory, aborting\n");
826 testResult = 1;
827 goto _end;
828 }
829 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) );
830 CHECK_VAR(cSize, ZSTD_compress2(cctx,
831 compressedBuffer, compressedBufferSize,
832 CNBuffer, CNBuffSize) );
833 ZSTD_freeCCtx(cctx);
834 }
835 { /* copy the compressed buffer and corrupt the checksum */
836 size_t r;
837 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
838 if (!dctx) {
839 DISPLAY("Not enough memory, aborting\n");
840 testResult = 1;
841 goto _end;
842 }
843
844 ((char*)compressedBuffer)[cSize-1] += 1;
845 r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
846 if (!ZSTD_isError(r)) goto _output_error;
847 if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error;
848
849 CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum));
850 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1);
851 if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */
852 r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
853 if (ZSTD_isError(r)) goto _output_error;
854
855 ZSTD_freeDCtx(dctx);
856 }
857 DISPLAYLEVEL(3, "OK \n");
858
859
860 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++);
861 { /* create compressed buffer with content size missing */
862 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
863 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
864 CHECK_VAR(cSize, ZSTD_compress2(cctx,
865 compressedBuffer, compressedBufferSize,
866 CNBuffer, CNBuffSize) );
867 ZSTD_freeCCtx(cctx);
868 }
869 { /* ensure frame content size is missing */
870 ZSTD_frameHeader zfh;
871 size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize);
872 if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
873 }
874 { /* ensure CNBuffSize <= decompressBound */
875 unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize);
876 if (CNBuffSize > bound) goto _output_error;
877 }
878 DISPLAYLEVEL(3, "OK \n");
879
880 DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++);
881 {
882 size_t const largeFrameSrcSize = 200;
883 size_t const smallFrameSrcSize = 10;
884 size_t const nbFrames = 256;
885
886 size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0;
887 int sizeReduced = 0;
888
889 BYTE* const dst = (BYTE*)compressedBuffer;
890 ZSTD_DCtx* dctx = ZSTD_createDCtx();
891
892 /* create a large frame and then a bunch of small frames */
893 size_t srcSize = ZSTD_compress((void*)dst,
894 compressedBufferSize, CNBuffer, largeFrameSrcSize, 3);
895 for (i = 0; i < nbFrames; i++)
896 srcSize += ZSTD_compress((void*)(dst + srcSize),
897 compressedBufferSize - srcSize, CNBuffer,
898 smallFrameSrcSize, 3);
899
900 /* decompressStream and make sure that dctx size was reduced at least once */
901 while (consumed < srcSize) {
902 ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0};
903 ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0};
904 ZSTD_decompressStream(dctx, &out, &in);
905 consumed += in.pos;
906 produced += out.pos;
907
908 /* success! size was reduced from the previous frame */
909 if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx))
910 sizeReduced = 1;
911
912 prevDCtxSize = ZSTD_sizeof_DCtx(dctx);
913 }
914
915 assert(sizeReduced);
916
917 ZSTD_freeDCtx(dctx);
918 }
919 DISPLAYLEVEL(3, "OK \n");
920
921 {
922 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
923 ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1);
924 ZSTD_parameters const params = ZSTD_getParams(1, 0, 0);
925 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
926
927 DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++);
928 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1));
929 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
930 DISPLAYLEVEL(3, "OK \n");
931
932 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++);
933 CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1));
934 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
935 DISPLAYLEVEL(3, "OK \n");
936
937 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++);
938 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict));
939 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
940 DISPLAYLEVEL(3, "OK \n");
941
942 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++);
943 CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params));
944 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
945 DISPLAYLEVEL(3, "OK \n");
946
947 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++);
948 CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams));
949 if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error;
950 DISPLAYLEVEL(3, "OK \n");
951
952 ZSTD_freeCDict(cdict);
953 ZSTD_freeCCtx(cctx);
954 }
955
956 DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++);
957 {
958 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
959
960 size_t const size = (1U << 10);
961 size_t const dstCapacity = ZSTD_compressBound(size);
962 void* dict = (void*)malloc(size);
963 void* src = (void*)malloc(size);
964 void* dst = (void*)malloc(dstCapacity);
965
966 RDG_genBuffer(dict, size, 0.5, 0.5, seed);
967 RDG_genBuffer(src, size, 0.5, 0.5, seed);
968
969 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
970 assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3)));
971
972 ZSTD_freeCCtx(cctx);
973 free(dict);
974 free(src);
975 free(dst);
976 }
977 DISPLAYLEVEL(3, "OK \n");
978
979 DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++);
980 {
981 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
982 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
983 void* dict = (void*)malloc(CNBuffSize);
984 int nbWorkers;
985
986 for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) {
987 RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed);
988 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
989
990 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers));
991 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
992 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1));
993 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
994 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize));
995 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
996 CHECK_Z(cSize);
997 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize));
998 }
999
1000 ZSTD_freeCCtx(cctx);
1001 ZSTD_freeDCtx(dctx);
1002 free(dict);
1003 }
1004 DISPLAYLEVEL(3, "OK \n");
1005
1006 DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++);
1007 {
1008 size_t const testSize = 1024;
1009 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1010 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1011 char* dict = (char*)malloc(2 * testSize);
1012 int ldmEnabled, level;
1013
1014 RDG_genBuffer(dict, testSize, 0.5, 0.5, seed);
1015 RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed);
1016 memcpy(dict + testSize, CNBuffer, testSize);
1017 for (level = 1; level <= 5; ++level) {
1018 for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) {
1019 size_t cSize0;
1020 XXH64_hash_t compressedChecksum0;
1021
1022 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1023 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level));
1024 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled));
1025 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1));
1026
1027 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1028 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize);
1029 CHECK_Z(cSize);
1030 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize));
1031
1032 cSize0 = cSize;
1033 compressedChecksum0 = XXH64(compressedBuffer, cSize, 0);
1034
1035 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize));
1036 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize);
1037 CHECK_Z(cSize);
1038
1039 if (cSize != cSize0) goto _output_error;
1040 if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error;
1041 }
1042 }
1043
1044 ZSTD_freeCCtx(cctx);
1045 ZSTD_freeDCtx(dctx);
1046 free(dict);
1047 }
1048 DISPLAYLEVEL(3, "OK \n");
1049
1050 DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++);
1051 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1052 ZSTD_DCtx* dctx = ZSTD_createDCtx();
1053 size_t const srcSize = 300 KB;
1054 size_t const flushSize = 128 KB + 5;
1055 size_t const dstSize = ZSTD_compressBound(srcSize);
1056 char* src = (char*)CNBuffer;
1057 char* dst = (char*)compressedBuffer;
1058
1059 ZSTD_outBuffer out = { dst, dstSize, 0 };
1060 ZSTD_inBuffer in = { src, flushSize, 0 };
1061
1062 if (!cctx || !dctx) {
1063 DISPLAY("Not enough memory, aborting\n");
1064 testResult = 1;
1065 goto _end;
1066 }
1067
1068 RDG_genBuffer(src, srcSize, 0.5, 0.5, seed);
1069 /* Force an LDM to exist that crosses block boundary into uncompressible block */
1070 memcpy(src + 125 KB, src, 3 KB + 5);
1071
1072 /* Enable MT, LDM, and opt parser */
1073 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1));
1074 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1075 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1077
1078 /* Flushes a block of 128 KB and block of 5 bytes */
1079 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1080
1081 /* Compress the rest */
1082 in.size = 300 KB;
1083 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1084
1085 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos));
1086
1087 ZSTD_freeCCtx(cctx);
1088 ZSTD_freeDCtx(dctx);
1089 }
1090 DISPLAYLEVEL(3, "OK \n");
1091
1092 DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++);
1093 {
1094 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1095 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1096 void* dict = (void*)malloc(CNBuffSize);
1097 size_t const kWindowLog = 10;
1098 size_t const kWindowSize = (size_t)1 << kWindowLog;
1099 size_t const dictSize = kWindowSize * 10;
1100 size_t const srcSize1 = kWindowSize / 2;
1101 size_t const srcSize2 = kWindowSize * 10;
1102
1103 if (CNBuffSize < dictSize) goto _output_error;
1104
1105 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
1106 RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed);
1107
1108 /* Enable checksum to verify round trip. */
1109 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
1110 /* Disable content size to skip single-pass decompression. */
1111 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0));
1112 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog));
1113 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
1114 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32));
1115 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1));
1116 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16));
1117 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3));
1118
1119 /* Round trip once with a dictionary. */
1120 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1121 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
1122 CHECK_Z(cSize);
1123 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1124 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
1125 /* Streaming decompression to catch out of bounds offsets. */
1126 {
1127 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1128 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1129 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1130 CHECK_Z(dSize);
1131 if (dSize != 0) goto _output_error;
1132 }
1133
1134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
1135 /* Round trip once with a dictionary. */
1136 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
1137 {
1138 ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
1139 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1140 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1141 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1142 cSize = out.pos;
1143 }
1144 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
1145 {
1146 ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
1147 ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
1148 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
1149 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1150 cSize = out.pos;
1151 }
1152 /* Streaming decompression to catch out of bounds offsets. */
1153 {
1154 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
1155 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
1156 size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
1157 CHECK_Z(dSize);
1158 if (dSize != 0) goto _output_error;
1159 }
1160
1161 ZSTD_freeCCtx(cctx);
1162 ZSTD_freeDCtx(dctx);
1163 free(dict);
1164 }
1165 DISPLAYLEVEL(3, "OK \n");
1166
1167 /* Note: this test takes 0.5 seconds to run */
1168 DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++);
1169 {
1170 /* test a big buffer so that ldm can take effect */
1171 size_t const size = 100 MB;
1172 int const windowLog = 27;
1173 size_t const dstSize = ZSTD_compressBound(size);
1174
1175 void* dict = (void*)malloc(size);
1176 void* src = (void*)malloc(size);
1177 void* dst = (void*)malloc(dstSize);
1178 void* recon = (void*)malloc(size);
1179
1180 size_t refPrefixCompressedSize = 0;
1181 size_t refPrefixLdmCompressedSize = 0;
1182 size_t reconSize = 0;
1183
1184 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1185 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
1186
1187 /* make dict and src the same uncompressible data */
1188 RDG_genBuffer(src, size, 0, 0, seed);
1189 memcpy(dict, src, size);
1190 assert(!memcmp(dict, src, size));
1191
1192 /* set level 1 and windowLog to cover src */
1193 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1));
1194 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog));
1195
1196 /* compress on level 1 using just refPrefix and no ldm */
1197 ZSTD_CCtx_refPrefix(cctx, dict, size);
1198 refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1199 assert(!ZSTD_isError(refPrefixCompressedSize));
1200
1201 /* test round trip just refPrefix */
1202 ZSTD_DCtx_refPrefix(dctx, dict, size);
1203 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize);
1204 assert(!ZSTD_isError(reconSize));
1205 assert(reconSize == size);
1206 assert(!memcmp(recon, src, size));
1207
1208 /* compress on level 1 using refPrefix and ldm */
1209 ZSTD_CCtx_refPrefix(cctx, dict, size);;
1210 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable))
1211 refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size);
1212 assert(!ZSTD_isError(refPrefixLdmCompressedSize));
1213
1214 /* test round trip refPrefix + ldm*/
1215 ZSTD_DCtx_refPrefix(dctx, dict, size);
1216 reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize);
1217 assert(!ZSTD_isError(reconSize));
1218 assert(reconSize == size);
1219 assert(!memcmp(recon, src, size));
1220
1221 /* make sure that refPrefixCompressedSize is significantly greater */
1222 assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize);
1223 /* make sure the ldm compressed size is less than 1% of original */
1224 assert((double)refPrefixLdmCompressedSize / (double)size < 0.01);
1225
1226 ZSTD_freeDCtx(dctx);
1227 ZSTD_freeCCtx(cctx);
1228 free(recon);
1229 free(dict);
1230 free(src);
1231 free(dst);
1232 }
1233 DISPLAYLEVEL(3, "OK \n");
1234
1235 DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++);
1236 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX);
1237 CHECK_Z(cSize);
1238 CHECK_LT(CNBuffSize, cSize);
1239 {
1240 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1241 size_t const outputSize = (CNBuffSize + margin);
1242 char* output = malloc(outputSize);
1243 char* input = output + outputSize - cSize;
1244 CHECK_LT(cSize, CNBuffSize + margin);
1245 CHECK(output != NULL);
1246 CHECK_Z(margin);
1247 CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX));
1248 memcpy(input, compressedBuffer, cSize);
1249
1250 {
1251 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1252 CHECK_Z(dSize);
1253 CHECK_EQ(dSize, CNBuffSize);
1254 }
1255 CHECK(!memcmp(output, CNBuffer, CNBuffSize));
1256 free(output);
1257 }
1258 DISPLAYLEVEL(3, "OK \n");
1259
1260 DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++);
1261 cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1262 CHECK_Z(cSize);
1263 {
1264 size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX);
1265 CHECK_Z(cSize2);
1266 cSize += cSize2;
1267 }
1268 {
1269 size_t const srcSize = (CNBuffSize / 3) * 2;
1270 size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize);
1271 size_t const outputSize = (CNBuffSize + margin);
1272 char* output = malloc(outputSize);
1273 char* input = output + outputSize - cSize;
1274 CHECK_LT(cSize, CNBuffSize + margin);
1275 CHECK(output != NULL);
1276 CHECK_Z(margin);
1277 memcpy(input, compressedBuffer, cSize);
1278
1279 {
1280 size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize);
1281 CHECK_Z(dSize);
1282 CHECK_EQ(dSize, srcSize);
1283 }
1284 CHECK(!memcmp(output, CNBuffer, srcSize));
1285 free(output);
1286 }
1287 DISPLAYLEVEL(3, "OK \n");
1288
1289 DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++);
1290 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
1291 size_t const srcSize = 256 * 1024;
1292 U32 const compressibleLenU32 = 32 * 1024 / 4;
1293 U32 const blockSizeU32 = 128 * 1024 / 4;
1294 U32 const litLenU32 = 64 * 1024 / 4;
1295 U32* data = (U32*)malloc(srcSize);
1296 size_t dSize;
1297
1298 if (data == NULL || cctx == NULL) goto _output_error;
1299
1300 /* Generate data without any matches */
1301 RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U);
1302 /* Generate 32K of compressible data */
1303 RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe);
1304
1305 /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */
1306 data[compressibleLenU32 + 0] = 0xFFFFFFFF;
1307 data[compressibleLenU32 + 1] = 0xEEEEEEEE;
1308 data[compressibleLenU32 + 4] = 0xFFFFFFFF;
1309 data[compressibleLenU32 + 5] = 0xEEEEEEEE;
1310
1311 /* Add a match of offset=16, length=8 at idx=64K + 64.
1312 * This generates a sequence with llen=64K, and repeat code 1.
1313 * The block splitter thought this was ll0, and corrupted the
1314 * repeat offset history.
1315 */
1316 data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD;
1317 data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC;
1318 data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD;
1319 data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC;
1320
1321 /* Add a match of offset=16, length=8 at idx=128K + 16.
1322 * This should generate a sequence with repeat code = 1.
1323 * But the block splitters mistake caused zstd to generate
1324 * repeat code = 2, corrupting the data.
1325 */
1326 data[blockSizeU32] = 0xBBBBBBBB;
1327 data[blockSizeU32 + 1] = 0xAAAAAAAA;
1328 data[blockSizeU32 + 4] = 0xBBBBBBBB;
1329 data[blockSizeU32 + 5] = 0xAAAAAAAA;
1330
1331 /* Generate a golden file from this data in case datagen changes and
1332 * doesn't generate the exact same data. We will also test this golden file.
1333 */
1334 if (0) {
1335 FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb");
1336 fwrite(data, 1, srcSize, f);
1337 fclose(f);
1338 }
1339
1340 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
1341 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7));
1342 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable));
1343
1344 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize);
1345 CHECK_Z(cSize);
1346 dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
1347 CHECK_Z(dSize);
1348 CHECK_EQ(dSize, srcSize);
1349 CHECK(!memcmp(decodedBuffer, data, srcSize));
1350
1351 free(data);
1352 ZSTD_freeCCtx(cctx);
1353 }
1354 DISPLAYLEVEL(3, "OK \n");
1355
1356 DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++);
1357 {
1358 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1359 const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer;
1360 size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize);
1361
1362 /* This is the number of bytes to stream before ending. This value
1363 * was obtained by trial and error :/. */
1364
1365 const size_t streamCompressThreshold = 161792;
1366 const size_t streamCompressDelta = 1024;
1367
1368 /* The first 1/5 of the buffer is compressible and the last 4/5 is
1369 * uncompressible. This is an approximation of the type of data
1370 * the fuzzer generated to catch this bug. Streams like this were making
1371 * zstd generate noCompress superblocks (which are larger than the src
1372 * they come from). Do this enough times, and we'll run out of room
1373 * and throw a dstSize_tooSmall error. */
1374
1375 const size_t compressiblePartSize = srcSize/5;
1376 const size_t uncompressiblePartSize = srcSize-compressiblePartSize;
1377 RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed);
1378 RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed);
1379
1380 /* Setting target block size so that superblock is used */
1381
1382 assert(cctx != NULL);
1383 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81);
1384
1385 { size_t read;
1386 for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) {
1387 ZSTD_inBuffer in = {src, streamCompressDelta, 0};
1388 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1389 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1390 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1391 src += streamCompressDelta; srcSize -= streamCompressDelta;
1392 dst += out.pos; dstCapacity -= out.pos;
1393 } }
1394
1395 /* This is trying to catch a dstSize_tooSmall error */
1396
1397 { ZSTD_inBuffer in = {src, srcSize, 0};
1398 ZSTD_outBuffer out = {dst, dstCapacity, 0};
1399 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
1400 }
1401 ZSTD_freeCCtx(cctx);
1402 }
1403 DISPLAYLEVEL(3, "OK \n");
1404
1405 DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++);
1406 /* Generate the same data 20 times over */
1407 { size_t const avgChunkSize = CNBuffSize / 20;
1408 size_t b;
1409 for (b = 0; b < CNBuffSize; b += avgChunkSize) {
1410 size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize);
1411 RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed);
1412 } }
1413 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1414 size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1415 size_t const allowedExpansion = (CNBuffSize * 3 / 1000);
1416 size_t superCSize;
1417 CHECK_Z(normalCSize);
1418 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
1419 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000);
1420 superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
1421 CHECK_Z(superCSize);
1422 if (superCSize > normalCSize + allowedExpansion) {
1423 DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion);
1424 goto _output_error;
1425 }
1426 ZSTD_freeCCtx(cctx);
1427 }
1428 DISPLAYLEVEL(3, "OK \n");
1429
1430 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed);
1431 DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++)
1432 /* This tests whether or not we leave enough room for the checksum at the end
1433 * of the dst buffer. The bug that motivated this test was found by the
1434 * stream_round_trip fuzzer but this crashes for the same reason and is
1435 * far more compact than re-creating the stream_round_trip fuzzer's code path */
1436 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1437 ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64);
1438 assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278)));
1439 ZSTD_freeCCtx(cctx);
1440 }
1441 DISPLAYLEVEL(3, "OK \n");
1442
1443 DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++);
1444 { int level = -1;
1445 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1446 if (!cctx) goto _output_error;
1447 for (level = -1; level <= ZSTD_maxCLevel(); ++level) {
1448 CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) );
1449 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) );
1450 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) );
1451 }
1452 ZSTD_freeCCtx(cctx);
1453 }
1454 DISPLAYLEVEL(3, "OK \n");
1455
1456 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++);
1457 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1458 size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19);
1459 if (ZSTD_isError(r)) goto _output_error;
1460 if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error;
1461 ZSTD_freeCCtx(cctx);
1462 cSize = r;
1463 }
1464 DISPLAYLEVEL(3, "OK \n");
1465
1466 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++);
1467 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize);
1468 if (ZSTD_isError(r)) goto _output_error;
1469 if (r != 0) goto _output_error;
1470 }
1471 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1472 ZSTD_outBuffer output;
1473 if (cctx==NULL) goto _output_error;
1474 output.dst = compressedBuffer;
1475 output.size = compressedBufferSize;
1476 output.pos = 0;
1477 CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */
1478 CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */
1479 CHECK_Z( ZSTD_endStream(cctx, &output) );
1480 ZSTD_freeCCtx(cctx);
1481 /* single scan decompression */
1482 { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos);
1483 if (ZSTD_isError(r)) goto _output_error;
1484 if (r != 0) goto _output_error;
1485 }
1486 /* streaming decompression */
1487 { ZSTD_DCtx* const dstream = ZSTD_createDStream();
1488 ZSTD_inBuffer dinput;
1489 ZSTD_outBuffer doutput;
1490 size_t ipos;
1491 if (dstream==NULL) goto _output_error;
1492 dinput.src = compressedBuffer;
1493 dinput.size = 0;
1494 dinput.pos = 0;
1495 doutput.dst = NULL;
1496 doutput.size = 0;
1497 doutput.pos = 0;
1498 CHECK_Z ( ZSTD_initDStream(dstream) );
1499 for (ipos=1; ipos<=output.pos; ipos++) {
1500 dinput.size = ipos;
1501 CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) );
1502 }
1503 if (doutput.pos != 0) goto _output_error;
1504 ZSTD_freeDStream(dstream);
1505 }
1506 }
1507 DISPLAYLEVEL(3, "OK \n");
1508
1509 DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++);
1510 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1511 ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1512 assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */
1513 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) );
1514 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */
1515
1516 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1517 { size_t const inSize = 2* 128 KB;
1518 size_t const outSize = ZSTD_compressBound(inSize);
1519 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) );
1520 /* will fail if blockSize is not resized */
1521 }
1522 ZSTD_freeCCtx(cctx);
1523 }
1524 DISPLAYLEVEL(3, "OK \n");
1525
1526 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++);
1527 { size_t const sampleSize = 30;
1528 int i;
1529 for (i=0; i<20; i++)
1530 ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */
1531 memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
1532 for (i=1; i<=19; i++) {
1533 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1534 size_t size1, size2;
1535 DISPLAYLEVEL(5, "l%i ", i);
1536 size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1537 CHECK_Z(size1);
1538
1539 size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i);
1540 CHECK_Z(size2);
1541 CHECK_EQ(size1, size2);
1542
1543 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) );
1544 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize);
1545 CHECK_Z(size2);
1546 CHECK_EQ(size1, size2);
1547
1548 size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */
1549 CHECK_Z(size2);
1550 CHECK_EQ(size1, size2);
1551
1552 { ZSTD_inBuffer inb;
1553 ZSTD_outBuffer outb;
1554 inb.src = CNBuffer;
1555 inb.pos = 0;
1556 inb.size = sampleSize;
1557 outb.dst = compressedBuffer;
1558 outb.pos = 0;
1559 outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */
1560 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) );
1561 assert(inb.pos == inb.size);
1562 CHECK_EQ(size1, outb.pos);
1563 }
1564
1565 ZSTD_freeCCtx(cctx);
1566 }
1567 }
1568 DISPLAYLEVEL(3, "OK \n");
1569
1570 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++);
1571 { size_t const sampleSize = 1024;
1572 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1573 ZSTD_inBuffer inb;
1574 ZSTD_outBuffer outb;
1575 inb.src = CNBuffer;
1576 inb.pos = 0;
1577 inb.size = 0;
1578 outb.dst = compressedBuffer;
1579 outb.pos = 0;
1580 outb.size = compressedBufferSize;
1581 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) );
1582
1583 inb.size = sampleSize; /* start with something, so that context is already used */
1584 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1585 assert(inb.pos == inb.size);
1586 outb.pos = 0; /* cancel output */
1587
1588 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) );
1589 inb.size = 4; /* too small size : compression will be skipped */
1590 inb.pos = 0;
1591 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1592 assert(inb.pos == inb.size);
1593
1594 inb.size += 5; /* too small size : compression will be skipped */
1595 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1596 assert(inb.pos == inb.size);
1597
1598 inb.size += 11; /* small enough to attempt compression */
1599 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) );
1600 assert(inb.pos == inb.size);
1601
1602 assert(inb.pos < sampleSize);
1603 inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */
1604 CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */
1605 assert(inb.pos == inb.size);
1606 ZSTD_freeCCtx(cctx);
1607 }
1608 DISPLAYLEVEL(3, "OK \n");
1609
1610 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++);
1611 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1612 ZSTD_outBuffer out = {NULL, 0, 0};
1613 ZSTD_inBuffer in = {NULL, 0, 0};
1614 int value;
1615
1616 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1617 CHECK_EQ(value, 3);
1618 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1619 CHECK_EQ(value, 0);
1620 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN));
1621 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1622 CHECK_EQ(value, 3);
1623 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1624 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1625 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7));
1626 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1627 CHECK_EQ(value, 7);
1628 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1629 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1630 /* Start a compression job */
1631 ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue);
1632 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1633 CHECK_EQ(value, 7);
1634 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1635 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1636 /* Reset the CCtx */
1637 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
1638 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1639 CHECK_EQ(value, 7);
1640 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1641 CHECK_EQ(value, ZSTD_HASHLOG_MIN);
1642 /* Reset the parameters */
1643 ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters);
1644 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value));
1645 CHECK_EQ(value, 3);
1646 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1647 CHECK_EQ(value, 0);
1648
1649 ZSTD_freeCCtx(cctx);
1650 }
1651 DISPLAYLEVEL(3, "OK \n");
1652
1653 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++);
1654 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1655 int value;
1656 ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0);
1657 cparams.strategy = -1;
1658 /* Set invalid cParams == no change. */
1659 CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams)));
1660
1661 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1662 CHECK_EQ(value, 0);
1663 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1664 CHECK_EQ(value, 0);
1665 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1666 CHECK_EQ(value, 0);
1667 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1668 CHECK_EQ(value, 0);
1669 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1670 CHECK_EQ(value, 0);
1671 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1672 CHECK_EQ(value, 0);
1673 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1674 CHECK_EQ(value, 0);
1675
1676 cparams = ZSTD_getCParams(12, 0, 0);
1677 CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams));
1678
1679 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1680 CHECK_EQ(value, (int)cparams.windowLog);
1681 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1682 CHECK_EQ(value, (int)cparams.chainLog);
1683 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1684 CHECK_EQ(value, (int)cparams.hashLog);
1685 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1686 CHECK_EQ(value, (int)cparams.searchLog);
1687 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1688 CHECK_EQ(value, (int)cparams.minMatch);
1689 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1690 CHECK_EQ(value, (int)cparams.targetLength);
1691 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1692 CHECK_EQ(value, (int)cparams.strategy);
1693
1694 ZSTD_freeCCtx(cctx);
1695 }
1696
1697 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++);
1698 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1699 int value;
1700 ZSTD_frameParameters fparams = {0, 1, 1};
1701
1702 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1703 CHECK_EQ(value, 1);
1704 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1705 CHECK_EQ(value, 0);
1706 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1707 CHECK_EQ(value, 1);
1708
1709 CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams));
1710
1711 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1712 CHECK_EQ(value, fparams.contentSizeFlag);
1713 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1714 CHECK_EQ(value, fparams.checksumFlag);
1715 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1716 CHECK_EQ(value, !fparams.noDictIDFlag);
1717
1718 ZSTD_freeCCtx(cctx);
1719 }
1720
1721 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCarams() : ", testNb++);
1722 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1723 int value;
1724 ZSTD_parameters params = ZSTD_getParams(1, 0, 0);
1725 params.cParams.strategy = -1;
1726 /* Set invalid params == no change. */
1727 CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params)));
1728
1729 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1730 CHECK_EQ(value, 0);
1731 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1732 CHECK_EQ(value, 0);
1733 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1734 CHECK_EQ(value, 0);
1735 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1736 CHECK_EQ(value, 0);
1737 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1738 CHECK_EQ(value, 0);
1739 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1740 CHECK_EQ(value, 0);
1741 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1742 CHECK_EQ(value, 0);
1743 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1744 CHECK_EQ(value, 1);
1745 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1746 CHECK_EQ(value, 0);
1747 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1748 CHECK_EQ(value, 1);
1749
1750 params = ZSTD_getParams(12, 0, 0);
1751 params.fParams.contentSizeFlag = 0;
1752 params.fParams.checksumFlag = 1;
1753 params.fParams.noDictIDFlag = 1;
1754 CHECK_Z(ZSTD_CCtx_setParams(cctx, params));
1755
1756 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value));
1757 CHECK_EQ(value, (int)params.cParams.windowLog);
1758 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value));
1759 CHECK_EQ(value, (int)params.cParams.chainLog);
1760 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value));
1761 CHECK_EQ(value, (int)params.cParams.hashLog);
1762 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value));
1763 CHECK_EQ(value, (int)params.cParams.searchLog);
1764 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value));
1765 CHECK_EQ(value, (int)params.cParams.minMatch);
1766 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value));
1767 CHECK_EQ(value, (int)params.cParams.targetLength);
1768 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value));
1769 CHECK_EQ(value, (int)params.cParams.strategy);
1770 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value));
1771 CHECK_EQ(value, params.fParams.contentSizeFlag);
1772 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value));
1773 CHECK_EQ(value, params.fParams.checksumFlag);
1774 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value));
1775 CHECK_EQ(value, !params.fParams.noDictIDFlag);
1776
1777 ZSTD_freeCCtx(cctx);
1778 }
1779
1780 DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++);
1781 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1782 ZSTD_outBuffer out = {NULL, 0, 0};
1783 ZSTD_inBuffer in = {NULL, 0, 0};
1784 int value;
1785
1786 /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt),
1787 * we should not modify the actual parameter specified by the user within the CCtx
1788 */
1789 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27));
1790 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt));
1791
1792 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue));
1793 CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value));
1794 CHECK_EQ(value, 0);
1795
1796 ZSTD_freeCCtx(cctx);
1797 }
1798 DISPLAYLEVEL(3, "OK \n");
1799
1800 /* this test is really too long, and should be made faster */
1801 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++);
1802 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
1803 ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0);
1804 size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */
1805 size_t cnb;
1806 assert(cctx != NULL);
1807 params.fParams.contentSizeFlag = 0;
1808 params.cParams.windowLog = ZSTD_WINDOWLOG_MAX;
1809 for (cnb = 0; cnb < nbCompressions; ++cnb) {
1810 DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions);
1811 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */
1812 CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) );
1813 }
1814 ZSTD_freeCCtx(cctx);
1815 }
1816 DISPLAYLEVEL(3, "OK \n");
1817
1818 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++);
1819 { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx();
1820 assert(largeCCtx != NULL);
1821 CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
1822 CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) );
1823 { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */
1824 { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx();
1825 assert(smallCCtx != NULL);
1826 CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1827 { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx);
1828 DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
1829 largeCCtxSize>>10, smallCCtxSize>>10);
1830 assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c .
1831 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
1832 }
1833 ZSTD_freeCCtx(smallCCtx);
1834 }
1835 { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
1836 * currently defined as 128x, but could be adjusted in the future.
1837 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
1838 unsigned u;
1839 for (u=0; u<maxNbAttempts; u++) {
1840 CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1));
1841 if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */
1842 }
1843 DISPLAYLEVEL(5, "size down after %u attempts : ", u);
1844 if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */
1845 }
1846 }
1847 ZSTD_freeCCtx(largeCCtx);
1848 }
1849 DISPLAYLEVEL(3, "OK \n");
1850
1851 /* Static CCtx tests */
1852#define STATIC_CCTX_LEVEL 4
1853 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1854 { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL);
1855 void* const staticCCtxBuffer = malloc(staticCStreamSize);
1856 size_t const staticDCtxSize = ZSTD_estimateDCtxSize();
1857 void* const staticDCtxBuffer = malloc(staticDCtxSize);
1858 DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize);
1859 if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) {
1860 free(staticCCtxBuffer);
1861 free(staticDCtxBuffer);
1862 DISPLAY("Not enough memory, aborting\n");
1863 testResult = 1;
1864 goto _end;
1865 }
1866 { size_t const smallInSize = 32 KB;
1867 ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0);
1868 size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small);
1869 size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL);
1870 ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize);
1871 ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize);
1872 DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize);
1873 DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize);
1874 if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error;
1875 DISPLAYLEVEL(3, "OK \n");
1876
1877 DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++);
1878 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1879 compressedBuffer, compressedBufferSize,
1880 CNBuffer, smallInSize, STATIC_CCTX_LEVEL) );
1881 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1882 (unsigned)cSize, (double)cSize/smallInSize*100);
1883
1884 DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++);
1885 { size_t const r = ZSTD_compressCCtx(staticCCtx,
1886 compressedBuffer, compressedBufferSize,
1887 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL);
1888 if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error;
1889 }
1890 DISPLAYLEVEL(3, "OK \n");
1891
1892 DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++);
1893 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize);
1894 DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx);
1895 if (staticCCtx == NULL) goto _output_error;
1896 DISPLAYLEVEL(3, "OK \n");
1897
1898 DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++);
1899 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1900 compressedBuffer, compressedBufferSize,
1901 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1902 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1903 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1904
1905 DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++);
1906 { int nbc;
1907 assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */
1908 assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3);
1909 for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) {
1910 CHECK_Z(ZSTD_compressCCtx(staticCCtx,
1911 compressedBuffer, compressedBufferSize,
1912 (char*)CNBuffer + nbc, smallInSize,
1913 STATIC_CCTX_LEVEL) );
1914 } }
1915 DISPLAYLEVEL(3, "OK \n")
1916
1917 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL);
1918 CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) );
1919 DISPLAYLEVEL(3, "OK \n");
1920
1921 DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++);
1922 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1923 compressedBuffer, compressedBufferSize,
1924 CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) );
1925 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
1926 (unsigned)cSize, (double)cSize/CNBuffSize*100);
1927
1928 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++);
1929 { size_t const r = ZSTD_decompressDCtx(staticDCtx,
1930 decodedBuffer, CNBuffSize,
1931 compressedBuffer, cSize);
1932 if (r != CNBuffSize) goto _output_error; }
1933 DISPLAYLEVEL(3, "OK \n");
1934
1935 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
1936 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
1937 DISPLAYLEVEL(3, "OK \n");
1938
1939 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++);
1940 { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel());
1941 if (!ZSTD_isError(r)) goto _output_error; }
1942 DISPLAYLEVEL(3, "OK \n");
1943
1944 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1);
1945 CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) );
1946 DISPLAYLEVEL(3, "OK \n");
1947
1948 DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++);
1949 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1950 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1951 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1952 size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */
1953 if (!ZSTD_isError(r)) goto _output_error;
1954 }
1955 DISPLAYLEVEL(3, "OK \n");
1956
1957 DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++);
1958 staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize);
1959 assert(staticCCtx != NULL);
1960 CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */
1961 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1962 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1963 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1964 }
1965 DISPLAYLEVEL(3, "OK \n");
1966
1967 DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1);
1968 CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */
1969 { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 };
1970 ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 };
1971 CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) );
1972 }
1973 DISPLAYLEVEL(3, "OK \n");
1974
1975 DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++);
1976 { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1);
1977 if (!ZSTD_isError(r)) goto _output_error; }
1978 DISPLAYLEVEL(3, "OK \n");
1979
1980 DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++);
1981 CHECK_Z( ZSTD_initDStream(staticDCtx) );
1982 { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 };
1983 ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 };
1984 size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input);
1985 if (!ZSTD_isError(r)) goto _output_error;
1986 }
1987 DISPLAYLEVEL(3, "OK \n");
1988
1989 DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++);
1990 {
1991 // Test ZSTD_estimateCCtxSize_usingCCtxParams
1992 {
1993 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
1994 size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
1995 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
1996 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
1997 compressedBuffer, compressedBufferSize,
1998 CNBuffer, CNBuffSize, 3));
1999
2000 {
2001 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2002 decodedBuffer, CNBuffSize,
2003 compressedBuffer, cSize);
2004 if (r != CNBuffSize) goto _output_error;
2005 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2006 }
2007 ZSTD_freeCCtxParams(params);
2008 }
2009
2010 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2011 {
2012 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2013 size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2014 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2015 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2016 compressedBuffer, compressedBufferSize,
2017 CNBuffer, CNBuffSize, 3) );
2018
2019 {
2020 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2021 decodedBuffer, CNBuffSize,
2022 compressedBuffer, cSize);
2023 if (r != CNBuffSize) goto _output_error;
2024 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2025 }
2026 ZSTD_freeCCtxParams(params);
2027 }
2028 }
2029 DISPLAYLEVEL(3, "OK \n");
2030
2031 DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++);
2032 {
2033 // Test ZSTD_estimateCCtxSize_usingCCtxParams
2034 {
2035 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2036 size_t cctxSizeDefault;
2037 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2038 cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params);
2039 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2040 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2041 compressedBuffer, compressedBufferSize,
2042 CNBuffer, CNBuffSize, 3) );
2043
2044 {
2045 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2046 decodedBuffer, CNBuffSize,
2047 compressedBuffer, cSize);
2048 if (r != CNBuffSize) goto _output_error;
2049 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2050 }
2051 ZSTD_freeCCtxParams(params);
2052 }
2053
2054 // Test ZSTD_estimateCStreamSize_usingCCtxParams
2055 {
2056 ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2057 size_t cctxSizeDefault;
2058 CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0));
2059 cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params);
2060 staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault);
2061 CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx,
2062 compressedBuffer, compressedBufferSize,
2063 CNBuffer, CNBuffSize, 3) );
2064
2065 {
2066 size_t const r = ZSTD_decompressDCtx(staticDCtx,
2067 decodedBuffer, CNBuffSize,
2068 compressedBuffer, cSize);
2069 if (r != CNBuffSize) goto _output_error;
2070 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error;
2071 }
2072 ZSTD_freeCCtxParams(params);
2073 }
2074 }
2075 DISPLAYLEVEL(3, "OK \n");
2076 }
2077 free(staticCCtxBuffer);
2078 free(staticDCtxBuffer);
2079 }
2080
2081 DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++);
2082 { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1);
2083 size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1);
2084 size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1);
2085 size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1);
2086
2087 if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error;
2088 if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error;
2089 }
2090 DISPLAYLEVEL(3, "OK \n");
2091
2092
2093 /* ZSTDMT simple MT compression test */
2094 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++);
2095 { ZSTD_CCtx* const mtctx = ZSTD_createCCtx();
2096 if (mtctx==NULL) {
2097 DISPLAY("mtctx : not enough memory, aborting \n");
2098 testResult = 1;
2099 goto _end;
2100 }
2101 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) );
2102 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) );
2103 DISPLAYLEVEL(3, "OK \n");
2104
2105 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize);
2106 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2107 compressedBuffer, compressedBufferSize,
2108 CNBuffer, CNBuffSize) );
2109 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2110
2111 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++);
2112 { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2113 if (rSize != CNBuffSize) {
2114 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize);
2115 goto _output_error;
2116 } }
2117 DISPLAYLEVEL(3, "OK \n");
2118
2119 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2120 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2121 if (r != CNBuffSize) goto _output_error; }
2122 DISPLAYLEVEL(3, "OK \n");
2123
2124 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2125 { size_t u;
2126 for (u=0; u<CNBuffSize; u++) {
2127 if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;
2128 } }
2129 DISPLAYLEVEL(3, "OK \n");
2130
2131 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++);
2132 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) );
2133 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) );
2134 CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) );
2135 CHECK_VAR(cSize, ZSTD_compress2(mtctx,
2136 compressedBuffer, compressedBufferSize,
2137 CNBuffer, CNBuffSize) );
2138 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2139
2140 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize);
2141 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize);
2142 if (r != CNBuffSize) goto _output_error; }
2143 DISPLAYLEVEL(3, "OK \n");
2144
2145 ZSTD_freeCCtx(mtctx);
2146 }
2147
2148 DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++);
2149 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2150 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2151 char out[32];
2152 if (cctx == NULL || dctx == NULL) goto _output_error;
2153 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) );
2154 CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) );
2155 DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize);
2156
2157 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) );
2158 { char const* outPtr = out;
2159 ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 };
2160 ZSTD_outBuffer outBuffer = { NULL, 0, 0 };
2161 size_t dSize;
2162 CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) );
2163 if (dSize != 0) goto _output_error;
2164 }
2165
2166 ZSTD_freeDCtx(dctx);
2167 ZSTD_freeCCtx(cctx);
2168 }
2169
2170 DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++)
2171 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2172 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) );
2173 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2174 CHECK_Z(cSize);
2175 ZSTD_freeCCtx(cctx);
2176 }
2177 DISPLAYLEVEL(3, "OK \n");
2178
2179 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++)
2180 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2181 size_t cSize1, cSize2;
2182 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2183 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) );
2184 cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2185 CHECK_Z(cSize1);
2186 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) );
2187 cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
2188 CHECK_Z(cSize2);
2189 CHECK_LT(cSize1, cSize2);
2190 ZSTD_freeCCtx(cctx);
2191 }
2192 DISPLAYLEVEL(3, "OK \n");
2193
2194 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++)
2195 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
2196 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
2197 * ZSTDMT is forced to not take the shortcut.
2198 */
2199 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2200 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) );
2201 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) );
2202 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) );
2203 ZSTD_freeCCtx(cctx);
2204 }
2205 DISPLAYLEVEL(3, "OK \n");
2206
2207 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++)
2208 { ZSTD_CCtx_params* params = ZSTD_createCCtxParams();
2209 int const jobSize = 512 KB;
2210 int value;
2211 /* Check that the overlap log and job size are unset. */
2212 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2213 CHECK_EQ(value, 0);
2214 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2215 CHECK_EQ(value, 0);
2216 /* Set and check the overlap log and job size. */
2217 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) );
2218 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) );
2219 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2220 CHECK_EQ(value, 5);
2221 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2222 CHECK_EQ(value, jobSize);
2223 /* Set the number of workers and check the overlap log and job size. */
2224 CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) );
2225 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) );
2226 CHECK_EQ(value, 5);
2227 CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) );
2228 CHECK_EQ(value, jobSize);
2229 ZSTD_freeCCtxParams(params);
2230 }
2231 DISPLAYLEVEL(3, "OK \n");
2232
2233 /* Simple API multiframe test */
2234 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++);
2235 { size_t off = 0;
2236 int i;
2237 int const segs = 4;
2238 /* only use the first half so we don't push against size limit of compressedBuffer */
2239 size_t const segSize = (CNBuffSize / 2) / segs;
2240
2241 const U32 skipLen = 129 KB;
2242 char* const skipBuff = (char*)malloc(skipLen);
2243 assert(skipBuff != NULL);
2244 memset(skipBuff, 0, skipLen);
2245 for (i = 0; i < segs; i++) {
2246 CHECK_NEWV(r, ZSTD_compress(
2247 (BYTE*)compressedBuffer + off, CNBuffSize - off,
2248 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
2249 5) );
2250 off += r;
2251 if (i == segs/2) {
2252 /* insert skippable frame */
2253 size_t const skippableSize =
2254 ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize,
2255 skipBuff, skipLen, seed % 15);
2256 CHECK_Z(skippableSize);
2257 off += skippableSize;
2258 }
2259 }
2260 cSize = off;
2261 free(skipBuff);
2262 }
2263 DISPLAYLEVEL(3, "OK \n");
2264
2265 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++);
2266 { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize);
2267 if (r != CNBuffSize / 2) goto _output_error; }
2268 DISPLAYLEVEL(3, "OK \n");
2269
2270 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++);
2271 { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize);
2272 if (bound != CNBuffSize / 2) goto _output_error; }
2273 DISPLAYLEVEL(3, "OK \n");
2274
2275 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++);
2276 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize));
2277 if (r != CNBuffSize / 2) goto _output_error; }
2278 DISPLAYLEVEL(3, "OK \n");
2279
2280 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++);
2281 if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error;
2282 DISPLAYLEVEL(3, "OK \n");
2283
2284 /* Simple API skippable frame test */
2285 DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++);
2286 { U32 i;
2287 unsigned readMagic;
2288 unsigned long long receivedSize;
2289 size_t skippableSize;
2290 const U32 skipLen = 129 KB;
2291 char* const skipBuff = (char*)malloc(skipLen);
2292 assert(skipBuff != NULL);
2293 for (i = 0; i < skipLen; i++)
2294 skipBuff[i] = (char) ((seed + i) % 256);
2295 skippableSize = ZSTD_writeSkippableFrame(
2296 compressedBuffer, compressedBufferSize,
2297 skipBuff, skipLen, seed % 15);
2298 CHECK_Z(skippableSize);
2299 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2300 receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize);
2301 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2302 CHECK_EQ(seed % 15, readMagic);
2303 if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error;
2304
2305 free(skipBuff);
2306 }
2307 DISPLAYLEVEL(3, "OK \n");
2308
2309 DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++);
2310 {
2311 unsigned readMagic;
2312 unsigned long long receivedSize;
2313 size_t skippableSize;
2314 skippableSize = ZSTD_writeSkippableFrame(
2315 compressedBuffer, compressedBufferSize,
2316 CNBuffer, 0, seed % 15);
2317 CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize);
2318 CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize));
2319 receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize);
2320 CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE);
2321 CHECK_EQ(seed % 15, readMagic);
2322 }
2323 DISPLAYLEVEL(3, "OK \n");
2324
2325 /* Dictionary and CCtx Duplication tests */
2326 { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx();
2327 ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx();
2328 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2329 static const size_t dictSize = 551;
2330 assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL);
2331
2332 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++);
2333 { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0);
2334 if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */
2335 DISPLAYLEVEL(3, "OK \n");
2336
2337 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++);
2338 CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) );
2339 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
2340 DISPLAYLEVEL(3, "OK \n");
2341
2342 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++);
2343 cSize = 0;
2344 CHECKPLUS(r, ZSTD_compressEnd(ctxOrig,
2345 compressedBuffer, compressedBufferSize,
2346 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2347 cSize += r);
2348 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2349
2350 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++);
2351 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2352 decodedBuffer, CNBuffSize,
2353 compressedBuffer, cSize,
2354 CNBuffer, dictSize),
2355 if (r != CNBuffSize - dictSize) goto _output_error);
2356 DISPLAYLEVEL(3, "OK \n");
2357
2358 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++);
2359 { size_t const cSizeOrig = cSize;
2360 cSize = 0;
2361 CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated,
2362 compressedBuffer, compressedBufferSize,
2363 (const char*)CNBuffer + dictSize, CNBuffSize - dictSize),
2364 cSize += r);
2365 if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */
2366 }
2367 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2368
2369 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++);
2370 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2371 decodedBuffer, CNBuffSize,
2372 compressedBuffer, cSize,
2373 CNBuffer, dictSize),
2374 if (r != CNBuffSize - dictSize) goto _output_error);
2375 DISPLAYLEVEL(3, "OK \n");
2376
2377 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++);
2378 { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize);
2379 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2380 if (r != CNBuffSize - dictSize) goto _output_error;
2381 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict));
2382 ZSTD_freeDDict(ddict);
2383 }
2384
2385 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++);
2386 { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy);
2387 void* const ddictBuffer = malloc(ddictBufferSize);
2388 if (ddictBuffer == NULL) goto _output_error;
2389 { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto);
2390 size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict);
2391 if (r != CNBuffSize - dictSize) goto _output_error;
2392 }
2393 free(ddictBuffer);
2394 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize);
2395 }
2396
2397 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++);
2398 { size_t const testSize = CNBuffSize / 3;
2399 CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) );
2400 CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) );
2401
2402 CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize),
2403 (const char*)CNBuffer + dictSize, testSize) );
2404 { ZSTD_frameHeader zfh;
2405 if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error;
2406 if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error;
2407 } }
2408 DISPLAYLEVEL(3, "OK \n");
2409
2410 /* Note : these tests should be replaced by proper regression tests,
2411 * but existing ones do not focus on small data + dictionary + all levels.
2412 */
2413 if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */
2414 size_t const flatdictSize = 22 KB;
2415 size_t const contentSize = 9 KB;
2416 const void* const dict = (const char*)CNBuffer;
2417 const void* const contentStart = (const char*)dict + flatdictSize;
2418 /* These upper bounds are generally within a few bytes of the compressed size */
2419 size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770,
2420 3770, 3770, 3770, 3750, 3750,
2421 3742, 3675, 3674, 3665, 3664,
2422 3663, 3662, 3661, 3660, 3660,
2423 3660, 3660, 3660 };
2424 size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940,
2425 2950, 2950, 2925, 2900, 2892,
2426 2910, 2910, 2910, 2780, 2775,
2427 2765, 2760, 2755, 2754, 2753,
2428 2753, 2753, 2753 };
2429 int l = 1;
2430 int const maxLevel = ZSTD_maxCLevel();
2431 /* clevels with strategies that support rowhash on small inputs */
2432 int rowLevel = 4;
2433 int const rowLevelEnd = 8;
2434
2435 DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++);
2436 assert(maxLevel == 22);
2437 RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed);
2438 DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n",
2439 (unsigned long long)XXH64(contentStart, contentSize, 0),
2440 (unsigned long long)XXH64(dict, flatdictSize, 0));
2441
2442 for ( ; l <= maxLevel; l++) {
2443 size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize,
2444 contentStart, contentSize, l);
2445 if (nodict_cSize > target_nodict_cSize[l]) {
2446 DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n",
2447 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2448 goto _output_error;
2449 }
2450 DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n",
2451 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2452 }
2453 for ( l=1 ; l <= maxLevel; l++) {
2454 size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig,
2455 compressedBuffer, compressedBufferSize,
2456 contentStart, contentSize,
2457 dict, flatdictSize,
2458 l);
2459 if (wdict_cSize > target_wdict_cSize[l]) {
2460 DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n",
2461 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2462 goto _output_error;
2463 }
2464 DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n",
2465 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2466 }
2467 /* Compression with ZSTD_compress2 and row match finder force enabled.
2468 * Give some slack for force-enabled row matchfinder since we're on a small input (9KB)
2469 */
2470 for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5;
2471 for (l=1 ; l <= maxLevel; l++) {
2472 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2473 size_t nodict_cSize;
2474 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l);
2475 ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable);
2476 nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize,
2477 contentStart, contentSize);
2478 if (nodict_cSize > target_nodict_cSize[l]) {
2479 DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n",
2480 l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]);
2481 ZSTD_freeCCtx(cctx);
2482 goto _output_error;
2483 }
2484 DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n",
2485 l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize);
2486 ZSTD_freeCCtx(cctx);
2487 }
2488 /* Dict compression with DMS */
2489 for ( l=1 ; l <= maxLevel; l++) {
2490 size_t wdict_cSize;
2491 CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) );
2492 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) );
2493 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) );
2494 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) );
2495 CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) );
2496 wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize);
2497 if (wdict_cSize > target_wdict_cSize[l]) {
2498 DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n",
2499 l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]);
2500 goto _output_error;
2501 }
2502 DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n",
2503 l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize);
2504 }
2505
2506 DISPLAYLEVEL(4, "compression efficiency tests OK \n");
2507 }
2508
2509 ZSTD_freeCCtx(ctxOrig);
2510 ZSTD_freeCCtx(ctxDuplicated);
2511 ZSTD_freeDCtx(dctx);
2512 }
2513
2514 /* Dictionary and dictBuilder tests */
2515 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
2516 size_t const dictBufferCapacity = 16 KB;
2517 void* const dictBuffer = malloc(dictBufferCapacity);
2518 size_t const totalSampleSize = 1 MB;
2519 size_t const sampleUnitSize = 8 KB;
2520 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
2521 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
2522 size_t dictSize;
2523 U32 dictID;
2524 size_t dictHeaderSize;
2525 size_t dictBufferFixedSize = 144;
2526 unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f,
2527 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2528 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01,
2529 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08,
2530 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2531 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08,
2532 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18,
2533 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c,
2534 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04,
2535 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61,
2536 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65,
2537 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69};
2538
2539 if (dictBuffer==NULL || samplesSizes==NULL) {
2540 free(dictBuffer);
2541 free(samplesSizes);
2542 goto _output_error;
2543 }
2544
2545 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++);
2546 assert(compressedBufferSize >= totalSampleSize);
2547 { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; }
2548 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2549 { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2550 decodedBuffer, samplesSizes, nbSamples);
2551 if (ZDICT_isError(sDictSize)) goto _output_error;
2552 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize);
2553 }
2554
2555 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++);
2556 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2557 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity,
2558 CNBuffer, samplesSizes, nbSamples);
2559 if (ZDICT_isError(dictSize)) goto _output_error;
2560 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2561
2562 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++);
2563 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2564 { ZDICT_cover_params_t coverParams;
2565 memset(&coverParams, 0, sizeof(coverParams));
2566 coverParams.steps = 8;
2567 coverParams.nbThreads = 4;
2568 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2569 dictBuffer, dictBufferCapacity,
2570 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2571 &coverParams);
2572 if (ZDICT_isError(dictSize)) goto _output_error;
2573 }
2574 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2575
2576 DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++);
2577 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2578 { ZDICT_cover_params_t coverParams;
2579 memset(&coverParams, 0, sizeof(coverParams));
2580 coverParams.steps = 8;
2581 coverParams.nbThreads = 4;
2582 coverParams.shrinkDict = 1;
2583 coverParams.shrinkDictMaxRegression = 1;
2584 dictSize = ZDICT_optimizeTrainFromBuffer_cover(
2585 dictBuffer, dictBufferCapacity,
2586 CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */
2587 &coverParams);
2588 if (ZDICT_isError(dictSize)) goto _output_error;
2589 }
2590 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2591
2592 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++);
2593 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2594 { ZDICT_fastCover_params_t fastCoverParams;
2595 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2596 fastCoverParams.steps = 8;
2597 fastCoverParams.nbThreads = 4;
2598 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2599 dictBuffer, dictBufferCapacity,
2600 CNBuffer, samplesSizes, nbSamples,
2601 &fastCoverParams);
2602 if (ZDICT_isError(dictSize)) goto _output_error;
2603 }
2604 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2605
2606 DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++);
2607 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
2608 { ZDICT_fastCover_params_t fastCoverParams;
2609 memset(&fastCoverParams, 0, sizeof(fastCoverParams));
2610 fastCoverParams.steps = 8;
2611 fastCoverParams.nbThreads = 4;
2612 fastCoverParams.shrinkDict = 1;
2613 fastCoverParams.shrinkDictMaxRegression = 1;
2614 dictSize = ZDICT_optimizeTrainFromBuffer_fastCover(
2615 dictBuffer, dictBufferCapacity,
2616 CNBuffer, samplesSizes, nbSamples,
2617 &fastCoverParams);
2618 if (ZDICT_isError(dictSize)) goto _output_error;
2619 }
2620 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
2621
2622 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
2623 dictID = ZDICT_getDictID(dictBuffer, dictSize);
2624 if (dictID==0) goto _output_error;
2625 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
2626
2627 DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++);
2628 dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize);
2629 if (dictHeaderSize==0) goto _output_error;
2630 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2631
2632 DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++);
2633 { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize);
2634 if (dictHeaderSize != 115) goto _output_error;
2635 }
2636 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize);
2637
2638 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++);
2639 cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize,
2640 CNBuffer, CNBuffSize,
2641 dictBuffer, dictSize, 4);
2642 if (ZSTD_isError(cSize)) goto _output_error;
2643 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2644
2645 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++);
2646 { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
2647 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2648 }
2649 DISPLAYLEVEL(3, "OK \n");
2650
2651 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2652 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2653 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2654 }
2655 DISPLAYLEVEL(3, "OK \n");
2656
2657 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2658 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2659 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2660 decodedBuffer, CNBuffSize,
2661 compressedBuffer, cSize,
2662 dictBuffer, dictSize),
2663 if (r != CNBuffSize) goto _output_error);
2664 ZSTD_freeDCtx(dctx);
2665 }
2666 DISPLAYLEVEL(3, "OK \n");
2667
2668 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++);
2669 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2670 size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef);
2671 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize);
2672 }
2673
2674 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++);
2675 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2676 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
2677 ZSTD_dlm_byRef, ZSTD_dct_auto,
2678 cParams, ZSTD_defaultCMem);
2679 assert(cdict != NULL);
2680 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict));
2681 assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict));
2682 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
2683 CNBuffer, CNBuffSize, cdict);
2684 ZSTD_freeCDict(cdict);
2685 if (ZSTD_isError(cSize)) goto _output_error;
2686 }
2687 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2688
2689 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++);
2690 { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
2691 if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
2692 }
2693 DISPLAYLEVEL(3, "OK \n");
2694
2695 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
2696 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2697 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2698 decodedBuffer, CNBuffSize,
2699 compressedBuffer, cSize,
2700 dictBuffer, dictSize),
2701 if (r != CNBuffSize) goto _output_error);
2702 ZSTD_freeDCtx(dctx);
2703 }
2704 DISPLAYLEVEL(3, "OK \n");
2705
2706 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++);
2707 { int const maxLevel = ZSTD_maxCLevel();
2708 int level;
2709 for (level = 1; level <= maxLevel; ++level) {
2710 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize);
2711 size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy);
2712 void* const cdictBuffer = malloc(cdictSize);
2713 if (cdictBuffer==NULL) goto _output_error;
2714 { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict(
2715 cdictBuffer, cdictSize,
2716 dictBuffer, dictSize,
2717 ZSTD_dlm_byCopy, ZSTD_dct_auto,
2718 cParams);
2719 if (cdict == NULL) {
2720 DISPLAY("ZSTD_initStaticCDict failed ");
2721 goto _output_error;
2722 }
2723 cSize = ZSTD_compress_usingCDict(cctx,
2724 compressedBuffer, compressedBufferSize,
2725 CNBuffer, MIN(10 KB, CNBuffSize), cdict);
2726 if (ZSTD_isError(cSize)) {
2727 DISPLAY("ZSTD_compress_usingCDict failed ");
2728 goto _output_error;
2729 } }
2730 free(cdictBuffer);
2731 } }
2732 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2733
2734 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++);
2735 { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
2736 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2737 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem);
2738 assert(cdict != NULL);
2739 cSize = ZSTD_compress_usingCDict_advanced(cctx,
2740 compressedBuffer, compressedBufferSize,
2741 CNBuffer, CNBuffSize,
2742 cdict, fParams);
2743 ZSTD_freeCDict(cdict);
2744 if (ZSTD_isError(cSize)) goto _output_error;
2745 }
2746 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2747
2748 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++);
2749 { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize);
2750 if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error;
2751 }
2752 DISPLAYLEVEL(3, "OK (unknown)\n");
2753
2754 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2755 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2756 assert(dctx != NULL);
2757 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2758 decodedBuffer, CNBuffSize,
2759 compressedBuffer, cSize,
2760 dictBuffer, dictSize),
2761 if (r != CNBuffSize) goto _output_error);
2762 ZSTD_freeDCtx(dctx);
2763 }
2764 DISPLAYLEVEL(3, "OK \n");
2765
2766 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++);
2767 { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize);
2768 p.fParams.noDictIDFlag = 1;
2769 cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize,
2770 CNBuffer, CNBuffSize,
2771 dictBuffer, dictSize, p);
2772 if (ZSTD_isError(cSize)) goto _output_error;
2773 }
2774 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100);
2775
2776 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++);
2777 { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL);
2778 CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
2779 decodedBuffer, CNBuffSize,
2780 compressedBuffer, cSize,
2781 dictBuffer, dictSize),
2782 if (r != CNBuffSize) goto _output_error);
2783 ZSTD_freeDCtx(dctx);
2784 }
2785 DISPLAYLEVEL(3, "OK \n");
2786
2787 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++);
2788 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2789 assert(dctx != NULL);
2790 { const size_t ret = ZSTD_decompress_usingDict(
2791 dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize,
2792 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
2793 if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted)
2794 goto _output_error;
2795 }
2796 ZSTD_freeDCtx(dctx);
2797 }
2798 DISPLAYLEVEL(3, "OK \n");
2799
2800 DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++);
2801 { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2802 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
2803 ZSTD_frameParameters const fParams = { 0, 1, 0 };
2804 size_t cBlockSize;
2805 cSize = 0;
2806 CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN));
2807 cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000);
2808 CHECK_Z(cBlockSize);
2809 cSize += cBlockSize;
2810 cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000);
2811 CHECK_Z(cBlockSize);
2812 cSize += cBlockSize;
2813
2814 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
2815
2816 ZSTD_freeCDict(cdict);
2817 ZSTD_freeDCtx(dctx);
2818 }
2819 DISPLAYLEVEL(3, "OK \n");
2820
2821 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++);
2822 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2823 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2824 if (cdict==NULL) goto _output_error;
2825 ZSTD_freeCDict(cdict);
2826 }
2827 DISPLAYLEVEL(3, "OK \n");
2828
2829 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++);
2830 { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize);
2831 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem);
2832 if (cdict!=NULL) goto _output_error;
2833 ZSTD_freeCDict(cdict);
2834 }
2835 DISPLAYLEVEL(3, "OK \n");
2836
2837 { char* rawDictBuffer = (char*)malloc(dictSize);
2838 assert(rawDictBuffer);
2839 memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2);
2840 memset(rawDictBuffer + dictSize - 2, 0, 2);
2841 MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY);
2842
2843 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++);
2844 {
2845 size_t ret;
2846 /* Either operation is allowed to fail, but one must fail. */
2847 ret = ZSTD_CCtx_loadDictionary_advanced(
2848 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto);
2849 if (!ZSTD_isError(ret)) {
2850 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2851 if (!ZSTD_isError(ret)) goto _output_error;
2852 }
2853 }
2854 DISPLAYLEVEL(3, "OK \n");
2855
2856 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++);
2857 {
2858 size_t ret;
2859 ret = ZSTD_CCtx_loadDictionary_advanced(
2860 cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent);
2861 if (ZSTD_isError(ret)) goto _output_error;
2862 ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100));
2863 if (ZSTD_isError(ret)) goto _output_error;
2864 }
2865 DISPLAYLEVEL(3, "OK \n");
2866
2867 DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++);
2868 { size_t const srcSize = MIN(CNBuffSize, 100);
2869 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2870 /* Force the dictionary to be reloaded in raw content mode */
2871 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad));
2872 CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent));
2873 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize);
2874 CHECK_Z(cSize);
2875 }
2876 DISPLAYLEVEL(3, "OK \n");
2877
2878 free(rawDictBuffer);
2879 }
2880
2881 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++);
2882 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1);
2883 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2884 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2885 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2886 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2887 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) );
2888 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 ));
2889 ZSTD_freeCDict(cdict);
2890 }
2891 DISPLAYLEVEL(3, "OK \n");
2892
2893 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++);
2894 {
2895 size_t size1, size2;
2896 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2897 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2898 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2899 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2900 if (ZSTD_isError(size1)) goto _output_error;
2901
2902 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2903 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2904 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) );
2905 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2906 if (ZSTD_isError(size2)) goto _output_error;
2907
2908 if (size1 != size2) goto _output_error;
2909 }
2910 DISPLAYLEVEL(3, "OK \n");
2911
2912 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++);
2913 {
2914 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2915 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2916 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2917 }
2918 DISPLAYLEVEL(3, "OK \n");
2919
2920 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++);
2921 {
2922 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2923 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2924 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2925 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2926 ZSTD_freeCDict(cdict);
2927 }
2928 DISPLAYLEVEL(3, "OK \n");
2929
2930 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++);
2931 {
2932 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2933 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2934 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2935 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2936 ZSTD_freeCDict(cdict);
2937 }
2938 DISPLAYLEVEL(3, "OK \n");
2939
2940 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++);
2941 {
2942 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2943 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2944 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2945 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2946 ZSTD_freeCDict(cdict);
2947 }
2948 DISPLAYLEVEL(3, "OK \n");
2949
2950 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++);
2951 {
2952 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) );
2953 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2954 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2955 }
2956 DISPLAYLEVEL(3, "OK \n");
2957
2958 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++);
2959 {
2960 ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1);
2961 CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) );
2962 CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) );
2963 CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) );
2964 ZSTD_freeCDict(cdict);
2965 }
2966 DISPLAYLEVEL(3, "OK \n");
2967
2968 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++);
2969 {
2970 size_t size1, size2;
2971 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2972 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2973 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2974 if (ZSTD_isError(size1)) goto _output_error;
2975
2976 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only);
2977 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2978 if (ZSTD_isError(size2)) goto _output_error;
2979
2980 if (size1 != size2) goto _output_error;
2981 }
2982 DISPLAYLEVEL(3, "OK \n");
2983
2984 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++);
2985 {
2986 size_t size1, size2;
2987 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2988 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) );
2989 size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2990 if (ZSTD_isError(size1)) goto _output_error;
2991
2992 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
2993 size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
2994 if (ZSTD_isError(size2)) goto _output_error;
2995
2996 if (size1 == size2) goto _output_error;
2997 }
2998 DISPLAYLEVEL(3, "OK \n");
2999
3000 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3001 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) );
3002 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB));
3003 CHECK_Z(cSize);
3004 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++);
3005 {
3006 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3007 size_t ret;
3008 /* We should fail to decompress without a dictionary. */
3009 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3010 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3011 if (!ZSTD_isError(ret)) goto _output_error;
3012 /* We should succeed to decompress with the dictionary. */
3013 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3014 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) );
3015 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3016 /* The dictionary should persist across calls. */
3017 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3018 /* When we reset the context the dictionary is cleared. */
3019 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3020 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3021 if (!ZSTD_isError(ret)) goto _output_error;
3022 ZSTD_freeDCtx(dctx);
3023 }
3024 DISPLAYLEVEL(3, "OK \n");
3025
3026 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++);
3027 {
3028 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3029 ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize);
3030 size_t ret;
3031 /* We should succeed to decompress with the ddict. */
3032 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3033 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) );
3034 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3035 /* The ddict should persist across calls. */
3036 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3037 /* When we reset the context the ddict is cleared. */
3038 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3039 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3040 if (!ZSTD_isError(ret)) goto _output_error;
3041 ZSTD_freeDCtx(dctx);
3042 ZSTD_freeDDict(ddict);
3043 }
3044 DISPLAYLEVEL(3, "OK \n");
3045
3046 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++);
3047 {
3048 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3049 size_t ret;
3050 /* We should succeed to decompress with the prefix. */
3051 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3052 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) );
3053 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3054 /* The prefix should be cleared after the first compression. */
3055 ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3056 if (!ZSTD_isError(ret)) goto _output_error;
3057 ZSTD_freeDCtx(dctx);
3058 }
3059 DISPLAYLEVEL(3, "OK \n");
3060
3061 DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3062 {
3063 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3064 ZSTD_customMem customMem = {NULL, NULL, NULL};
3065 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3066 ZSTD_CDict* cdict;
3067 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast));
3068 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3069 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3070 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3071 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3072 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3073 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3074 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3075 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3076 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3077 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3078 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3079 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3080 CHECK_Z(cSize);
3081 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3082 ZSTD_freeCDict(cdict);
3083 ZSTD_freeDCtx(dctx);
3084 ZSTD_freeCCtxParams(cctxParams);
3085 }
3086 DISPLAYLEVEL(3, "OK \n");
3087
3088 DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++);
3089 {
3090 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3091 ZSTD_customMem customMem = {NULL, NULL, NULL};
3092 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3093 ZSTD_CDict* cdict;
3094 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast));
3095 /* Set windowLog to 25 so hash/chain logs don't get sized down */
3096 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25));
3097 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25));
3098 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25));
3099 /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */
3100 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25));
3101 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3102 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3103 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3104 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3105 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3106 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3107 CHECK_Z(cSize);
3108 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3109 ZSTD_freeCDict(cdict);
3110 ZSTD_freeDCtx(dctx);
3111 ZSTD_freeCCtxParams(cctxParams);
3112 }
3113 DISPLAYLEVEL(3, "OK \n");
3114
3115 DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++);
3116 if (MEM_64bits()) {
3117 ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams();
3118 ZSTD_customMem customMem = {NULL, NULL, NULL};
3119 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3120 ZSTD_CDict* cdict;
3121 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy));
3122 /* Force enable row based match finder, and disable dedicated dict search. */
3123 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable));
3124 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0));
3125 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4));
3126 /* Set windowLog to 29 so hash/chain logs don't get sized down */
3127 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29));
3128 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29));
3129 /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */
3130 CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29));
3131 cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem);
3132 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
3133 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach));
3134 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
3135 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
3136 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
3137 CHECK_Z(cSize);
3138 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize));
3139 ZSTD_freeCDict(cdict);
3140 ZSTD_freeDCtx(dctx);
3141 ZSTD_freeCCtxParams(cctxParams);
3142 }
3143 DISPLAYLEVEL(3, "OK \n");
3144
3145 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++);
3146 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3147 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize,
3148 CNBuffer, samplesSizes, nbSamples);
3149 if (ZDICT_isError(dictSize)) goto _output_error;
3150 /* Set all the repcodes to non-default */
3151 {
3152 BYTE* dictPtr = (BYTE*)dictBuffer;
3153 BYTE* dictLimit = dictPtr + dictSize - 12;
3154 /* Find the repcodes */
3155 while (dictPtr < dictLimit &&
3156 (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 ||
3157 MEM_readLE32(dictPtr + 8) != 8)) {
3158 ++dictPtr;
3159 }
3160 if (dictPtr >= dictLimit) goto _output_error;
3161 MEM_writeLE32(dictPtr + 0, 10);
3162 MEM_writeLE32(dictPtr + 4, 10);
3163 MEM_writeLE32(dictPtr + 8, 10);
3164 /* Set the last 8 bytes to 'x' */
3165 memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8);
3166 }
3167 /* The optimal parser checks all the repcodes.
3168 * Make sure at least one is a match >= targetLength so that it is
3169 * immediately chosen. This will make sure that the compressor and
3170 * decompressor agree on at least one of the repcodes.
3171 */
3172 { size_t dSize;
3173 BYTE data[1024];
3174 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3175 ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize);
3176 ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize,
3177 ZSTD_dlm_byRef, ZSTD_dct_auto,
3178 cParams, ZSTD_defaultCMem);
3179 assert(dctx != NULL); assert(cdict != NULL);
3180 memset(data, 'x', sizeof(data));
3181 cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize,
3182 data, sizeof(data), cdict);
3183 ZSTD_freeCDict(cdict);
3184 if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; }
3185 dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize);
3186 if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; }
3187 if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; }
3188 ZSTD_freeDCtx(dctx);
3189 }
3190 DISPLAYLEVEL(3, "OK \n");
3191
3192 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++);
3193 {
3194 const size_t numDicts = 128;
3195 const size_t numFrames = 4;
3196 size_t i;
3197 ZSTD_DCtx* dctx = ZSTD_createDCtx();
3198 ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts);
3199 ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts);
3200 U32 dictIDSeed = seed;
3201 /* Create new compressed buffer that will hold frames with differing dictIDs */
3202 char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */
3203
3204 ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize);
3205 /* Create a bunch of DDicts with random dict IDs */
3206 for (i = 0; i < numDicts; ++i) {
3207 U32 currDictID = FUZ_rand(&dictIDSeed);
3208 MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID);
3209 ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize);
3210 cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3);
3211 if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) {
3212 goto _output_error;
3213 }
3214 }
3215 /* Compress a few frames using random CDicts */
3216 {
3217 size_t off = 0;
3218 /* only use the first half so we don't push against size limit of compressedBuffer */
3219 size_t const segSize = (CNBuffSize / 2) / numFrames;
3220 for (i = 0; i < numFrames; i++) {
3221 size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts;
3222 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3223 { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx,
3224 (BYTE*)compressedBuffer + off, CNBuffSize - off,
3225 (BYTE*)CNBuffer + segSize * (size_t)i, segSize,
3226 cdictTable[dictIdx]));
3227 off += r;
3228 }
3229 }
3230 cSize = off;
3231 }
3232
3233 /* We should succeed to decompression even though different dicts were used on different frames */
3234 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3235 ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts);
3236 /* Reference every single ddict we made */
3237 for (i = 0; i < numDicts; ++i) {
3238 CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i]));
3239 }
3240 CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3241 /* Streaming decompression should also work */
3242 {
3243 ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
3244 ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
3245 while (in.pos < in.size) {
3246 CHECK_Z(ZSTD_decompressStream(dctx, &out, &in));
3247 }
3248 }
3249 ZSTD_freeDCtx(dctx);
3250 for (i = 0; i < numDicts; ++i) {
3251 ZSTD_freeCDict(cdictTable[i]);
3252 ZSTD_freeDDict(ddictTable[i]);
3253 }
3254 free(dictBufferMulti);
3255 free(ddictTable);
3256 free(cdictTable);
3257 }
3258 DISPLAYLEVEL(3, "OK \n");
3259
3260 ZSTD_freeCCtx(cctx);
3261 free(dictBuffer);
3262 free(samplesSizes);
3263 }
3264
3265 /* COVER dictionary builder tests */
3266 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3267 size_t dictSize = 16 KB;
3268 size_t optDictSize = dictSize;
3269 void* dictBuffer = malloc(dictSize);
3270 size_t const totalSampleSize = 1 MB;
3271 size_t const sampleUnitSize = 8 KB;
3272 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3273 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3274 U32 seed32 = seed;
3275 ZDICT_cover_params_t params;
3276 U32 dictID;
3277
3278 if (dictBuffer==NULL || samplesSizes==NULL) {
3279 free(dictBuffer);
3280 free(samplesSizes);
3281 goto _output_error;
3282 }
3283
3284 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++);
3285 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3286 memset(&params, 0, sizeof(params));
3287 params.d = 1 + (FUZ_rand(&seed32) % 16);
3288 params.k = params.d + (FUZ_rand(&seed32) % 256);
3289 dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize,
3290 CNBuffer, samplesSizes, nbSamples,
3291 params);
3292 if (ZDICT_isError(dictSize)) goto _output_error;
3293 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize);
3294
3295 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3296 dictID = ZDICT_getDictID(dictBuffer, dictSize);
3297 if (dictID==0) goto _output_error;
3298 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3299
3300 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++);
3301 memset(&params, 0, sizeof(params));
3302 params.steps = 4;
3303 optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize,
3304 CNBuffer, samplesSizes,
3305 nbSamples / 4, &params);
3306 if (ZDICT_isError(optDictSize)) goto _output_error;
3307 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize);
3308
3309 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++);
3310 dictID = ZDICT_getDictID(dictBuffer, optDictSize);
3311 if (dictID==0) goto _output_error;
3312 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID);
3313
3314 ZSTD_freeCCtx(cctx);
3315 free(dictBuffer);
3316 free(samplesSizes);
3317 }
3318
3319 /* Decompression defense tests */
3320 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++);
3321 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */
3322 if (!ZSTD_isError(r)) goto _output_error;
3323 if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; }
3324 DISPLAYLEVEL(3, "OK \n");
3325
3326 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++);
3327 ((char*)(CNBuffer))[0] = 1;
3328 { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4);
3329 if (!ZSTD_isError(r)) goto _output_error; }
3330 DISPLAYLEVEL(3, "OK \n");
3331
3332 /* content size verification test */
3333 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++);
3334 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3335 size_t const srcSize = 5000;
3336 size_t const wrongSrcSize = (srcSize + 1000);
3337 ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0);
3338 params.fParams.contentSizeFlag = 1;
3339 CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) );
3340 { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize);
3341 if (!ZSTD_isError(result)) goto _output_error;
3342 if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error;
3343 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result));
3344 }
3345 ZSTD_freeCCtx(cctx);
3346 }
3347
3348 /* negative compression level test : ensure simple API and advanced API produce same result */
3349 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++);
3350 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3351 size_t const srcSize = CNBuffSize / 5;
3352 int const compressionLevel = -1;
3353
3354 assert(cctx != NULL);
3355 { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize,
3356 CNBuffer, srcSize, compressionLevel);
3357 if (ZSTD_isError(cSize_1pass)) goto _output_error;
3358
3359 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) );
3360 { size_t const compressionResult = ZSTD_compress2(cctx,
3361 compressedBuffer, compressedBufferSize,
3362 CNBuffer, srcSize);
3363 DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult);
3364 if (ZSTD_isError(compressionResult)) goto _output_error;
3365 if (compressionResult != cSize_1pass) goto _output_error;
3366 } }
3367 ZSTD_freeCCtx(cctx);
3368 }
3369 DISPLAYLEVEL(3, "OK \n");
3370
3371 /* parameters order test */
3372 { size_t const inputSize = CNBuffSize / 2;
3373 U64 xxh64;
3374
3375 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3376 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++);
3377 assert(cctx != NULL);
3378 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3379 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3380 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3381 { size_t const compressedSize = ZSTD_compress2(cctx,
3382 compressedBuffer, ZSTD_compressBound(inputSize),
3383 CNBuffer, inputSize);
3384 CHECK_Z(compressedSize);
3385 cSize = compressedSize;
3386 xxh64 = XXH64(compressedBuffer, compressedSize, 0);
3387 }
3388 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3389 ZSTD_freeCCtx(cctx);
3390 }
3391
3392 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
3393 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++);
3394 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) );
3395 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) );
3396 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) );
3397 { size_t const result = ZSTD_compress2(cctx,
3398 compressedBuffer, ZSTD_compressBound(inputSize),
3399 CNBuffer, inputSize);
3400 CHECK_Z(result);
3401 if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */
3402 if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */
3403 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result);
3404 }
3405 ZSTD_freeCCtx(cctx);
3406 }
3407 }
3408
3409 /* advanced parameters for decompression */
3410 { ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3411 assert(dctx != NULL);
3412
3413 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++);
3414 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax);
3415 CHECK_Z(bounds.error);
3416 }
3417 DISPLAYLEVEL(3, "OK \n");
3418
3419 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++);
3420 { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0);
3421 if (!ZSTD_isError(sr)) goto _output_error;
3422 }
3423 { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998);
3424 if (!ZSTD_isError(bounds.error)) goto _output_error;
3425 }
3426 DISPLAYLEVEL(3, "OK \n");
3427
3428 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++);
3429 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999);
3430 if (!ZSTD_isError(sr)) goto _output_error;
3431 }
3432 { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888);
3433 if (!ZSTD_isError(sr)) goto _output_error;
3434 }
3435 DISPLAYLEVEL(3, "OK \n");
3436
3437 ZSTD_freeDCtx(dctx);
3438 }
3439
3440
3441 /* custom formats tests */
3442 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3443 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3444 size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */
3445 assert(dctx != NULL); assert(cctx != NULL);
3446
3447 /* basic block compression */
3448 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++);
3449 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3450 { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 };
3451 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 };
3452 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3453 if (result != 0) goto _output_error;
3454 if (in.pos != in.size) goto _output_error;
3455 cSize = out.pos;
3456 }
3457 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize);
3458
3459 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++);
3460 { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3461 if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error;
3462 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult));
3463 }
3464
3465 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++);
3466 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3467 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3468 { ZSTD_frameHeader zfh;
3469 size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless);
3470 if (zfhrt != 0) goto _output_error;
3471 }
3472 /* one shot */
3473 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3474 if (result != inputSize) goto _output_error;
3475 DISPLAYLEVEL(3, "one-shot OK, ");
3476 }
3477 /* streaming */
3478 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3479 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3480 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3481 if (result != 0) goto _output_error;
3482 if (in.pos != in.size) goto _output_error;
3483 if (out.pos != inputSize) goto _output_error;
3484 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3485 }
3486
3487 /* basic block compression */
3488 DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++);
3489 CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) );
3490 { ZSTD_inBuffer in = { CNBuffer, 0, 0 };
3491 ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 };
3492 size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end);
3493 if (result != 0) goto _output_error;
3494 if (in.pos != in.size) goto _output_error;
3495 cSize = out.pos;
3496 }
3497 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize);
3498
3499 DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++);
3500 ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters);
3501 CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) );
3502 /* one shot */
3503 { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize);
3504 if (result != 0) goto _output_error;
3505 DISPLAYLEVEL(3, "one-shot OK, ");
3506 }
3507 /* streaming */
3508 { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 };
3509 ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 };
3510 size_t const result = ZSTD_decompressStream(dctx, &out, &in);
3511 if (result != 0) goto _output_error;
3512 if (in.pos != in.size) goto _output_error;
3513 if (out.pos != 0) goto _output_error;
3514 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos);
3515 }
3516
3517 ZSTD_freeCCtx(cctx);
3518 ZSTD_freeDCtx(dctx);
3519 }
3520
3521 DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++);
3522 {
3523 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3524 /* Attempt to future proof this to new parameters. */
3525 int const maxParam = 2000;
3526 int param;
3527 if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error;
3528 for (param = 0; param < maxParam; ++param) {
3529 ZSTD_dParameter dParam = (ZSTD_dParameter)param;
3530 ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam);
3531 int value1;
3532 int value2;
3533 int check;
3534 if (ZSTD_isError(bounds.error))
3535 continue;
3536 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1));
3537 value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound;
3538 CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2));
3539 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3540 if (check != value2) goto _output_error;
3541 CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters));
3542 CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check));
3543 if (check != value1) goto _output_error;
3544 }
3545 ZSTD_freeDCtx(dctx);
3546 }
3547 DISPLAYLEVEL(3, "OK \n");
3548
3549 /* block API tests */
3550 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3551 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
3552 static const size_t dictSize = 65 KB;
3553 static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */
3554 size_t cSize2;
3555 assert(cctx != NULL); assert(dctx != NULL);
3556
3557 /* basic block compression */
3558 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++);
3559 CHECK_Z( ZSTD_compressBegin(cctx, 5) );
3560 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3561 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) );
3562 DISPLAYLEVEL(3, "OK \n");
3563
3564 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++);
3565 CHECK_Z( ZSTD_decompressBegin(dctx) );
3566 { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) );
3567 if (r != blockSize) goto _output_error; }
3568 DISPLAYLEVEL(3, "OK \n");
3569
3570 /* very long stream of block compression */
3571 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++);
3572 CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */
3573 CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize);
3574 { U64 const toCompress = 5000000000ULL; /* > 4 GB */
3575 U64 compressed = 0;
3576 while (compressed < toCompress) {
3577 size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize);
3578 assert(blockCSize != 0);
3579 if (ZSTD_isError(blockCSize)) goto _output_error;
3580 compressed += blockCSize;
3581 } }
3582 DISPLAYLEVEL(3, "OK \n");
3583
3584 /* dictionary block compression */
3585 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++);
3586 CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) );
3587 CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize));
3588 RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */
3589 { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */
3590 assert(r == 0); /* non-compressible block */ }
3591 memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */
3592 CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize),
3593 (char*)CNBuffer+dictSize+2*blockSize, blockSize));
3594 DISPLAYLEVEL(3, "OK \n");
3595
3596 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++);
3597 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) );
3598 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) );
3599 if (r != blockSize) {
3600 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3601 goto _output_error;
3602 } }
3603 memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize);
3604 ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */
3605 { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) );
3606 if (r != blockSize) {
3607 DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize);
3608 goto _output_error;
3609 } }
3610 assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */
3611 DISPLAYLEVEL(3, "OK \n");
3612
3613 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++);
3614 { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3);
3615 if (cdict==NULL) goto _output_error;
3616 CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) );
3617 CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) );
3618 ZSTD_freeCDict(cdict);
3619 }
3620 DISPLAYLEVEL(3, "OK \n");
3621
3622 ZSTD_freeCCtx(cctx);
3623 ZSTD_freeDCtx(dctx);
3624 }
3625
3626 /* long rle test */
3627 { size_t sampleSize = 0;
3628 size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */
3629 DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++);
3630 memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1);
3631 sampleSize += 256 KB - 1;
3632 memset((char*)CNBuffer+sampleSize, 'A', 96 KB);
3633 sampleSize += 96 KB;
3634 cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1);
3635 if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error;
3636 { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize));
3637 if (regenSize!=sampleSize) goto _output_error; }
3638 DISPLAYLEVEL(3, "OK \n");
3639 }
3640
3641 DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++);
3642 {
3643 size_t srcSize = 150 KB;
3644 BYTE* src = (BYTE*)CNBuffer;
3645 BYTE* decoded = (BYTE*)compressedBuffer;
3646
3647 ZSTD_CCtx* cctx = ZSTD_createCCtx();
3648 ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3649 size_t seqsSize;
3650
3651 if (seqs == NULL) goto _output_error;
3652 assert(cctx != NULL);
3653 ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19);
3654 /* Populate src with random data */
3655 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed);
3656
3657 /* Test with block delimiters roundtrip */
3658 seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3659 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters);
3660 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3661
3662 /* Test no block delimiters roundtrip */
3663 seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize);
3664 FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters);
3665 assert(!memcmp(CNBuffer, compressedBuffer, srcSize));
3666
3667 ZSTD_freeCCtx(cctx);
3668 free(seqs);
3669 }
3670 DISPLAYLEVEL(3, "OK \n");
3671
3672 DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++);
3673 {
3674 const size_t srcSize = 500 KB;
3675 const BYTE* const src = (BYTE*)CNBuffer;
3676 BYTE* const dst = (BYTE*)compressedBuffer;
3677 const size_t dstCapacity = ZSTD_compressBound(srcSize);
3678 const size_t decompressSize = srcSize;
3679 char* const decompressBuffer = (char*)malloc(decompressSize);
3680 size_t compressedSize;
3681
3682 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3683 ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence));
3684 size_t nbSeqs;
3685
3686 if (seqs == NULL) goto _output_error;
3687 assert(cctx != NULL);
3688
3689 /* Populate src with random data */
3690 RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed);
3691
3692 /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */
3693 nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize);
3694 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3695 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
3696 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize);
3697 if (ZSTD_isError(compressedSize)) {
3698 DISPLAY("Error in sequence compression with block delims\n");
3699 goto _output_error;
3700 }
3701 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3702 if (ZSTD_isError(dSize)) {
3703 DISPLAY("Error in sequence compression roundtrip with block delims\n");
3704 goto _output_error;
3705 } }
3706 assert(!memcmp(decompressBuffer, src, srcSize));
3707
3708 /* Roundtrip Test with no block delimiters */
3709 { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs);
3710 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
3711 ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters);
3712 compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize);
3713 }
3714 if (ZSTD_isError(compressedSize)) {
3715 DISPLAY("Error in sequence compression with no block delims\n");
3716 goto _output_error;
3717 }
3718 { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize);
3719 if (ZSTD_isError(dSize)) {
3720 DISPLAY("Error in sequence compression roundtrip with no block delims\n");
3721 goto _output_error;
3722 } }
3723 assert(!memcmp(decompressBuffer, src, srcSize));
3724
3725 ZSTD_freeCCtx(cctx);
3726 free(decompressBuffer);
3727 free(seqs);
3728 }
3729 DISPLAYLEVEL(3, "OK \n");
3730
3731 /* Multiple blocks of zeros test */
3732 #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */
3733 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3734 memset(CNBuffer, 0, LONGZEROSLENGTH);
3735 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) );
3736 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100);
3737
3738 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH);
3739 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) );
3740 if (r != LONGZEROSLENGTH) goto _output_error; }
3741 DISPLAYLEVEL(3, "OK \n");
3742
3743 /* All zeroes test (test bug #137) */
3744 #define ZEROESLENGTH 100
3745 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH);
3746 memset(CNBuffer, 0, ZEROESLENGTH);
3747 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) );
3748 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100);
3749
3750 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH);
3751 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) );
3752 if (r != ZEROESLENGTH) goto _output_error; }
3753 DISPLAYLEVEL(3, "OK \n");
3754
3755 /* nbSeq limit test */
3756 #define _3BYTESTESTLENGTH 131000
3757 #define NB3BYTESSEQLOG 9
3758 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
3759 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
3760 /* creates a buffer full of 3-bytes sequences */
3761 { BYTE _3BytesSeqs[NB3BYTESSEQ][3];
3762 U32 rSeed = 1;
3763
3764 /* create batch of 3-bytes sequences */
3765 { int i;
3766 for (i=0; i < NB3BYTESSEQ; i++) {
3767 _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255);
3768 _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255);
3769 _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255);
3770 } }
3771
3772 /* randomly fills CNBuffer with prepared 3-bytes sequences */
3773 { int i;
3774 for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
3775 U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK;
3776 ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0];
3777 ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1];
3778 ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2];
3779 } } }
3780 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++);
3781 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3782 size_t const maxNbSeq = _3BYTESTESTLENGTH / 3;
3783 size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH);
3784 size_t nbSeq = 1;
3785 while (nbSeq <= maxNbSeq) {
3786 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19));
3787 /* Check every sequence for the first 100, then skip more rapidly. */
3788 if (nbSeq < 100) {
3789 ++nbSeq;
3790 } else {
3791 nbSeq += (nbSeq >> 2);
3792 }
3793 }
3794 ZSTD_freeCCtx(cctx);
3795 }
3796 DISPLAYLEVEL(3, "OK \n");
3797
3798 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++);
3799 CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH),
3800 CNBuffer, _3BYTESTESTLENGTH, 19) );
3801 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100);
3802
3803 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++);
3804 { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) );
3805 if (r != _3BYTESTESTLENGTH) goto _output_error; }
3806 DISPLAYLEVEL(3, "OK \n");
3807
3808
3809 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++);
3810 RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed);
3811 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3812 size_t const bound = ZSTD_compressBound(CNBuffSize);
3813 size_t size = 1;
3814 while (size <= CNBuffSize) {
3815 CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3));
3816 /* Check every size for the first 100, then skip more rapidly. */
3817 if (size < 100) {
3818 ++size;
3819 } else {
3820 size += (size >> 2);
3821 }
3822 }
3823 ZSTD_freeCCtx(cctx);
3824 }
3825 DISPLAYLEVEL(3, "OK \n");
3826
3827 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++);
3828 { /* Train a dictionary on low characters */
3829 size_t dictSize = 16 KB;
3830 void* const dictBuffer = malloc(dictSize);
3831 size_t const totalSampleSize = 1 MB;
3832 size_t const sampleUnitSize = 8 KB;
3833 U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
3834 size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
3835 if (!dictBuffer || !samplesSizes) goto _output_error;
3836 { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; }
3837 dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples);
3838 if (ZDICT_isError(dictSize)) goto _output_error;
3839 /* Reverse the characters to make the dictionary ill suited */
3840 { U32 u;
3841 for (u = 0; u < CNBuffSize; ++u) {
3842 ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u];
3843 } }
3844 { /* Compress the data */
3845 size_t const inputSize = 500;
3846 size_t const outputSize = ZSTD_compressBound(inputSize);
3847 void* const outputBuffer = malloc(outputSize);
3848 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3849 if (!outputBuffer || !cctx) goto _output_error;
3850 CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1));
3851 free(outputBuffer);
3852 ZSTD_freeCCtx(cctx);
3853 }
3854
3855 free(dictBuffer);
3856 free(samplesSizes);
3857 }
3858 DISPLAYLEVEL(3, "OK \n");
3859
3860
3861 /* findFrameCompressedSize on skippable frames */
3862 DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++);
3863 { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
3864 size_t const frameSrcSize = 13;
3865 if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; }
3866 DISPLAYLEVEL(3, "OK \n");
3867
3868 /* error string tests */
3869 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++);
3870 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error;
3871 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error;
3872 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error;
3873 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error;
3874 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error;
3875 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error;
3876 DISPLAYLEVEL(3, "OK \n");
3877
3878 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++);
3879 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
3880 {
3881 size_t const size = MIN(128 KB, CNBuffSize);
3882 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
3883 ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1);
3884 ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1);
3885 ZSTD_frameHeader lgHeader;
3886 ZSTD_frameHeader smHeader;
3887
3888 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict));
3889 CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize));
3890 CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict));
3891 CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize));
3892
3893 if (lgHeader.windowSize != smHeader.windowSize) goto _output_error;
3894
3895 ZSTD_freeCDict(smCDict);
3896 ZSTD_freeCDict(lgCDict);
3897 ZSTD_freeCCtx(cctx);
3898 }
3899 DISPLAYLEVEL(3, "OK \n");
3900
3901 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++);
3902 {
3903 short norm[32];
3904 unsigned count[32];
3905 unsigned const tableLog = 5;
3906 size_t const nbSeq = 32;
3907 unsigned const maxSymbolValue = 31;
3908 size_t i;
3909
3910 for (i = 0; i < 32; ++i)
3911 count[i] = 1;
3912 /* Calling FSE_normalizeCount() on a uniform distribution should not
3913 * cause a division by zero.
3914 */
3915 FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1);
3916 }
3917 DISPLAYLEVEL(3, "OK \n");
3918
3919 DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++);
3920 {
3921 size_t const outBufSize = 9;
3922 short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18};
3923 unsigned const tableLog = 5;
3924 unsigned const maxSymbolValue = 10;
3925 BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE));
3926
3927 /* Ensure that this write doesn't write out of bounds, and that
3928 * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1.
3929 */
3930 FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog);
3931 free(outBuf);
3932 }
3933 DISPLAYLEVEL(3, "OK \n");
3934
3935 DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++);
3936 {
3937 U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */
3938 U32 rand32 = FUZ_rand(&seed_copy);
3939 U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy);
3940 U32 lowbit_only_32 = 1;
3941 U64 lowbit_only_64 = 1;
3942 U32 highbit_only_32 = (U32)1 << 31;
3943 U64 highbit_only_64 = (U64)1 << 63;
3944 U32 i;
3945 if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */
3946 if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */
3947
3948 /* Test ZSTD_countTrailingZeros32 */
3949 CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u);
3950 CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u);
3951 CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32));
3952
3953 /* Test ZSTD_countLeadingZeros32 */
3954 CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u);
3955 CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u);
3956 CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32));
3957
3958 /* Test ZSTD_countTrailingZeros64 */
3959 CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u);
3960 CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u);
3961
3962 /* Test ZSTD_countLeadingZeros64 */
3963 CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u);
3964 CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u);
3965
3966 /* Test ZSTD_highbit32 */
3967 CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u);
3968 CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u);
3969
3970 /* Test ZSTD_NbCommonBytes */
3971 if (MEM_isLittleEndian()) {
3972 if (MEM_64bits()) {
3973 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
3974 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
3975 } else {
3976 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u);
3977 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u);
3978 }
3979 } else {
3980 if (MEM_64bits()) {
3981 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u);
3982 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u);
3983 } else {
3984 CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u);
3985 CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u);
3986 }
3987 }
3988
3989 /* Test MEM_ intrinsics */
3990 CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32));
3991 CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64));
3992
3993 /* Test fallbacks vs intrinsics on a range of small integers */
3994 for (i=1; i <= 1000; i++) {
3995 CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i));
3996 CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i));
3997 CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i));
3998 CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i));
3999 }
4000 }
4001 DISPLAYLEVEL(3, "OK \n");
4002
4003#ifdef ZSTD_MULTITHREAD
4004 DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++);
4005 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4006 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4007 size_t const dstSize = ZSTD_compressBound(srcSize);
4008 void* const src = CNBuffer;
4009 void* const dst = compressedBuffer;
4010 void* dict = (void*)malloc(srcSize);
4011
4012 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4013 RDG_genBuffer(dict, srcSize, compressibility, 0., seed);
4014
4015 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4016 memset(dict, 0, sizeof(U32));
4017
4018 /* something more than 1 */
4019 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4020 /* lie and claim this is a full dict */
4021 CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict));
4022
4023 { ZSTD_outBuffer out = {dst, dstSize, 0};
4024 ZSTD_inBuffer in = {src, srcSize, 0};
4025 /* should fail because its not a full dict like we said it was */
4026 assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)));
4027 }
4028
4029 ZSTD_freeCCtx(cctx);
4030 free(dict);
4031 }
4032 DISPLAYLEVEL(3, "OK \n");
4033
4034 DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++);
4035 { ZSTD_CCtx* cctx = ZSTD_createCCtx();
4036 size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */
4037 size_t const dictSize = 10;
4038 size_t const dstSize = ZSTD_compressBound(srcSize);
4039 void* const src = CNBuffer;
4040 void* const dst = compressedBuffer;
4041 void* dict = (void*)malloc(dictSize);
4042
4043 RDG_genBuffer(src, srcSize, compressibility, 0.5, seed);
4044 RDG_genBuffer(dict, dictSize, compressibility, 0., seed);
4045
4046 /* Make sure there is no ZSTD_MAGIC_NUMBER */
4047 memset(dict, 0, sizeof(U32));
4048
4049 /* Enable MT, LDM, and use refPrefix() for a small dict */
4050 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
4051 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4052 CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
4053
4054 CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize));
4055
4056 ZSTD_freeCCtx(cctx);
4057 free(dict);
4058 }
4059 DISPLAYLEVEL(3, "OK \n");
4060
4061 DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++);
4062 {
4063 ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0);
4064 ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0);
4065 ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400);
4066 ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000);
4067 ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000);
4068
4069 assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict)));
4070 assert(!memcmp(&medium, &mediumDict, sizeof(medium)));
4071 assert(!memcmp(&large, &largeDict, sizeof(large)));
4072 }
4073 DISPLAYLEVEL(3, "OK \n");
4074
4075 DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++);
4076 {
4077 ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0);
4078 ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400);
4079 ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400);
4080
4081 assert(smallSrcAndDict.windowLog == 10);
4082 assert(!memcmp(&cParams, &smallDict, sizeof(cParams)));
4083 }
4084 DISPLAYLEVEL(3, "OK \n");
4085
4086 DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++);
4087 {
4088 int level = 1;
4089 size_t prevSize = 0;
4090 for (; level < ZSTD_maxCLevel(); ++level) {
4091 size_t const currSize = ZSTD_estimateCCtxSize(level);
4092 if (prevSize > currSize) {
4093 DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d",
4094 prevSize, level-1, currSize, level);
4095 goto _output_error;
4096 }
4097 prevSize = currSize;
4098 }
4099 }
4100 DISPLAYLEVEL(3, "OK \n");
4101
4102 DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++);
4103 {
4104 size_t const kSizeIncrement = 2 KB;
4105 int level = -3;
4106
4107 for (; level <= ZSTD_maxCLevel(); ++level) {
4108 size_t dictSize = 0;
4109 for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) {
4110 size_t srcSize = 2 KB;
4111 for (; srcSize < 300 KB; srcSize += kSizeIncrement) {
4112 ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize);
4113 size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams);
4114 size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level);
4115 if (cctxSizeUsingLevel < cctxSizeUsingCParams
4116 || ZSTD_isError(cctxSizeUsingCParams)
4117 || ZSTD_isError(cctxSizeUsingLevel)) {
4118 DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n",
4119 level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel);
4120 goto _output_error;
4121 } } } } }
4122 DISPLAYLEVEL(3, "OK \n");
4123
4124 DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++)
4125 {
4126 int const threadPoolTestResult = threadPoolTests();
4127 if (threadPoolTestResult) {
4128 goto _output_error;
4129 }
4130 }
4131 DISPLAYLEVEL(3, "thread pool tests OK \n");
4132
4133#endif /* ZSTD_MULTITHREAD */
4134
4135_end:
4136 free(CNBuffer);
4137 free(compressedBuffer);
4138 free(decodedBuffer);
4139 return testResult;
4140
4141_output_error:
4142 testResult = 1;
4143 DISPLAY("Error detected in Unit tests ! \n");
4144 goto _end;
4145}
4146
4147static int longUnitTests(U32 const seed, double compressibility)
4148{
4149 size_t const CNBuffSize = 5 MB;
4150 void* const CNBuffer = malloc(CNBuffSize);
4151 size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize);
4152 void* const compressedBuffer = malloc(compressedBufferSize);
4153 void* const decodedBuffer = malloc(CNBuffSize);
4154 int testResult = 0;
4155 unsigned testNb=0;
4156 size_t cSize;
4157
4158 /* Create compressible noise */
4159 if (!CNBuffer || !compressedBuffer || !decodedBuffer) {
4160 DISPLAY("Not enough memory, aborting\n");
4161 testResult = 1;
4162 goto _end;
4163 }
4164 RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed);
4165
4166 /* note : this test is rather long, it would be great to find a way to speed up its execution */
4167 DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++);
4168 { int cLevel;
4169 size_t approxIndex = 0;
4170 size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */
4171
4172 /* Provision enough space in a static context so that we can do all
4173 * this without ever reallocating, which would reset the indices. */
4174 size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22);
4175 void* const staticCCtxBuffer = malloc(staticCCtxSize);
4176 ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize);
4177
4178 /* bump the indices so the following compressions happen at high
4179 * indices. */
4180 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4181 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4182 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4183 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4184 while (approxIndex <= (maxIndex / 4) * 3) {
4185 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4186 approxIndex += in.pos;
4187 CHECK_Z(in.pos == in.size);
4188 in.pos = 0;
4189 out.pos = 0;
4190 }
4191 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4192 }
4193
4194 /* spew a bunch of stuff into the table area */
4195 for (cLevel = 1; cLevel <= 22; cLevel++) {
4196 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4197 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4198 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4199 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4200 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4201 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4202 approxIndex += in.pos;
4203 }
4204
4205 /* now crank the indices so we overflow */
4206 { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 };
4207 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4208 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4209 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500));
4210 while (approxIndex <= maxIndex) {
4211 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4212 approxIndex += in.pos;
4213 CHECK_Z(in.pos == in.size);
4214 in.pos = 0;
4215 out.pos = 0;
4216 }
4217 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4218 }
4219
4220 /* do a bunch of compressions again in low indices and ensure we don't
4221 * hit untracked invalid indices */
4222 for (cLevel = 1; cLevel <= 22; cLevel++) {
4223 ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 };
4224 ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 };
4225 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4226 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel));
4227 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
4228 CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
4229 approxIndex += in.pos;
4230 }
4231
4232 free(staticCCtxBuffer);
4233 }
4234 DISPLAYLEVEL(3, "OK \n");
4235
4236 DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++);
4237 { size_t cSizeLdm;
4238 size_t cSizeNoLdm;
4239 ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4240
4241 RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed);
4242
4243 /* Enable checksum to verify round trip. */
4244 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4245 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable));
4246 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4247
4248 /* Round trip once with ldm. */
4249 cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4250 CHECK_Z(cSizeLdm);
4251 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm));
4252
4253 ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters);
4254 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1));
4255 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable));
4256 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19));
4257
4258 /* Round trip once without ldm. */
4259 cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4260 CHECK_Z(cSizeNoLdm);
4261 CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm));
4262
4263 if (cSizeLdm > cSizeNoLdm) {
4264 DISPLAY("Using long mode should not cause regressions for btopt+\n");
4265 testResult = 1;
4266 goto _end;
4267 }
4268
4269 ZSTD_freeCCtx(cctx);
4270 }
4271 DISPLAYLEVEL(3, "OK \n");
4272
4273 DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++);
4274 { ZSTD_CCtx* const cctx = ZSTD_createCCtx();
4275 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4276 size_t dictSize = CNBuffSize;
4277 void* dict = (void*)malloc(dictSize);
4278 ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams();
4279 ZSTD_dictAttachPref_e const attachPrefs[] = {
4280 ZSTD_dictDefaultAttach,
4281 ZSTD_dictForceAttach,
4282 ZSTD_dictForceCopy,
4283 ZSTD_dictForceLoad,
4284 ZSTD_dictDefaultAttach,
4285 ZSTD_dictForceAttach,
4286 ZSTD_dictForceCopy,
4287 ZSTD_dictForceLoad
4288 };
4289 int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1};
4290 int cLevel;
4291 int i;
4292
4293 RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
4294 RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed);
4295
4296 CHECK_Z(cctx_params != NULL);
4297
4298 for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) {
4299 DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize);
4300 for (cLevel = 4; cLevel < 13; cLevel++) {
4301 for (i = 0; i < 8; ++i) {
4302 ZSTD_dictAttachPref_e const attachPref = attachPrefs[i];
4303 int const enableDDS = enableDedicatedDictSearch[i];
4304 ZSTD_CDict* cdict;
4305
4306 DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i);
4307
4308 ZSTD_CCtxParams_init(cctx_params, cLevel);
4309 CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS));
4310
4311 cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem);
4312 CHECK(cdict != NULL);
4313
4314 CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict));
4315 CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref));
4316
4317 cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize);
4318 CHECK_Z(cSize);
4319 CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
4320
4321 DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize);
4322
4323 CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters));
4324 ZSTD_freeCDict(cdict);
4325 } } }
4326
4327 ZSTD_freeCCtx(cctx);
4328 ZSTD_freeDCtx(dctx);
4329 ZSTD_freeCCtxParams(cctx_params);
4330 free(dict);
4331 }
4332 DISPLAYLEVEL(3, "OK \n");
4333
4334_end:
4335 free(CNBuffer);
4336 free(compressedBuffer);
4337 free(decodedBuffer);
4338 return testResult;
4339}
4340
4341
4342static size_t findDiff(const void* buf1, const void* buf2, size_t max)
4343{
4344 const BYTE* b1 = (const BYTE*)buf1;
4345 const BYTE* b2 = (const BYTE*)buf2;
4346 size_t u;
4347 for (u=0; u<max; u++) {
4348 if (b1[u] != b2[u]) break;
4349 }
4350 return u;
4351}
4352
4353
4354static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams)
4355{
4356 ZSTD_parameters params;
4357 params.cParams = cParams;
4358 params.fParams = fParams;
4359 return params;
4360}
4361
4362static size_t FUZ_rLogLength(U32* seed, U32 logLength)
4363{
4364 size_t const lengthMask = ((size_t)1 << logLength) - 1;
4365 return (lengthMask+1) + (FUZ_rand(seed) & lengthMask);
4366}
4367
4368static size_t FUZ_randomLength(U32* seed, U32 maxLog)
4369{
4370 U32 const logLength = FUZ_rand(seed) % maxLog;
4371 return FUZ_rLogLength(seed, logLength);
4372}
4373
4374#undef CHECK
4375#define CHECK(cond, ...) { \
4376 if (cond) { \
4377 DISPLAY("Error => "); \
4378 DISPLAY(__VA_ARGS__); \
4379 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4380 goto _output_error; \
4381} }
4382
4383#undef CHECK_Z
4384#define CHECK_Z(f) { \
4385 size_t const err = f; \
4386 if (ZSTD_isError(err)) { \
4387 DISPLAY("Error => %s : %s ", \
4388 #f, ZSTD_getErrorName(err)); \
4389 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
4390 goto _output_error; \
4391} }
4392
4393
4394static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests)
4395{
4396 static const U32 maxSrcLog = 23;
4397 static const U32 maxSampleLog = 22;
4398 size_t const srcBufferSize = (size_t)1<<maxSrcLog;
4399 size_t const dstBufferSize = (size_t)1<<maxSampleLog;
4400 size_t const cBufferSize = ZSTD_compressBound(dstBufferSize);
4401 BYTE* cNoiseBuffer[5];
4402 BYTE* const cBuffer = (BYTE*) malloc (cBufferSize);
4403 BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize);
4404 BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize);
4405 ZSTD_CCtx* const refCtx = ZSTD_createCCtx();
4406 ZSTD_CCtx* const ctx = ZSTD_createCCtx();
4407 ZSTD_DCtx* const dctx = ZSTD_createDCtx();
4408 U32 result = 0;
4409 unsigned testNb = 0;
4410 U32 coreSeed = seed;
4411 UTIL_time_t const startClock = UTIL_getTime();
4412 U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO;
4413 int const cLevelLimiter = bigTests ? 3 : 2;
4414
4415 /* allocation */
4416 cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize);
4417 cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize);
4418 cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize);
4419 cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize);
4420 cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize);
4421 CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4]
4422 || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx,
4423 "Not enough memory, fuzzer tests cancelled");
4424
4425 /* Create initial samples */
4426 RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */
4427 RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */
4428 RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed);
4429 RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */
4430 RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */
4431
4432 /* catch up testNb */
4433 for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed);
4434
4435 /* main test loop */
4436 for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) {
4437 BYTE* srcBuffer; /* jumping pointer */
4438 U32 lseed;
4439 size_t sampleSize, maxTestSize, totalTestSize;
4440 size_t cSize, totalCSize, totalGenSize;
4441 U64 crcOrig;
4442 BYTE* sampleBuffer;
4443 const BYTE* dict;
4444 size_t dictSize;
4445
4446 /* notification */
4447 if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); }
4448 else { DISPLAYUPDATE(2, "\r%6u ", testNb); }
4449
4450 FUZ_rand(&coreSeed);
4451 { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; }
4452
4453 /* srcBuffer selection [0-4] */
4454 { U32 buffNb = FUZ_rand(&lseed) & 0x7F;
4455 if (buffNb & 7) buffNb=2; /* most common : compressible (P) */
4456 else {
4457 buffNb >>= 3;
4458 if (buffNb & 7) {
4459 const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */
4460 buffNb = tnb[buffNb >> 3];
4461 } else {
4462 const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */
4463 buffNb = tnb[buffNb >> 3];
4464 } }
4465 srcBuffer = cNoiseBuffer[buffNb];
4466 }
4467
4468 /* select src segment */
4469 sampleSize = FUZ_randomLength(&lseed, maxSampleLog);
4470
4471 /* create sample buffer (to catch read error with valgrind & sanitizers) */
4472 sampleBuffer = (BYTE*)malloc(sampleSize);
4473 CHECK(sampleBuffer==NULL, "not enough memory for sample buffer");
4474 { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize);
4475 memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); }
4476 crcOrig = XXH64(sampleBuffer, sampleSize, 0);
4477
4478 /* compression tests */
4479 { int const cLevelPositive = (int)
4480 ( FUZ_rand(&lseed) %
4481 ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) )
4482 + 1;
4483 int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ?
4484 - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */
4485 cLevelPositive;
4486 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel);
4487 cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel);
4488 CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize));
4489
4490 /* compression failure test : too small dest buffer */
4491 assert(cSize > 3);
4492 { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1;
4493 const size_t tooSmallSize = cSize - missing;
4494 const unsigned endMark = 0x4DC2B1A9;
4495 memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark));
4496 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
4497 testNb, (unsigned)tooSmallSize, (unsigned)missing);
4498 { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel);
4499 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); }
4500 { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck));
4501 CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); }
4502 } }
4503
4504 /* frame header decompression test */
4505 { ZSTD_frameHeader zfh;
4506 CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) );
4507 CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect");
4508 }
4509
4510 /* Decompressed size test */
4511 { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize);
4512 CHECK(rSize != sampleSize, "decompressed size incorrect");
4513 }
4514
4515 /* successful decompression test */
4516 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb);
4517 { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1;
4518 size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize);
4519 CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize);
4520 { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0);
4521 CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize);
4522 } }
4523
4524 free(sampleBuffer); /* no longer useful after this point */
4525
4526 /* truncated src decompression test */
4527 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb);
4528 { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4529 size_t const tooSmallSize = cSize - missing;
4530 void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */
4531 CHECK(cBufferTooSmall == NULL, "not enough memory !");
4532 memcpy(cBufferTooSmall, cBuffer, tooSmallSize);
4533 { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize);
4534 CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
4535 free(cBufferTooSmall);
4536 }
4537
4538 /* too small dst decompression test */
4539 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb);
4540 if (sampleSize > 3) {
4541 size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
4542 size_t const tooSmallSize = sampleSize - missing;
4543 static const BYTE token = 0xA9;
4544 dstBuffer[tooSmallSize] = token;
4545 { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize);
4546 CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); }
4547 CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow");
4548 }
4549
4550 /* noisy src decompression test */
4551 if (cSize > 6) {
4552 /* insert noise into src */
4553 { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4));
4554 size_t pos = 4; /* preserve magic number (too easy to detect) */
4555 for (;;) {
4556 /* keep some original src */
4557 { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits;
4558 size_t const mask = (1<<nbBits) - 1;
4559 size_t const skipLength = FUZ_rand(&lseed) & mask;
4560 pos += skipLength;
4561 }
4562 if (pos >= cSize) break;
4563 /* add noise */
4564 { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits;
4565 U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0;
4566 size_t const mask = (1<<nbBits) - 1;
4567 size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1;
4568 size_t const noiseLength = MIN(rNoiseLength, cSize-pos);
4569 size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength);
4570 memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength);
4571 pos += noiseLength;
4572 } } }
4573
4574 /* decompress noisy source */
4575 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb);
4576 { U32 const endMark = 0xA9B1C3D6;
4577 memcpy(dstBuffer+sampleSize, &endMark, 4);
4578 { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize);
4579 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
4580 CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize),
4581 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize);
4582 }
4583 { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4);
4584 CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow");
4585 } } } /* noisy src decompression test */
4586
4587 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
4588 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb);
4589 { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog;
4590 U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog;
4591 int const cLevel = (int)(FUZ_rand(&lseed) %
4592 ((U32)ZSTD_maxCLevel() -
4593 (MAX(testLog, dictLog) / (U32)cLevelLimiter))) +
4594 1;
4595 maxTestSize = FUZ_rLogLength(&lseed, testLog);
4596 if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1;
4597
4598 dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */
4599 dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize));
4600
4601 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
4602 testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize);
4603
4604 if (FUZ_rand(&lseed) & 0xF) {
4605 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) );
4606 } else {
4607 ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize);
4608 ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */,
4609 !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/,
4610 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
4611 ZSTD_parameters const p = FUZ_makeParams(cPar, fPar);
4612 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) );
4613 }
4614 CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) );
4615 }
4616
4617 { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2;
4618 U32 n;
4619 XXH64_state_t xxhState;
4620 XXH64_reset(&xxhState, 0);
4621 for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) {
4622 size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog);
4623 size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize);
4624
4625 if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */
4626 if (totalTestSize+segmentSize > maxTestSize) break;
4627
4628 { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize);
4629 CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult));
4630 cSize += compressResult;
4631 }
4632 XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize);
4633 memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize);
4634 totalTestSize += segmentSize;
4635 }
4636
4637 { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0);
4638 CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult));
4639 cSize += flushResult;
4640 }
4641 crcOrig = XXH64_digest(&xxhState);
4642 }
4643
4644 /* streaming decompression test */
4645 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb);
4646 /* ensure memory requirement is good enough (should always be true) */
4647 { ZSTD_frameHeader zfh;
4648 CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX),
4649 "ZSTD_getFrameHeader(): error retrieving frame information");
4650 { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize);
4651 CHECK_Z(roundBuffSize);
4652 CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN),
4653 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)",
4654 (unsigned)roundBuffSize, (unsigned)totalTestSize );
4655 } }
4656 if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */
4657 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) );
4658 totalCSize = 0;
4659 totalGenSize = 0;
4660 while (totalCSize < cSize) {
4661 size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx);
4662 size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize);
4663 CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize));
4664 totalGenSize += genSize;
4665 totalCSize += inSize;
4666 }
4667 CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded");
4668 CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size")
4669 CHECK (totalCSize != cSize, "compressed data should be fully read")
4670 { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0);
4671 CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)",
4672 (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize);
4673 }
4674 } /* for ( ; (testNb <= nbTests) */
4675 DISPLAY("\r%u fuzzer tests completed \n", testNb-1);
4676
4677_cleanup:
4678 ZSTD_freeCCtx(refCtx);
4679 ZSTD_freeCCtx(ctx);
4680 ZSTD_freeDCtx(dctx);
4681 free(cNoiseBuffer[0]);
4682 free(cNoiseBuffer[1]);
4683 free(cNoiseBuffer[2]);
4684 free(cNoiseBuffer[3]);
4685 free(cNoiseBuffer[4]);
4686 free(cBuffer);
4687 free(dstBuffer);
4688 free(mirrorBuffer);
4689 return (int)result;
4690
4691_output_error:
4692 result = 1;
4693 goto _cleanup;
4694}
4695
4696
4697/*_*******************************************************
4698* Command line
4699*********************************************************/
4700static int FUZ_usage(const char* programName)
4701{
4702 DISPLAY( "Usage :\n");
4703 DISPLAY( " %s [args]\n", programName);
4704 DISPLAY( "\n");
4705 DISPLAY( "Arguments :\n");
4706 DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault);
4707 DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n");
4708 DISPLAY( " -s# : Select seed (default:prompt user)\n");
4709 DISPLAY( " -t# : Select starting test number (default:0)\n");
4710 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default);
4711 DISPLAY( " -v : verbose\n");
4712 DISPLAY( " -p : pause at the end\n");
4713 DISPLAY( " -h : display help and exit\n");
4714 return 0;
4715}
4716
4717/*! readU32FromChar() :
4718 @return : unsigned integer value read from input in `char` format
4719 allows and interprets K, KB, KiB, M, MB and MiB suffix.
4720 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
4721 Note : function result can overflow if digit string > MAX_UINT */
4722static unsigned readU32FromChar(const char** stringPtr)
4723{
4724 unsigned result = 0;
4725 while ((**stringPtr >='0') && (**stringPtr <='9'))
4726 result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ;
4727 if ((**stringPtr=='K') || (**stringPtr=='M')) {
4728 result <<= 10;
4729 if (**stringPtr=='M') result <<= 10;
4730 (*stringPtr)++ ;
4731 if (**stringPtr=='i') (*stringPtr)++;
4732 if (**stringPtr=='B') (*stringPtr)++;
4733 }
4734 return result;
4735}
4736
4737/** longCommandWArg() :
4738 * check if *stringPtr is the same as longCommand.
4739 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
4740 * @return 0 and doesn't modify *stringPtr otherwise.
4741 */
4742static int longCommandWArg(const char** stringPtr, const char* longCommand)
4743{
4744 size_t const comSize = strlen(longCommand);
4745 int const result = !strncmp(*stringPtr, longCommand, comSize);
4746 if (result) *stringPtr += comSize;
4747 return result;
4748}
4749
4750int main(int argc, const char** argv)
4751{
4752 U32 seed = 0;
4753 int seedset = 0;
4754 int argNb;
4755 int nbTests = nbTestsDefault;
4756 int testNb = 0;
4757 int proba = FUZ_compressibility_default;
4758 double probfloat;
4759 int result = 0;
4760 U32 mainPause = 0;
4761 U32 maxDuration = 0;
4762 int bigTests = 1;
4763 int longTests = 0;
4764 U32 memTestsOnly = 0;
4765 const char* const programName = argv[0];
4766
4767 /* Check command line */
4768 for (argNb=1; argNb<argc; argNb++) {
4769 const char* argument = argv[argNb];
4770 if(!argument) continue; /* Protection if argument empty */
4771
4772 /* Handle commands. Aggregated commands are allowed */
4773 if (argument[0]=='-') {
4774
4775 if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; }
4776
4777 if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; }
4778 if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; }
4779 if (!strcmp(argument, "--long-tests")) { longTests=1; continue; }
4780 if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; }
4781
4782 argument++;
4783 while (*argument!=0) {
4784 switch(*argument)
4785 {
4786 case 'h':
4787 return FUZ_usage(programName);
4788
4789 case 'v':
4790 argument++;
4791 g_displayLevel++;
4792 break;
4793
4794 case 'q':
4795 argument++;
4796 g_displayLevel--;
4797 break;
4798
4799 case 'p': /* pause at the end */
4800 argument++;
4801 mainPause = 1;
4802 break;
4803
4804 case 'i':
4805 argument++; maxDuration = 0;
4806 nbTests = (int)readU32FromChar(&argument);
4807 break;
4808
4809 case 'T':
4810 argument++;
4811 nbTests = 0;
4812 maxDuration = readU32FromChar(&argument);
4813 if (*argument=='s') argument++; /* seconds */
4814 if (*argument=='m') maxDuration *= 60, argument++; /* minutes */
4815 if (*argument=='n') argument++;
4816 break;
4817
4818 case 's':
4819 argument++;
4820 seedset = 1;
4821 seed = readU32FromChar(&argument);
4822 break;
4823
4824 case 't':
4825 argument++;
4826 testNb = (int)readU32FromChar(&argument);
4827 break;
4828
4829 case 'P': /* compressibility % */
4830 argument++;
4831 proba = (int)readU32FromChar(&argument);
4832 if (proba>100) proba = 100;
4833 break;
4834
4835 default:
4836 return (FUZ_usage(programName), 1);
4837 } } } } /* for (argNb=1; argNb<argc; argNb++) */
4838
4839 /* Get Seed */
4840 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING);
4841
4842 if (!seedset) {
4843 time_t const t = time(NULL);
4844 U32 const h = XXH32(&t, sizeof(t), 1);
4845 seed = h % 10000;
4846 }
4847
4848 DISPLAY("Seed = %u\n", (unsigned)seed);
4849 if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba);
4850
4851 probfloat = ((double)proba) / 100;
4852
4853 if (memTestsOnly) {
4854 g_displayLevel = MAX(3, g_displayLevel);
4855 return FUZ_mallocTests(seed, probfloat, memTestsOnly);
4856 }
4857
4858 if (nbTests < testNb) nbTests = testNb;
4859
4860 if (testNb==0) {
4861 result = basicUnitTests(0, probfloat); /* constant seed for predictability */
4862
4863 if (!result && longTests) {
4864 result = longUnitTests(0, probfloat);
4865 }
4866 }
4867 if (!result)
4868 result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests);
4869 if (mainPause) {
4870 int unused;
4871 DISPLAY("Press Enter \n");
4872 unused = getchar();
4873 (void)unused;
4874 }
4875 return result;
4876}