648db22b |
1 | /* |
2 | * Copyright (c) Meta Platforms, Inc. and affiliates. |
3 | * All rights reserved. |
4 | * |
5 | * This source code is licensed under both the BSD-style license (found in the |
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found |
7 | * in the COPYING file in the root directory of this source tree). |
8 | * You may select, at your option, one of the above-listed licenses. |
9 | */ |
10 | |
11 | |
12 | /*-************************************ |
13 | * Compiler specific |
14 | **************************************/ |
15 | #ifdef _MSC_VER /* Visual Studio */ |
16 | # define _CRT_SECURE_NO_WARNINGS /* fgets */ |
17 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ |
18 | # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ |
19 | #endif |
20 | |
21 | |
22 | /*-************************************ |
23 | * Includes |
24 | **************************************/ |
25 | #include <stdlib.h> /* free */ |
26 | #include <stdio.h> /* fgets, sscanf */ |
27 | #include <string.h> /* strcmp */ |
28 | #include <time.h> /* time(), time_t */ |
29 | #undef NDEBUG /* always enable assert() */ |
30 | #include <assert.h> |
31 | #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ |
32 | #include "debug.h" /* DEBUG_STATIC_ASSERT */ |
33 | #include "fse.h" |
34 | #define ZSTD_DISABLE_DEPRECATE_WARNINGS /* No deprecation warnings, we still test some deprecated functions */ |
35 | #include "zstd.h" /* ZSTD_VERSION_STRING */ |
36 | #include "zstd_errors.h" /* ZSTD_getErrorCode */ |
37 | #define ZDICT_STATIC_LINKING_ONLY |
38 | #include "zdict.h" /* ZDICT_trainFromBuffer */ |
39 | #include "mem.h" |
40 | #include "datagen.h" /* RDG_genBuffer */ |
41 | #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ |
42 | #include "xxhash.h" /* XXH64 */ |
43 | #include "util.h" |
44 | #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ |
45 | /* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */ |
46 | #include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ |
47 | #include "threading.h" /* ZSTD_pthread_create, ZSTD_pthread_join */ |
48 | |
49 | |
50 | /*-************************************ |
51 | * Constants |
52 | **************************************/ |
53 | #define GB *(1U<<30) |
54 | |
55 | static const int FUZ_compressibility_default = 50; |
56 | static const int nbTestsDefault = 30000; |
57 | |
58 | |
59 | /*-************************************ |
60 | * Display Macros |
61 | **************************************/ |
62 | #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) |
63 | #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } |
64 | static U32 g_displayLevel = 2; |
65 | |
66 | static const U64 g_refreshRate = SEC_TO_MICRO / 6; |
67 | static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; |
68 | |
69 | #define DISPLAYUPDATE(l, ...) \ |
70 | if (g_displayLevel>=l) { \ |
71 | if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ |
72 | { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ |
73 | if (g_displayLevel>=4) fflush(stderr); } \ |
74 | } |
75 | |
76 | |
77 | /*-******************************************************* |
78 | * Compile time test |
79 | *********************************************************/ |
80 | #undef MIN |
81 | #undef MAX |
82 | /* Declaring the function, to avoid -Wmissing-prototype */ |
83 | void FUZ_bug976(void); |
84 | void FUZ_bug976(void) |
85 | { /* these constants shall not depend on MIN() macro */ |
86 | DEBUG_STATIC_ASSERT(ZSTD_HASHLOG_MAX < 31); |
87 | DEBUG_STATIC_ASSERT(ZSTD_CHAINLOG_MAX < 31); |
88 | } |
89 | |
90 | |
91 | /*-******************************************************* |
92 | * Internal functions |
93 | *********************************************************/ |
94 | #define MIN(a,b) ((a)<(b)?(a):(b)) |
95 | #define MAX(a,b) ((a)>(b)?(a):(b)) |
96 | |
97 | #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) |
98 | static U32 FUZ_rand(U32* src) |
99 | { |
100 | static const U32 prime1 = 2654435761U; |
101 | static const U32 prime2 = 2246822519U; |
102 | U32 rand32 = *src; |
103 | rand32 *= prime1; |
104 | rand32 += prime2; |
105 | rand32 = FUZ_rotl32(rand32, 13); |
106 | *src = rand32; |
107 | return rand32 >> 5; |
108 | } |
109 | |
110 | static U32 FUZ_highbit32(U32 v32) |
111 | { |
112 | unsigned nbBits = 0; |
113 | if (v32==0) return 0; |
114 | while (v32) v32 >>= 1, nbBits++; |
115 | return nbBits; |
116 | } |
117 | |
118 | |
119 | /*============================================= |
120 | * Test macros |
121 | =============================================*/ |
122 | #define CHECK(fn) { if(!(fn)) { DISPLAYLEVEL(1, "Error : test (%s) failed \n", #fn); exit(1); } } |
123 | |
124 | #define CHECK_Z(f) { \ |
125 | size_t const err = f; \ |
126 | if (ZSTD_isError(err)) { \ |
127 | DISPLAY("Error => %s : %s ", \ |
128 | #f, ZSTD_getErrorName(err)); \ |
129 | exit(1); \ |
130 | } } |
131 | |
132 | #define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); exit(1); } |
133 | #define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn) |
134 | #define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; } |
135 | |
136 | #define CHECK_OP(op, lhs, rhs) { \ |
137 | if (!((lhs) op (rhs))) { \ |
138 | DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \ |
139 | exit(1); \ |
140 | } \ |
141 | } |
142 | #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs) |
143 | #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs) |
144 | |
145 | |
146 | /*============================================= |
147 | * Memory Tests |
148 | =============================================*/ |
149 | #if defined(__APPLE__) && defined(__MACH__) |
150 | |
151 | #include <malloc/malloc.h> /* malloc_size */ |
152 | |
153 | typedef struct { |
154 | unsigned long long totalMalloc; |
155 | size_t currentMalloc; |
156 | size_t peakMalloc; |
157 | unsigned nbMalloc; |
158 | unsigned nbFree; |
159 | } mallocCounter_t; |
160 | |
161 | static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 }; |
162 | |
163 | static void* FUZ_mallocDebug(void* counter, size_t size) |
164 | { |
165 | mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; |
166 | void* const ptr = malloc(size); |
167 | if (ptr==NULL) return NULL; |
168 | DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n", |
169 | (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */ |
170 | mcPtr->totalMalloc += size; |
171 | mcPtr->currentMalloc += size; |
172 | if (mcPtr->currentMalloc > mcPtr->peakMalloc) |
173 | mcPtr->peakMalloc = mcPtr->currentMalloc; |
174 | mcPtr->nbMalloc += 1; |
175 | return ptr; |
176 | } |
177 | |
178 | static void FUZ_freeDebug(void* counter, void* address) |
179 | { |
180 | mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; |
181 | DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10)); |
182 | mcPtr->nbFree += 1; |
183 | mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */ |
184 | free(address); |
185 | } |
186 | |
187 | static void FUZ_displayMallocStats(mallocCounter_t count) |
188 | { |
189 | DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n", |
190 | (unsigned)(count.peakMalloc >> 10), |
191 | count.nbMalloc, |
192 | (unsigned)(count.totalMalloc >> 10)); |
193 | } |
194 | |
195 | static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part, |
196 | void* inBuffer, size_t inSize, void* outBuffer, size_t outSize) |
197 | { |
198 | /* test only played in verbose mode, as they are long */ |
199 | if (g_displayLevel<3) return 0; |
200 | |
201 | /* Create compressible noise */ |
202 | if (!inBuffer || !outBuffer) { |
203 | DISPLAY("Not enough memory, aborting\n"); |
204 | exit(1); |
205 | } |
206 | RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed); |
207 | |
208 | /* simple compression tests */ |
209 | if (part <= 1) |
210 | { int compressionLevel; |
211 | for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { |
212 | mallocCounter_t malcount = INIT_MALLOC_COUNTER; |
213 | ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; |
214 | ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); |
215 | CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) ); |
216 | ZSTD_freeCCtx(cctx); |
217 | DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel); |
218 | FUZ_displayMallocStats(malcount); |
219 | } } |
220 | |
221 | /* streaming compression tests */ |
222 | if (part <= 2) |
223 | { int compressionLevel; |
224 | for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { |
225 | mallocCounter_t malcount = INIT_MALLOC_COUNTER; |
226 | ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; |
227 | ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem); |
228 | ZSTD_outBuffer out = { outBuffer, outSize, 0 }; |
229 | ZSTD_inBuffer in = { inBuffer, inSize, 0 }; |
230 | CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) ); |
231 | CHECK_Z( ZSTD_compressStream(cstream, &out, &in) ); |
232 | CHECK_Z( ZSTD_endStream(cstream, &out) ); |
233 | ZSTD_freeCStream(cstream); |
234 | DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel); |
235 | FUZ_displayMallocStats(malcount); |
236 | } } |
237 | |
238 | /* advanced MT API test */ |
239 | if (part <= 3) |
240 | { int nbThreads; |
241 | for (nbThreads=1; nbThreads<=4; nbThreads++) { |
242 | int compressionLevel; |
243 | for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { |
244 | mallocCounter_t malcount = INIT_MALLOC_COUNTER; |
245 | ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; |
246 | ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); |
247 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); |
248 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); |
249 | CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) ); |
250 | ZSTD_freeCCtx(cctx); |
251 | DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ", |
252 | nbThreads, compressionLevel); |
253 | FUZ_displayMallocStats(malcount); |
254 | } } } |
255 | |
256 | /* advanced MT streaming API test */ |
257 | if (part <= 4) |
258 | { int nbThreads; |
259 | for (nbThreads=1; nbThreads<=4; nbThreads++) { |
260 | int compressionLevel; |
261 | for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { |
262 | mallocCounter_t malcount = INIT_MALLOC_COUNTER; |
263 | ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; |
264 | ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); |
265 | ZSTD_outBuffer out = { outBuffer, outSize, 0 }; |
266 | ZSTD_inBuffer in = { inBuffer, inSize, 0 }; |
267 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); |
268 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); |
269 | CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) ); |
270 | while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {} |
271 | ZSTD_freeCCtx(cctx); |
272 | DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ", |
273 | nbThreads, compressionLevel); |
274 | FUZ_displayMallocStats(malcount); |
275 | } } } |
276 | |
277 | return 0; |
278 | } |
279 | |
280 | static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) |
281 | { |
282 | size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */ |
283 | size_t const outSize = ZSTD_compressBound(inSize); |
284 | void* const inBuffer = malloc(inSize); |
285 | void* const outBuffer = malloc(outSize); |
286 | int result; |
287 | |
288 | /* Create compressible noise */ |
289 | if (!inBuffer || !outBuffer) { |
290 | DISPLAY("Not enough memory, aborting \n"); |
291 | exit(1); |
292 | } |
293 | |
294 | result = FUZ_mallocTests_internal(seed, compressibility, part, |
295 | inBuffer, inSize, outBuffer, outSize); |
296 | |
297 | free(inBuffer); |
298 | free(outBuffer); |
299 | return result; |
300 | } |
301 | |
302 | #else |
303 | |
304 | static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) |
305 | { |
306 | (void)seed; (void)compressibility; (void)part; |
307 | return 0; |
308 | } |
309 | |
310 | #endif |
311 | |
312 | static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, |
313 | BYTE* src, size_t size, ZSTD_sequenceFormat_e format) |
314 | { |
315 | size_t i; |
316 | size_t j; |
317 | for(i = 0; i < seqsSize; ++i) { |
318 | assert(dst + seqs[i].litLength + seqs[i].matchLength <= dst + size); |
319 | assert(src + seqs[i].litLength + seqs[i].matchLength <= src + size); |
320 | if (format == ZSTD_sf_noBlockDelimiters) { |
321 | assert(seqs[i].matchLength != 0 || seqs[i].offset != 0); |
322 | } |
323 | |
324 | memcpy(dst, src, seqs[i].litLength); |
325 | dst += seqs[i].litLength; |
326 | src += seqs[i].litLength; |
327 | size -= seqs[i].litLength; |
328 | |
329 | if (seqs[i].offset != 0) { |
330 | for (j = 0; j < seqs[i].matchLength; ++j) |
f535537f |
331 | dst[j] = dst[(ptrdiff_t)(j - seqs[i].offset)]; |
648db22b |
332 | dst += seqs[i].matchLength; |
333 | src += seqs[i].matchLength; |
334 | size -= seqs[i].matchLength; |
335 | } |
336 | } |
337 | if (format == ZSTD_sf_noBlockDelimiters) { |
338 | memcpy(dst, src, size); |
339 | } |
340 | } |
341 | |
342 | #ifdef ZSTD_MULTITHREAD |
343 | |
344 | typedef struct { |
345 | ZSTD_CCtx* cctx; |
346 | ZSTD_threadPool* pool; |
347 | void* CNBuffer; |
348 | size_t CNBuffSize; |
349 | void* compressedBuffer; |
350 | size_t compressedBufferSize; |
351 | void* decodedBuffer; |
352 | int err; |
353 | } threadPoolTests_compressionJob_payload; |
354 | |
355 | static void* threadPoolTests_compressionJob(void* payload) { |
356 | threadPoolTests_compressionJob_payload* args = (threadPoolTests_compressionJob_payload*)payload; |
357 | size_t cSize; |
358 | if (ZSTD_isError(ZSTD_CCtx_refThreadPool(args->cctx, args->pool))) args->err = 1; |
359 | cSize = ZSTD_compress2(args->cctx, args->compressedBuffer, args->compressedBufferSize, args->CNBuffer, args->CNBuffSize); |
360 | if (ZSTD_isError(cSize)) args->err = 1; |
361 | if (ZSTD_isError(ZSTD_decompress(args->decodedBuffer, args->CNBuffSize, args->compressedBuffer, cSize))) args->err = 1; |
362 | return payload; |
363 | } |
364 | |
365 | static int threadPoolTests(void) { |
366 | int testResult = 0; |
367 | size_t err; |
368 | |
369 | size_t const CNBuffSize = 5 MB; |
370 | void* const CNBuffer = malloc(CNBuffSize); |
371 | size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); |
372 | void* const compressedBuffer = malloc(compressedBufferSize); |
373 | void* const decodedBuffer = malloc(CNBuffSize); |
374 | |
375 | size_t const kPoolNumThreads = 8; |
376 | |
377 | RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, 0); |
378 | |
f535537f |
379 | DISPLAYLEVEL(3, "thread pool test : threadPool reuse roundtrips: "); |
648db22b |
380 | { |
381 | ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
382 | ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); |
383 | |
384 | size_t nbThreads = 1; |
385 | for (; nbThreads <= kPoolNumThreads; ++nbThreads) { |
386 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
387 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, (int)nbThreads); |
388 | err = ZSTD_CCtx_refThreadPool(cctx, pool); |
389 | if (ZSTD_isError(err)) { |
390 | DISPLAYLEVEL(3, "refThreadPool error!\n"); |
391 | ZSTD_freeCCtx(cctx); |
392 | goto _output_error; |
393 | } |
394 | err = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
395 | if (ZSTD_isError(err)) { |
396 | DISPLAYLEVEL(3, "Compression error!\n"); |
397 | ZSTD_freeCCtx(cctx); |
398 | goto _output_error; |
399 | } |
400 | err = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, err); |
401 | if (ZSTD_isError(err)) { |
402 | DISPLAYLEVEL(3, "Decompression error!\n"); |
403 | ZSTD_freeCCtx(cctx); |
404 | goto _output_error; |
405 | } |
406 | } |
407 | |
408 | ZSTD_freeCCtx(cctx); |
409 | ZSTD_freeThreadPool(pool); |
410 | } |
411 | DISPLAYLEVEL(3, "OK \n"); |
412 | |
413 | DISPLAYLEVEL(3, "thread pool test : threadPool simultaneous usage: "); |
414 | { |
415 | void* const decodedBuffer2 = malloc(CNBuffSize); |
416 | void* const compressedBuffer2 = malloc(compressedBufferSize); |
417 | ZSTD_threadPool* pool = ZSTD_createThreadPool(kPoolNumThreads); |
418 | ZSTD_CCtx* cctx1 = ZSTD_createCCtx(); |
419 | ZSTD_CCtx* cctx2 = ZSTD_createCCtx(); |
420 | |
421 | ZSTD_pthread_t t1; |
422 | ZSTD_pthread_t t2; |
423 | threadPoolTests_compressionJob_payload p1 = {cctx1, pool, CNBuffer, CNBuffSize, |
424 | compressedBuffer, compressedBufferSize, decodedBuffer, 0 /* err */}; |
425 | threadPoolTests_compressionJob_payload p2 = {cctx2, pool, CNBuffer, CNBuffSize, |
426 | compressedBuffer2, compressedBufferSize, decodedBuffer2, 0 /* err */}; |
427 | |
428 | ZSTD_CCtx_setParameter(cctx1, ZSTD_c_nbWorkers, 2); |
429 | ZSTD_CCtx_setParameter(cctx2, ZSTD_c_nbWorkers, 2); |
430 | ZSTD_CCtx_refThreadPool(cctx1, pool); |
431 | ZSTD_CCtx_refThreadPool(cctx2, pool); |
432 | |
433 | ZSTD_pthread_create(&t1, NULL, threadPoolTests_compressionJob, &p1); |
434 | ZSTD_pthread_create(&t2, NULL, threadPoolTests_compressionJob, &p2); |
435 | ZSTD_pthread_join(t1); |
436 | ZSTD_pthread_join(t2); |
437 | |
438 | assert(!memcmp(decodedBuffer, decodedBuffer2, CNBuffSize)); |
439 | free(decodedBuffer2); |
440 | free(compressedBuffer2); |
441 | |
442 | ZSTD_freeThreadPool(pool); |
443 | ZSTD_freeCCtx(cctx1); |
444 | ZSTD_freeCCtx(cctx2); |
445 | |
446 | if (p1.err || p2.err) goto _output_error; |
447 | } |
448 | DISPLAYLEVEL(3, "OK \n"); |
449 | |
450 | _end: |
451 | free(CNBuffer); |
452 | free(compressedBuffer); |
453 | free(decodedBuffer); |
454 | return testResult; |
455 | |
456 | _output_error: |
457 | testResult = 1; |
458 | DISPLAY("Error detected in Unit tests ! \n"); |
459 | goto _end; |
460 | } |
461 | #endif /* ZSTD_MULTITHREAD */ |
462 | |
463 | /*============================================= |
464 | * Unit tests |
465 | =============================================*/ |
466 | |
467 | static void test_compressBound(unsigned tnb) |
468 | { |
469 | DISPLAYLEVEL(3, "test%3u : compressBound : ", tnb); |
470 | |
471 | /* check ZSTD_compressBound == ZSTD_COMPRESSBOUND |
472 | * for a large range of known valid values */ |
473 | DEBUG_STATIC_ASSERT(sizeof(size_t) >= 4); |
474 | { int s; |
475 | for (s=0; s<30; s++) { |
476 | size_t const w = (size_t)1 << s; |
477 | CHECK_EQ(ZSTD_compressBound(w), ZSTD_COMPRESSBOUND(w)); |
478 | } } |
479 | |
480 | /* Ensure error if srcSize too big */ |
481 | { size_t const w = ZSTD_MAX_INPUT_SIZE + 1; |
482 | CHECK(ZSTD_isError(ZSTD_compressBound(w))); /* must fail */ |
483 | CHECK_EQ(ZSTD_COMPRESSBOUND(w), 0); |
484 | } |
485 | |
486 | DISPLAYLEVEL(3, "OK \n"); |
487 | } |
488 | |
489 | static void test_decompressBound(unsigned tnb) |
490 | { |
491 | DISPLAYLEVEL(3, "test%3u : decompressBound : ", tnb); |
492 | |
493 | /* Simple compression, with size : should provide size; */ |
494 | { const char example[] = "abcd"; |
495 | char cBuffer[ZSTD_COMPRESSBOUND(sizeof(example))]; |
496 | size_t const cSize = ZSTD_compress(cBuffer, sizeof(cBuffer), example, sizeof(example), 0); |
497 | CHECK_Z(cSize); |
498 | CHECK_EQ(ZSTD_decompressBound(cBuffer, cSize), (unsigned long long)sizeof(example)); |
499 | } |
500 | |
501 | /* Simple small compression without size : should provide 1 block size */ |
502 | { char cBuffer[ZSTD_COMPRESSBOUND(0)]; |
503 | ZSTD_outBuffer out = { cBuffer, sizeof(cBuffer), 0 }; |
504 | ZSTD_inBuffer in = { NULL, 0, 0 }; |
505 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
506 | assert(cctx); |
507 | CHECK_Z( ZSTD_initCStream(cctx, 0) ); |
508 | CHECK_Z( ZSTD_compressStream(cctx, &out, &in) ); |
509 | CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); |
510 | CHECK_EQ( ZSTD_decompressBound(cBuffer, out.pos), ZSTD_BLOCKSIZE_MAX ); |
511 | ZSTD_freeCCtx(cctx); |
512 | } |
513 | |
514 | /* Attempt to overflow 32-bit intermediate multiplication result |
515 | * This requires dBound >= 4 GB, aka 2^32. |
516 | * This requires 2^32 / 2^17 = 2^15 blocks |
517 | * => create 2^15 blocks (can be empty, or just 1 byte). */ |
518 | { const char input[] = "a"; |
519 | size_t const nbBlocks = (1 << 15) + 1; |
520 | size_t blockNb; |
521 | size_t const outCapacity = 1 << 18; /* large margin */ |
522 | char* const outBuffer = malloc (outCapacity); |
523 | ZSTD_outBuffer out = { outBuffer, outCapacity, 0 }; |
524 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
525 | assert(cctx); |
526 | assert(outBuffer); |
527 | CHECK_Z( ZSTD_initCStream(cctx, 0) ); |
528 | for (blockNb=0; blockNb<nbBlocks; blockNb++) { |
529 | ZSTD_inBuffer in = { input, sizeof(input), 0 }; |
530 | CHECK_Z( ZSTD_compressStream(cctx, &out, &in) ); |
531 | CHECK_EQ( ZSTD_flushStream(cctx, &out), 0 ); |
532 | } |
533 | CHECK_EQ( ZSTD_endStream(cctx, &out), 0 ); |
f535537f |
534 | CHECK( ZSTD_decompressBound(outBuffer, out.pos) > 0x100000000ULL /* 4 GB */ ); |
648db22b |
535 | ZSTD_freeCCtx(cctx); |
536 | free(outBuffer); |
537 | } |
538 | |
539 | DISPLAYLEVEL(3, "OK \n"); |
540 | } |
541 | |
542 | static void test_setCParams(unsigned tnb) |
543 | { |
544 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
545 | ZSTD_compressionParameters cparams; |
546 | assert(cctx); |
547 | |
548 | DISPLAYLEVEL(3, "test%3u : ZSTD_CCtx_setCParams : ", tnb); |
549 | |
550 | /* valid cparams */ |
551 | cparams = ZSTD_getCParams(1, 0, 0); |
552 | CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams)); |
553 | |
554 | /* invalid cparams (must fail) */ |
555 | cparams.windowLog = 99; |
556 | CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams))); |
557 | |
558 | free(cctx); |
559 | DISPLAYLEVEL(3, "OK \n"); |
560 | } |
561 | |
562 | static int basicUnitTests(U32 const seed, double compressibility) |
563 | { |
564 | size_t const CNBuffSize = 5 MB; |
565 | void* const CNBuffer = malloc(CNBuffSize); |
566 | size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); |
567 | void* const compressedBuffer = malloc(compressedBufferSize); |
568 | void* const decodedBuffer = malloc(CNBuffSize); |
569 | int testResult = 0; |
570 | unsigned testNb=0; |
571 | size_t cSize; |
572 | |
573 | /* Create compressible noise */ |
574 | if (!CNBuffer || !compressedBuffer || !decodedBuffer) { |
575 | DISPLAY("Not enough memory, aborting\n"); |
576 | testResult = 1; |
577 | goto _end; |
578 | } |
579 | RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); |
580 | |
581 | /* Basic tests */ |
582 | DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++); |
583 | { const char* errorString = ZSTD_getErrorName(0); |
584 | DISPLAYLEVEL(3, "OK : %s \n", errorString); |
585 | } |
586 | |
587 | DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++); |
588 | { const char* errorString = ZSTD_getErrorName(499); |
589 | DISPLAYLEVEL(3, "OK : %s \n", errorString); |
590 | } |
591 | |
592 | DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++); |
593 | { int const mcl = ZSTD_minCLevel(); |
594 | DISPLAYLEVEL(3, "%i (OK) \n", mcl); |
595 | } |
596 | |
597 | DISPLAYLEVEL(3, "test%3u : default compression level : ", testNb++); |
598 | { int const defaultCLevel = ZSTD_defaultCLevel(); |
599 | if (defaultCLevel != ZSTD_CLEVEL_DEFAULT) goto _output_error; |
600 | DISPLAYLEVEL(3, "%i (OK) \n", defaultCLevel); |
601 | } |
602 | |
603 | DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++); |
604 | { unsigned const vn = ZSTD_versionNumber(); |
605 | DISPLAYLEVEL(3, "%u (OK) \n", vn); |
606 | } |
607 | |
608 | test_compressBound(testNb++); |
609 | |
610 | test_decompressBound(testNb++); |
611 | |
612 | test_setCParams(testNb++); |
613 | |
614 | DISPLAYLEVEL(3, "test%3u : ZSTD_adjustCParams : ", testNb++); |
615 | { |
616 | ZSTD_compressionParameters params; |
617 | memset(¶ms, 0, sizeof(params)); |
618 | params.windowLog = 10; |
619 | params.hashLog = 19; |
620 | params.chainLog = 19; |
621 | params = ZSTD_adjustCParams(params, 1000, 100000); |
622 | if (params.hashLog != 18) goto _output_error; |
623 | if (params.chainLog != 17) goto _output_error; |
624 | } |
625 | DISPLAYLEVEL(3, "OK \n"); |
626 | |
627 | DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); |
628 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
629 | if (cctx==NULL) goto _output_error; |
630 | CHECK_VAR(cSize, ZSTD_compressCCtx(cctx, |
631 | compressedBuffer, compressedBufferSize, |
632 | CNBuffer, CNBuffSize, 1) ); |
633 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
634 | |
635 | DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++); |
636 | { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); |
637 | DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize); |
638 | } |
639 | ZSTD_freeCCtx(cctx); |
640 | } |
641 | |
642 | DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++); |
643 | { |
644 | char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff"; |
645 | size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8); |
646 | if (!ZSTD_isError(size)) goto _output_error; |
647 | } |
648 | DISPLAYLEVEL(3, "OK \n"); |
649 | |
650 | DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); |
651 | { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); |
652 | if (rSize != CNBuffSize) goto _output_error; |
653 | } |
654 | DISPLAYLEVEL(3, "OK \n"); |
655 | |
656 | DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++); |
657 | { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize); |
658 | if (rSize != CNBuffSize) goto _output_error; |
659 | } |
660 | DISPLAYLEVEL(3, "OK \n"); |
661 | |
662 | DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); |
663 | { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize); |
664 | if (rSize != CNBuffSize) goto _output_error; |
665 | } |
666 | DISPLAYLEVEL(3, "OK \n"); |
667 | |
668 | DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++); |
669 | { |
670 | unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize); |
671 | if (bound != CNBuffSize) goto _output_error; |
672 | } |
673 | DISPLAYLEVEL(3, "OK \n"); |
674 | |
675 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++); |
676 | { |
677 | unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1); |
678 | if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error; |
679 | } |
680 | DISPLAYLEVEL(3, "OK \n"); |
681 | |
682 | DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); |
683 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
684 | if (r != CNBuffSize) goto _output_error; } |
685 | DISPLAYLEVEL(3, "OK \n"); |
686 | |
687 | DISPLAYLEVEL(3, "test%3i : decompress %u bytes with Huffman assembly disabled : ", testNb++, (unsigned)CNBuffSize); |
688 | { |
689 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
690 | size_t r; |
691 | CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_disableHuffmanAssembly, 1)); |
692 | r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
693 | if (r != CNBuffSize || memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; |
694 | ZSTD_freeDCtx(dctx); |
695 | } |
696 | DISPLAYLEVEL(3, "OK \n"); |
697 | |
698 | DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); |
699 | { size_t u; |
700 | for (u=0; u<CNBuffSize; u++) { |
701 | if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; |
702 | } } |
703 | DISPLAYLEVEL(3, "OK \n"); |
704 | |
705 | DISPLAYLEVEL(3, "test%3u : invalid endDirective : ", testNb++); |
706 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
707 | ZSTD_inBuffer inb = { CNBuffer, CNBuffSize, 0 }; |
708 | ZSTD_outBuffer outb = { compressedBuffer, compressedBufferSize, 0 }; |
709 | if (cctx==NULL) goto _output_error; |
710 | CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective) 3) ) ); /* must fail */ |
711 | CHECK( ZSTD_isError( ZSTD_compressStream2(cctx, &outb, &inb, (ZSTD_EndDirective)-1) ) ); /* must fail */ |
712 | ZSTD_freeCCtx(cctx); |
713 | } |
714 | DISPLAYLEVEL(3, "OK \n"); |
715 | |
716 | DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++); |
717 | { |
718 | ZSTD_parameters params = ZSTD_getParams(3, 0, 0); |
719 | assert(!ZSTD_checkCParams(params.cParams)); |
720 | } |
721 | DISPLAYLEVEL(3, "OK \n"); |
722 | |
723 | DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++); |
724 | { |
725 | ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem); |
726 | assert(dctx != NULL); |
727 | assert(ZSTD_sizeof_DCtx(dctx) != 0); |
728 | ZSTD_freeDCtx(dctx); |
729 | } |
730 | DISPLAYLEVEL(3, "OK \n"); |
731 | |
732 | DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++); |
733 | { |
734 | /* %p takes a void*. In ISO C, it's illegal to cast a function pointer |
735 | * to a data pointer. (Although in POSIX you're required to be allowed |
736 | * to do it...) So we have to fall back to our trusty friend memcpy. */ |
737 | unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) = |
738 | ZSTD_getDictID_fromDDict; |
739 | ZSTD_DStream* (* const funcptr_createDStream)( |
740 | ZSTD_customMem customMem) = ZSTD_createDStream_advanced; |
741 | void (* const funcptr_copyDCtx)( |
742 | ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx; |
743 | ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) = |
744 | ZSTD_nextInputType; |
745 | const void *voidptr_getDictID; |
746 | const void *voidptr_createDStream; |
747 | const void *voidptr_copyDCtx; |
748 | const void *voidptr_nextInputType; |
749 | DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID)); |
750 | memcpy( |
751 | (void*)&voidptr_getDictID, |
752 | (const void*)&funcptr_getDictID, |
753 | sizeof(void*)); |
754 | memcpy( |
755 | (void*)&voidptr_createDStream, |
756 | (const void*)&funcptr_createDStream, |
757 | sizeof(void*)); |
758 | memcpy( |
759 | (void*)&voidptr_copyDCtx, |
760 | (const void*)&funcptr_copyDCtx, |
761 | sizeof(void*)); |
762 | memcpy( |
763 | (void*)&voidptr_nextInputType, |
764 | (const void*)&funcptr_nextInputType, |
765 | sizeof(void*)); |
766 | DISPLAYLEVEL(3, "%p ", voidptr_getDictID); |
767 | DISPLAYLEVEL(3, "%p ", voidptr_createDStream); |
768 | DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx); |
769 | DISPLAYLEVEL(3, "%p ", voidptr_nextInputType); |
770 | } |
771 | DISPLAYLEVEL(3, ": OK \n"); |
772 | |
773 | DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++); |
774 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); |
775 | { size_t const r = ZSTD_decompress_usingDict(dctx, |
776 | decodedBuffer, CNBuffSize, |
777 | compressedBuffer, cSize, |
778 | NULL, 0); |
779 | if (r != CNBuffSize) goto _output_error; |
780 | } |
781 | ZSTD_freeDCtx(dctx); |
782 | } |
783 | DISPLAYLEVEL(3, "OK \n"); |
784 | |
785 | DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++); |
786 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); |
787 | { size_t const r = ZSTD_decompress_usingDDict(dctx, |
788 | decodedBuffer, CNBuffSize, |
789 | compressedBuffer, cSize, |
790 | NULL); |
791 | if (r != CNBuffSize) goto _output_error; |
792 | } |
793 | ZSTD_freeDCtx(dctx); |
794 | } |
795 | DISPLAYLEVEL(3, "OK \n"); |
796 | |
797 | DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++); |
798 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); |
799 | if (!ZSTD_isError(r)) goto _output_error; |
800 | if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; } |
801 | DISPLAYLEVEL(3, "OK \n"); |
802 | |
803 | DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++); |
804 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1); |
805 | if (!ZSTD_isError(r)) goto _output_error; |
806 | if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } |
807 | DISPLAYLEVEL(3, "OK \n"); |
808 | |
809 | DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++); |
810 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize); |
811 | if (!ZSTD_isError(r)) goto _output_error; |
812 | if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } |
813 | DISPLAYLEVEL(3, "OK \n"); |
814 | |
815 | DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++); |
816 | { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize); |
817 | if (!ZSTD_isError(r)) goto _output_error; |
818 | if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; } |
819 | DISPLAYLEVEL(3, "OK \n"); |
820 | |
821 | DISPLAYLEVEL(3, "test%3i : decompress with corrupted checksum : ", testNb++); |
822 | { /* create compressed buffer with checksumming enabled */ |
823 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
824 | if (!cctx) { |
825 | DISPLAY("Not enough memory, aborting\n"); |
826 | testResult = 1; |
827 | goto _end; |
828 | } |
829 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1) ); |
830 | CHECK_VAR(cSize, ZSTD_compress2(cctx, |
831 | compressedBuffer, compressedBufferSize, |
832 | CNBuffer, CNBuffSize) ); |
833 | ZSTD_freeCCtx(cctx); |
834 | } |
835 | { /* copy the compressed buffer and corrupt the checksum */ |
836 | size_t r; |
837 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
838 | if (!dctx) { |
839 | DISPLAY("Not enough memory, aborting\n"); |
840 | testResult = 1; |
841 | goto _end; |
842 | } |
843 | |
844 | ((char*)compressedBuffer)[cSize-1] += 1; |
845 | r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
846 | if (!ZSTD_isError(r)) goto _output_error; |
847 | if (ZSTD_getErrorCode(r) != ZSTD_error_checksum_wrong) goto _output_error; |
848 | |
849 | CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_forceIgnoreChecksum, ZSTD_d_ignoreChecksum)); |
850 | r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); |
851 | if (!ZSTD_isError(r)) goto _output_error; /* wrong checksum size should still throw error */ |
852 | r = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
853 | if (ZSTD_isError(r)) goto _output_error; |
854 | |
855 | ZSTD_freeDCtx(dctx); |
856 | } |
857 | DISPLAYLEVEL(3, "OK \n"); |
858 | |
859 | |
860 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++); |
861 | { /* create compressed buffer with content size missing */ |
862 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
863 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); |
864 | CHECK_VAR(cSize, ZSTD_compress2(cctx, |
865 | compressedBuffer, compressedBufferSize, |
866 | CNBuffer, CNBuffSize) ); |
867 | ZSTD_freeCCtx(cctx); |
868 | } |
869 | { /* ensure frame content size is missing */ |
870 | ZSTD_frameHeader zfh; |
871 | size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize); |
872 | if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; |
873 | } |
874 | { /* ensure CNBuffSize <= decompressBound */ |
875 | unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize); |
876 | if (CNBuffSize > bound) goto _output_error; |
877 | } |
878 | DISPLAYLEVEL(3, "OK \n"); |
879 | |
880 | DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++); |
881 | { |
882 | size_t const largeFrameSrcSize = 200; |
883 | size_t const smallFrameSrcSize = 10; |
884 | size_t const nbFrames = 256; |
885 | |
886 | size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0; |
887 | int sizeReduced = 0; |
888 | |
889 | BYTE* const dst = (BYTE*)compressedBuffer; |
890 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
891 | |
892 | /* create a large frame and then a bunch of small frames */ |
893 | size_t srcSize = ZSTD_compress((void*)dst, |
894 | compressedBufferSize, CNBuffer, largeFrameSrcSize, 3); |
895 | for (i = 0; i < nbFrames; i++) |
896 | srcSize += ZSTD_compress((void*)(dst + srcSize), |
897 | compressedBufferSize - srcSize, CNBuffer, |
898 | smallFrameSrcSize, 3); |
899 | |
900 | /* decompressStream and make sure that dctx size was reduced at least once */ |
901 | while (consumed < srcSize) { |
902 | ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0}; |
903 | ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0}; |
904 | ZSTD_decompressStream(dctx, &out, &in); |
905 | consumed += in.pos; |
906 | produced += out.pos; |
907 | |
908 | /* success! size was reduced from the previous frame */ |
909 | if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx)) |
910 | sizeReduced = 1; |
911 | |
912 | prevDCtxSize = ZSTD_sizeof_DCtx(dctx); |
913 | } |
914 | |
915 | assert(sizeReduced); |
916 | |
917 | ZSTD_freeDCtx(dctx); |
918 | } |
919 | DISPLAYLEVEL(3, "OK \n"); |
920 | |
921 | { |
922 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
923 | ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, 100, 1); |
924 | ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); |
925 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); |
926 | |
927 | DISPLAYLEVEL(3, "test%3i : ZSTD_compressCCtx() doesn't use advanced parameters", testNb++); |
928 | CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 1)); |
929 | if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; |
930 | DISPLAYLEVEL(3, "OK \n"); |
931 | |
932 | DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingDict() doesn't use advanced parameters: ", testNb++); |
933 | CHECK_Z(ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, 1)); |
934 | if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; |
935 | DISPLAYLEVEL(3, "OK \n"); |
936 | |
937 | DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict() doesn't use advanced parameters: ", testNb++); |
938 | CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict)); |
939 | if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; |
940 | DISPLAYLEVEL(3, "OK \n"); |
941 | |
942 | DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced() doesn't use advanced parameters: ", testNb++); |
943 | CHECK_Z(ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, NULL, 0, params)); |
944 | if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; |
945 | DISPLAYLEVEL(3, "OK \n"); |
946 | |
947 | DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced() doesn't use advanced parameters: ", testNb++); |
948 | CHECK_Z(ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, NULL, 0, cdict, params.fParams)); |
949 | if (MEM_readLE32(compressedBuffer) != ZSTD_MAGICNUMBER) goto _output_error; |
950 | DISPLAYLEVEL(3, "OK \n"); |
951 | |
952 | ZSTD_freeCDict(cdict); |
953 | ZSTD_freeCCtx(cctx); |
954 | } |
955 | |
f535537f |
956 | DISPLAYLEVEL(3, "test%3i : maxBlockSize = 2K", testNb++); |
957 | { |
958 | ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
959 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
960 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
961 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_maxBlockSize, 2048)); |
962 | CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 2048)); |
963 | |
964 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
965 | CHECK_Z(cSize); |
966 | CHECK_Z(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize)); |
967 | |
968 | CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_maxBlockSize, 1024)); |
969 | CHECK(ZSTD_isError(ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize))); |
970 | |
971 | ZSTD_freeDCtx(dctx); |
972 | ZSTD_freeCCtx(cctx); |
973 | } |
974 | |
648db22b |
975 | DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++); |
976 | { |
977 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
978 | |
979 | size_t const size = (1U << 10); |
980 | size_t const dstCapacity = ZSTD_compressBound(size); |
981 | void* dict = (void*)malloc(size); |
982 | void* src = (void*)malloc(size); |
983 | void* dst = (void*)malloc(dstCapacity); |
984 | |
985 | RDG_genBuffer(dict, size, 0.5, 0.5, seed); |
986 | RDG_genBuffer(src, size, 0.5, 0.5, seed); |
987 | |
988 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); |
989 | assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3))); |
990 | |
991 | ZSTD_freeCCtx(cctx); |
992 | free(dict); |
993 | free(src); |
994 | free(dst); |
995 | } |
996 | DISPLAYLEVEL(3, "OK \n"); |
997 | |
998 | DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++); |
999 | { |
1000 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1001 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
1002 | void* dict = (void*)malloc(CNBuffSize); |
1003 | int nbWorkers; |
1004 | |
1005 | for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) { |
1006 | RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed); |
1007 | RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); |
1008 | |
1009 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); |
1010 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
1011 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1)); |
1012 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); |
1013 | CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize)); |
1014 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
1015 | CHECK_Z(cSize); |
1016 | CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize)); |
1017 | } |
1018 | |
1019 | ZSTD_freeCCtx(cctx); |
1020 | ZSTD_freeDCtx(dctx); |
1021 | free(dict); |
1022 | } |
1023 | DISPLAYLEVEL(3, "OK \n"); |
1024 | |
1025 | DISPLAYLEVEL(3, "test%3i : testing dict compression for determinism : ", testNb++); |
1026 | { |
1027 | size_t const testSize = 1024; |
1028 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1029 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
1030 | char* dict = (char*)malloc(2 * testSize); |
1031 | int ldmEnabled, level; |
1032 | |
1033 | RDG_genBuffer(dict, testSize, 0.5, 0.5, seed); |
1034 | RDG_genBuffer(CNBuffer, testSize, 0.6, 0.6, seed); |
1035 | memcpy(dict + testSize, CNBuffer, testSize); |
1036 | for (level = 1; level <= 5; ++level) { |
1037 | for (ldmEnabled = ZSTD_ps_enable; ldmEnabled <= ZSTD_ps_disable; ++ldmEnabled) { |
1038 | size_t cSize0; |
1039 | XXH64_hash_t compressedChecksum0; |
1040 | |
1041 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
1042 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level)); |
1043 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ldmEnabled)); |
1044 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_deterministicRefPrefix, 1)); |
1045 | |
1046 | CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); |
1047 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, testSize); |
1048 | CHECK_Z(cSize); |
1049 | CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, testSize, compressedBuffer, cSize, dict, testSize)); |
1050 | |
1051 | cSize0 = cSize; |
1052 | compressedChecksum0 = XXH64(compressedBuffer, cSize, 0); |
1053 | |
1054 | CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, testSize)); |
1055 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, dict + testSize, testSize); |
1056 | CHECK_Z(cSize); |
1057 | |
1058 | if (cSize != cSize0) goto _output_error; |
1059 | if (XXH64(compressedBuffer, cSize, 0) != compressedChecksum0) goto _output_error; |
1060 | } |
1061 | } |
1062 | |
1063 | ZSTD_freeCCtx(cctx); |
1064 | ZSTD_freeDCtx(dctx); |
1065 | free(dict); |
1066 | } |
1067 | DISPLAYLEVEL(3, "OK \n"); |
1068 | |
1069 | DISPLAYLEVEL(3, "test%3i : LDM + opt parser with small uncompressible block ", testNb++); |
1070 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
1071 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
1072 | size_t const srcSize = 300 KB; |
1073 | size_t const flushSize = 128 KB + 5; |
1074 | size_t const dstSize = ZSTD_compressBound(srcSize); |
1075 | char* src = (char*)CNBuffer; |
1076 | char* dst = (char*)compressedBuffer; |
1077 | |
1078 | ZSTD_outBuffer out = { dst, dstSize, 0 }; |
1079 | ZSTD_inBuffer in = { src, flushSize, 0 }; |
1080 | |
1081 | if (!cctx || !dctx) { |
1082 | DISPLAY("Not enough memory, aborting\n"); |
1083 | testResult = 1; |
1084 | goto _end; |
1085 | } |
1086 | |
1087 | RDG_genBuffer(src, srcSize, 0.5, 0.5, seed); |
1088 | /* Force an LDM to exist that crosses block boundary into uncompressible block */ |
1089 | memcpy(src + 125 KB, src, 3 KB + 5); |
1090 | |
1091 | /* Enable MT, LDM, and opt parser */ |
1092 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1)); |
1093 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); |
1094 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
1095 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); |
1096 | |
1097 | /* Flushes a block of 128 KB and block of 5 bytes */ |
1098 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); |
1099 | |
1100 | /* Compress the rest */ |
1101 | in.size = 300 KB; |
1102 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); |
1103 | |
1104 | CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, dst, out.pos)); |
1105 | |
1106 | ZSTD_freeCCtx(cctx); |
1107 | ZSTD_freeDCtx(dctx); |
1108 | } |
1109 | DISPLAYLEVEL(3, "OK \n"); |
1110 | |
1111 | DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++); |
1112 | { |
1113 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1114 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
1115 | void* dict = (void*)malloc(CNBuffSize); |
1116 | size_t const kWindowLog = 10; |
1117 | size_t const kWindowSize = (size_t)1 << kWindowLog; |
1118 | size_t const dictSize = kWindowSize * 10; |
1119 | size_t const srcSize1 = kWindowSize / 2; |
1120 | size_t const srcSize2 = kWindowSize * 10; |
1121 | |
f535537f |
1122 | CHECK(cctx!=NULL); |
1123 | CHECK(dctx!=NULL); |
1124 | CHECK(dict!=NULL); |
648db22b |
1125 | if (CNBuffSize < dictSize) goto _output_error; |
1126 | |
1127 | RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); |
1128 | RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed); |
1129 | |
1130 | /* Enable checksum to verify round trip. */ |
1131 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
1132 | /* Disable content size to skip single-pass decompression. */ |
1133 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0)); |
1134 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog)); |
1135 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); |
1136 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32)); |
1137 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1)); |
1138 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16)); |
1139 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3)); |
1140 | |
1141 | /* Round trip once with a dictionary. */ |
1142 | CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); |
1143 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); |
1144 | CHECK_Z(cSize); |
1145 | CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); |
f535537f |
1146 | |
648db22b |
1147 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); |
1148 | /* Streaming decompression to catch out of bounds offsets. */ |
1149 | { |
1150 | ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; |
1151 | ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; |
1152 | size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); |
1153 | CHECK_Z(dSize); |
1154 | if (dSize != 0) goto _output_error; |
1155 | } |
1156 | |
1157 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); |
1158 | /* Round trip once with a dictionary. */ |
1159 | CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); |
f535537f |
1160 | { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; |
648db22b |
1161 | ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; |
1162 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); |
1163 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); |
1164 | cSize = out.pos; |
1165 | } |
1166 | CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); |
f535537f |
1167 | |
1168 | { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; |
648db22b |
1169 | ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; |
1170 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); |
1171 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); |
1172 | cSize = out.pos; |
1173 | } |
1174 | /* Streaming decompression to catch out of bounds offsets. */ |
f535537f |
1175 | { ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; |
648db22b |
1176 | ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; |
1177 | size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); |
1178 | CHECK_Z(dSize); |
1179 | if (dSize != 0) goto _output_error; |
1180 | } |
1181 | |
1182 | ZSTD_freeCCtx(cctx); |
1183 | ZSTD_freeDCtx(dctx); |
1184 | free(dict); |
1185 | } |
1186 | DISPLAYLEVEL(3, "OK \n"); |
1187 | |
1188 | /* Note: this test takes 0.5 seconds to run */ |
1189 | DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++); |
1190 | { |
1191 | /* test a big buffer so that ldm can take effect */ |
1192 | size_t const size = 100 MB; |
1193 | int const windowLog = 27; |
1194 | size_t const dstSize = ZSTD_compressBound(size); |
1195 | |
1196 | void* dict = (void*)malloc(size); |
1197 | void* src = (void*)malloc(size); |
1198 | void* dst = (void*)malloc(dstSize); |
1199 | void* recon = (void*)malloc(size); |
1200 | |
1201 | size_t refPrefixCompressedSize = 0; |
1202 | size_t refPrefixLdmCompressedSize = 0; |
1203 | size_t reconSize = 0; |
1204 | |
1205 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1206 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
1207 | |
1208 | /* make dict and src the same uncompressible data */ |
1209 | RDG_genBuffer(src, size, 0, 0, seed); |
1210 | memcpy(dict, src, size); |
1211 | assert(!memcmp(dict, src, size)); |
1212 | |
1213 | /* set level 1 and windowLog to cover src */ |
1214 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1)); |
1215 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog)); |
1216 | |
1217 | /* compress on level 1 using just refPrefix and no ldm */ |
1218 | ZSTD_CCtx_refPrefix(cctx, dict, size); |
1219 | refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); |
1220 | assert(!ZSTD_isError(refPrefixCompressedSize)); |
1221 | |
1222 | /* test round trip just refPrefix */ |
1223 | ZSTD_DCtx_refPrefix(dctx, dict, size); |
1224 | reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize); |
1225 | assert(!ZSTD_isError(reconSize)); |
1226 | assert(reconSize == size); |
1227 | assert(!memcmp(recon, src, size)); |
1228 | |
1229 | /* compress on level 1 using refPrefix and ldm */ |
1230 | ZSTD_CCtx_refPrefix(cctx, dict, size);; |
1231 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)) |
1232 | refPrefixLdmCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); |
1233 | assert(!ZSTD_isError(refPrefixLdmCompressedSize)); |
1234 | |
1235 | /* test round trip refPrefix + ldm*/ |
1236 | ZSTD_DCtx_refPrefix(dctx, dict, size); |
1237 | reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmCompressedSize); |
1238 | assert(!ZSTD_isError(reconSize)); |
1239 | assert(reconSize == size); |
1240 | assert(!memcmp(recon, src, size)); |
1241 | |
1242 | /* make sure that refPrefixCompressedSize is significantly greater */ |
1243 | assert(refPrefixCompressedSize > 10 * refPrefixLdmCompressedSize); |
1244 | /* make sure the ldm compressed size is less than 1% of original */ |
1245 | assert((double)refPrefixLdmCompressedSize / (double)size < 0.01); |
1246 | |
1247 | ZSTD_freeDCtx(dctx); |
1248 | ZSTD_freeCCtx(cctx); |
1249 | free(recon); |
1250 | free(dict); |
1251 | free(src); |
1252 | free(dst); |
1253 | } |
1254 | DISPLAYLEVEL(3, "OK \n"); |
1255 | |
1256 | DISPLAYLEVEL(3, "test%3i : in-place decompression : ", testNb++); |
1257 | cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize, -ZSTD_BLOCKSIZE_MAX); |
1258 | CHECK_Z(cSize); |
1259 | CHECK_LT(CNBuffSize, cSize); |
1260 | { |
1261 | size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize); |
1262 | size_t const outputSize = (CNBuffSize + margin); |
1263 | char* output = malloc(outputSize); |
1264 | char* input = output + outputSize - cSize; |
1265 | CHECK_LT(cSize, CNBuffSize + margin); |
1266 | CHECK(output != NULL); |
1267 | CHECK_Z(margin); |
1268 | CHECK(margin <= ZSTD_DECOMPRESSION_MARGIN(CNBuffSize, ZSTD_BLOCKSIZE_MAX)); |
1269 | memcpy(input, compressedBuffer, cSize); |
1270 | |
1271 | { |
1272 | size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize); |
1273 | CHECK_Z(dSize); |
1274 | CHECK_EQ(dSize, CNBuffSize); |
1275 | } |
1276 | CHECK(!memcmp(output, CNBuffer, CNBuffSize)); |
1277 | free(output); |
1278 | } |
1279 | DISPLAYLEVEL(3, "OK \n"); |
1280 | |
1281 | DISPLAYLEVEL(3, "test%3i : in-place decompression with 2 frames : ", testNb++); |
1282 | cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX); |
1283 | CHECK_Z(cSize); |
1284 | { |
1285 | size_t const cSize2 = ZSTD_compress((char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + (CNBuffSize / 3), CNBuffSize / 3, -ZSTD_BLOCKSIZE_MAX); |
1286 | CHECK_Z(cSize2); |
1287 | cSize += cSize2; |
1288 | } |
1289 | { |
1290 | size_t const srcSize = (CNBuffSize / 3) * 2; |
1291 | size_t const margin = ZSTD_decompressionMargin(compressedBuffer, cSize); |
1292 | size_t const outputSize = (CNBuffSize + margin); |
1293 | char* output = malloc(outputSize); |
1294 | char* input = output + outputSize - cSize; |
1295 | CHECK_LT(cSize, CNBuffSize + margin); |
1296 | CHECK(output != NULL); |
1297 | CHECK_Z(margin); |
1298 | memcpy(input, compressedBuffer, cSize); |
1299 | |
1300 | { |
1301 | size_t const dSize = ZSTD_decompress(output, outputSize, input, cSize); |
1302 | CHECK_Z(dSize); |
1303 | CHECK_EQ(dSize, srcSize); |
1304 | } |
1305 | CHECK(!memcmp(output, CNBuffer, srcSize)); |
1306 | free(output); |
1307 | } |
1308 | DISPLAYLEVEL(3, "OK \n"); |
1309 | |
1310 | DISPLAYLEVEL(3, "test%3i : Check block splitter with 64K literal length : ", testNb++); |
1311 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
1312 | size_t const srcSize = 256 * 1024; |
1313 | U32 const compressibleLenU32 = 32 * 1024 / 4; |
1314 | U32 const blockSizeU32 = 128 * 1024 / 4; |
1315 | U32 const litLenU32 = 64 * 1024 / 4; |
1316 | U32* data = (U32*)malloc(srcSize); |
1317 | size_t dSize; |
1318 | |
1319 | if (data == NULL || cctx == NULL) goto _output_error; |
1320 | |
1321 | /* Generate data without any matches */ |
1322 | RDG_genBuffer(data, srcSize, 0.0, 0.01, 2654435761U); |
1323 | /* Generate 32K of compressible data */ |
1324 | RDG_genBuffer(data, compressibleLenU32 * 4, 0.5, 0.5, 0xcafebabe); |
1325 | |
1326 | /* Add a match of offset=12, length=8 at idx=16, 32, 48, 64 */ |
1327 | data[compressibleLenU32 + 0] = 0xFFFFFFFF; |
1328 | data[compressibleLenU32 + 1] = 0xEEEEEEEE; |
1329 | data[compressibleLenU32 + 4] = 0xFFFFFFFF; |
1330 | data[compressibleLenU32 + 5] = 0xEEEEEEEE; |
1331 | |
1332 | /* Add a match of offset=16, length=8 at idx=64K + 64. |
1333 | * This generates a sequence with llen=64K, and repeat code 1. |
1334 | * The block splitter thought this was ll0, and corrupted the |
1335 | * repeat offset history. |
1336 | */ |
1337 | data[compressibleLenU32 + litLenU32 + 2 + 0] = 0xDDDDDDDD; |
1338 | data[compressibleLenU32 + litLenU32 + 2 + 1] = 0xCCCCCCCC; |
1339 | data[compressibleLenU32 + litLenU32 + 2 + 4] = 0xDDDDDDDD; |
1340 | data[compressibleLenU32 + litLenU32 + 2 + 5] = 0xCCCCCCCC; |
1341 | |
1342 | /* Add a match of offset=16, length=8 at idx=128K + 16. |
1343 | * This should generate a sequence with repeat code = 1. |
1344 | * But the block splitters mistake caused zstd to generate |
1345 | * repeat code = 2, corrupting the data. |
1346 | */ |
1347 | data[blockSizeU32] = 0xBBBBBBBB; |
1348 | data[blockSizeU32 + 1] = 0xAAAAAAAA; |
1349 | data[blockSizeU32 + 4] = 0xBBBBBBBB; |
1350 | data[blockSizeU32 + 5] = 0xAAAAAAAA; |
1351 | |
1352 | /* Generate a golden file from this data in case datagen changes and |
1353 | * doesn't generate the exact same data. We will also test this golden file. |
1354 | */ |
1355 | if (0) { |
1356 | FILE* f = fopen("golden-compression/PR-3517-block-splitter-corruption-test", "wb"); |
1357 | fwrite(data, 1, srcSize, f); |
1358 | fclose(f); |
1359 | } |
1360 | |
1361 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); |
1362 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7)); |
1363 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable)); |
1364 | |
1365 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, data, srcSize); |
1366 | CHECK_Z(cSize); |
1367 | dSize = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
1368 | CHECK_Z(dSize); |
1369 | CHECK_EQ(dSize, srcSize); |
1370 | CHECK(!memcmp(decodedBuffer, data, srcSize)); |
1371 | |
1372 | free(data); |
1373 | ZSTD_freeCCtx(cctx); |
1374 | } |
1375 | DISPLAYLEVEL(3, "OK \n"); |
1376 | |
f535537f |
1377 | DISPLAYLEVEL(3, "test%3d : superblock uncompressible data: too many nocompress superblocks : ", testNb++); |
648db22b |
1378 | { |
1379 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1380 | const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer; |
1381 | size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize); |
1382 | |
1383 | /* This is the number of bytes to stream before ending. This value |
1384 | * was obtained by trial and error :/. */ |
1385 | |
1386 | const size_t streamCompressThreshold = 161792; |
1387 | const size_t streamCompressDelta = 1024; |
1388 | |
1389 | /* The first 1/5 of the buffer is compressible and the last 4/5 is |
1390 | * uncompressible. This is an approximation of the type of data |
1391 | * the fuzzer generated to catch this bug. Streams like this were making |
1392 | * zstd generate noCompress superblocks (which are larger than the src |
1393 | * they come from). Do this enough times, and we'll run out of room |
1394 | * and throw a dstSize_tooSmall error. */ |
1395 | |
1396 | const size_t compressiblePartSize = srcSize/5; |
1397 | const size_t uncompressiblePartSize = srcSize-compressiblePartSize; |
1398 | RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed); |
1399 | RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed); |
1400 | |
1401 | /* Setting target block size so that superblock is used */ |
1402 | |
1403 | assert(cctx != NULL); |
1404 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81); |
1405 | |
1406 | { size_t read; |
1407 | for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) { |
1408 | ZSTD_inBuffer in = {src, streamCompressDelta, 0}; |
1409 | ZSTD_outBuffer out = {dst, dstCapacity, 0}; |
1410 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); |
1411 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); |
1412 | src += streamCompressDelta; srcSize -= streamCompressDelta; |
1413 | dst += out.pos; dstCapacity -= out.pos; |
1414 | } } |
1415 | |
1416 | /* This is trying to catch a dstSize_tooSmall error */ |
1417 | |
1418 | { ZSTD_inBuffer in = {src, srcSize, 0}; |
1419 | ZSTD_outBuffer out = {dst, dstCapacity, 0}; |
1420 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); |
1421 | } |
1422 | ZSTD_freeCCtx(cctx); |
1423 | } |
1424 | DISPLAYLEVEL(3, "OK \n"); |
1425 | |
1426 | DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++); |
1427 | /* Generate the same data 20 times over */ |
1428 | { size_t const avgChunkSize = CNBuffSize / 20; |
1429 | size_t b; |
1430 | for (b = 0; b < CNBuffSize; b += avgChunkSize) { |
1431 | size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize); |
1432 | RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed); |
1433 | } } |
1434 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1435 | size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
1436 | size_t const allowedExpansion = (CNBuffSize * 3 / 1000); |
1437 | size_t superCSize; |
1438 | CHECK_Z(normalCSize); |
1439 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); |
1440 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000); |
1441 | superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
1442 | CHECK_Z(superCSize); |
1443 | if (superCSize > normalCSize + allowedExpansion) { |
1444 | DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion); |
1445 | goto _output_error; |
1446 | } |
1447 | ZSTD_freeCCtx(cctx); |
1448 | } |
1449 | DISPLAYLEVEL(3, "OK \n"); |
1450 | |
1451 | RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed); |
1452 | DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++) |
1453 | /* This tests whether or not we leave enough room for the checksum at the end |
1454 | * of the dst buffer. The bug that motivated this test was found by the |
1455 | * stream_round_trip fuzzer but this crashes for the same reason and is |
1456 | * far more compact than re-creating the stream_round_trip fuzzer's code path */ |
1457 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1458 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64); |
1459 | assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278))); |
1460 | ZSTD_freeCCtx(cctx); |
1461 | } |
1462 | DISPLAYLEVEL(3, "OK \n"); |
1463 | |
1464 | DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++); |
1465 | { int level = -1; |
1466 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1467 | if (!cctx) goto _output_error; |
1468 | for (level = -1; level <= ZSTD_maxCLevel(); ++level) { |
1469 | CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) ); |
1470 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) ); |
1471 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) ); |
1472 | } |
1473 | ZSTD_freeCCtx(cctx); |
1474 | } |
1475 | DISPLAYLEVEL(3, "OK \n"); |
1476 | |
1477 | DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++); |
1478 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1479 | size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19); |
1480 | if (ZSTD_isError(r)) goto _output_error; |
1481 | if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error; |
1482 | ZSTD_freeCCtx(cctx); |
1483 | cSize = r; |
1484 | } |
1485 | DISPLAYLEVEL(3, "OK \n"); |
1486 | |
1487 | DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++); |
1488 | { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize); |
1489 | if (ZSTD_isError(r)) goto _output_error; |
1490 | if (r != 0) goto _output_error; |
1491 | } |
1492 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1493 | ZSTD_outBuffer output; |
1494 | if (cctx==NULL) goto _output_error; |
1495 | output.dst = compressedBuffer; |
1496 | output.size = compressedBufferSize; |
1497 | output.pos = 0; |
1498 | CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */ |
1499 | CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */ |
1500 | CHECK_Z( ZSTD_endStream(cctx, &output) ); |
1501 | ZSTD_freeCCtx(cctx); |
1502 | /* single scan decompression */ |
1503 | { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos); |
1504 | if (ZSTD_isError(r)) goto _output_error; |
1505 | if (r != 0) goto _output_error; |
1506 | } |
1507 | /* streaming decompression */ |
1508 | { ZSTD_DCtx* const dstream = ZSTD_createDStream(); |
1509 | ZSTD_inBuffer dinput; |
1510 | ZSTD_outBuffer doutput; |
1511 | size_t ipos; |
1512 | if (dstream==NULL) goto _output_error; |
1513 | dinput.src = compressedBuffer; |
1514 | dinput.size = 0; |
1515 | dinput.pos = 0; |
1516 | doutput.dst = NULL; |
1517 | doutput.size = 0; |
1518 | doutput.pos = 0; |
1519 | CHECK_Z ( ZSTD_initDStream(dstream) ); |
1520 | for (ipos=1; ipos<=output.pos; ipos++) { |
1521 | dinput.size = ipos; |
1522 | CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) ); |
1523 | } |
1524 | if (doutput.pos != 0) goto _output_error; |
1525 | ZSTD_freeDStream(dstream); |
1526 | } |
1527 | } |
1528 | DISPLAYLEVEL(3, "OK \n"); |
1529 | |
f535537f |
1530 | DISPLAYLEVEL(3, "test%3d : reuse CCtx with expanding block size : ", testNb++); |
648db22b |
1531 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1532 | ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); |
1533 | assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */ |
1534 | CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) ); |
1535 | CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */ |
1536 | |
f535537f |
1537 | CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ |
648db22b |
1538 | { size_t const inSize = 2* 128 KB; |
1539 | size_t const outSize = ZSTD_compressBound(inSize); |
1540 | CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) ); |
1541 | /* will fail if blockSize is not resized */ |
1542 | } |
1543 | ZSTD_freeCCtx(cctx); |
1544 | } |
1545 | DISPLAYLEVEL(3, "OK \n"); |
1546 | |
1547 | DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); |
1548 | { size_t const sampleSize = 30; |
1549 | int i; |
1550 | for (i=0; i<20; i++) |
1551 | ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ |
1552 | memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ |
1553 | for (i=1; i<=19; i++) { |
1554 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1555 | size_t size1, size2; |
1556 | DISPLAYLEVEL(5, "l%i ", i); |
1557 | size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); |
1558 | CHECK_Z(size1); |
1559 | |
1560 | size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); |
1561 | CHECK_Z(size2); |
1562 | CHECK_EQ(size1, size2); |
1563 | |
1564 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) ); |
1565 | size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize); |
1566 | CHECK_Z(size2); |
1567 | CHECK_EQ(size1, size2); |
1568 | |
1569 | size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */ |
1570 | CHECK_Z(size2); |
1571 | CHECK_EQ(size1, size2); |
1572 | |
1573 | { ZSTD_inBuffer inb; |
1574 | ZSTD_outBuffer outb; |
1575 | inb.src = CNBuffer; |
1576 | inb.pos = 0; |
1577 | inb.size = sampleSize; |
1578 | outb.dst = compressedBuffer; |
1579 | outb.pos = 0; |
1580 | outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */ |
1581 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); |
1582 | assert(inb.pos == inb.size); |
1583 | CHECK_EQ(size1, outb.pos); |
1584 | } |
1585 | |
1586 | ZSTD_freeCCtx(cctx); |
1587 | } |
1588 | } |
1589 | DISPLAYLEVEL(3, "OK \n"); |
1590 | |
1591 | DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++); |
1592 | { size_t const sampleSize = 1024; |
1593 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1594 | ZSTD_inBuffer inb; |
1595 | ZSTD_outBuffer outb; |
1596 | inb.src = CNBuffer; |
1597 | inb.pos = 0; |
1598 | inb.size = 0; |
1599 | outb.dst = compressedBuffer; |
1600 | outb.pos = 0; |
1601 | outb.size = compressedBufferSize; |
1602 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) ); |
1603 | |
1604 | inb.size = sampleSize; /* start with something, so that context is already used */ |
1605 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ |
1606 | assert(inb.pos == inb.size); |
1607 | outb.pos = 0; /* cancel output */ |
1608 | |
1609 | CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) ); |
1610 | inb.size = 4; /* too small size : compression will be skipped */ |
1611 | inb.pos = 0; |
1612 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); |
1613 | assert(inb.pos == inb.size); |
1614 | |
1615 | inb.size += 5; /* too small size : compression will be skipped */ |
1616 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); |
1617 | assert(inb.pos == inb.size); |
1618 | |
1619 | inb.size += 11; /* small enough to attempt compression */ |
1620 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); |
1621 | assert(inb.pos == inb.size); |
1622 | |
1623 | assert(inb.pos < sampleSize); |
1624 | inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */ |
1625 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ |
1626 | assert(inb.pos == inb.size); |
1627 | ZSTD_freeCCtx(cctx); |
1628 | } |
1629 | DISPLAYLEVEL(3, "OK \n"); |
1630 | |
1631 | DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); |
1632 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1633 | ZSTD_outBuffer out = {NULL, 0, 0}; |
1634 | ZSTD_inBuffer in = {NULL, 0, 0}; |
1635 | int value; |
1636 | |
1637 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); |
1638 | CHECK_EQ(value, 3); |
1639 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1640 | CHECK_EQ(value, 0); |
1641 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN)); |
1642 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); |
1643 | CHECK_EQ(value, 3); |
1644 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1645 | CHECK_EQ(value, ZSTD_HASHLOG_MIN); |
1646 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7)); |
1647 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); |
1648 | CHECK_EQ(value, 7); |
1649 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1650 | CHECK_EQ(value, ZSTD_HASHLOG_MIN); |
1651 | /* Start a compression job */ |
1652 | ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); |
1653 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); |
1654 | CHECK_EQ(value, 7); |
1655 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1656 | CHECK_EQ(value, ZSTD_HASHLOG_MIN); |
1657 | /* Reset the CCtx */ |
1658 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); |
1659 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); |
1660 | CHECK_EQ(value, 7); |
1661 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1662 | CHECK_EQ(value, ZSTD_HASHLOG_MIN); |
1663 | /* Reset the parameters */ |
1664 | ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); |
1665 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); |
1666 | CHECK_EQ(value, 3); |
1667 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1668 | CHECK_EQ(value, 0); |
1669 | |
1670 | ZSTD_freeCCtx(cctx); |
1671 | } |
1672 | DISPLAYLEVEL(3, "OK \n"); |
1673 | |
1674 | DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCParams() : ", testNb++); |
1675 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1676 | int value; |
1677 | ZSTD_compressionParameters cparams = ZSTD_getCParams(1, 0, 0); |
1678 | cparams.strategy = -1; |
1679 | /* Set invalid cParams == no change. */ |
1680 | CHECK(ZSTD_isError(ZSTD_CCtx_setCParams(cctx, cparams))); |
1681 | |
1682 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); |
1683 | CHECK_EQ(value, 0); |
1684 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); |
1685 | CHECK_EQ(value, 0); |
1686 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1687 | CHECK_EQ(value, 0); |
1688 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); |
1689 | CHECK_EQ(value, 0); |
1690 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); |
1691 | CHECK_EQ(value, 0); |
1692 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); |
1693 | CHECK_EQ(value, 0); |
1694 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); |
1695 | CHECK_EQ(value, 0); |
1696 | |
1697 | cparams = ZSTD_getCParams(12, 0, 0); |
1698 | CHECK_Z(ZSTD_CCtx_setCParams(cctx, cparams)); |
1699 | |
1700 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); |
1701 | CHECK_EQ(value, (int)cparams.windowLog); |
1702 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); |
1703 | CHECK_EQ(value, (int)cparams.chainLog); |
1704 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1705 | CHECK_EQ(value, (int)cparams.hashLog); |
1706 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); |
1707 | CHECK_EQ(value, (int)cparams.searchLog); |
1708 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); |
1709 | CHECK_EQ(value, (int)cparams.minMatch); |
1710 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); |
1711 | CHECK_EQ(value, (int)cparams.targetLength); |
1712 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); |
1713 | CHECK_EQ(value, (int)cparams.strategy); |
1714 | |
1715 | ZSTD_freeCCtx(cctx); |
1716 | } |
1717 | |
1718 | DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setFParams() : ", testNb++); |
1719 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1720 | int value; |
1721 | ZSTD_frameParameters fparams = {0, 1, 1}; |
1722 | |
1723 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); |
1724 | CHECK_EQ(value, 1); |
1725 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); |
1726 | CHECK_EQ(value, 0); |
1727 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); |
1728 | CHECK_EQ(value, 1); |
1729 | |
1730 | CHECK_Z(ZSTD_CCtx_setFParams(cctx, fparams)); |
1731 | |
1732 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); |
1733 | CHECK_EQ(value, fparams.contentSizeFlag); |
1734 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); |
1735 | CHECK_EQ(value, fparams.checksumFlag); |
1736 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); |
1737 | CHECK_EQ(value, !fparams.noDictIDFlag); |
1738 | |
1739 | ZSTD_freeCCtx(cctx); |
1740 | } |
1741 | |
1742 | DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_setCarams() : ", testNb++); |
1743 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1744 | int value; |
1745 | ZSTD_parameters params = ZSTD_getParams(1, 0, 0); |
1746 | params.cParams.strategy = -1; |
1747 | /* Set invalid params == no change. */ |
1748 | CHECK(ZSTD_isError(ZSTD_CCtx_setParams(cctx, params))); |
1749 | |
1750 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); |
1751 | CHECK_EQ(value, 0); |
1752 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); |
1753 | CHECK_EQ(value, 0); |
1754 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1755 | CHECK_EQ(value, 0); |
1756 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); |
1757 | CHECK_EQ(value, 0); |
1758 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); |
1759 | CHECK_EQ(value, 0); |
1760 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); |
1761 | CHECK_EQ(value, 0); |
1762 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); |
1763 | CHECK_EQ(value, 0); |
1764 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); |
1765 | CHECK_EQ(value, 1); |
1766 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); |
1767 | CHECK_EQ(value, 0); |
1768 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); |
1769 | CHECK_EQ(value, 1); |
1770 | |
1771 | params = ZSTD_getParams(12, 0, 0); |
1772 | params.fParams.contentSizeFlag = 0; |
1773 | params.fParams.checksumFlag = 1; |
1774 | params.fParams.noDictIDFlag = 1; |
1775 | CHECK_Z(ZSTD_CCtx_setParams(cctx, params)); |
1776 | |
1777 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_windowLog, &value)); |
1778 | CHECK_EQ(value, (int)params.cParams.windowLog); |
1779 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_chainLog, &value)); |
1780 | CHECK_EQ(value, (int)params.cParams.chainLog); |
1781 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); |
1782 | CHECK_EQ(value, (int)params.cParams.hashLog); |
1783 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_searchLog, &value)); |
1784 | CHECK_EQ(value, (int)params.cParams.searchLog); |
1785 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_minMatch, &value)); |
1786 | CHECK_EQ(value, (int)params.cParams.minMatch); |
1787 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetLength, &value)); |
1788 | CHECK_EQ(value, (int)params.cParams.targetLength); |
1789 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_strategy, &value)); |
1790 | CHECK_EQ(value, (int)params.cParams.strategy); |
1791 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_contentSizeFlag, &value)); |
1792 | CHECK_EQ(value, params.fParams.contentSizeFlag); |
1793 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_checksumFlag, &value)); |
1794 | CHECK_EQ(value, params.fParams.checksumFlag); |
1795 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_dictIDFlag, &value)); |
1796 | CHECK_EQ(value, !params.fParams.noDictIDFlag); |
1797 | |
1798 | ZSTD_freeCCtx(cctx); |
1799 | } |
1800 | |
1801 | DISPLAYLEVEL(3, "test%3d : ldm conditionally enabled by default doesn't change cctx params: ", testNb++); |
1802 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1803 | ZSTD_outBuffer out = {NULL, 0, 0}; |
1804 | ZSTD_inBuffer in = {NULL, 0, 0}; |
1805 | int value; |
1806 | |
1807 | /* Even if LDM will be enabled by default in the applied params (since wlog >= 27 and strategy >= btopt), |
1808 | * we should not modify the actual parameter specified by the user within the CCtx |
1809 | */ |
1810 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 27)); |
1811 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt)); |
1812 | |
1813 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); |
1814 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_enableLongDistanceMatching, &value)); |
1815 | CHECK_EQ(value, 0); |
1816 | |
1817 | ZSTD_freeCCtx(cctx); |
1818 | } |
1819 | DISPLAYLEVEL(3, "OK \n"); |
1820 | |
1821 | /* this test is really too long, and should be made faster */ |
1822 | DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); |
1823 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1824 | ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0); |
1825 | size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */ |
1826 | size_t cnb; |
1827 | assert(cctx != NULL); |
1828 | params.fParams.contentSizeFlag = 0; |
1829 | params.cParams.windowLog = ZSTD_WINDOWLOG_MAX; |
1830 | for (cnb = 0; cnb < nbCompressions; ++cnb) { |
1831 | DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions); |
f535537f |
1832 | CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* reuse same parameters */ |
648db22b |
1833 | CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) ); |
1834 | } |
1835 | ZSTD_freeCCtx(cctx); |
1836 | } |
1837 | DISPLAYLEVEL(3, "OK \n"); |
1838 | |
1839 | DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++); |
1840 | { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx(); |
1841 | assert(largeCCtx != NULL); |
1842 | CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */ |
1843 | CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); |
1844 | { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */ |
1845 | { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx(); |
1846 | assert(smallCCtx != NULL); |
1847 | CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); |
1848 | { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx); |
1849 | DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ", |
1850 | largeCCtxSize>>10, smallCCtxSize>>10); |
1851 | assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c . |
1852 | * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */ |
1853 | } |
1854 | ZSTD_freeCCtx(smallCCtx); |
1855 | } |
1856 | { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c. |
1857 | * currently defined as 128x, but could be adjusted in the future. |
1858 | * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */ |
1859 | unsigned u; |
1860 | for (u=0; u<maxNbAttempts; u++) { |
1861 | CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); |
1862 | if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */ |
1863 | } |
1864 | DISPLAYLEVEL(5, "size down after %u attempts : ", u); |
1865 | if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */ |
1866 | } |
1867 | } |
1868 | ZSTD_freeCCtx(largeCCtx); |
1869 | } |
1870 | DISPLAYLEVEL(3, "OK \n"); |
1871 | |
1872 | /* Static CCtx tests */ |
1873 | #define STATIC_CCTX_LEVEL 4 |
1874 | DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); |
1875 | { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL); |
1876 | void* const staticCCtxBuffer = malloc(staticCStreamSize); |
1877 | size_t const staticDCtxSize = ZSTD_estimateDCtxSize(); |
1878 | void* const staticDCtxBuffer = malloc(staticDCtxSize); |
1879 | DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize); |
1880 | if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) { |
1881 | free(staticCCtxBuffer); |
1882 | free(staticDCtxBuffer); |
1883 | DISPLAY("Not enough memory, aborting\n"); |
1884 | testResult = 1; |
1885 | goto _end; |
1886 | } |
1887 | { size_t const smallInSize = 32 KB; |
1888 | ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0); |
1889 | size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small); |
1890 | size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL); |
1891 | ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize); |
1892 | ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize); |
1893 | DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize); |
1894 | DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize); |
1895 | if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error; |
1896 | DISPLAYLEVEL(3, "OK \n"); |
1897 | |
1898 | DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++); |
1899 | CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, |
1900 | compressedBuffer, compressedBufferSize, |
1901 | CNBuffer, smallInSize, STATIC_CCTX_LEVEL) ); |
1902 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", |
1903 | (unsigned)cSize, (double)cSize/smallInSize*100); |
1904 | |
1905 | DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++); |
1906 | { size_t const r = ZSTD_compressCCtx(staticCCtx, |
1907 | compressedBuffer, compressedBufferSize, |
1908 | CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL); |
1909 | if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error; |
1910 | } |
1911 | DISPLAYLEVEL(3, "OK \n"); |
1912 | |
1913 | DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++); |
1914 | staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize); |
1915 | DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, (void*)staticCCtx); |
1916 | if (staticCCtx == NULL) goto _output_error; |
1917 | DISPLAYLEVEL(3, "OK \n"); |
1918 | |
1919 | DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++); |
1920 | CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, |
1921 | compressedBuffer, compressedBufferSize, |
1922 | CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); |
1923 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", |
1924 | (unsigned)cSize, (double)cSize/CNBuffSize*100); |
1925 | |
1926 | DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++); |
1927 | { int nbc; |
1928 | assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */ |
1929 | assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3); |
1930 | for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) { |
1931 | CHECK_Z(ZSTD_compressCCtx(staticCCtx, |
1932 | compressedBuffer, compressedBufferSize, |
1933 | (char*)CNBuffer + nbc, smallInSize, |
1934 | STATIC_CCTX_LEVEL) ); |
1935 | } } |
1936 | DISPLAYLEVEL(3, "OK \n") |
1937 | |
1938 | DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); |
1939 | CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) ); |
1940 | DISPLAYLEVEL(3, "OK \n"); |
1941 | |
1942 | DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++); |
1943 | CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, |
1944 | compressedBuffer, compressedBufferSize, |
1945 | CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); |
1946 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", |
1947 | (unsigned)cSize, (double)cSize/CNBuffSize*100); |
1948 | |
1949 | DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++); |
1950 | { size_t const r = ZSTD_decompressDCtx(staticDCtx, |
1951 | decodedBuffer, CNBuffSize, |
1952 | compressedBuffer, cSize); |
1953 | if (r != CNBuffSize) goto _output_error; } |
1954 | DISPLAYLEVEL(3, "OK \n"); |
1955 | |
1956 | DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); |
1957 | if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; |
1958 | DISPLAYLEVEL(3, "OK \n"); |
1959 | |
1960 | DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++); |
1961 | { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel()); |
1962 | if (!ZSTD_isError(r)) goto _output_error; } |
1963 | DISPLAYLEVEL(3, "OK \n"); |
1964 | |
1965 | DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1); |
1966 | CHECK_Z( ZSTD_compressBegin(staticCCtx, 1) ); |
1967 | DISPLAYLEVEL(3, "OK \n"); |
1968 | |
1969 | DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++); |
1970 | CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ |
1971 | { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; |
1972 | ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; |
1973 | size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */ |
1974 | if (!ZSTD_isError(r)) goto _output_error; |
1975 | } |
1976 | DISPLAYLEVEL(3, "OK \n"); |
1977 | |
1978 | DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++); |
1979 | staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize); |
1980 | assert(staticCCtx != NULL); |
1981 | CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ |
1982 | { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; |
1983 | ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; |
1984 | CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); |
1985 | } |
1986 | DISPLAYLEVEL(3, "OK \n"); |
1987 | |
1988 | DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1); |
1989 | CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */ |
1990 | { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; |
1991 | ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; |
1992 | CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); |
1993 | } |
1994 | DISPLAYLEVEL(3, "OK \n"); |
1995 | |
1996 | DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++); |
1997 | { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1); |
1998 | if (!ZSTD_isError(r)) goto _output_error; } |
1999 | DISPLAYLEVEL(3, "OK \n"); |
2000 | |
2001 | DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++); |
2002 | CHECK_Z( ZSTD_initDStream(staticDCtx) ); |
2003 | { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 }; |
2004 | ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 }; |
2005 | size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input); |
2006 | if (!ZSTD_isError(r)) goto _output_error; |
2007 | } |
2008 | DISPLAYLEVEL(3, "OK \n"); |
2009 | |
2010 | DISPLAYLEVEL(3, "test%3i : test estimation functions with default cctx params : ", testNb++); |
2011 | { |
2012 | // Test ZSTD_estimateCCtxSize_usingCCtxParams |
2013 | { |
2014 | ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); |
2015 | size_t const cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); |
2016 | staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); |
2017 | CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, |
2018 | compressedBuffer, compressedBufferSize, |
2019 | CNBuffer, CNBuffSize, 3)); |
2020 | |
2021 | { |
2022 | size_t const r = ZSTD_decompressDCtx(staticDCtx, |
2023 | decodedBuffer, CNBuffSize, |
2024 | compressedBuffer, cSize); |
2025 | if (r != CNBuffSize) goto _output_error; |
2026 | if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; |
2027 | } |
2028 | ZSTD_freeCCtxParams(params); |
2029 | } |
2030 | |
2031 | // Test ZSTD_estimateCStreamSize_usingCCtxParams |
2032 | { |
2033 | ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); |
2034 | size_t const cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); |
2035 | staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); |
2036 | CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, |
2037 | compressedBuffer, compressedBufferSize, |
2038 | CNBuffer, CNBuffSize, 3) ); |
2039 | |
2040 | { |
2041 | size_t const r = ZSTD_decompressDCtx(staticDCtx, |
2042 | decodedBuffer, CNBuffSize, |
2043 | compressedBuffer, cSize); |
2044 | if (r != CNBuffSize) goto _output_error; |
2045 | if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; |
2046 | } |
2047 | ZSTD_freeCCtxParams(params); |
2048 | } |
2049 | } |
2050 | DISPLAYLEVEL(3, "OK \n"); |
2051 | |
2052 | DISPLAYLEVEL(3, "test%3i : test estimation functions with maxBlockSize = 0 : ", testNb++); |
2053 | { |
2054 | // Test ZSTD_estimateCCtxSize_usingCCtxParams |
2055 | { |
2056 | ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); |
2057 | size_t cctxSizeDefault; |
2058 | CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); |
2059 | cctxSizeDefault = ZSTD_estimateCCtxSize_usingCCtxParams(params); |
2060 | staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); |
2061 | CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, |
2062 | compressedBuffer, compressedBufferSize, |
2063 | CNBuffer, CNBuffSize, 3) ); |
2064 | |
2065 | { |
2066 | size_t const r = ZSTD_decompressDCtx(staticDCtx, |
2067 | decodedBuffer, CNBuffSize, |
2068 | compressedBuffer, cSize); |
2069 | if (r != CNBuffSize) goto _output_error; |
2070 | if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; |
2071 | } |
2072 | ZSTD_freeCCtxParams(params); |
2073 | } |
2074 | |
2075 | // Test ZSTD_estimateCStreamSize_usingCCtxParams |
2076 | { |
2077 | ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); |
2078 | size_t cctxSizeDefault; |
2079 | CHECK_Z(ZSTD_CCtxParams_setParameter(params, ZSTD_c_maxBlockSize, 0)); |
2080 | cctxSizeDefault = ZSTD_estimateCStreamSize_usingCCtxParams(params); |
2081 | staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, cctxSizeDefault); |
2082 | CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, |
2083 | compressedBuffer, compressedBufferSize, |
2084 | CNBuffer, CNBuffSize, 3) ); |
2085 | |
2086 | { |
2087 | size_t const r = ZSTD_decompressDCtx(staticDCtx, |
2088 | decodedBuffer, CNBuffSize, |
2089 | compressedBuffer, cSize); |
2090 | if (r != CNBuffSize) goto _output_error; |
2091 | if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; |
2092 | } |
2093 | ZSTD_freeCCtxParams(params); |
2094 | } |
2095 | } |
2096 | DISPLAYLEVEL(3, "OK \n"); |
2097 | } |
2098 | free(staticCCtxBuffer); |
2099 | free(staticDCtxBuffer); |
2100 | } |
2101 | |
2102 | DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++); |
2103 | { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1); |
2104 | size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1); |
2105 | size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1); |
2106 | size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1); |
2107 | |
2108 | if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error; |
2109 | if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error; |
2110 | } |
2111 | DISPLAYLEVEL(3, "OK \n"); |
2112 | |
2113 | |
2114 | /* ZSTDMT simple MT compression test */ |
2115 | DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++); |
2116 | { ZSTD_CCtx* const mtctx = ZSTD_createCCtx(); |
2117 | if (mtctx==NULL) { |
2118 | DISPLAY("mtctx : not enough memory, aborting \n"); |
2119 | testResult = 1; |
2120 | goto _end; |
2121 | } |
2122 | CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_nbWorkers, 2) ); |
2123 | CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_compressionLevel, 1) ); |
2124 | DISPLAYLEVEL(3, "OK \n"); |
2125 | |
2126 | DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize); |
2127 | CHECK_VAR(cSize, ZSTD_compress2(mtctx, |
2128 | compressedBuffer, compressedBufferSize, |
2129 | CNBuffer, CNBuffSize) ); |
2130 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
2131 | |
2132 | DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++); |
2133 | { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); |
2134 | if (rSize != CNBuffSize) { |
2135 | DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize); |
2136 | goto _output_error; |
2137 | } } |
2138 | DISPLAYLEVEL(3, "OK \n"); |
2139 | |
2140 | DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); |
2141 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
2142 | if (r != CNBuffSize) goto _output_error; } |
2143 | DISPLAYLEVEL(3, "OK \n"); |
2144 | |
2145 | DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); |
2146 | { size_t u; |
2147 | for (u=0; u<CNBuffSize; u++) { |
2148 | if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; |
2149 | } } |
2150 | DISPLAYLEVEL(3, "OK \n"); |
2151 | |
2152 | DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++); |
2153 | CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_checksumFlag, 1) ); |
2154 | CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_contentSizeFlag, 1) ); |
2155 | CHECK_Z( ZSTD_CCtx_setParameter(mtctx, ZSTD_c_overlapLog, 3) ); |
2156 | CHECK_VAR(cSize, ZSTD_compress2(mtctx, |
2157 | compressedBuffer, compressedBufferSize, |
2158 | CNBuffer, CNBuffSize) ); |
2159 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
2160 | |
2161 | DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); |
2162 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
2163 | if (r != CNBuffSize) goto _output_error; } |
2164 | DISPLAYLEVEL(3, "OK \n"); |
2165 | |
2166 | ZSTD_freeCCtx(mtctx); |
2167 | } |
2168 | |
2169 | DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++); |
2170 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
2171 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
2172 | char out[32]; |
2173 | if (cctx == NULL || dctx == NULL) goto _output_error; |
2174 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); |
2175 | CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) ); |
2176 | DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize); |
2177 | |
2178 | CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) ); |
2179 | { char const* outPtr = out; |
2180 | ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 }; |
2181 | ZSTD_outBuffer outBuffer = { NULL, 0, 0 }; |
2182 | size_t dSize; |
2183 | CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) ); |
2184 | if (dSize != 0) goto _output_error; |
2185 | } |
2186 | |
2187 | ZSTD_freeDCtx(dctx); |
2188 | ZSTD_freeCCtx(cctx); |
2189 | } |
2190 | |
2191 | DISPLAYLEVEL(3, "test%3i : compress with block splitting : ", testNb++) |
2192 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
2193 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_useBlockSplitter, ZSTD_ps_enable) ); |
2194 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
2195 | CHECK_Z(cSize); |
2196 | ZSTD_freeCCtx(cctx); |
2197 | } |
2198 | DISPLAYLEVEL(3, "OK \n"); |
2199 | |
2200 | DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++) |
2201 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
2202 | size_t cSize1, cSize2; |
2203 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); |
2204 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) ); |
2205 | cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
2206 | CHECK_Z(cSize1); |
2207 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_ps_disable) ); |
2208 | cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
2209 | CHECK_Z(cSize2); |
2210 | CHECK_LT(cSize1, cSize2); |
2211 | ZSTD_freeCCtx(cctx); |
2212 | } |
2213 | DISPLAYLEVEL(3, "OK \n"); |
2214 | |
2215 | DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++) |
2216 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
2217 | /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so |
2218 | * ZSTDMT is forced to not take the shortcut. |
2219 | */ |
2220 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); |
2221 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) ); |
2222 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) ); |
2223 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) ); |
2224 | ZSTD_freeCCtx(cctx); |
2225 | } |
2226 | DISPLAYLEVEL(3, "OK \n"); |
2227 | |
2228 | DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++) |
2229 | { ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); |
2230 | int const jobSize = 512 KB; |
2231 | int value; |
2232 | /* Check that the overlap log and job size are unset. */ |
2233 | CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); |
2234 | CHECK_EQ(value, 0); |
2235 | CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); |
2236 | CHECK_EQ(value, 0); |
2237 | /* Set and check the overlap log and job size. */ |
2238 | CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) ); |
2239 | CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, jobSize) ); |
2240 | CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); |
2241 | CHECK_EQ(value, 5); |
2242 | CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); |
2243 | CHECK_EQ(value, jobSize); |
2244 | /* Set the number of workers and check the overlap log and job size. */ |
2245 | CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) ); |
2246 | CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); |
2247 | CHECK_EQ(value, 5); |
2248 | CHECK_Z( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); |
2249 | CHECK_EQ(value, jobSize); |
2250 | ZSTD_freeCCtxParams(params); |
2251 | } |
2252 | DISPLAYLEVEL(3, "OK \n"); |
2253 | |
2254 | /* Simple API multiframe test */ |
2255 | DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++); |
2256 | { size_t off = 0; |
2257 | int i; |
2258 | int const segs = 4; |
2259 | /* only use the first half so we don't push against size limit of compressedBuffer */ |
2260 | size_t const segSize = (CNBuffSize / 2) / segs; |
2261 | |
2262 | const U32 skipLen = 129 KB; |
2263 | char* const skipBuff = (char*)malloc(skipLen); |
2264 | assert(skipBuff != NULL); |
2265 | memset(skipBuff, 0, skipLen); |
2266 | for (i = 0; i < segs; i++) { |
2267 | CHECK_NEWV(r, ZSTD_compress( |
2268 | (BYTE*)compressedBuffer + off, CNBuffSize - off, |
2269 | (BYTE*)CNBuffer + segSize * (size_t)i, segSize, |
2270 | 5) ); |
2271 | off += r; |
2272 | if (i == segs/2) { |
2273 | /* insert skippable frame */ |
2274 | size_t const skippableSize = |
2275 | ZSTD_writeSkippableFrame((BYTE*)compressedBuffer + off, compressedBufferSize, |
2276 | skipBuff, skipLen, seed % 15); |
2277 | CHECK_Z(skippableSize); |
2278 | off += skippableSize; |
2279 | } |
2280 | } |
2281 | cSize = off; |
2282 | free(skipBuff); |
2283 | } |
2284 | DISPLAYLEVEL(3, "OK \n"); |
2285 | |
2286 | DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++); |
2287 | { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize); |
2288 | if (r != CNBuffSize / 2) goto _output_error; } |
2289 | DISPLAYLEVEL(3, "OK \n"); |
2290 | |
2291 | DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++); |
2292 | { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize); |
2293 | if (bound != CNBuffSize / 2) goto _output_error; } |
2294 | DISPLAYLEVEL(3, "OK \n"); |
2295 | |
2296 | DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++); |
2297 | { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize)); |
2298 | if (r != CNBuffSize / 2) goto _output_error; } |
2299 | DISPLAYLEVEL(3, "OK \n"); |
2300 | |
2301 | DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); |
2302 | if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error; |
2303 | DISPLAYLEVEL(3, "OK \n"); |
2304 | |
2305 | /* Simple API skippable frame test */ |
2306 | DISPLAYLEVEL(3, "test%3i : read/write a skippable frame : ", testNb++); |
2307 | { U32 i; |
2308 | unsigned readMagic; |
2309 | unsigned long long receivedSize; |
2310 | size_t skippableSize; |
2311 | const U32 skipLen = 129 KB; |
2312 | char* const skipBuff = (char*)malloc(skipLen); |
2313 | assert(skipBuff != NULL); |
2314 | for (i = 0; i < skipLen; i++) |
2315 | skipBuff[i] = (char) ((seed + i) % 256); |
2316 | skippableSize = ZSTD_writeSkippableFrame( |
2317 | compressedBuffer, compressedBufferSize, |
2318 | skipBuff, skipLen, seed % 15); |
2319 | CHECK_Z(skippableSize); |
2320 | CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); |
2321 | receivedSize = ZSTD_readSkippableFrame(decodedBuffer, CNBuffSize, &readMagic, compressedBuffer, skippableSize); |
2322 | CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); |
2323 | CHECK_EQ(seed % 15, readMagic); |
2324 | if (memcmp(decodedBuffer, skipBuff, skipLen) != 0) goto _output_error; |
2325 | |
2326 | free(skipBuff); |
2327 | } |
2328 | DISPLAYLEVEL(3, "OK \n"); |
2329 | |
2330 | DISPLAYLEVEL(3, "test%3i : read/write an empty skippable frame : ", testNb++); |
2331 | { |
2332 | unsigned readMagic; |
2333 | unsigned long long receivedSize; |
2334 | size_t skippableSize; |
2335 | skippableSize = ZSTD_writeSkippableFrame( |
2336 | compressedBuffer, compressedBufferSize, |
2337 | CNBuffer, 0, seed % 15); |
2338 | CHECK_EQ(ZSTD_SKIPPABLEHEADERSIZE, skippableSize); |
2339 | CHECK_EQ(1, ZSTD_isSkippableFrame(compressedBuffer, skippableSize)); |
2340 | receivedSize = ZSTD_readSkippableFrame(NULL, 0, &readMagic, compressedBuffer, skippableSize); |
2341 | CHECK_EQ(skippableSize, receivedSize + ZSTD_SKIPPABLEHEADERSIZE); |
2342 | CHECK_EQ(seed % 15, readMagic); |
2343 | } |
2344 | DISPLAYLEVEL(3, "OK \n"); |
2345 | |
2346 | /* Dictionary and CCtx Duplication tests */ |
2347 | { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); |
2348 | ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); |
2349 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
2350 | static const size_t dictSize = 551; |
2351 | assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL); |
2352 | |
2353 | DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++); |
2354 | { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0); |
2355 | if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */ |
2356 | DISPLAYLEVEL(3, "OK \n"); |
2357 | |
2358 | DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++); |
2359 | CHECK_Z( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) ); |
2360 | CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */ |
2361 | DISPLAYLEVEL(3, "OK \n"); |
2362 | |
2363 | DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++); |
2364 | cSize = 0; |
2365 | CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, |
2366 | compressedBuffer, compressedBufferSize, |
2367 | (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), |
2368 | cSize += r); |
2369 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
2370 | |
2371 | DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); |
2372 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, |
2373 | decodedBuffer, CNBuffSize, |
2374 | compressedBuffer, cSize, |
2375 | CNBuffer, dictSize), |
2376 | if (r != CNBuffSize - dictSize) goto _output_error); |
2377 | DISPLAYLEVEL(3, "OK \n"); |
2378 | |
2379 | DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++); |
2380 | { size_t const cSizeOrig = cSize; |
2381 | cSize = 0; |
2382 | CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, |
2383 | compressedBuffer, compressedBufferSize, |
2384 | (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), |
2385 | cSize += r); |
2386 | if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */ |
2387 | } |
2388 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
2389 | |
2390 | DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); |
2391 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, |
2392 | decodedBuffer, CNBuffSize, |
2393 | compressedBuffer, cSize, |
2394 | CNBuffer, dictSize), |
2395 | if (r != CNBuffSize - dictSize) goto _output_error); |
2396 | DISPLAYLEVEL(3, "OK \n"); |
2397 | |
2398 | DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++); |
2399 | { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize); |
2400 | size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); |
2401 | if (r != CNBuffSize - dictSize) goto _output_error; |
2402 | DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict)); |
2403 | ZSTD_freeDDict(ddict); |
2404 | } |
2405 | |
2406 | DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++); |
2407 | { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); |
2408 | void* const ddictBuffer = malloc(ddictBufferSize); |
2409 | if (ddictBuffer == NULL) goto _output_error; |
2410 | { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); |
2411 | size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); |
2412 | if (r != CNBuffSize - dictSize) goto _output_error; |
2413 | } |
2414 | free(ddictBuffer); |
2415 | DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize); |
2416 | } |
2417 | |
2418 | DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); |
2419 | { size_t const testSize = CNBuffSize / 3; |
2420 | CHECK_Z( ZSTD_compressBegin(ctxOrig, ZSTD_defaultCLevel()) ); |
2421 | CHECK_Z( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); |
2422 | |
2423 | CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), |
2424 | (const char*)CNBuffer + dictSize, testSize) ); |
2425 | { ZSTD_frameHeader zfh; |
2426 | if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error; |
2427 | if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error; |
2428 | } } |
2429 | DISPLAYLEVEL(3, "OK \n"); |
2430 | |
f535537f |
2431 | #if !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ |
2432 | && !defined(ZSTD_EXCLUDE_GREEDY_BLOCK_COMPRESSOR) \ |
2433 | && !defined(ZSTD_EXCLUDE_DFAST_BLOCK_COMPRESSOR) \ |
2434 | && !defined(ZSTD_EXCLUDE_LAZY_BLOCK_COMPRESSOR) \ |
2435 | && !defined(ZSTD_EXCLUDE_LAZY2_BLOCK_COMPRESSOR) \ |
2436 | && !defined(ZSTD_EXCLUDE_BTLAZY2_BLOCK_COMPRESSOR) \ |
2437 | && !defined(ZSTD_EXCLUDE_BTOPT_BLOCK_COMPRESSOR) \ |
2438 | && !defined(ZSTD_EXCLUDE_BTULTRA_BLOCK_COMPRESSOR) |
648db22b |
2439 | /* Note : these tests should be replaced by proper regression tests, |
2440 | * but existing ones do not focus on small data + dictionary + all levels. |
2441 | */ |
2442 | if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */ |
2443 | size_t const flatdictSize = 22 KB; |
2444 | size_t const contentSize = 9 KB; |
2445 | const void* const dict = (const char*)CNBuffer; |
2446 | const void* const contentStart = (const char*)dict + flatdictSize; |
2447 | /* These upper bounds are generally within a few bytes of the compressed size */ |
2448 | size_t target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770, |
2449 | 3770, 3770, 3770, 3750, 3750, |
2450 | 3742, 3675, 3674, 3665, 3664, |
2451 | 3663, 3662, 3661, 3660, 3660, |
2452 | 3660, 3660, 3660 }; |
2453 | size_t const target_wdict_cSize[22+1] = { 2830, 2896, 2893, 2820, 2940, |
2454 | 2950, 2950, 2925, 2900, 2892, |
2455 | 2910, 2910, 2910, 2780, 2775, |
2456 | 2765, 2760, 2755, 2754, 2753, |
2457 | 2753, 2753, 2753 }; |
2458 | int l = 1; |
2459 | int const maxLevel = ZSTD_maxCLevel(); |
2460 | /* clevels with strategies that support rowhash on small inputs */ |
2461 | int rowLevel = 4; |
2462 | int const rowLevelEnd = 8; |
2463 | |
2464 | DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++); |
2465 | assert(maxLevel == 22); |
2466 | RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed); |
2467 | DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n", |
2468 | (unsigned long long)XXH64(contentStart, contentSize, 0), |
2469 | (unsigned long long)XXH64(dict, flatdictSize, 0)); |
2470 | |
2471 | for ( ; l <= maxLevel; l++) { |
2472 | size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, |
2473 | contentStart, contentSize, l); |
2474 | if (nodict_cSize > target_nodict_cSize[l]) { |
2475 | DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n", |
2476 | l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]); |
2477 | goto _output_error; |
2478 | } |
2479 | DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n", |
2480 | l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize); |
2481 | } |
2482 | for ( l=1 ; l <= maxLevel; l++) { |
2483 | size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig, |
2484 | compressedBuffer, compressedBufferSize, |
2485 | contentStart, contentSize, |
2486 | dict, flatdictSize, |
2487 | l); |
2488 | if (wdict_cSize > target_wdict_cSize[l]) { |
2489 | DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n", |
2490 | l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]); |
2491 | goto _output_error; |
2492 | } |
2493 | DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n", |
2494 | l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize); |
2495 | } |
2496 | /* Compression with ZSTD_compress2 and row match finder force enabled. |
2497 | * Give some slack for force-enabled row matchfinder since we're on a small input (9KB) |
2498 | */ |
2499 | for ( ; rowLevel <= rowLevelEnd; ++rowLevel) target_nodict_cSize[rowLevel] += 5; |
2500 | for (l=1 ; l <= maxLevel; l++) { |
2501 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
2502 | size_t nodict_cSize; |
2503 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, l); |
2504 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable); |
2505 | nodict_cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, |
2506 | contentStart, contentSize); |
2507 | if (nodict_cSize > target_nodict_cSize[l]) { |
2508 | DISPLAYLEVEL(1, "error : compression with compress2 at level %i worse than expected (%u > %u) \n", |
2509 | l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]); |
2510 | ZSTD_freeCCtx(cctx); |
2511 | goto _output_error; |
2512 | } |
2513 | DISPLAYLEVEL(4, "level %i with compress2 : max expected %u >= reached %u \n", |
2514 | l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize); |
2515 | ZSTD_freeCCtx(cctx); |
2516 | } |
2517 | /* Dict compression with DMS */ |
2518 | for ( l=1 ; l <= maxLevel; l++) { |
2519 | size_t wdict_cSize; |
2520 | CHECK_Z( ZSTD_CCtx_loadDictionary(ctxOrig, dict, flatdictSize) ); |
2521 | CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_compressionLevel, l) ); |
2522 | CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_enableDedicatedDictSearch, 0) ); |
2523 | CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach) ); |
2524 | CHECK_Z( ZSTD_CCtx_setParameter(ctxOrig, ZSTD_c_prefetchCDictTables, seed % 3) ); |
2525 | wdict_cSize = ZSTD_compress2(ctxOrig, compressedBuffer, compressedBufferSize, contentStart, contentSize); |
2526 | if (wdict_cSize > target_wdict_cSize[l]) { |
2527 | DISPLAYLEVEL(1, "error : compression with dictionary and compress2 at level %i worse than expected (%u > %u) \n", |
2528 | l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]); |
2529 | goto _output_error; |
2530 | } |
2531 | DISPLAYLEVEL(4, "level %i with dictionary and compress2 : max expected %u >= reached %u \n", |
2532 | l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize); |
2533 | } |
2534 | |
2535 | DISPLAYLEVEL(4, "compression efficiency tests OK \n"); |
2536 | } |
f535537f |
2537 | #endif |
648db22b |
2538 | |
2539 | ZSTD_freeCCtx(ctxOrig); |
2540 | ZSTD_freeCCtx(ctxDuplicated); |
2541 | ZSTD_freeDCtx(dctx); |
2542 | } |
2543 | |
2544 | /* Dictionary and dictBuilder tests */ |
2545 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
2546 | size_t const dictBufferCapacity = 16 KB; |
2547 | void* const dictBuffer = malloc(dictBufferCapacity); |
2548 | size_t const totalSampleSize = 1 MB; |
2549 | size_t const sampleUnitSize = 8 KB; |
2550 | U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); |
2551 | size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); |
2552 | size_t dictSize; |
2553 | U32 dictID; |
2554 | size_t dictHeaderSize; |
2555 | size_t dictBufferFixedSize = 144; |
2556 | unsigned char const dictBufferFixed[144] = {0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f, |
2557 | 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, |
2558 | 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01, |
2559 | 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08, |
2560 | 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, |
2561 | 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, |
2562 | 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18, |
2563 | 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c, |
2564 | 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04, |
2565 | 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61, |
2566 | 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65, |
2567 | 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69}; |
2568 | |
2569 | if (dictBuffer==NULL || samplesSizes==NULL) { |
2570 | free(dictBuffer); |
2571 | free(samplesSizes); |
2572 | goto _output_error; |
2573 | } |
2574 | |
2575 | DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++); |
2576 | assert(compressedBufferSize >= totalSampleSize); |
2577 | { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; } |
2578 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
2579 | { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, |
2580 | decodedBuffer, samplesSizes, nbSamples); |
2581 | if (ZDICT_isError(sDictSize)) goto _output_error; |
2582 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize); |
2583 | } |
2584 | |
2585 | DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++); |
2586 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
2587 | dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, |
2588 | CNBuffer, samplesSizes, nbSamples); |
2589 | if (ZDICT_isError(dictSize)) goto _output_error; |
2590 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); |
2591 | |
2592 | DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++); |
2593 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
2594 | { ZDICT_cover_params_t coverParams; |
2595 | memset(&coverParams, 0, sizeof(coverParams)); |
2596 | coverParams.steps = 8; |
2597 | coverParams.nbThreads = 4; |
2598 | dictSize = ZDICT_optimizeTrainFromBuffer_cover( |
2599 | dictBuffer, dictBufferCapacity, |
2600 | CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ |
2601 | &coverParams); |
2602 | if (ZDICT_isError(dictSize)) goto _output_error; |
2603 | } |
2604 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); |
2605 | |
2606 | DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++); |
2607 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
2608 | { ZDICT_cover_params_t coverParams; |
2609 | memset(&coverParams, 0, sizeof(coverParams)); |
2610 | coverParams.steps = 8; |
2611 | coverParams.nbThreads = 4; |
2612 | coverParams.shrinkDict = 1; |
2613 | coverParams.shrinkDictMaxRegression = 1; |
2614 | dictSize = ZDICT_optimizeTrainFromBuffer_cover( |
2615 | dictBuffer, dictBufferCapacity, |
2616 | CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ |
2617 | &coverParams); |
2618 | if (ZDICT_isError(dictSize)) goto _output_error; |
2619 | } |
2620 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); |
2621 | |
2622 | DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++); |
2623 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
2624 | { ZDICT_fastCover_params_t fastCoverParams; |
2625 | memset(&fastCoverParams, 0, sizeof(fastCoverParams)); |
2626 | fastCoverParams.steps = 8; |
2627 | fastCoverParams.nbThreads = 4; |
2628 | dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( |
2629 | dictBuffer, dictBufferCapacity, |
2630 | CNBuffer, samplesSizes, nbSamples, |
2631 | &fastCoverParams); |
2632 | if (ZDICT_isError(dictSize)) goto _output_error; |
2633 | } |
2634 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); |
2635 | |
2636 | DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++); |
2637 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
2638 | { ZDICT_fastCover_params_t fastCoverParams; |
2639 | memset(&fastCoverParams, 0, sizeof(fastCoverParams)); |
2640 | fastCoverParams.steps = 8; |
2641 | fastCoverParams.nbThreads = 4; |
2642 | fastCoverParams.shrinkDict = 1; |
2643 | fastCoverParams.shrinkDictMaxRegression = 1; |
2644 | dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( |
2645 | dictBuffer, dictBufferCapacity, |
2646 | CNBuffer, samplesSizes, nbSamples, |
2647 | &fastCoverParams); |
2648 | if (ZDICT_isError(dictSize)) goto _output_error; |
2649 | } |
2650 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); |
2651 | |
2652 | DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); |
2653 | dictID = ZDICT_getDictID(dictBuffer, dictSize); |
2654 | if (dictID==0) goto _output_error; |
2655 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); |
2656 | |
2657 | DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++); |
2658 | dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize); |
2659 | if (dictHeaderSize==0) goto _output_error; |
2660 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); |
2661 | |
2662 | DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++); |
2663 | { dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, dictBufferFixedSize); |
2664 | if (dictHeaderSize != 115) goto _output_error; |
2665 | } |
2666 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); |
2667 | |
2668 | DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++); |
2669 | cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, |
2670 | CNBuffer, CNBuffSize, |
2671 | dictBuffer, dictSize, 4); |
2672 | if (ZSTD_isError(cSize)) goto _output_error; |
2673 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
2674 | |
2675 | DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++); |
2676 | { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize); |
2677 | if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ |
2678 | } |
2679 | DISPLAYLEVEL(3, "OK \n"); |
2680 | |
2681 | DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); |
2682 | { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); |
2683 | if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ |
2684 | } |
2685 | DISPLAYLEVEL(3, "OK \n"); |
2686 | |
2687 | DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); |
2688 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); |
2689 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, |
2690 | decodedBuffer, CNBuffSize, |
2691 | compressedBuffer, cSize, |
2692 | dictBuffer, dictSize), |
2693 | if (r != CNBuffSize) goto _output_error); |
2694 | ZSTD_freeDCtx(dctx); |
2695 | } |
2696 | DISPLAYLEVEL(3, "OK \n"); |
2697 | |
2698 | DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++); |
2699 | { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); |
2700 | size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef); |
2701 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize); |
2702 | } |
2703 | |
2704 | DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++); |
2705 | { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); |
2706 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, |
2707 | ZSTD_dlm_byRef, ZSTD_dct_auto, |
2708 | cParams, ZSTD_defaultCMem); |
2709 | assert(cdict != NULL); |
2710 | DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict)); |
2711 | assert(ZSTD_getDictID_fromDict(dictBuffer, dictSize) == ZSTD_getDictID_fromCDict(cdict)); |
2712 | cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, |
2713 | CNBuffer, CNBuffSize, cdict); |
2714 | ZSTD_freeCDict(cdict); |
2715 | if (ZSTD_isError(cSize)) goto _output_error; |
2716 | } |
2717 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
2718 | |
2719 | DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); |
2720 | { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); |
2721 | if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ |
2722 | } |
2723 | DISPLAYLEVEL(3, "OK \n"); |
2724 | |
2725 | DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); |
2726 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); |
2727 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, |
2728 | decodedBuffer, CNBuffSize, |
2729 | compressedBuffer, cSize, |
2730 | dictBuffer, dictSize), |
2731 | if (r != CNBuffSize) goto _output_error); |
2732 | ZSTD_freeDCtx(dctx); |
2733 | } |
2734 | DISPLAYLEVEL(3, "OK \n"); |
2735 | |
2736 | DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++); |
2737 | { int const maxLevel = ZSTD_maxCLevel(); |
2738 | int level; |
2739 | for (level = 1; level <= maxLevel; ++level) { |
2740 | ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize); |
2741 | size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); |
2742 | void* const cdictBuffer = malloc(cdictSize); |
2743 | if (cdictBuffer==NULL) goto _output_error; |
2744 | { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict( |
2745 | cdictBuffer, cdictSize, |
2746 | dictBuffer, dictSize, |
2747 | ZSTD_dlm_byCopy, ZSTD_dct_auto, |
2748 | cParams); |
2749 | if (cdict == NULL) { |
2750 | DISPLAY("ZSTD_initStaticCDict failed "); |
2751 | goto _output_error; |
2752 | } |
2753 | cSize = ZSTD_compress_usingCDict(cctx, |
2754 | compressedBuffer, compressedBufferSize, |
2755 | CNBuffer, MIN(10 KB, CNBuffSize), cdict); |
2756 | if (ZSTD_isError(cSize)) { |
2757 | DISPLAY("ZSTD_compress_usingCDict failed "); |
2758 | goto _output_error; |
2759 | } } |
2760 | free(cdictBuffer); |
2761 | } } |
2762 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
2763 | |
2764 | DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); |
2765 | { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; |
2766 | ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); |
2767 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); |
2768 | assert(cdict != NULL); |
2769 | cSize = ZSTD_compress_usingCDict_advanced(cctx, |
2770 | compressedBuffer, compressedBufferSize, |
2771 | CNBuffer, CNBuffSize, |
2772 | cdict, fParams); |
2773 | ZSTD_freeCDict(cdict); |
2774 | if (ZSTD_isError(cSize)) goto _output_error; |
2775 | } |
2776 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
2777 | |
2778 | DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++); |
2779 | { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); |
2780 | if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; |
2781 | } |
2782 | DISPLAYLEVEL(3, "OK (unknown)\n"); |
2783 | |
2784 | DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); |
2785 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
2786 | assert(dctx != NULL); |
2787 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, |
2788 | decodedBuffer, CNBuffSize, |
2789 | compressedBuffer, cSize, |
2790 | dictBuffer, dictSize), |
2791 | if (r != CNBuffSize) goto _output_error); |
2792 | ZSTD_freeDCtx(dctx); |
2793 | } |
2794 | DISPLAYLEVEL(3, "OK \n"); |
2795 | |
2796 | DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); |
2797 | { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize); |
2798 | p.fParams.noDictIDFlag = 1; |
2799 | cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, |
2800 | CNBuffer, CNBuffSize, |
2801 | dictBuffer, dictSize, p); |
2802 | if (ZSTD_isError(cSize)) goto _output_error; |
2803 | } |
2804 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
2805 | |
2806 | DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); |
2807 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); |
2808 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, |
2809 | decodedBuffer, CNBuffSize, |
2810 | compressedBuffer, cSize, |
2811 | dictBuffer, dictSize), |
2812 | if (r != CNBuffSize) goto _output_error); |
2813 | ZSTD_freeDCtx(dctx); |
2814 | } |
2815 | DISPLAYLEVEL(3, "OK \n"); |
2816 | |
2817 | DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++); |
2818 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
2819 | assert(dctx != NULL); |
2820 | { const size_t ret = ZSTD_decompress_usingDict( |
2821 | dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, |
2822 | "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); |
2823 | if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) |
2824 | goto _output_error; |
2825 | } |
2826 | ZSTD_freeDCtx(dctx); |
2827 | } |
2828 | DISPLAYLEVEL(3, "OK \n"); |
2829 | |
2830 | DISPLAYLEVEL(3, "test%3d : bufferless api with cdict : ", testNb++); |
2831 | { ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); |
2832 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
2833 | ZSTD_frameParameters const fParams = { 0, 1, 0 }; |
2834 | size_t cBlockSize; |
2835 | cSize = 0; |
2836 | CHECK_Z(ZSTD_compressBegin_usingCDict_advanced(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN)); |
2837 | cBlockSize = ZSTD_compressContinue(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, CNBuffer, 1000); |
2838 | CHECK_Z(cBlockSize); |
2839 | cSize += cBlockSize; |
2840 | cBlockSize = ZSTD_compressEnd(cctx, (char*)compressedBuffer + cSize, compressedBufferSize - cSize, (char const*)CNBuffer + 2000, 1000); |
2841 | CHECK_Z(cBlockSize); |
2842 | cSize += cBlockSize; |
2843 | |
2844 | CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); |
2845 | |
2846 | ZSTD_freeCDict(cdict); |
2847 | ZSTD_freeDCtx(dctx); |
2848 | } |
2849 | DISPLAYLEVEL(3, "OK \n"); |
2850 | |
2851 | DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++); |
2852 | { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); |
2853 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); |
2854 | if (cdict==NULL) goto _output_error; |
2855 | ZSTD_freeCDict(cdict); |
2856 | } |
2857 | DISPLAYLEVEL(3, "OK \n"); |
2858 | |
2859 | DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++); |
2860 | { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); |
2861 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); |
2862 | if (cdict!=NULL) goto _output_error; |
2863 | ZSTD_freeCDict(cdict); |
2864 | } |
2865 | DISPLAYLEVEL(3, "OK \n"); |
2866 | |
2867 | { char* rawDictBuffer = (char*)malloc(dictSize); |
2868 | assert(rawDictBuffer); |
2869 | memcpy(rawDictBuffer, (char*)dictBuffer + 2, dictSize - 2); |
2870 | memset(rawDictBuffer + dictSize - 2, 0, 2); |
2871 | MEM_writeLE32((char*)rawDictBuffer, ZSTD_MAGIC_DICTIONARY); |
2872 | |
2873 | DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++); |
2874 | { |
2875 | size_t ret; |
2876 | /* Either operation is allowed to fail, but one must fail. */ |
2877 | ret = ZSTD_CCtx_loadDictionary_advanced( |
2878 | cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); |
2879 | if (!ZSTD_isError(ret)) { |
2880 | ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); |
2881 | if (!ZSTD_isError(ret)) goto _output_error; |
2882 | } |
2883 | } |
2884 | DISPLAYLEVEL(3, "OK \n"); |
2885 | |
2886 | DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++); |
2887 | { |
2888 | size_t ret; |
2889 | ret = ZSTD_CCtx_loadDictionary_advanced( |
2890 | cctx, (const char*)rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent); |
2891 | if (ZSTD_isError(ret)) goto _output_error; |
2892 | ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); |
2893 | if (ZSTD_isError(ret)) goto _output_error; |
2894 | } |
2895 | DISPLAYLEVEL(3, "OK \n"); |
2896 | |
2897 | DISPLAYLEVEL(3, "test%3i : Testing non-attached CDict with ZSTD_dct_rawContent : ", testNb++); |
2898 | { size_t const srcSize = MIN(CNBuffSize, 100); |
2899 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
2900 | /* Force the dictionary to be reloaded in raw content mode */ |
2901 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceLoad)); |
2902 | CHECK_Z(ZSTD_CCtx_loadDictionary_advanced(cctx, rawDictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_rawContent)); |
2903 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize); |
2904 | CHECK_Z(cSize); |
2905 | } |
2906 | DISPLAYLEVEL(3, "OK \n"); |
2907 | |
2908 | free(rawDictBuffer); |
2909 | } |
2910 | |
2911 | DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++); |
2912 | { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1); |
2913 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
2914 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); |
2915 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); |
2916 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); |
2917 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); |
2918 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); |
2919 | ZSTD_freeCDict(cdict); |
2920 | } |
2921 | DISPLAYLEVEL(3, "OK \n"); |
2922 | |
2923 | DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++); |
2924 | { |
2925 | size_t size1, size2; |
2926 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
2927 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); |
2928 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); |
2929 | size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); |
2930 | if (ZSTD_isError(size1)) goto _output_error; |
2931 | |
2932 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
2933 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); |
2934 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); |
2935 | size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); |
2936 | if (ZSTD_isError(size2)) goto _output_error; |
2937 | |
2938 | if (size1 != size2) goto _output_error; |
2939 | } |
2940 | DISPLAYLEVEL(3, "OK \n"); |
2941 | |
2942 | DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++); |
2943 | { |
2944 | CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); |
2945 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); |
2946 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); |
2947 | } |
2948 | DISPLAYLEVEL(3, "OK \n"); |
2949 | |
2950 | DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++); |
2951 | { |
2952 | ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); |
2953 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); |
2954 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); |
2955 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); |
2956 | ZSTD_freeCDict(cdict); |
2957 | } |
2958 | DISPLAYLEVEL(3, "OK \n"); |
2959 | |
2960 | DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++); |
2961 | { |
2962 | ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); |
2963 | CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); |
2964 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); |
2965 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); |
2966 | ZSTD_freeCDict(cdict); |
2967 | } |
2968 | DISPLAYLEVEL(3, "OK \n"); |
2969 | |
2970 | DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++); |
2971 | { |
2972 | ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); |
2973 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); |
2974 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); |
2975 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); |
2976 | ZSTD_freeCDict(cdict); |
2977 | } |
2978 | DISPLAYLEVEL(3, "OK \n"); |
2979 | |
2980 | DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++); |
2981 | { |
2982 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); |
2983 | CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); |
2984 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); |
2985 | } |
2986 | DISPLAYLEVEL(3, "OK \n"); |
2987 | |
2988 | DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++); |
2989 | { |
2990 | ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); |
2991 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); |
2992 | CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); |
2993 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); |
2994 | ZSTD_freeCDict(cdict); |
2995 | } |
2996 | DISPLAYLEVEL(3, "OK \n"); |
2997 | |
2998 | DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++); |
2999 | { |
3000 | size_t size1, size2; |
3001 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
3002 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); |
3003 | size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); |
3004 | if (ZSTD_isError(size1)) goto _output_error; |
3005 | |
3006 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); |
3007 | size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); |
3008 | if (ZSTD_isError(size2)) goto _output_error; |
3009 | |
3010 | if (size1 != size2) goto _output_error; |
3011 | } |
3012 | DISPLAYLEVEL(3, "OK \n"); |
3013 | |
3014 | DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++); |
3015 | { |
3016 | size_t size1, size2; |
3017 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
3018 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); |
3019 | size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); |
3020 | if (ZSTD_isError(size1)) goto _output_error; |
3021 | |
3022 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
3023 | size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); |
3024 | if (ZSTD_isError(size2)) goto _output_error; |
3025 | |
3026 | if (size1 == size2) goto _output_error; |
3027 | } |
3028 | DISPLAYLEVEL(3, "OK \n"); |
3029 | |
3030 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
3031 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) ); |
3032 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); |
3033 | CHECK_Z(cSize); |
3034 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++); |
3035 | { |
3036 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
3037 | size_t ret; |
3038 | /* We should fail to decompress without a dictionary. */ |
3039 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); |
3040 | ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
3041 | if (!ZSTD_isError(ret)) goto _output_error; |
3042 | /* We should succeed to decompress with the dictionary. */ |
3043 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); |
3044 | CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) ); |
3045 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); |
3046 | /* The dictionary should persist across calls. */ |
3047 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); |
3048 | /* When we reset the context the dictionary is cleared. */ |
3049 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); |
3050 | ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
3051 | if (!ZSTD_isError(ret)) goto _output_error; |
3052 | ZSTD_freeDCtx(dctx); |
3053 | } |
3054 | DISPLAYLEVEL(3, "OK \n"); |
3055 | |
3056 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++); |
3057 | { |
3058 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
3059 | ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize); |
3060 | size_t ret; |
3061 | /* We should succeed to decompress with the ddict. */ |
3062 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); |
3063 | CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) ); |
3064 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); |
3065 | /* The ddict should persist across calls. */ |
3066 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); |
3067 | /* When we reset the context the ddict is cleared. */ |
3068 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); |
3069 | ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
3070 | if (!ZSTD_isError(ret)) goto _output_error; |
3071 | ZSTD_freeDCtx(dctx); |
3072 | ZSTD_freeDDict(ddict); |
3073 | } |
3074 | DISPLAYLEVEL(3, "OK \n"); |
3075 | |
3076 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++); |
3077 | { |
3078 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
3079 | size_t ret; |
3080 | /* We should succeed to decompress with the prefix. */ |
3081 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); |
3082 | CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) ); |
3083 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); |
3084 | /* The prefix should be cleared after the first compression. */ |
3085 | ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
3086 | if (!ZSTD_isError(ret)) goto _output_error; |
3087 | ZSTD_freeDCtx(dctx); |
3088 | } |
3089 | DISPLAYLEVEL(3, "OK \n"); |
3090 | |
3091 | DISPLAYLEVEL(3, "test%3i : ZSTD_fast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++); |
3092 | { |
3093 | ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); |
3094 | ZSTD_customMem customMem = {NULL, NULL, NULL}; |
3095 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
3096 | ZSTD_CDict* cdict; |
3097 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_fast)); |
3098 | /* Set windowLog to 25 so hash/chain logs don't get sized down */ |
3099 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25)); |
3100 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25)); |
3101 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25)); |
3102 | /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */ |
3103 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25)); |
3104 | cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); |
3105 | CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); |
3106 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); |
3107 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
3108 | CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); |
3109 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
3110 | CHECK_Z(cSize); |
3111 | CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); |
3112 | ZSTD_freeCDict(cdict); |
3113 | ZSTD_freeDCtx(dctx); |
3114 | ZSTD_freeCCtxParams(cctxParams); |
3115 | } |
3116 | DISPLAYLEVEL(3, "OK \n"); |
3117 | |
3118 | DISPLAYLEVEL(3, "test%3i : ZSTD_dfast attach dictionary with hashLog = 25 and chainLog = 25 : ", testNb++); |
3119 | { |
3120 | ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); |
3121 | ZSTD_customMem customMem = {NULL, NULL, NULL}; |
3122 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
3123 | ZSTD_CDict* cdict; |
3124 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_dfast)); |
3125 | /* Set windowLog to 25 so hash/chain logs don't get sized down */ |
3126 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 25)); |
3127 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 25)); |
3128 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_chainLog, 25)); |
3129 | /* Set srcSizeHint to 2^25 so hash/chain logs don't get sized down */ |
3130 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 25)); |
3131 | cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); |
3132 | CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); |
3133 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); |
3134 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
3135 | CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); |
3136 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
3137 | CHECK_Z(cSize); |
3138 | CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); |
3139 | ZSTD_freeCDict(cdict); |
3140 | ZSTD_freeDCtx(dctx); |
3141 | ZSTD_freeCCtxParams(cctxParams); |
3142 | } |
3143 | DISPLAYLEVEL(3, "OK \n"); |
3144 | |
3145 | DISPLAYLEVEL(3, "test%3i : ZSTD_lazy attach dictionary with hashLog = 29 and searchLog = 4 : ", testNb++); |
3146 | if (MEM_64bits()) { |
3147 | ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); |
3148 | ZSTD_customMem customMem = {NULL, NULL, NULL}; |
3149 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
3150 | ZSTD_CDict* cdict; |
3151 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_strategy, ZSTD_lazy)); |
3152 | /* Force enable row based match finder, and disable dedicated dict search. */ |
3153 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_useRowMatchFinder, ZSTD_ps_enable)); |
3154 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_enableDedicatedDictSearch, 0)); |
3155 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_searchLog, 4)); |
3156 | /* Set windowLog to 29 so hash/chain logs don't get sized down */ |
3157 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_windowLog, 29)); |
3158 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_hashLog, 29)); |
3159 | /* Set srcSizeHint to 2^29 so hash/chain logs don't get sized down */ |
3160 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_srcSizeHint, 1u << 29)); |
3161 | cdict = ZSTD_createCDict_advanced2(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctxParams, customMem); |
3162 | CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); |
3163 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, ZSTD_dictForceAttach)); |
3164 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
3165 | CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); |
3166 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
3167 | CHECK_Z(cSize); |
3168 | CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dictBuffer, dictSize)); |
3169 | ZSTD_freeCDict(cdict); |
3170 | ZSTD_freeDCtx(dctx); |
3171 | ZSTD_freeCCtxParams(cctxParams); |
3172 | } |
3173 | DISPLAYLEVEL(3, "OK \n"); |
3174 | |
3175 | DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++); |
3176 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
3177 | dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, |
3178 | CNBuffer, samplesSizes, nbSamples); |
3179 | if (ZDICT_isError(dictSize)) goto _output_error; |
3180 | /* Set all the repcodes to non-default */ |
3181 | { |
3182 | BYTE* dictPtr = (BYTE*)dictBuffer; |
3183 | BYTE* dictLimit = dictPtr + dictSize - 12; |
3184 | /* Find the repcodes */ |
3185 | while (dictPtr < dictLimit && |
3186 | (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 || |
3187 | MEM_readLE32(dictPtr + 8) != 8)) { |
3188 | ++dictPtr; |
3189 | } |
3190 | if (dictPtr >= dictLimit) goto _output_error; |
3191 | MEM_writeLE32(dictPtr + 0, 10); |
3192 | MEM_writeLE32(dictPtr + 4, 10); |
3193 | MEM_writeLE32(dictPtr + 8, 10); |
3194 | /* Set the last 8 bytes to 'x' */ |
3195 | memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8); |
3196 | } |
3197 | /* The optimal parser checks all the repcodes. |
3198 | * Make sure at least one is a match >= targetLength so that it is |
3199 | * immediately chosen. This will make sure that the compressor and |
3200 | * decompressor agree on at least one of the repcodes. |
3201 | */ |
3202 | { size_t dSize; |
3203 | BYTE data[1024]; |
3204 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
3205 | ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); |
3206 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, |
3207 | ZSTD_dlm_byRef, ZSTD_dct_auto, |
3208 | cParams, ZSTD_defaultCMem); |
3209 | assert(dctx != NULL); assert(cdict != NULL); |
3210 | memset(data, 'x', sizeof(data)); |
3211 | cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, |
3212 | data, sizeof(data), cdict); |
3213 | ZSTD_freeCDict(cdict); |
3214 | if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; } |
3215 | dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize); |
3216 | if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; } |
3217 | if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; } |
3218 | ZSTD_freeDCtx(dctx); |
3219 | } |
3220 | DISPLAYLEVEL(3, "OK \n"); |
3221 | |
3222 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with multiple ddicts : ", testNb++); |
3223 | { |
3224 | const size_t numDicts = 128; |
3225 | const size_t numFrames = 4; |
3226 | size_t i; |
3227 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); |
3228 | ZSTD_DDict** ddictTable = (ZSTD_DDict**)malloc(sizeof(ZSTD_DDict*)*numDicts); |
3229 | ZSTD_CDict** cdictTable = (ZSTD_CDict**)malloc(sizeof(ZSTD_CDict*)*numDicts); |
3230 | U32 dictIDSeed = seed; |
3231 | /* Create new compressed buffer that will hold frames with differing dictIDs */ |
3232 | char* dictBufferMulti = (char*)malloc(sizeof(char) * dictBufferFixedSize); /* Modifiable copy of fixed full dict buffer */ |
3233 | |
3234 | ZSTD_memcpy(dictBufferMulti, dictBufferFixed, dictBufferFixedSize); |
3235 | /* Create a bunch of DDicts with random dict IDs */ |
3236 | for (i = 0; i < numDicts; ++i) { |
3237 | U32 currDictID = FUZ_rand(&dictIDSeed); |
3238 | MEM_writeLE32(dictBufferMulti+ZSTD_FRAMEIDSIZE, currDictID); |
3239 | ddictTable[i] = ZSTD_createDDict(dictBufferMulti, dictBufferFixedSize); |
3240 | cdictTable[i] = ZSTD_createCDict(dictBufferMulti, dictBufferFixedSize, 3); |
3241 | if (!ddictTable[i] || !cdictTable[i] || ZSTD_getDictID_fromCDict(cdictTable[i]) != ZSTD_getDictID_fromDDict(ddictTable[i])) { |
3242 | goto _output_error; |
3243 | } |
3244 | } |
3245 | /* Compress a few frames using random CDicts */ |
3246 | { |
3247 | size_t off = 0; |
3248 | /* only use the first half so we don't push against size limit of compressedBuffer */ |
3249 | size_t const segSize = (CNBuffSize / 2) / numFrames; |
3250 | for (i = 0; i < numFrames; i++) { |
3251 | size_t dictIdx = FUZ_rand(&dictIDSeed) % numDicts; |
3252 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
3253 | { CHECK_NEWV(r, ZSTD_compress_usingCDict(cctx, |
3254 | (BYTE*)compressedBuffer + off, CNBuffSize - off, |
3255 | (BYTE*)CNBuffer + segSize * (size_t)i, segSize, |
3256 | cdictTable[dictIdx])); |
3257 | off += r; |
3258 | } |
3259 | } |
3260 | cSize = off; |
3261 | } |
3262 | |
3263 | /* We should succeed to decompression even though different dicts were used on different frames */ |
3264 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); |
3265 | ZSTD_DCtx_setParameter(dctx, ZSTD_d_refMultipleDDicts, ZSTD_rmd_refMultipleDDicts); |
3266 | /* Reference every single ddict we made */ |
3267 | for (i = 0; i < numDicts; ++i) { |
3268 | CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddictTable[i])); |
3269 | } |
3270 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); |
3271 | /* Streaming decompression should also work */ |
3272 | { |
3273 | ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; |
3274 | ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; |
3275 | while (in.pos < in.size) { |
3276 | CHECK_Z(ZSTD_decompressStream(dctx, &out, &in)); |
3277 | } |
3278 | } |
3279 | ZSTD_freeDCtx(dctx); |
3280 | for (i = 0; i < numDicts; ++i) { |
3281 | ZSTD_freeCDict(cdictTable[i]); |
3282 | ZSTD_freeDDict(ddictTable[i]); |
3283 | } |
3284 | free(dictBufferMulti); |
3285 | free(ddictTable); |
3286 | free(cdictTable); |
3287 | } |
3288 | DISPLAYLEVEL(3, "OK \n"); |
3289 | |
3290 | ZSTD_freeCCtx(cctx); |
3291 | free(dictBuffer); |
3292 | free(samplesSizes); |
3293 | } |
3294 | |
3295 | /* COVER dictionary builder tests */ |
3296 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3297 | size_t dictSize = 16 KB; |
3298 | size_t optDictSize = dictSize; |
3299 | void* dictBuffer = malloc(dictSize); |
3300 | size_t const totalSampleSize = 1 MB; |
3301 | size_t const sampleUnitSize = 8 KB; |
3302 | U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); |
3303 | size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); |
3304 | U32 seed32 = seed; |
3305 | ZDICT_cover_params_t params; |
3306 | U32 dictID; |
3307 | |
3308 | if (dictBuffer==NULL || samplesSizes==NULL) { |
3309 | free(dictBuffer); |
3310 | free(samplesSizes); |
3311 | goto _output_error; |
3312 | } |
3313 | |
3314 | DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++); |
3315 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
3316 | memset(¶ms, 0, sizeof(params)); |
3317 | params.d = 1 + (FUZ_rand(&seed32) % 16); |
3318 | params.k = params.d + (FUZ_rand(&seed32) % 256); |
3319 | dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize, |
3320 | CNBuffer, samplesSizes, nbSamples, |
3321 | params); |
3322 | if (ZDICT_isError(dictSize)) goto _output_error; |
3323 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); |
3324 | |
3325 | DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); |
3326 | dictID = ZDICT_getDictID(dictBuffer, dictSize); |
3327 | if (dictID==0) goto _output_error; |
3328 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); |
3329 | |
3330 | DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++); |
3331 | memset(¶ms, 0, sizeof(params)); |
3332 | params.steps = 4; |
3333 | optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize, |
3334 | CNBuffer, samplesSizes, |
3335 | nbSamples / 4, ¶ms); |
3336 | if (ZDICT_isError(optDictSize)) goto _output_error; |
3337 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize); |
3338 | |
3339 | DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); |
3340 | dictID = ZDICT_getDictID(dictBuffer, optDictSize); |
3341 | if (dictID==0) goto _output_error; |
3342 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); |
3343 | |
3344 | ZSTD_freeCCtx(cctx); |
3345 | free(dictBuffer); |
3346 | free(samplesSizes); |
3347 | } |
3348 | |
3349 | /* Decompression defense tests */ |
3350 | DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++); |
3351 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */ |
3352 | if (!ZSTD_isError(r)) goto _output_error; |
3353 | if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } |
3354 | DISPLAYLEVEL(3, "OK \n"); |
3355 | |
3356 | DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++); |
3357 | ((char*)(CNBuffer))[0] = 1; |
3358 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4); |
3359 | if (!ZSTD_isError(r)) goto _output_error; } |
3360 | DISPLAYLEVEL(3, "OK \n"); |
3361 | |
3362 | /* content size verification test */ |
3363 | DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++); |
3364 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3365 | size_t const srcSize = 5000; |
3366 | size_t const wrongSrcSize = (srcSize + 1000); |
3367 | ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0); |
3368 | params.fParams.contentSizeFlag = 1; |
3369 | CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) ); |
3370 | { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize); |
3371 | if (!ZSTD_isError(result)) goto _output_error; |
3372 | if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error; |
3373 | DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result)); |
3374 | } |
3375 | ZSTD_freeCCtx(cctx); |
3376 | } |
3377 | |
3378 | /* negative compression level test : ensure simple API and advanced API produce same result */ |
3379 | DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++); |
3380 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3381 | size_t const srcSize = CNBuffSize / 5; |
3382 | int const compressionLevel = -1; |
3383 | |
3384 | assert(cctx != NULL); |
3385 | { size_t const cSize_1pass = ZSTD_compress(compressedBuffer, compressedBufferSize, |
3386 | CNBuffer, srcSize, compressionLevel); |
3387 | if (ZSTD_isError(cSize_1pass)) goto _output_error; |
3388 | |
3389 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); |
3390 | { size_t const compressionResult = ZSTD_compress2(cctx, |
3391 | compressedBuffer, compressedBufferSize, |
3392 | CNBuffer, srcSize); |
3393 | DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult); |
3394 | if (ZSTD_isError(compressionResult)) goto _output_error; |
3395 | if (compressionResult != cSize_1pass) goto _output_error; |
3396 | } } |
3397 | ZSTD_freeCCtx(cctx); |
3398 | } |
3399 | DISPLAYLEVEL(3, "OK \n"); |
3400 | |
3401 | /* parameters order test */ |
3402 | { size_t const inputSize = CNBuffSize / 2; |
3403 | U64 xxh64; |
3404 | |
3405 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3406 | DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); |
3407 | assert(cctx != NULL); |
3408 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); |
3409 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); |
3410 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); |
3411 | { size_t const compressedSize = ZSTD_compress2(cctx, |
3412 | compressedBuffer, ZSTD_compressBound(inputSize), |
3413 | CNBuffer, inputSize); |
3414 | CHECK_Z(compressedSize); |
3415 | cSize = compressedSize; |
3416 | xxh64 = XXH64(compressedBuffer, compressedSize, 0); |
3417 | } |
3418 | DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); |
3419 | ZSTD_freeCCtx(cctx); |
3420 | } |
3421 | |
3422 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
3423 | DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); |
3424 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); |
3425 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable) ); |
3426 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); |
3427 | { size_t const result = ZSTD_compress2(cctx, |
3428 | compressedBuffer, ZSTD_compressBound(inputSize), |
3429 | CNBuffer, inputSize); |
3430 | CHECK_Z(result); |
3431 | if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */ |
3432 | if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ |
3433 | DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result); |
3434 | } |
3435 | ZSTD_freeCCtx(cctx); |
3436 | } |
3437 | } |
3438 | |
3439 | /* advanced parameters for decompression */ |
3440 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
3441 | assert(dctx != NULL); |
3442 | |
3443 | DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++); |
3444 | { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); |
3445 | CHECK_Z(bounds.error); |
3446 | } |
3447 | DISPLAYLEVEL(3, "OK \n"); |
3448 | |
3449 | DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++); |
3450 | { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0); |
3451 | if (!ZSTD_isError(sr)) goto _output_error; |
3452 | } |
3453 | { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998); |
3454 | if (!ZSTD_isError(bounds.error)) goto _output_error; |
3455 | } |
3456 | DISPLAYLEVEL(3, "OK \n"); |
3457 | |
3458 | DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++); |
3459 | { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999); |
3460 | if (!ZSTD_isError(sr)) goto _output_error; |
3461 | } |
3462 | { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888); |
3463 | if (!ZSTD_isError(sr)) goto _output_error; |
3464 | } |
3465 | DISPLAYLEVEL(3, "OK \n"); |
3466 | |
3467 | ZSTD_freeDCtx(dctx); |
3468 | } |
3469 | |
3470 | |
3471 | /* custom formats tests */ |
3472 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3473 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
3474 | size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ |
3475 | assert(dctx != NULL); assert(cctx != NULL); |
3476 | |
3477 | /* basic block compression */ |
3478 | DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); |
3479 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); |
3480 | { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; |
3481 | ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; |
3482 | size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); |
3483 | if (result != 0) goto _output_error; |
3484 | if (in.pos != in.size) goto _output_error; |
3485 | cSize = out.pos; |
3486 | } |
3487 | DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); |
3488 | |
3489 | DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++); |
3490 | { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
3491 | if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error; |
3492 | DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult)); |
3493 | } |
3494 | |
3495 | DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++); |
3496 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); |
3497 | CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); |
3498 | { ZSTD_frameHeader zfh; |
3499 | size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless); |
3500 | if (zfhrt != 0) goto _output_error; |
3501 | } |
3502 | /* one shot */ |
3503 | { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
3504 | if (result != inputSize) goto _output_error; |
3505 | DISPLAYLEVEL(3, "one-shot OK, "); |
3506 | } |
3507 | /* streaming */ |
3508 | { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; |
3509 | ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; |
3510 | size_t const result = ZSTD_decompressStream(dctx, &out, &in); |
3511 | if (result != 0) goto _output_error; |
3512 | if (in.pos != in.size) goto _output_error; |
3513 | if (out.pos != inputSize) goto _output_error; |
3514 | DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); |
3515 | } |
3516 | |
3517 | /* basic block compression */ |
3518 | DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++); |
3519 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); |
3520 | { ZSTD_inBuffer in = { CNBuffer, 0, 0 }; |
3521 | ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 }; |
3522 | size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); |
3523 | if (result != 0) goto _output_error; |
3524 | if (in.pos != in.size) goto _output_error; |
3525 | cSize = out.pos; |
3526 | } |
3527 | DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize); |
3528 | |
3529 | DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++); |
3530 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); |
3531 | CHECK_Z( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); |
3532 | /* one shot */ |
3533 | { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
3534 | if (result != 0) goto _output_error; |
3535 | DISPLAYLEVEL(3, "one-shot OK, "); |
3536 | } |
3537 | /* streaming */ |
3538 | { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; |
3539 | ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; |
3540 | size_t const result = ZSTD_decompressStream(dctx, &out, &in); |
3541 | if (result != 0) goto _output_error; |
3542 | if (in.pos != in.size) goto _output_error; |
3543 | if (out.pos != 0) goto _output_error; |
3544 | DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); |
3545 | } |
3546 | |
3547 | ZSTD_freeCCtx(cctx); |
3548 | ZSTD_freeDCtx(dctx); |
3549 | } |
3550 | |
3551 | DISPLAYLEVEL(3, "test%3i : Decompression parameter reset test : ", testNb++); |
3552 | { |
3553 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
3554 | /* Attempt to future proof this to new parameters. */ |
3555 | int const maxParam = 2000; |
3556 | int param; |
3557 | if (ZSTD_d_experimentalParam3 > maxParam) goto _output_error; |
3558 | for (param = 0; param < maxParam; ++param) { |
3559 | ZSTD_dParameter dParam = (ZSTD_dParameter)param; |
3560 | ZSTD_bounds bounds = ZSTD_dParam_getBounds(dParam); |
3561 | int value1; |
3562 | int value2; |
3563 | int check; |
3564 | if (ZSTD_isError(bounds.error)) |
3565 | continue; |
3566 | CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &value1)); |
3567 | value2 = (value1 != bounds.lowerBound) ? bounds.lowerBound : bounds.upperBound; |
3568 | CHECK_Z(ZSTD_DCtx_setParameter(dctx, dParam, value2)); |
3569 | CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check)); |
3570 | if (check != value2) goto _output_error; |
3571 | CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_parameters)); |
3572 | CHECK_Z(ZSTD_DCtx_getParameter(dctx, dParam, &check)); |
3573 | if (check != value1) goto _output_error; |
3574 | } |
3575 | ZSTD_freeDCtx(dctx); |
3576 | } |
3577 | DISPLAYLEVEL(3, "OK \n"); |
3578 | |
3579 | /* block API tests */ |
3580 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3581 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
3582 | static const size_t dictSize = 65 KB; |
3583 | static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ |
3584 | size_t cSize2; |
3585 | assert(cctx != NULL); assert(dctx != NULL); |
3586 | |
3587 | /* basic block compression */ |
3588 | DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); |
3589 | CHECK_Z( ZSTD_compressBegin(cctx, 5) ); |
3590 | CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize); |
3591 | CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) ); |
3592 | DISPLAYLEVEL(3, "OK \n"); |
3593 | |
3594 | DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++); |
3595 | CHECK_Z( ZSTD_decompressBegin(dctx) ); |
3596 | { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); |
3597 | if (r != blockSize) goto _output_error; } |
3598 | DISPLAYLEVEL(3, "OK \n"); |
3599 | |
3600 | /* very long stream of block compression */ |
3601 | DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); |
3602 | CHECK_Z( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */ |
3603 | CHECK_Z( ZSTD_getBlockSize(cctx) >= blockSize); |
3604 | { U64 const toCompress = 5000000000ULL; /* > 4 GB */ |
3605 | U64 compressed = 0; |
3606 | while (compressed < toCompress) { |
3607 | size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); |
3608 | assert(blockCSize != 0); |
3609 | if (ZSTD_isError(blockCSize)) goto _output_error; |
3610 | compressed += blockCSize; |
3611 | } } |
3612 | DISPLAYLEVEL(3, "OK \n"); |
3613 | |
3614 | /* dictionary block compression */ |
3615 | DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); |
3616 | CHECK_Z( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); |
3617 | CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize)); |
3618 | RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */ |
3619 | { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */ |
3620 | assert(r == 0); /* non-compressible block */ } |
3621 | memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */ |
3622 | CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize), |
3623 | (char*)CNBuffer+dictSize+2*blockSize, blockSize)); |
3624 | DISPLAYLEVEL(3, "OK \n"); |
3625 | |
3626 | DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++); |
3627 | CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); |
3628 | { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) ); |
3629 | if (r != blockSize) { |
3630 | DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); |
3631 | goto _output_error; |
3632 | } } |
3633 | memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize); |
3634 | ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */ |
3635 | { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) ); |
3636 | if (r != blockSize) { |
3637 | DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); |
3638 | goto _output_error; |
3639 | } } |
3640 | assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */ |
3641 | DISPLAYLEVEL(3, "OK \n"); |
3642 | |
3643 | DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++); |
3644 | { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3); |
3645 | if (cdict==NULL) goto _output_error; |
3646 | CHECK_Z( ZSTD_compressBegin_usingCDict(cctx, cdict) ); |
3647 | CHECK_Z( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) ); |
3648 | ZSTD_freeCDict(cdict); |
3649 | } |
3650 | DISPLAYLEVEL(3, "OK \n"); |
3651 | |
3652 | ZSTD_freeCCtx(cctx); |
3653 | ZSTD_freeDCtx(dctx); |
3654 | } |
3655 | |
3656 | /* long rle test */ |
3657 | { size_t sampleSize = 0; |
3658 | size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */ |
3659 | DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); |
3660 | memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); |
3661 | sampleSize += 256 KB - 1; |
3662 | memset((char*)CNBuffer+sampleSize, 'A', 96 KB); |
3663 | sampleSize += 96 KB; |
3664 | cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); |
3665 | if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error; |
3666 | { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); |
3667 | if (regenSize!=sampleSize) goto _output_error; } |
3668 | DISPLAYLEVEL(3, "OK \n"); |
3669 | } |
3670 | |
3671 | DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences decode from sequences test : ", testNb++); |
3672 | { |
3673 | size_t srcSize = 150 KB; |
3674 | BYTE* src = (BYTE*)CNBuffer; |
3675 | BYTE* decoded = (BYTE*)compressedBuffer; |
3676 | |
3677 | ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
3678 | ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); |
3679 | size_t seqsSize; |
3680 | |
3681 | if (seqs == NULL) goto _output_error; |
3682 | assert(cctx != NULL); |
3683 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); |
3684 | /* Populate src with random data */ |
3685 | RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); |
3686 | |
3687 | /* Test with block delimiters roundtrip */ |
3688 | seqsSize = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); |
f535537f |
3689 | CHECK_Z(seqsSize); |
648db22b |
3690 | FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_explicitBlockDelimiters); |
3691 | assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); |
3692 | |
3693 | /* Test no block delimiters roundtrip */ |
3694 | seqsSize = ZSTD_mergeBlockDelimiters(seqs, seqsSize); |
f535537f |
3695 | CHECK_Z(seqsSize); |
648db22b |
3696 | FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize, ZSTD_sf_noBlockDelimiters); |
3697 | assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); |
3698 | |
3699 | ZSTD_freeCCtx(cctx); |
3700 | free(seqs); |
3701 | } |
3702 | DISPLAYLEVEL(3, "OK \n"); |
3703 | |
f535537f |
3704 | DISPLAYLEVEL(3, "test%3i : ZSTD_generateSequences too small output buffer : ", testNb++); |
3705 | { |
3706 | const size_t seqsCapacity = 10; |
3707 | const size_t srcSize = 150 KB; |
3708 | const BYTE* src = (BYTE*)CNBuffer; |
3709 | |
3710 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3711 | ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(seqsCapacity * sizeof(ZSTD_Sequence)); |
3712 | |
3713 | if (seqs == NULL) goto _output_error; |
3714 | if (cctx == NULL) goto _output_error; |
3715 | /* Populate src with random data */ |
3716 | RDG_genBuffer(CNBuffer, srcSize, compressibility, 0.5, seed); |
3717 | |
3718 | /* Test with block delimiters roundtrip */ |
3719 | { |
3720 | size_t const seqsSize = ZSTD_generateSequences(cctx, seqs, seqsCapacity, src, srcSize); |
3721 | if (!ZSTD_isError(seqsSize)) goto _output_error; |
3722 | } |
3723 | |
3724 | ZSTD_freeCCtx(cctx); |
3725 | free(seqs); |
3726 | } |
3727 | DISPLAYLEVEL(3, "OK \n"); |
3728 | |
648db22b |
3729 | DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences followed by ZSTD_compressSequences : ", testNb++); |
3730 | { |
3731 | const size_t srcSize = 500 KB; |
3732 | const BYTE* const src = (BYTE*)CNBuffer; |
3733 | BYTE* const dst = (BYTE*)compressedBuffer; |
3734 | const size_t dstCapacity = ZSTD_compressBound(srcSize); |
3735 | const size_t decompressSize = srcSize; |
3736 | char* const decompressBuffer = (char*)malloc(decompressSize); |
3737 | size_t compressedSize; |
3738 | |
3739 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3740 | ZSTD_Sequence* const seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); |
3741 | size_t nbSeqs; |
3742 | |
3743 | if (seqs == NULL) goto _output_error; |
3744 | assert(cctx != NULL); |
3745 | |
3746 | /* Populate src with random data */ |
3747 | RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); |
3748 | |
3749 | /* Roundtrip Test with block delimiters generated by ZSTD_generateSequences() */ |
3750 | nbSeqs = ZSTD_generateSequences(cctx, seqs, srcSize, src, srcSize); |
3751 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
3752 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters); |
3753 | compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqs, src, srcSize); |
3754 | if (ZSTD_isError(compressedSize)) { |
3755 | DISPLAY("Error in sequence compression with block delims\n"); |
3756 | goto _output_error; |
3757 | } |
3758 | { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); |
3759 | if (ZSTD_isError(dSize)) { |
3760 | DISPLAY("Error in sequence compression roundtrip with block delims\n"); |
3761 | goto _output_error; |
3762 | } } |
3763 | assert(!memcmp(decompressBuffer, src, srcSize)); |
3764 | |
3765 | /* Roundtrip Test with no block delimiters */ |
3766 | { size_t const nbSeqsAfterMerge = ZSTD_mergeBlockDelimiters(seqs, nbSeqs); |
3767 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
3768 | ZSTD_CCtx_setParameter(cctx, ZSTD_c_blockDelimiters, ZSTD_sf_noBlockDelimiters); |
3769 | compressedSize = ZSTD_compressSequences(cctx, dst, dstCapacity, seqs, nbSeqsAfterMerge, src, srcSize); |
3770 | } |
3771 | if (ZSTD_isError(compressedSize)) { |
3772 | DISPLAY("Error in sequence compression with no block delims\n"); |
3773 | goto _output_error; |
3774 | } |
3775 | { size_t const dSize = ZSTD_decompress(decompressBuffer, decompressSize, dst, compressedSize); |
3776 | if (ZSTD_isError(dSize)) { |
3777 | DISPLAY("Error in sequence compression roundtrip with no block delims\n"); |
3778 | goto _output_error; |
3779 | } } |
3780 | assert(!memcmp(decompressBuffer, src, srcSize)); |
3781 | |
3782 | ZSTD_freeCCtx(cctx); |
3783 | free(decompressBuffer); |
3784 | free(seqs); |
3785 | } |
3786 | DISPLAYLEVEL(3, "OK \n"); |
3787 | |
3788 | /* Multiple blocks of zeros test */ |
3789 | #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */ |
3790 | DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH); |
3791 | memset(CNBuffer, 0, LONGZEROSLENGTH); |
3792 | CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) ); |
3793 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100); |
3794 | |
3795 | DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH); |
3796 | { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) ); |
3797 | if (r != LONGZEROSLENGTH) goto _output_error; } |
3798 | DISPLAYLEVEL(3, "OK \n"); |
3799 | |
3800 | /* All zeroes test (test bug #137) */ |
3801 | #define ZEROESLENGTH 100 |
3802 | DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); |
3803 | memset(CNBuffer, 0, ZEROESLENGTH); |
3804 | CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); |
3805 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100); |
3806 | |
3807 | DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); |
3808 | { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); |
3809 | if (r != ZEROESLENGTH) goto _output_error; } |
3810 | DISPLAYLEVEL(3, "OK \n"); |
3811 | |
3812 | /* nbSeq limit test */ |
3813 | #define _3BYTESTESTLENGTH 131000 |
3814 | #define NB3BYTESSEQLOG 9 |
3815 | #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG) |
3816 | #define NB3BYTESSEQMASK (NB3BYTESSEQ-1) |
3817 | /* creates a buffer full of 3-bytes sequences */ |
3818 | { BYTE _3BytesSeqs[NB3BYTESSEQ][3]; |
3819 | U32 rSeed = 1; |
3820 | |
3821 | /* create batch of 3-bytes sequences */ |
3822 | { int i; |
3823 | for (i=0; i < NB3BYTESSEQ; i++) { |
3824 | _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); |
3825 | _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); |
3826 | _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); |
3827 | } } |
3828 | |
3829 | /* randomly fills CNBuffer with prepared 3-bytes sequences */ |
3830 | { int i; |
3831 | for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ |
3832 | U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; |
3833 | ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; |
3834 | ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; |
3835 | ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; |
3836 | } } } |
3837 | DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++); |
3838 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3839 | size_t const maxNbSeq = _3BYTESTESTLENGTH / 3; |
3840 | size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH); |
3841 | size_t nbSeq = 1; |
3842 | while (nbSeq <= maxNbSeq) { |
3843 | CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19)); |
3844 | /* Check every sequence for the first 100, then skip more rapidly. */ |
3845 | if (nbSeq < 100) { |
3846 | ++nbSeq; |
3847 | } else { |
3848 | nbSeq += (nbSeq >> 2); |
3849 | } |
3850 | } |
3851 | ZSTD_freeCCtx(cctx); |
3852 | } |
3853 | DISPLAYLEVEL(3, "OK \n"); |
3854 | |
3855 | DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++); |
3856 | CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), |
3857 | CNBuffer, _3BYTESTESTLENGTH, 19) ); |
3858 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100); |
3859 | |
3860 | DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++); |
3861 | { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); |
3862 | if (r != _3BYTESTESTLENGTH) goto _output_error; } |
3863 | DISPLAYLEVEL(3, "OK \n"); |
3864 | |
3865 | |
3866 | DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++); |
3867 | RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed); |
3868 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3869 | size_t const bound = ZSTD_compressBound(CNBuffSize); |
3870 | size_t size = 1; |
3871 | while (size <= CNBuffSize) { |
3872 | CHECK_Z(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3)); |
3873 | /* Check every size for the first 100, then skip more rapidly. */ |
3874 | if (size < 100) { |
3875 | ++size; |
3876 | } else { |
3877 | size += (size >> 2); |
3878 | } |
3879 | } |
3880 | ZSTD_freeCCtx(cctx); |
3881 | } |
3882 | DISPLAYLEVEL(3, "OK \n"); |
3883 | |
3884 | DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++); |
3885 | { /* Train a dictionary on low characters */ |
3886 | size_t dictSize = 16 KB; |
3887 | void* const dictBuffer = malloc(dictSize); |
3888 | size_t const totalSampleSize = 1 MB; |
3889 | size_t const sampleUnitSize = 8 KB; |
3890 | U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); |
3891 | size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); |
3892 | if (!dictBuffer || !samplesSizes) goto _output_error; |
3893 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
3894 | dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples); |
3895 | if (ZDICT_isError(dictSize)) goto _output_error; |
3896 | /* Reverse the characters to make the dictionary ill suited */ |
3897 | { U32 u; |
3898 | for (u = 0; u < CNBuffSize; ++u) { |
3899 | ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u]; |
3900 | } } |
3901 | { /* Compress the data */ |
3902 | size_t const inputSize = 500; |
3903 | size_t const outputSize = ZSTD_compressBound(inputSize); |
3904 | void* const outputBuffer = malloc(outputSize); |
3905 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3906 | if (!outputBuffer || !cctx) goto _output_error; |
3907 | CHECK_Z(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1)); |
3908 | free(outputBuffer); |
3909 | ZSTD_freeCCtx(cctx); |
3910 | } |
3911 | |
3912 | free(dictBuffer); |
3913 | free(samplesSizes); |
3914 | } |
3915 | DISPLAYLEVEL(3, "OK \n"); |
3916 | |
3917 | |
3918 | /* findFrameCompressedSize on skippable frames */ |
3919 | DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++); |
3920 | { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde"; |
3921 | size_t const frameSrcSize = 13; |
3922 | if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; } |
3923 | DISPLAYLEVEL(3, "OK \n"); |
3924 | |
3925 | /* error string tests */ |
3926 | DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++); |
3927 | if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error; |
3928 | if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error; |
3929 | if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error; |
3930 | if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error; |
3931 | if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error; |
3932 | if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error; |
3933 | DISPLAYLEVEL(3, "OK \n"); |
3934 | |
3935 | DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++); |
3936 | RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); |
3937 | { |
3938 | size_t const size = MIN(128 KB, CNBuffSize); |
3939 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
3940 | ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1); |
3941 | ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1); |
3942 | ZSTD_frameHeader lgHeader; |
3943 | ZSTD_frameHeader smHeader; |
3944 | |
3945 | CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict)); |
3946 | CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize)); |
3947 | CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict)); |
3948 | CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize)); |
3949 | |
3950 | if (lgHeader.windowSize != smHeader.windowSize) goto _output_error; |
3951 | |
3952 | ZSTD_freeCDict(smCDict); |
3953 | ZSTD_freeCDict(lgCDict); |
3954 | ZSTD_freeCCtx(cctx); |
3955 | } |
3956 | DISPLAYLEVEL(3, "OK \n"); |
3957 | |
3958 | DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++); |
3959 | { |
3960 | short norm[32]; |
3961 | unsigned count[32]; |
3962 | unsigned const tableLog = 5; |
3963 | size_t const nbSeq = 32; |
3964 | unsigned const maxSymbolValue = 31; |
3965 | size_t i; |
3966 | |
3967 | for (i = 0; i < 32; ++i) |
3968 | count[i] = 1; |
3969 | /* Calling FSE_normalizeCount() on a uniform distribution should not |
3970 | * cause a division by zero. |
3971 | */ |
3972 | FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue, /* useLowProbCount */ 1); |
3973 | } |
3974 | DISPLAYLEVEL(3, "OK \n"); |
3975 | |
3976 | DISPLAYLEVEL(3, "test%3i : testing FSE_writeNCount() PR#2779: ", testNb++); |
3977 | { |
3978 | size_t const outBufSize = 9; |
3979 | short const count[11] = {1, 0, 1, 0, 1, 0, 1, 0, 1, 9, 18}; |
3980 | unsigned const tableLog = 5; |
3981 | unsigned const maxSymbolValue = 10; |
3982 | BYTE* outBuf = (BYTE*)malloc(outBufSize*sizeof(BYTE)); |
3983 | |
3984 | /* Ensure that this write doesn't write out of bounds, and that |
3985 | * FSE_writeNCount_generic() is *not* called with writeIsSafe == 1. |
3986 | */ |
3987 | FSE_writeNCount(outBuf, outBufSize, count, maxSymbolValue, tableLog); |
3988 | free(outBuf); |
3989 | } |
3990 | DISPLAYLEVEL(3, "OK \n"); |
3991 | |
3992 | DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++); |
3993 | { |
3994 | U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */ |
3995 | U32 rand32 = FUZ_rand(&seed_copy); |
3996 | U64 rand64 = ((U64)FUZ_rand(&seed_copy) << 32) | FUZ_rand(&seed_copy); |
3997 | U32 lowbit_only_32 = 1; |
3998 | U64 lowbit_only_64 = 1; |
3999 | U32 highbit_only_32 = (U32)1 << 31; |
4000 | U64 highbit_only_64 = (U64)1 << 63; |
4001 | U32 i; |
4002 | if (rand32 == 0) rand32 = 1; /* CLZ and CTZ are undefined on 0 */ |
4003 | if (rand64 == 0) rand64 = 1; /* CLZ and CTZ are undefined on 0 */ |
4004 | |
4005 | /* Test ZSTD_countTrailingZeros32 */ |
4006 | CHECK_EQ(ZSTD_countTrailingZeros32(lowbit_only_32), 0u); |
4007 | CHECK_EQ(ZSTD_countTrailingZeros32(highbit_only_32), 31u); |
4008 | CHECK_EQ(ZSTD_countTrailingZeros32(rand32), ZSTD_countTrailingZeros32_fallback(rand32)); |
4009 | |
4010 | /* Test ZSTD_countLeadingZeros32 */ |
4011 | CHECK_EQ(ZSTD_countLeadingZeros32(lowbit_only_32), 31u); |
4012 | CHECK_EQ(ZSTD_countLeadingZeros32(highbit_only_32), 0u); |
4013 | CHECK_EQ(ZSTD_countLeadingZeros32(rand32), ZSTD_countLeadingZeros32_fallback(rand32)); |
4014 | |
4015 | /* Test ZSTD_countTrailingZeros64 */ |
4016 | CHECK_EQ(ZSTD_countTrailingZeros64(lowbit_only_64), 0u); |
4017 | CHECK_EQ(ZSTD_countTrailingZeros64(highbit_only_64), 63u); |
4018 | |
4019 | /* Test ZSTD_countLeadingZeros64 */ |
4020 | CHECK_EQ(ZSTD_countLeadingZeros64(lowbit_only_64), 63u); |
4021 | CHECK_EQ(ZSTD_countLeadingZeros64(highbit_only_64), 0u); |
4022 | |
4023 | /* Test ZSTD_highbit32 */ |
4024 | CHECK_EQ(ZSTD_highbit32(lowbit_only_32), 0u); |
4025 | CHECK_EQ(ZSTD_highbit32(highbit_only_32), 31u); |
4026 | |
4027 | /* Test ZSTD_NbCommonBytes */ |
4028 | if (MEM_isLittleEndian()) { |
4029 | if (MEM_64bits()) { |
4030 | CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u); |
4031 | CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u); |
4032 | } else { |
4033 | CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 0u); |
4034 | CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 3u); |
4035 | } |
4036 | } else { |
4037 | if (MEM_64bits()) { |
4038 | CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 7u); |
4039 | CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 4u); |
4040 | } else { |
4041 | CHECK_EQ(ZSTD_NbCommonBytes(lowbit_only_32), 3u); |
4042 | CHECK_EQ(ZSTD_NbCommonBytes(highbit_only_32), 0u); |
4043 | } |
4044 | } |
4045 | |
4046 | /* Test MEM_ intrinsics */ |
4047 | CHECK_EQ(MEM_swap32(rand32), MEM_swap32_fallback(rand32)); |
4048 | CHECK_EQ(MEM_swap64(rand64), MEM_swap64_fallback(rand64)); |
4049 | |
4050 | /* Test fallbacks vs intrinsics on a range of small integers */ |
4051 | for (i=1; i <= 1000; i++) { |
4052 | CHECK_EQ(MEM_swap32(i), MEM_swap32_fallback(i)); |
4053 | CHECK_EQ(MEM_swap64((U64)i), MEM_swap64_fallback((U64)i)); |
4054 | CHECK_EQ(ZSTD_countTrailingZeros32(i), ZSTD_countTrailingZeros32_fallback(i)); |
4055 | CHECK_EQ(ZSTD_countLeadingZeros32(i), ZSTD_countLeadingZeros32_fallback(i)); |
4056 | } |
4057 | } |
4058 | DISPLAYLEVEL(3, "OK \n"); |
4059 | |
4060 | #ifdef ZSTD_MULTITHREAD |
4061 | DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++); |
4062 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
4063 | size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ |
4064 | size_t const dstSize = ZSTD_compressBound(srcSize); |
4065 | void* const src = CNBuffer; |
4066 | void* const dst = compressedBuffer; |
4067 | void* dict = (void*)malloc(srcSize); |
4068 | |
4069 | RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); |
4070 | RDG_genBuffer(dict, srcSize, compressibility, 0., seed); |
4071 | |
4072 | /* Make sure there is no ZSTD_MAGIC_NUMBER */ |
4073 | memset(dict, 0, sizeof(U32)); |
4074 | |
4075 | /* something more than 1 */ |
4076 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); |
4077 | /* lie and claim this is a full dict */ |
4078 | CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict)); |
4079 | |
4080 | { ZSTD_outBuffer out = {dst, dstSize, 0}; |
4081 | ZSTD_inBuffer in = {src, srcSize, 0}; |
4082 | /* should fail because its not a full dict like we said it was */ |
4083 | assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush))); |
4084 | } |
4085 | |
4086 | ZSTD_freeCCtx(cctx); |
4087 | free(dict); |
4088 | } |
4089 | DISPLAYLEVEL(3, "OK \n"); |
4090 | |
4091 | DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++); |
4092 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); |
4093 | size_t const srcSize = 1 MB + 5; /* A little more than ZSTDMT_JOBSIZE_MIN */ |
4094 | size_t const dictSize = 10; |
4095 | size_t const dstSize = ZSTD_compressBound(srcSize); |
4096 | void* const src = CNBuffer; |
4097 | void* const dst = compressedBuffer; |
4098 | void* dict = (void*)malloc(dictSize); |
4099 | |
4100 | RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); |
4101 | RDG_genBuffer(dict, dictSize, compressibility, 0., seed); |
4102 | |
4103 | /* Make sure there is no ZSTD_MAGIC_NUMBER */ |
4104 | memset(dict, 0, sizeof(U32)); |
4105 | |
4106 | /* Enable MT, LDM, and use refPrefix() for a small dict */ |
4107 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); |
4108 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); |
4109 | CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); |
4110 | |
4111 | CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize)); |
4112 | |
4113 | ZSTD_freeCCtx(cctx); |
4114 | free(dict); |
4115 | } |
4116 | DISPLAYLEVEL(3, "OK \n"); |
4117 | |
4118 | DISPLAYLEVEL(3, "test%3i : ZSTD_getCParams() + dictionary ", testNb++); |
4119 | { |
4120 | ZSTD_compressionParameters const medium = ZSTD_getCParams(1, 16*1024-1, 0); |
4121 | ZSTD_compressionParameters const large = ZSTD_getCParams(1, 128*1024-1, 0); |
4122 | ZSTD_compressionParameters const smallDict = ZSTD_getCParams(1, 0, 400); |
4123 | ZSTD_compressionParameters const mediumDict = ZSTD_getCParams(1, 0, 10000); |
4124 | ZSTD_compressionParameters const largeDict = ZSTD_getCParams(1, 0, 100000); |
4125 | |
4126 | assert(!memcmp(&smallDict, &mediumDict, sizeof(smallDict))); |
4127 | assert(!memcmp(&medium, &mediumDict, sizeof(medium))); |
4128 | assert(!memcmp(&large, &largeDict, sizeof(large))); |
4129 | } |
4130 | DISPLAYLEVEL(3, "OK \n"); |
4131 | |
4132 | DISPLAYLEVEL(3, "test%3i : ZSTD_adjustCParams() + dictionary ", testNb++); |
4133 | { |
4134 | ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 0, 0); |
4135 | ZSTD_compressionParameters const smallDict = ZSTD_adjustCParams(cParams, 0, 400); |
4136 | ZSTD_compressionParameters const smallSrcAndDict = ZSTD_adjustCParams(cParams, 500, 400); |
4137 | |
4138 | assert(smallSrcAndDict.windowLog == 10); |
4139 | assert(!memcmp(&cParams, &smallDict, sizeof(cParams))); |
4140 | } |
4141 | DISPLAYLEVEL(3, "OK \n"); |
4142 | |
4143 | DISPLAYLEVEL(3, "test%3i : check compression mem usage monotonicity over levels for estimateCCtxSize() : ", testNb++); |
4144 | { |
4145 | int level = 1; |
4146 | size_t prevSize = 0; |
4147 | for (; level < ZSTD_maxCLevel(); ++level) { |
4148 | size_t const currSize = ZSTD_estimateCCtxSize(level); |
4149 | if (prevSize > currSize) { |
4150 | DISPLAYLEVEL(3, "Error! previous cctx size: %zu at level: %d is larger than current cctx size: %zu at level: %d", |
4151 | prevSize, level-1, currSize, level); |
4152 | goto _output_error; |
4153 | } |
4154 | prevSize = currSize; |
4155 | } |
4156 | } |
4157 | DISPLAYLEVEL(3, "OK \n"); |
4158 | |
4159 | DISPLAYLEVEL(3, "test%3i : check estimateCCtxSize() always larger or equal to ZSTD_estimateCCtxSize_usingCParams() : ", testNb++); |
4160 | { |
4161 | size_t const kSizeIncrement = 2 KB; |
4162 | int level = -3; |
4163 | |
4164 | for (; level <= ZSTD_maxCLevel(); ++level) { |
4165 | size_t dictSize = 0; |
4166 | for (; dictSize <= 256 KB; dictSize += 8 * kSizeIncrement) { |
4167 | size_t srcSize = 2 KB; |
4168 | for (; srcSize < 300 KB; srcSize += kSizeIncrement) { |
4169 | ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, srcSize, dictSize); |
4170 | size_t const cctxSizeUsingCParams = ZSTD_estimateCCtxSize_usingCParams(cParams); |
4171 | size_t const cctxSizeUsingLevel = ZSTD_estimateCCtxSize(level); |
4172 | if (cctxSizeUsingLevel < cctxSizeUsingCParams |
4173 | || ZSTD_isError(cctxSizeUsingCParams) |
4174 | || ZSTD_isError(cctxSizeUsingLevel)) { |
4175 | DISPLAYLEVEL(3, "error! l: %d dict: %zu srcSize: %zu cctx size cpar: %zu, cctx size level: %zu\n", |
4176 | level, dictSize, srcSize, cctxSizeUsingCParams, cctxSizeUsingLevel); |
4177 | goto _output_error; |
4178 | } } } } } |
4179 | DISPLAYLEVEL(3, "OK \n"); |
4180 | |
4181 | DISPLAYLEVEL(3, "test%3i : thread pool API tests : \n", testNb++) |
4182 | { |
4183 | int const threadPoolTestResult = threadPoolTests(); |
4184 | if (threadPoolTestResult) { |
4185 | goto _output_error; |
4186 | } |
4187 | } |
4188 | DISPLAYLEVEL(3, "thread pool tests OK \n"); |
4189 | |
4190 | #endif /* ZSTD_MULTITHREAD */ |
4191 | |
4192 | _end: |
4193 | free(CNBuffer); |
4194 | free(compressedBuffer); |
4195 | free(decodedBuffer); |
4196 | return testResult; |
4197 | |
4198 | _output_error: |
4199 | testResult = 1; |
4200 | DISPLAY("Error detected in Unit tests ! \n"); |
4201 | goto _end; |
4202 | } |
4203 | |
4204 | static int longUnitTests(U32 const seed, double compressibility) |
4205 | { |
4206 | size_t const CNBuffSize = 5 MB; |
4207 | void* const CNBuffer = malloc(CNBuffSize); |
4208 | size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); |
4209 | void* const compressedBuffer = malloc(compressedBufferSize); |
4210 | void* const decodedBuffer = malloc(CNBuffSize); |
4211 | int testResult = 0; |
4212 | unsigned testNb=0; |
4213 | size_t cSize; |
4214 | |
4215 | /* Create compressible noise */ |
4216 | if (!CNBuffer || !compressedBuffer || !decodedBuffer) { |
4217 | DISPLAY("Not enough memory, aborting\n"); |
4218 | testResult = 1; |
4219 | goto _end; |
4220 | } |
4221 | RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); |
4222 | |
4223 | /* note : this test is rather long, it would be great to find a way to speed up its execution */ |
4224 | DISPLAYLEVEL(3, "longtest%3i : table cleanliness through index reduction : ", testNb++); |
4225 | { int cLevel; |
4226 | size_t approxIndex = 0; |
4227 | size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ |
4228 | |
4229 | /* Provision enough space in a static context so that we can do all |
4230 | * this without ever reallocating, which would reset the indices. */ |
4231 | size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22); |
4232 | void* const staticCCtxBuffer = malloc(staticCCtxSize); |
4233 | ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); |
4234 | |
4235 | /* bump the indices so the following compressions happen at high |
4236 | * indices. */ |
4237 | { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; |
4238 | ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; |
4239 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
4240 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); |
4241 | while (approxIndex <= (maxIndex / 4) * 3) { |
4242 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); |
4243 | approxIndex += in.pos; |
4244 | CHECK_Z(in.pos == in.size); |
4245 | in.pos = 0; |
4246 | out.pos = 0; |
4247 | } |
4248 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); |
4249 | } |
4250 | |
4251 | /* spew a bunch of stuff into the table area */ |
4252 | for (cLevel = 1; cLevel <= 22; cLevel++) { |
4253 | ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; |
4254 | ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; |
4255 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
4256 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); |
4257 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); |
4258 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); |
4259 | approxIndex += in.pos; |
4260 | } |
4261 | |
4262 | /* now crank the indices so we overflow */ |
4263 | { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; |
4264 | ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; |
4265 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
4266 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); |
4267 | while (approxIndex <= maxIndex) { |
4268 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); |
4269 | approxIndex += in.pos; |
4270 | CHECK_Z(in.pos == in.size); |
4271 | in.pos = 0; |
4272 | out.pos = 0; |
4273 | } |
4274 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); |
4275 | } |
4276 | |
4277 | /* do a bunch of compressions again in low indices and ensure we don't |
4278 | * hit untracked invalid indices */ |
4279 | for (cLevel = 1; cLevel <= 22; cLevel++) { |
4280 | ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; |
4281 | ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; |
4282 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
4283 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); |
4284 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); |
4285 | CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); |
4286 | approxIndex += in.pos; |
4287 | } |
4288 | |
4289 | free(staticCCtxBuffer); |
4290 | } |
4291 | DISPLAYLEVEL(3, "OK \n"); |
4292 | |
4293 | DISPLAYLEVEL(3, "longtest%3i : testing ldm no regressions in size for opt parser : ", testNb++); |
4294 | { size_t cSizeLdm; |
4295 | size_t cSizeNoLdm; |
4296 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
4297 | |
4298 | RDG_genBuffer(CNBuffer, CNBuffSize, 0.5, 0.5, seed); |
4299 | |
4300 | /* Enable checksum to verify round trip. */ |
4301 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
4302 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable)); |
4303 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); |
4304 | |
4305 | /* Round trip once with ldm. */ |
4306 | cSizeLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
4307 | CHECK_Z(cSizeLdm); |
4308 | CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeLdm)); |
4309 | |
4310 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); |
4311 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); |
4312 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable)); |
4313 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19)); |
4314 | |
4315 | /* Round trip once without ldm. */ |
4316 | cSizeNoLdm = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
4317 | CHECK_Z(cSizeNoLdm); |
4318 | CHECK_Z(ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSizeNoLdm)); |
4319 | |
4320 | if (cSizeLdm > cSizeNoLdm) { |
4321 | DISPLAY("Using long mode should not cause regressions for btopt+\n"); |
4322 | testResult = 1; |
4323 | goto _end; |
4324 | } |
4325 | |
4326 | ZSTD_freeCCtx(cctx); |
4327 | } |
4328 | DISPLAYLEVEL(3, "OK \n"); |
4329 | |
4330 | DISPLAYLEVEL(3, "longtest%3i : testing cdict compression with different attachment strategies : ", testNb++); |
4331 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
4332 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
4333 | size_t dictSize = CNBuffSize; |
4334 | void* dict = (void*)malloc(dictSize); |
4335 | ZSTD_CCtx_params* cctx_params = ZSTD_createCCtxParams(); |
4336 | ZSTD_dictAttachPref_e const attachPrefs[] = { |
4337 | ZSTD_dictDefaultAttach, |
4338 | ZSTD_dictForceAttach, |
4339 | ZSTD_dictForceCopy, |
4340 | ZSTD_dictForceLoad, |
4341 | ZSTD_dictDefaultAttach, |
4342 | ZSTD_dictForceAttach, |
4343 | ZSTD_dictForceCopy, |
4344 | ZSTD_dictForceLoad |
4345 | }; |
4346 | int const enableDedicatedDictSearch[] = {0, 0, 0, 0, 1, 1, 1, 1}; |
4347 | int cLevel; |
4348 | int i; |
4349 | |
4350 | RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); |
4351 | RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); |
4352 | |
4353 | CHECK_Z(cctx_params != NULL); |
4354 | |
4355 | for (dictSize = CNBuffSize; dictSize; dictSize = dictSize >> 3) { |
4356 | DISPLAYLEVEL(3, "\n Testing with dictSize %u ", (U32)dictSize); |
4357 | for (cLevel = 4; cLevel < 13; cLevel++) { |
4358 | for (i = 0; i < 8; ++i) { |
4359 | ZSTD_dictAttachPref_e const attachPref = attachPrefs[i]; |
4360 | int const enableDDS = enableDedicatedDictSearch[i]; |
4361 | ZSTD_CDict* cdict; |
4362 | |
4363 | DISPLAYLEVEL(5, "\n dictSize %u cLevel %d iter %d ", (U32)dictSize, cLevel, i); |
4364 | |
4365 | ZSTD_CCtxParams_init(cctx_params, cLevel); |
4366 | CHECK_Z(ZSTD_CCtxParams_setParameter(cctx_params, ZSTD_c_enableDedicatedDictSearch, enableDDS)); |
4367 | |
4368 | cdict = ZSTD_createCDict_advanced2(dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cctx_params, ZSTD_defaultCMem); |
4369 | CHECK(cdict != NULL); |
4370 | |
4371 | CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); |
4372 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceAttachDict, (int)attachPref)); |
4373 | |
4374 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); |
4375 | CHECK_Z(cSize); |
4376 | CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); |
4377 | |
4378 | DISPLAYLEVEL(5, "compressed to %u bytes ", (U32)cSize); |
4379 | |
4380 | CHECK_Z(ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters)); |
4381 | ZSTD_freeCDict(cdict); |
4382 | } } } |
4383 | |
4384 | ZSTD_freeCCtx(cctx); |
4385 | ZSTD_freeDCtx(dctx); |
4386 | ZSTD_freeCCtxParams(cctx_params); |
4387 | free(dict); |
4388 | } |
4389 | DISPLAYLEVEL(3, "OK \n"); |
4390 | |
4391 | _end: |
4392 | free(CNBuffer); |
4393 | free(compressedBuffer); |
4394 | free(decodedBuffer); |
4395 | return testResult; |
4396 | } |
4397 | |
4398 | |
4399 | static size_t findDiff(const void* buf1, const void* buf2, size_t max) |
4400 | { |
4401 | const BYTE* b1 = (const BYTE*)buf1; |
4402 | const BYTE* b2 = (const BYTE*)buf2; |
4403 | size_t u; |
4404 | for (u=0; u<max; u++) { |
4405 | if (b1[u] != b2[u]) break; |
4406 | } |
4407 | return u; |
4408 | } |
4409 | |
4410 | |
4411 | static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams) |
4412 | { |
4413 | ZSTD_parameters params; |
4414 | params.cParams = cParams; |
4415 | params.fParams = fParams; |
4416 | return params; |
4417 | } |
4418 | |
4419 | static size_t FUZ_rLogLength(U32* seed, U32 logLength) |
4420 | { |
4421 | size_t const lengthMask = ((size_t)1 << logLength) - 1; |
4422 | return (lengthMask+1) + (FUZ_rand(seed) & lengthMask); |
4423 | } |
4424 | |
4425 | static size_t FUZ_randomLength(U32* seed, U32 maxLog) |
4426 | { |
4427 | U32 const logLength = FUZ_rand(seed) % maxLog; |
4428 | return FUZ_rLogLength(seed, logLength); |
4429 | } |
4430 | |
4431 | #undef CHECK |
4432 | #define CHECK(cond, ...) { \ |
4433 | if (cond) { \ |
4434 | DISPLAY("Error => "); \ |
4435 | DISPLAY(__VA_ARGS__); \ |
4436 | DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ |
4437 | goto _output_error; \ |
4438 | } } |
4439 | |
4440 | #undef CHECK_Z |
4441 | #define CHECK_Z(f) { \ |
4442 | size_t const err = f; \ |
4443 | if (ZSTD_isError(err)) { \ |
4444 | DISPLAY("Error => %s : %s ", \ |
4445 | #f, ZSTD_getErrorName(err)); \ |
4446 | DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ |
4447 | goto _output_error; \ |
4448 | } } |
4449 | |
4450 | |
4451 | static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) |
4452 | { |
4453 | static const U32 maxSrcLog = 23; |
4454 | static const U32 maxSampleLog = 22; |
4455 | size_t const srcBufferSize = (size_t)1<<maxSrcLog; |
4456 | size_t const dstBufferSize = (size_t)1<<maxSampleLog; |
4457 | size_t const cBufferSize = ZSTD_compressBound(dstBufferSize); |
4458 | BYTE* cNoiseBuffer[5]; |
4459 | BYTE* const cBuffer = (BYTE*) malloc (cBufferSize); |
4460 | BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize); |
4461 | BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize); |
4462 | ZSTD_CCtx* const refCtx = ZSTD_createCCtx(); |
4463 | ZSTD_CCtx* const ctx = ZSTD_createCCtx(); |
4464 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
4465 | U32 result = 0; |
4466 | unsigned testNb = 0; |
4467 | U32 coreSeed = seed; |
4468 | UTIL_time_t const startClock = UTIL_getTime(); |
4469 | U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO; |
4470 | int const cLevelLimiter = bigTests ? 3 : 2; |
4471 | |
4472 | /* allocation */ |
4473 | cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); |
4474 | cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); |
4475 | cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); |
4476 | cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); |
4477 | cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); |
4478 | CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] |
4479 | || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, |
4480 | "Not enough memory, fuzzer tests cancelled"); |
4481 | |
4482 | /* Create initial samples */ |
4483 | RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ |
4484 | RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ |
4485 | RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); |
4486 | RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ |
4487 | RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ |
4488 | |
4489 | /* catch up testNb */ |
4490 | for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); |
4491 | |
4492 | /* main test loop */ |
4493 | for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) { |
4494 | BYTE* srcBuffer; /* jumping pointer */ |
4495 | U32 lseed; |
4496 | size_t sampleSize, maxTestSize, totalTestSize; |
4497 | size_t cSize, totalCSize, totalGenSize; |
4498 | U64 crcOrig; |
4499 | BYTE* sampleBuffer; |
4500 | const BYTE* dict; |
4501 | size_t dictSize; |
4502 | |
4503 | /* notification */ |
4504 | if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } |
4505 | else { DISPLAYUPDATE(2, "\r%6u ", testNb); } |
4506 | |
4507 | FUZ_rand(&coreSeed); |
4508 | { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; } |
4509 | |
4510 | /* srcBuffer selection [0-4] */ |
4511 | { U32 buffNb = FUZ_rand(&lseed) & 0x7F; |
4512 | if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ |
4513 | else { |
4514 | buffNb >>= 3; |
4515 | if (buffNb & 7) { |
4516 | const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ |
4517 | buffNb = tnb[buffNb >> 3]; |
4518 | } else { |
4519 | const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ |
4520 | buffNb = tnb[buffNb >> 3]; |
4521 | } } |
4522 | srcBuffer = cNoiseBuffer[buffNb]; |
4523 | } |
4524 | |
4525 | /* select src segment */ |
4526 | sampleSize = FUZ_randomLength(&lseed, maxSampleLog); |
4527 | |
4528 | /* create sample buffer (to catch read error with valgrind & sanitizers) */ |
4529 | sampleBuffer = (BYTE*)malloc(sampleSize); |
4530 | CHECK(sampleBuffer==NULL, "not enough memory for sample buffer"); |
4531 | { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); |
4532 | memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); } |
4533 | crcOrig = XXH64(sampleBuffer, sampleSize, 0); |
4534 | |
4535 | /* compression tests */ |
4536 | { int const cLevelPositive = (int) |
4537 | ( FUZ_rand(&lseed) % |
4538 | ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) ) |
4539 | + 1; |
4540 | int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ? |
4541 | - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */ |
4542 | cLevelPositive; |
4543 | DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel); |
4544 | cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); |
4545 | CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize)); |
4546 | |
4547 | /* compression failure test : too small dest buffer */ |
4548 | assert(cSize > 3); |
4549 | { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; |
4550 | const size_t tooSmallSize = cSize - missing; |
4551 | const unsigned endMark = 0x4DC2B1A9; |
4552 | memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark)); |
4553 | DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n", |
4554 | testNb, (unsigned)tooSmallSize, (unsigned)missing); |
4555 | { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); |
4556 | CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); } |
4557 | { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck)); |
4558 | CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); } |
4559 | } } |
4560 | |
4561 | /* frame header decompression test */ |
4562 | { ZSTD_frameHeader zfh; |
4563 | CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) ); |
4564 | CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect"); |
4565 | } |
4566 | |
4567 | /* Decompressed size test */ |
4568 | { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize); |
4569 | CHECK(rSize != sampleSize, "decompressed size incorrect"); |
4570 | } |
4571 | |
4572 | /* successful decompression test */ |
4573 | DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb); |
4574 | { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; |
4575 | size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); |
4576 | CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize); |
4577 | { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0); |
4578 | CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize); |
4579 | } } |
4580 | |
4581 | free(sampleBuffer); /* no longer useful after this point */ |
4582 | |
4583 | /* truncated src decompression test */ |
4584 | DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb); |
4585 | { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ |
4586 | size_t const tooSmallSize = cSize - missing; |
4587 | void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ |
4588 | CHECK(cBufferTooSmall == NULL, "not enough memory !"); |
4589 | memcpy(cBufferTooSmall, cBuffer, tooSmallSize); |
4590 | { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); |
4591 | CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); } |
4592 | free(cBufferTooSmall); |
4593 | } |
4594 | |
4595 | /* too small dst decompression test */ |
4596 | DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb); |
4597 | if (sampleSize > 3) { |
4598 | size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ |
4599 | size_t const tooSmallSize = sampleSize - missing; |
4600 | static const BYTE token = 0xA9; |
4601 | dstBuffer[tooSmallSize] = token; |
4602 | { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); |
4603 | CHECK(ZSTD_getErrorCode(errorCode) != ZSTD_error_dstSize_tooSmall, "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); } |
4604 | CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); |
4605 | } |
4606 | |
4607 | /* noisy src decompression test */ |
4608 | if (cSize > 6) { |
4609 | /* insert noise into src */ |
4610 | { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4)); |
4611 | size_t pos = 4; /* preserve magic number (too easy to detect) */ |
4612 | for (;;) { |
4613 | /* keep some original src */ |
4614 | { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits; |
4615 | size_t const mask = (1<<nbBits) - 1; |
4616 | size_t const skipLength = FUZ_rand(&lseed) & mask; |
4617 | pos += skipLength; |
4618 | } |
4619 | if (pos >= cSize) break; |
4620 | /* add noise */ |
4621 | { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits; |
4622 | U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; |
4623 | size_t const mask = (1<<nbBits) - 1; |
4624 | size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1; |
4625 | size_t const noiseLength = MIN(rNoiseLength, cSize-pos); |
4626 | size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); |
4627 | memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); |
4628 | pos += noiseLength; |
4629 | } } } |
4630 | |
4631 | /* decompress noisy source */ |
4632 | DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb); |
4633 | { U32 const endMark = 0xA9B1C3D6; |
4634 | memcpy(dstBuffer+sampleSize, &endMark, 4); |
4635 | { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); |
4636 | /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */ |
4637 | CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize), |
4638 | "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize); |
4639 | } |
4640 | { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); |
4641 | CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); |
4642 | } } } /* noisy src decompression test */ |
4643 | |
4644 | /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/ |
4645 | DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb); |
4646 | { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; |
4647 | U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; |
4648 | int const cLevel = (int)(FUZ_rand(&lseed) % |
4649 | ((U32)ZSTD_maxCLevel() - |
4650 | (MAX(testLog, dictLog) / (U32)cLevelLimiter))) + |
4651 | 1; |
4652 | maxTestSize = FUZ_rLogLength(&lseed, testLog); |
4653 | if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; |
4654 | |
4655 | dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */ |
4656 | dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); |
4657 | |
4658 | DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n", |
4659 | testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize); |
4660 | |
4661 | if (FUZ_rand(&lseed) & 0xF) { |
4662 | CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) ); |
4663 | } else { |
4664 | ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); |
4665 | ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, |
4666 | !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, |
4667 | 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ |
4668 | ZSTD_parameters const p = FUZ_makeParams(cPar, fPar); |
4669 | CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) ); |
4670 | } |
4671 | CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) ); |
4672 | } |
4673 | |
4674 | { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; |
4675 | U32 n; |
4676 | XXH64_state_t xxhState; |
4677 | XXH64_reset(&xxhState, 0); |
4678 | for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { |
4679 | size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog); |
4680 | size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize); |
4681 | |
4682 | if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */ |
4683 | if (totalTestSize+segmentSize > maxTestSize) break; |
4684 | |
4685 | { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); |
4686 | CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); |
4687 | cSize += compressResult; |
4688 | } |
4689 | XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); |
4690 | memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); |
4691 | totalTestSize += segmentSize; |
4692 | } |
4693 | |
4694 | { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); |
4695 | CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); |
4696 | cSize += flushResult; |
4697 | } |
4698 | crcOrig = XXH64_digest(&xxhState); |
4699 | } |
4700 | |
4701 | /* streaming decompression test */ |
4702 | DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb); |
4703 | /* ensure memory requirement is good enough (should always be true) */ |
4704 | { ZSTD_frameHeader zfh; |
4705 | CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX), |
4706 | "ZSTD_getFrameHeader(): error retrieving frame information"); |
4707 | { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize); |
4708 | CHECK_Z(roundBuffSize); |
4709 | CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN), |
4710 | "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)", |
4711 | (unsigned)roundBuffSize, (unsigned)totalTestSize ); |
4712 | } } |
4713 | if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ |
4714 | CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) ); |
4715 | totalCSize = 0; |
4716 | totalGenSize = 0; |
4717 | while (totalCSize < cSize) { |
4718 | size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx); |
4719 | size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); |
4720 | CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize)); |
4721 | totalGenSize += genSize; |
4722 | totalCSize += inSize; |
4723 | } |
4724 | CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); |
4725 | CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size") |
4726 | CHECK (totalCSize != cSize, "compressed data should be fully read") |
4727 | { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); |
4728 | CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)", |
4729 | (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize); |
4730 | } |
4731 | } /* for ( ; (testNb <= nbTests) */ |
4732 | DISPLAY("\r%u fuzzer tests completed \n", testNb-1); |
4733 | |
4734 | _cleanup: |
4735 | ZSTD_freeCCtx(refCtx); |
4736 | ZSTD_freeCCtx(ctx); |
4737 | ZSTD_freeDCtx(dctx); |
4738 | free(cNoiseBuffer[0]); |
4739 | free(cNoiseBuffer[1]); |
4740 | free(cNoiseBuffer[2]); |
4741 | free(cNoiseBuffer[3]); |
4742 | free(cNoiseBuffer[4]); |
4743 | free(cBuffer); |
4744 | free(dstBuffer); |
4745 | free(mirrorBuffer); |
4746 | return (int)result; |
4747 | |
4748 | _output_error: |
4749 | result = 1; |
4750 | goto _cleanup; |
4751 | } |
4752 | |
4753 | |
4754 | /*_******************************************************* |
4755 | * Command line |
4756 | *********************************************************/ |
4757 | static int FUZ_usage(const char* programName) |
4758 | { |
4759 | DISPLAY( "Usage :\n"); |
4760 | DISPLAY( " %s [args]\n", programName); |
4761 | DISPLAY( "\n"); |
4762 | DISPLAY( "Arguments :\n"); |
4763 | DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault); |
4764 | DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n"); |
4765 | DISPLAY( " -s# : Select seed (default:prompt user)\n"); |
4766 | DISPLAY( " -t# : Select starting test number (default:0)\n"); |
4767 | DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default); |
4768 | DISPLAY( " -v : verbose\n"); |
4769 | DISPLAY( " -p : pause at the end\n"); |
4770 | DISPLAY( " -h : display help and exit\n"); |
4771 | return 0; |
4772 | } |
4773 | |
4774 | /*! readU32FromChar() : |
4775 | @return : unsigned integer value read from input in `char` format |
4776 | allows and interprets K, KB, KiB, M, MB and MiB suffix. |
4777 | Will also modify `*stringPtr`, advancing it to position where it stopped reading. |
4778 | Note : function result can overflow if digit string > MAX_UINT */ |
4779 | static unsigned readU32FromChar(const char** stringPtr) |
4780 | { |
4781 | unsigned result = 0; |
4782 | while ((**stringPtr >='0') && (**stringPtr <='9')) |
4783 | result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ; |
4784 | if ((**stringPtr=='K') || (**stringPtr=='M')) { |
4785 | result <<= 10; |
4786 | if (**stringPtr=='M') result <<= 10; |
4787 | (*stringPtr)++ ; |
4788 | if (**stringPtr=='i') (*stringPtr)++; |
4789 | if (**stringPtr=='B') (*stringPtr)++; |
4790 | } |
4791 | return result; |
4792 | } |
4793 | |
4794 | /** longCommandWArg() : |
4795 | * check if *stringPtr is the same as longCommand. |
4796 | * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. |
4797 | * @return 0 and doesn't modify *stringPtr otherwise. |
4798 | */ |
4799 | static int longCommandWArg(const char** stringPtr, const char* longCommand) |
4800 | { |
4801 | size_t const comSize = strlen(longCommand); |
4802 | int const result = !strncmp(*stringPtr, longCommand, comSize); |
4803 | if (result) *stringPtr += comSize; |
4804 | return result; |
4805 | } |
4806 | |
4807 | int main(int argc, const char** argv) |
4808 | { |
4809 | U32 seed = 0; |
4810 | int seedset = 0; |
4811 | int argNb; |
4812 | int nbTests = nbTestsDefault; |
4813 | int testNb = 0; |
4814 | int proba = FUZ_compressibility_default; |
4815 | double probfloat; |
4816 | int result = 0; |
4817 | U32 mainPause = 0; |
4818 | U32 maxDuration = 0; |
4819 | int bigTests = 1; |
4820 | int longTests = 0; |
4821 | U32 memTestsOnly = 0; |
4822 | const char* const programName = argv[0]; |
4823 | |
4824 | /* Check command line */ |
4825 | for (argNb=1; argNb<argc; argNb++) { |
4826 | const char* argument = argv[argNb]; |
4827 | if(!argument) continue; /* Protection if argument empty */ |
4828 | |
4829 | /* Handle commands. Aggregated commands are allowed */ |
4830 | if (argument[0]=='-') { |
4831 | |
4832 | if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; } |
4833 | |
4834 | if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; } |
4835 | if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } |
4836 | if (!strcmp(argument, "--long-tests")) { longTests=1; continue; } |
4837 | if (!strcmp(argument, "--no-long-tests")) { longTests=0; continue; } |
4838 | |
4839 | argument++; |
4840 | while (*argument!=0) { |
4841 | switch(*argument) |
4842 | { |
4843 | case 'h': |
4844 | return FUZ_usage(programName); |
4845 | |
4846 | case 'v': |
4847 | argument++; |
4848 | g_displayLevel++; |
4849 | break; |
4850 | |
4851 | case 'q': |
4852 | argument++; |
4853 | g_displayLevel--; |
4854 | break; |
4855 | |
4856 | case 'p': /* pause at the end */ |
4857 | argument++; |
4858 | mainPause = 1; |
4859 | break; |
4860 | |
4861 | case 'i': |
4862 | argument++; maxDuration = 0; |
4863 | nbTests = (int)readU32FromChar(&argument); |
4864 | break; |
4865 | |
4866 | case 'T': |
4867 | argument++; |
4868 | nbTests = 0; |
4869 | maxDuration = readU32FromChar(&argument); |
4870 | if (*argument=='s') argument++; /* seconds */ |
4871 | if (*argument=='m') maxDuration *= 60, argument++; /* minutes */ |
4872 | if (*argument=='n') argument++; |
4873 | break; |
4874 | |
4875 | case 's': |
4876 | argument++; |
4877 | seedset = 1; |
4878 | seed = readU32FromChar(&argument); |
4879 | break; |
4880 | |
4881 | case 't': |
4882 | argument++; |
4883 | testNb = (int)readU32FromChar(&argument); |
4884 | break; |
4885 | |
4886 | case 'P': /* compressibility % */ |
4887 | argument++; |
4888 | proba = (int)readU32FromChar(&argument); |
4889 | if (proba>100) proba = 100; |
4890 | break; |
4891 | |
4892 | default: |
4893 | return (FUZ_usage(programName), 1); |
4894 | } } } } /* for (argNb=1; argNb<argc; argNb++) */ |
4895 | |
4896 | /* Get Seed */ |
4897 | DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); |
4898 | |
4899 | if (!seedset) { |
4900 | time_t const t = time(NULL); |
4901 | U32 const h = XXH32(&t, sizeof(t), 1); |
4902 | seed = h % 10000; |
4903 | } |
4904 | |
4905 | DISPLAY("Seed = %u\n", (unsigned)seed); |
4906 | if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba); |
4907 | |
4908 | probfloat = ((double)proba) / 100; |
4909 | |
4910 | if (memTestsOnly) { |
4911 | g_displayLevel = MAX(3, g_displayLevel); |
4912 | return FUZ_mallocTests(seed, probfloat, memTestsOnly); |
4913 | } |
4914 | |
4915 | if (nbTests < testNb) nbTests = testNb; |
4916 | |
4917 | if (testNb==0) { |
4918 | result = basicUnitTests(0, probfloat); /* constant seed for predictability */ |
4919 | |
4920 | if (!result && longTests) { |
4921 | result = longUnitTests(0, probfloat); |
4922 | } |
4923 | } |
4924 | if (!result) |
4925 | result = fuzzerTests(seed, (unsigned)nbTests, (unsigned)testNb, maxDuration, ((double)proba) / 100, bigTests); |
4926 | if (mainPause) { |
4927 | int unused; |
4928 | DISPLAY("Press Enter \n"); |
4929 | unused = getchar(); |
4930 | (void)unused; |
4931 | } |
4932 | return result; |
4933 | } |