648db22b |
1 | #!/bin/sh |
2 | |
3 | set -e |
4 | |
5 | unset ZSTD_CLEVEL |
6 | unset ZSTD_NBTHREADS |
7 | |
8 | |
9 | die() { |
10 | println "$@" 1>&2 |
11 | exit 1 |
12 | } |
13 | |
14 | datagen() { |
15 | "$DATAGEN_BIN" "$@" |
16 | } |
17 | |
18 | zstd() { |
19 | if [ -z "$EXEC_PREFIX" ]; then |
20 | "$ZSTD_BIN" "$@" |
21 | else |
22 | "$EXEC_PREFIX" "$ZSTD_BIN" "$@" |
23 | fi |
24 | } |
25 | |
26 | sudoZstd() { |
27 | if [ -z "$EXEC_PREFIX" ]; then |
28 | sudo "$ZSTD_BIN" "$@" |
29 | else |
30 | sudo "$EXEC_PREFIX" "$ZSTD_BIN" "$@" |
31 | fi |
32 | } |
33 | |
34 | roundTripTest() { |
35 | if [ -n "$3" ]; then |
36 | cLevel="$3" |
37 | proba="$2" |
38 | else |
39 | cLevel="$2" |
40 | proba="" |
41 | fi |
42 | if [ -n "$4" ]; then |
43 | dLevel="$4" |
44 | else |
45 | dLevel="$cLevel" |
46 | fi |
47 | |
48 | rm -f tmp1 tmp2 |
49 | println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel" |
50 | datagen $1 $proba | $MD5SUM > tmp1 |
51 | datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 |
52 | $DIFF -q tmp1 tmp2 |
53 | } |
54 | |
55 | fileRoundTripTest() { |
56 | if [ -n "$3" ]; then |
57 | local_c="$3" |
58 | local_p="$2" |
59 | else |
60 | local_c="$2" |
61 | local_p="" |
62 | fi |
63 | if [ -n "$4" ]; then |
64 | local_d="$4" |
65 | else |
66 | local_d="$local_c" |
67 | fi |
68 | |
69 | rm -f tmp.zst tmp.md5.1 tmp.md5.2 |
70 | println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d" |
71 | datagen $1 $local_p > tmp |
72 | < tmp $MD5SUM > tmp.md5.1 |
73 | zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 |
74 | $DIFF -q tmp.md5.1 tmp.md5.2 |
75 | } |
76 | |
77 | truncateLastByte() { |
78 | dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" |
79 | } |
80 | |
81 | println() { |
82 | printf '%b\n' "${*}" |
83 | } |
84 | |
85 | if [ -z "${size}" ]; then |
86 | size= |
87 | else |
88 | size=${size} |
89 | fi |
90 | |
91 | SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) |
92 | PRGDIR="$SCRIPT_DIR/../programs" |
93 | TESTDIR="$SCRIPT_DIR/../tests" |
94 | UNAME=$(uname) |
95 | |
96 | detectedTerminal=false |
97 | if [ -t 0 ] && [ -t 1 ] |
98 | then |
99 | detectedTerminal=true |
100 | fi |
101 | isTerminal=${isTerminal:-$detectedTerminal} |
102 | |
103 | isWindows=false |
104 | INTOVOID="/dev/null" |
105 | case "$UNAME" in |
106 | GNU) DEVDEVICE="/dev/random" ;; |
107 | *) DEVDEVICE="/dev/zero" ;; |
108 | esac |
109 | case "$OS" in |
110 | Windows*) |
111 | isWindows=true |
112 | INTOVOID="NUL" |
113 | DEVDEVICE="NUL" |
114 | ;; |
115 | esac |
116 | |
117 | case "$UNAME" in |
118 | Darwin) MD5SUM="md5 -r" ;; |
119 | FreeBSD) MD5SUM="gmd5sum" ;; |
120 | NetBSD) MD5SUM="md5 -n" ;; |
121 | OpenBSD) MD5SUM="md5" ;; |
122 | *) MD5SUM="md5sum" ;; |
123 | esac |
124 | |
125 | MTIME="stat -c %Y" |
126 | case "$UNAME" in |
127 | Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; |
128 | esac |
129 | |
130 | assertSameMTime() { |
131 | MT1=$($MTIME "$1") |
132 | MT2=$($MTIME "$2") |
133 | echo MTIME $MT1 $MT2 |
134 | [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)" |
135 | } |
136 | |
137 | GET_PERMS="stat -c %a" |
138 | case "$UNAME" in |
139 | Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; |
140 | esac |
141 | |
142 | assertFilePermissions() { |
143 | STAT1=$($GET_PERMS "$1") |
144 | STAT2=$2 |
145 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" |
146 | } |
147 | |
148 | assertSamePermissions() { |
149 | STAT1=$($GET_PERMS "$1") |
150 | STAT2=$($GET_PERMS "$2") |
151 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" |
152 | } |
153 | |
154 | DIFF="diff" |
155 | case "$UNAME" in |
156 | SunOS) DIFF="gdiff" ;; |
157 | esac |
158 | |
159 | |
160 | # check if ZSTD_BIN is defined. if not, use the default value |
161 | if [ -z "${ZSTD_BIN}" ]; then |
162 | println "\nZSTD_BIN is not set. Using the default value..." |
163 | ZSTD_BIN="$PRGDIR/zstd" |
164 | fi |
165 | |
166 | # check if DATAGEN_BIN is defined. if not, use the default value |
167 | if [ -z "${DATAGEN_BIN}" ]; then |
168 | println "\nDATAGEN_BIN is not set. Using the default value..." |
169 | DATAGEN_BIN="$TESTDIR/datagen" |
170 | fi |
171 | |
172 | # Why was this line here ? Generates a strange ZSTD_BIN when EXE_PREFIX is non empty |
173 | # ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN" |
174 | |
175 | # assertions |
176 | [ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd following the instructions in README.md" |
177 | [ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. " |
178 | println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" |
179 | |
180 | if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' |
181 | then |
182 | hasMT="" |
183 | else |
184 | hasMT="true" |
185 | fi |
186 | |
187 | |
188 | zstd -vvV |
189 | |
190 | println "\n===> simple tests " |
191 | |
192 | datagen > tmp |
193 | zstd -h |
194 | zstd -H |
195 | zstd -V |
196 | println "test : basic compression " |
197 | zstd -f tmp # trivial compression case, creates tmp.zst |
198 | zstd -f -z tmp |
199 | zstd -f -k tmp |
200 | zstd -f -C tmp |
201 | println "test : basic decompression" |
202 | zstd -df tmp.zst # trivial decompression case (overwrites tmp) |
203 | println "test : too large compression level => auto-fix" |
204 | zstd -99 -f tmp # too large compression level, automatic sized down |
205 | zstd -5000000000 -f tmp && die "too large numeric value : must fail" |
206 | println "test : --fast aka negative compression levels" |
207 | zstd --fast -f tmp # == -1 |
208 | zstd --fast=3 -f tmp # == -3 |
209 | zstd --fast=200000 -f tmp # too low compression level, automatic fixed |
210 | zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail" |
211 | zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" |
212 | println "test : too large numeric argument" |
213 | zstd --fast=9999999999 -f tmp && die "should have refused numeric value" |
214 | println "test : set compression level with environment variable ZSTD_CLEVEL" |
215 | |
216 | ZSTD_CLEVEL=12 zstd -f tmp # positive compression level |
217 | ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level |
218 | ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign |
219 | ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting |
220 | ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting |
221 | ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting |
222 | ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting |
223 | ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting |
224 | ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting |
225 | println "test : override ZSTD_CLEVEL with command line option" |
226 | ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option |
227 | |
228 | # temporary envvar changes in the above tests would actually persist in macos /bin/sh |
229 | unset ZSTD_CLEVEL |
230 | |
231 | |
232 | println "test : compress to stdout" |
233 | zstd tmp -c > tmpCompressed |
234 | zstd tmp --stdout > tmpCompressed # long command format |
235 | println "test : compress to named file" |
236 | rm -f tmpCompressed |
237 | zstd tmp -o tmpCompressed |
238 | test -f tmpCompressed # file must be created |
239 | println "test : force write, correct order" |
240 | zstd tmp -fo tmpCompressed |
241 | println "test : forgotten argument" |
242 | cp tmp tmp2 |
243 | zstd tmp2 -fo && die "-o must be followed by filename " |
244 | println "test : implied stdout when input is stdin" |
245 | println bob | zstd | zstd -d |
246 | if [ "$isTerminal" = true ]; then |
247 | println "test : compressed data to terminal" |
248 | println bob | zstd && die "should have refused : compressed data to terminal" |
249 | println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" |
250 | zstd -d > $INTOVOID && die "should have refused : compressed data from terminal" |
251 | fi |
252 | println "test : null-length file roundtrip" |
253 | println -n '' | zstd - --stdout | zstd -d --stdout |
254 | println "test : ensure small file doesn't add 3-bytes null block" |
255 | datagen -g1 > tmp1 |
256 | zstd tmp1 -c | wc -c | grep "14" |
257 | zstd < tmp1 | wc -c | grep "14" |
258 | println "test : decompress file with wrong suffix (must fail)" |
259 | zstd -d tmpCompressed && die "wrong suffix error not detected!" |
260 | zstd -df tmp && die "should have refused : wrong extension" |
261 | println "test : decompress into stdout" |
262 | zstd -d tmpCompressed -c > tmpResult # decompression using stdout |
263 | zstd --decompress tmpCompressed -c > tmpResult |
264 | zstd --decompress tmpCompressed --stdout > tmpResult |
265 | println "test : decompress from stdin into stdout" |
266 | zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout |
267 | zstd -dc - < tmp.zst > $INTOVOID |
268 | zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used |
269 | zstd -d - < tmp.zst > $INTOVOID |
270 | println "test : impose memory limitation (must fail)" |
271 | datagen -g500K > tmplimit |
272 | zstd -f tmplimit |
273 | zstd -d -f tmplimit.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" |
274 | zstd -d -f tmplimit.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command |
275 | zstd -d -f tmplimit.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command |
276 | zstd -d -f tmplimit.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command |
277 | rm -f tmplimit tmplimit.zst |
278 | println "test : overwrite protection" |
279 | zstd -q tmp && die "overwrite check failed!" |
280 | println "test : force overwrite" |
281 | zstd -q -f tmp |
282 | zstd -q --force tmp |
283 | println "test : overwrite readonly file" |
284 | rm -f tmpro tmpro.zst |
285 | println foo > tmpro.zst |
286 | println foo > tmpro |
287 | chmod 400 tmpro.zst |
288 | zstd -q tmpro && die "should have refused to overwrite read-only file" |
289 | zstd -q -f tmpro |
290 | println "test: --no-progress flag" |
291 | zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress |
292 | zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress |
293 | println "test: --progress flag" |
294 | zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes" |
295 | zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes" |
296 | zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes" |
297 | rm -f tmpro tmpro.zst |
298 | println "test: overwrite input file (must fail)" |
299 | zstd tmp -fo tmp && die "zstd compression overwrote the input file" |
300 | zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" |
301 | println "test: detect that input file does not exist" |
302 | zstd nothere && die "zstd hasn't detected that input file does not exist" |
303 | println "test: --[no-]compress-literals" |
304 | zstd tmp -c --no-compress-literals -1 | zstd -t |
305 | zstd tmp -c --no-compress-literals --fast=1 | zstd -t |
306 | zstd tmp -c --no-compress-literals -19 | zstd -t |
307 | zstd tmp -c --compress-literals -1 | zstd -t |
308 | zstd tmp -c --compress-literals --fast=1 | zstd -t |
309 | zstd tmp -c --compress-literals -19 | zstd -t |
310 | zstd -b --fast=1 -i0e1 tmp --compress-literals |
311 | zstd -b --fast=1 -i0e1 tmp --no-compress-literals |
312 | println "test: --no-check for decompression" |
313 | zstd -f tmp -o tmp_corrupt.zst --check |
314 | zstd -f tmp -o tmp.zst --no-check |
315 | printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp |
316 | zstd -d -f tmp_corrupt.zst --no-check |
317 | zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides |
318 | zstd -d -f tmp.zst --no-check |
319 | |
320 | if [ "$isWindows" = false ] && [ "$UNAME" != "AIX" ]; then |
321 | if [ -n "$(which readelf)" ]; then |
322 | println "test: check if binary has executable stack (#2963)" |
323 | readelf -lW "$ZSTD_BIN" | grep 'GNU_STACK .* RW ' || die "zstd binary has executable stack!" |
324 | fi |
325 | fi |
326 | |
327 | println "\n===> --exclude-compressed flag" |
328 | rm -rf precompressedFilterTestDir |
329 | mkdir -p precompressedFilterTestDir |
330 | datagen $size > precompressedFilterTestDir/input.5 |
331 | datagen $size > precompressedFilterTestDir/input.6 |
332 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
333 | datagen $size > precompressedFilterTestDir/input.7 |
334 | datagen $size > precompressedFilterTestDir/input.8 |
335 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
336 | test ! -f precompressedFilterTestDir/input.5.zst.zst |
337 | test ! -f precompressedFilterTestDir/input.6.zst.zst |
338 | file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst` |
339 | file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst` |
340 | if [ $file2timestamp -ge $file1timestamp ]; then |
341 | println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again." |
342 | else |
343 | println "Test is not successful" |
344 | fi |
345 | # File Extension check. |
346 | datagen $size > precompressedFilterTestDir/input.zstbar |
347 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
348 | # zstd should compress input.zstbar |
349 | test -f precompressedFilterTestDir/input.zstbar.zst |
350 | # Check without the --exclude-compressed flag |
351 | zstd --long --rm -r precompressedFilterTestDir |
352 | # Files should get compressed again without the --exclude-compressed flag. |
353 | test -f precompressedFilterTestDir/input.5.zst.zst |
354 | test -f precompressedFilterTestDir/input.6.zst.zst |
355 | rm -rf precompressedFilterTestDir |
356 | println "Test completed" |
357 | |
358 | |
359 | |
360 | println "\n===> warning prompts should not occur if stdin is an input" |
361 | println "y" > tmpPrompt |
362 | println "hello world" >> tmpPrompt |
363 | zstd tmpPrompt -f |
364 | zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" |
365 | zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f |
366 | zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated |
367 | $DIFF tmpPromptRegenerated tmpPrompt # the first 'y' character should not be swallowed |
368 | |
369 | echo 'yes' | zstd tmpPrompt -v -o tmpPrompt.zst # accept piped "y" input to force overwrite when using files |
370 | echo 'yes' | zstd < tmpPrompt -v -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" |
371 | zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove" |
372 | |
373 | println "Test completed" |
374 | |
375 | |
376 | println "\n===> recursive mode test " |
377 | # combination of -r with empty list of input file |
378 | zstd -c -r < tmp > tmp.zst |
379 | |
380 | # combination of -r with empty folder |
381 | mkdir -p tmpEmptyDir |
382 | zstd -r tmpEmptyDir |
383 | rm -rf tmpEmptyDir |
384 | |
385 | |
386 | println "\n===> file removal" |
387 | zstd -f --rm tmp |
388 | test ! -f tmp # tmp should no longer be present |
389 | zstd -f -d --rm tmp.zst |
390 | test ! -f tmp.zst # tmp.zst should no longer be present |
391 | println "test: --rm is disabled when output is stdout" |
392 | test -f tmp |
393 | zstd --rm tmp -c > $INTOVOID |
394 | test -f tmp # tmp shall still be there |
395 | zstd -f --rm tmp -c > $INTOVOID |
396 | test -f tmp # tmp shall still be there |
397 | zstd -f tmp -c > $INTOVOID --rm |
398 | test -f tmp # tmp shall still be there |
399 | println "test: --rm is disabled when multiple inputs are concatenated into a single output" |
400 | cp tmp tmp2 |
401 | zstd --rm tmp tmp2 -c > $INTOVOID |
402 | test -f tmp |
403 | test -f tmp2 |
404 | rm -f tmp3.zst |
405 | echo 'y' | zstd -v tmp tmp2 -o tmp3.zst --rm # prompt for confirmation |
406 | test -f tmp |
407 | test -f tmp2 |
408 | zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt |
409 | test -f tmp |
410 | test -f tmp2 |
411 | zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate" |
412 | |
413 | println "test : should quietly not remove non-regular file" |
414 | println hello > tmp |
415 | zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" |
416 | grep "Refusing to remove non-regular file" tmplog && die |
417 | rm -f tmplog |
418 | zstd tmp -f -o "$INTOVOID" 2>&1 | grep "Refusing to remove non-regular file" && die |
419 | println "test : --rm on stdin" |
420 | println a | zstd --rm > $INTOVOID # --rm should remain silent |
421 | rm -f tmp |
422 | zstd -f tmp && die "tmp not present : should have failed" |
423 | test ! -f tmp.zst # tmp.zst should not be created |
424 | println "test : -d -f do not delete destination when source is not present" |
425 | touch tmp # create destination file |
426 | zstd -d -f tmp.zst && die "attempt to decompress a non existing file" |
427 | test -f tmp # destination file should still be present |
428 | println "test : -f do not delete destination when source is not present" |
429 | rm -f tmp # erase source file |
430 | touch tmp.zst # create destination file |
431 | zstd -f tmp && die "attempt to compress a non existing file" |
432 | test -f tmp.zst # destination file should still be present |
433 | rm -rf tmp* # may also erase tmp* directory from previous failed run |
434 | |
435 | |
436 | println "\n===> decompression only tests " |
437 | # the following test verifies that the decoder is compatible with RLE as first block |
438 | # older versions of zstd cli are not able to decode such corner case. |
439 | # As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions. |
440 | dd bs=1048576 count=1 if=/dev/zero of=tmp |
441 | zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" |
442 | $DIFF -s tmp1 tmp |
443 | |
444 | touch tmp_empty |
445 | zstd -d -o tmp2 "$TESTDIR/golden-decompression/empty-block.zst" |
446 | $DIFF -s tmp2 tmp_empty |
447 | rm -f tmp* |
448 | |
449 | println "\n===> compress multiple files" |
450 | println hello > tmp1 |
451 | println world > tmp2 |
452 | zstd tmp1 tmp2 -o "$INTOVOID" -f |
453 | zstd tmp1 tmp2 -c | zstd -t |
454 | echo 'y' | zstd -v tmp1 tmp2 -o tmp.zst |
455 | test ! -f tmp1.zst |
456 | test ! -f tmp2.zst |
457 | zstd tmp1 tmp2 |
458 | zstd -t tmp1.zst tmp2.zst |
459 | zstd -dc tmp1.zst tmp2.zst |
460 | zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f |
461 | echo 'y' | zstd -v -d tmp1.zst tmp2.zst -o tmp |
462 | touch tmpexists |
463 | zstd tmp1 tmp2 -f -o tmpexists |
464 | zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite" |
465 | println gooder > tmp_rm1 |
466 | println boi > tmp_rm2 |
467 | println worldly > tmp_rm3 |
468 | echo 'y' | zstd -v tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst |
469 | test -f tmp_rm1 |
470 | test -f tmp_rm2 |
471 | cp tmp_rm3.zst tmp_rm4.zst |
472 | echo 'Y' | zstd -v -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm |
473 | test -f tmp_rm3.zst |
474 | test -f tmp_rm4.zst |
475 | println gooder > tmpexists1 |
476 | zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID |
477 | # Bug: PR #972 |
478 | if [ "$?" -eq 139 ]; then |
479 | die "should not have segfaulted" |
480 | fi |
481 | test -f tmpexists1 |
482 | test -f tmpexists |
483 | println "\n===> multiple files and shell completion " |
484 | datagen -s1 > tmp1 2> $INTOVOID |
485 | datagen -s2 -g100K > tmp2 2> $INTOVOID |
486 | datagen -s3 -g1M > tmp3 2> $INTOVOID |
487 | println "compress tmp* : " |
488 | zstd -f tmp* |
489 | test -f tmp1.zst |
490 | test -f tmp2.zst |
491 | test -f tmp3.zst |
492 | rm -f tmp1 tmp2 tmp3 |
493 | println "decompress tmp* : " |
494 | zstd -df ./*.zst |
495 | test -f tmp1 |
496 | test -f tmp2 |
497 | test -f tmp3 |
498 | println "compress tmp* into stdout > tmpall : " |
499 | zstd -c tmp1 tmp2 tmp3 > tmpall |
500 | test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) |
501 | println "decompress tmpall* into stdout > tmpdec : " |
502 | cp tmpall tmpall2 |
503 | zstd -dc tmpall* > tmpdec |
504 | test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) |
505 | println "compress multiple files including a missing one (notHere) : " |
506 | zstd -f tmp1 notHere tmp2 && die "missing file not detected!" |
507 | rm -f tmp* |
508 | |
509 | |
510 | if [ "$isWindows" = false ] ; then |
511 | println "\n===> zstd fifo named pipe test " |
512 | echo "Hello World!" > tmp_original |
513 | mkfifo tmp_named_pipe |
514 | # note : fifo test doesn't work in combination with `dd` or `cat` |
515 | echo "Hello World!" > tmp_named_pipe & |
516 | zstd tmp_named_pipe -o tmp_compressed |
517 | zstd -d -o tmp_decompressed tmp_compressed |
518 | $DIFF -s tmp_original tmp_decompressed |
519 | rm -rf tmp* |
520 | fi |
521 | |
522 | println "\n===> zstd created file permissions tests" |
523 | if [ "$isWindows" = false ] ; then |
524 | rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove |
525 | |
526 | ORIGINAL_UMASK=$(umask) |
527 | umask 0000 |
528 | |
529 | datagen > tmp1 |
530 | datagen > tmp2 |
531 | assertFilePermissions tmp1 666 |
532 | assertFilePermissions tmp2 666 |
533 | |
534 | println "test : copy 666 permissions in file -> file compression " |
535 | zstd -f tmp1 -o tmp1.zst |
536 | assertSamePermissions tmp1 tmp1.zst |
537 | println "test : copy 666 permissions in file -> file decompression " |
538 | zstd -f -d tmp1.zst -o tmp1.out |
539 | assertSamePermissions tmp1.zst tmp1.out |
540 | |
541 | rm -f tmp1.zst tmp1.out |
542 | |
543 | println "test : copy 400 permissions in file -> file compression (write to a read-only file) " |
544 | chmod 0400 tmp1 |
545 | assertFilePermissions tmp1 400 |
546 | zstd -f tmp1 -o tmp1.zst |
547 | assertSamePermissions tmp1 tmp1.zst |
548 | println "test : copy 400 permissions in file -> file decompression (write to a read-only file) " |
549 | zstd -f -d tmp1.zst -o tmp1 |
550 | assertSamePermissions tmp1.zst tmp1 |
551 | |
552 | rm -f tmp1.zst tmp1.out |
553 | |
554 | println "test : check created permissions from stdin input in compression " |
555 | zstd -f -o tmp1.zst < tmp1 |
556 | assertFilePermissions tmp1.zst 666 |
557 | println "test : check created permissions from stdin input in decompression " |
558 | zstd -f -d -o tmp1.out < tmp1.zst |
559 | assertFilePermissions tmp1.out 666 |
560 | |
561 | rm -f tmp1.zst tmp1.out |
562 | |
563 | println "test : check created permissions from multiple inputs in compression " |
564 | zstd -f tmp1 tmp2 -o tmp1.zst |
565 | assertFilePermissions tmp1.zst 666 |
566 | println "test : check created permissions from multiple inputs in decompression " |
567 | cp tmp1.zst tmp2.zst |
568 | zstd -f -d tmp1.zst tmp2.zst -o tmp1.out |
569 | assertFilePermissions tmp1.out 666 |
570 | |
571 | rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out |
572 | |
573 | println "test : check permissions on pre-existing output file in compression " |
574 | chmod 0600 tmp1 |
575 | touch tmp1.zst |
576 | chmod 0400 tmp1.zst |
577 | zstd -f tmp1 -o tmp1.zst |
578 | assertFilePermissions tmp1.zst 600 |
579 | println "test : check permissions on pre-existing output file in decompression " |
580 | chmod 0400 tmp1.zst |
581 | touch tmp1.out |
582 | chmod 0200 tmp1.out |
583 | zstd -f -d tmp1.zst -o tmp1.out |
584 | assertFilePermissions tmp1.out 400 |
585 | |
586 | umask 0666 |
587 | chmod 0666 tmp1 tmp2 |
588 | |
589 | rm -f tmp1.zst tmp1.out |
590 | |
591 | println "test : respect umask when compressing from stdin input " |
592 | zstd -f -o tmp1.zst < tmp1 |
593 | assertFilePermissions tmp1.zst 0 |
594 | println "test : respect umask when decompressing from stdin input " |
595 | chmod 0666 tmp1.zst |
596 | zstd -f -d -o tmp1.out < tmp1.zst |
597 | assertFilePermissions tmp1.out 0 |
598 | |
599 | rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out |
600 | umask $ORIGINAL_UMASK |
601 | fi |
602 | |
603 | if [ -n "$DEVNULLRIGHTS" ] ; then |
604 | # these tests requires sudo rights, which is uncommon. |
605 | # they are only triggered if DEVNULLRIGHTS macro is defined. |
606 | println "\n===> checking /dev/null permissions are unaltered " |
607 | datagen > tmp |
608 | sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions |
609 | sudoZstd tmp -c > $INTOVOID |
610 | zstd tmp -f -o tmp.zst |
611 | sudoZstd -d tmp.zst -c > $INTOVOID |
612 | sudoZstd -d tmp.zst -o $INTOVOID |
613 | ls -las $INTOVOID | grep "rw-rw-rw-" |
614 | fi |
615 | |
616 | if [ -n "$READFROMBLOCKDEVICE" ] ; then |
617 | # This creates a temporary block device, which is only possible on unix-y |
618 | # systems, is somewhat invasive, and requires sudo. For these reasons, you |
619 | # have to specifically ask for this test. |
620 | println "\n===> checking that zstd can read from a block device" |
621 | datagen -g65536 > tmp.img |
622 | sudo losetup -fP tmp.img |
623 | LOOP_DEV=$(losetup -a | grep 'tmp\.img' | cut -f1 -d:) |
624 | [ -z "$LOOP_DEV" ] && die "failed to get loopback device" |
625 | sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f" |
626 | sudoZstd -f $LOOP_DEV -c > tmp.img.zst |
627 | zstd -d tmp.img.zst -o tmp.img.copy |
628 | sudo losetup -d $LOOP_DEV |
629 | $DIFF -s tmp.img tmp.img.copy || die "round trip failed" |
630 | rm -f tmp.img tmp.img.zst tmp.img.copy |
631 | fi |
632 | |
633 | println "\n===> zstd created file timestamp tests" |
634 | datagen > tmp |
635 | touch -m -t 200001010000.00 tmp |
636 | println "test : copy mtime in file -> file compression " |
637 | zstd -f tmp -o tmp.zst |
638 | assertSameMTime tmp tmp.zst |
639 | println "test : copy mtime in file -> file decompression " |
640 | zstd -f -d tmp.zst -o tmp.out |
641 | assertSameMTime tmp.zst tmp.out |
642 | rm -f tmp |
643 | |
644 | println "\n===> compress multiple files into an output directory, --output-dir-flat" |
645 | println henlo > tmp1 |
646 | mkdir tmpInputTestDir |
647 | mkdir tmpInputTestDir/we |
648 | mkdir tmpInputTestDir/we/must |
649 | mkdir tmpInputTestDir/we/must/go |
650 | mkdir tmpInputTestDir/we/must/go/deeper |
651 | println cool > tmpInputTestDir/we/must/go/deeper/tmp2 |
652 | mkdir tmpOutDir |
653 | zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir |
654 | test -f tmpOutDir/tmp1.zst |
655 | test -f tmpOutDir/tmp2.zst |
656 | println "test : decompress multiple files into an output directory, --output-dir-flat" |
657 | mkdir tmpOutDirDecomp |
658 | zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp |
659 | test -f tmpOutDirDecomp/tmp2 |
660 | test -f tmpOutDirDecomp/tmp1 |
661 | rm -f tmpOutDirDecomp/* |
662 | zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp |
663 | test -f tmpOutDirDecomp/tmp2 |
664 | test -f tmpOutDirDecomp/tmp1 |
665 | rm -rf tmp* |
666 | |
667 | if [ "$isWindows" = false ] ; then |
668 | println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror" |
669 | println "test --output-dir-mirror" > tmp1 |
670 | mkdir -p tmpInputTestDir/we/.../..must/go/deeper.. |
671 | println cool > tmpInputTestDir/we/.../..must/go/deeper../tmp2 |
672 | zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir |
673 | test -f tmpOutDir/tmp1.zst |
674 | test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst |
675 | |
676 | println "test: compress input dir will be ignored if it has '..'" |
677 | zstd -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" |
678 | zstd -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input cannot contain '..'" |
679 | zstd -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" |
680 | test ! -d non-exist |
681 | |
682 | println "test: compress input dir should succeed with benign uses of '..'" |
683 | zstd -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout |
684 | test -d tmpout |
685 | |
686 | println "test : decompress multiple files into an output directory, --output-dir-mirror" |
687 | zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp |
688 | test -f tmpOutDirDecomp/tmpOutDir/tmp1 |
689 | test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2 |
690 | |
691 | println "test: decompress input dir will be ignored if it has '..'" |
692 | zstd -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "input cannot contain '..'" |
693 | test ! -d non-exist |
694 | |
695 | rm -rf tmp* |
696 | fi |
697 | |
698 | |
699 | println "test : compress multiple files reading them from a file, --filelist=FILE" |
700 | println "Hello world!, file1" > tmp1 |
701 | println "Hello world!, file2" > tmp2 |
702 | println tmp1 > tmp_fileList |
703 | println tmp2 >> tmp_fileList |
704 | zstd -f --filelist=tmp_fileList |
705 | test -f tmp2.zst |
706 | test -f tmp1.zst |
707 | |
708 | println "test : alternate syntax: --filelist FILE" |
709 | zstd -f --filelist tmp_fileList |
710 | test -f tmp2.zst |
711 | test -f tmp1.zst |
712 | |
713 | println "test : reading file list from a symlink, --filelist=FILE" |
714 | rm -f *.zst |
715 | ln -s tmp_fileList tmp_symLink |
716 | zstd -f --filelist=tmp_symLink |
717 | test -f tmp2.zst |
718 | test -f tmp1.zst |
719 | |
720 | println "test : compress multiple files reading them from multiple files, --filelist=FILE" |
721 | rm -f *.zst |
722 | println "Hello world!, file3" > tmp3 |
723 | println "Hello world!, file4" > tmp4 |
724 | println tmp3 > tmp_fileList2 |
725 | println tmp4 >> tmp_fileList2 |
726 | zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2 |
727 | test -f tmp1.zst |
728 | test -f tmp2.zst |
729 | test -f tmp3.zst |
730 | test -f tmp4.zst |
731 | |
732 | println "test : decompress multiple files reading them from a file, --filelist=FILE" |
733 | rm -f tmp1 tmp2 |
734 | println tmp1.zst > tmpZst |
735 | println tmp2.zst >> tmpZst |
736 | zstd -d -f --filelist=tmpZst |
737 | test -f tmp1 |
738 | test -f tmp2 |
739 | |
740 | println "test : decompress multiple files reading them from multiple files, --filelist=FILE" |
741 | rm -f tmp1 tmp2 tmp3 tmp4 |
742 | println tmp3.zst > tmpZst2 |
743 | println tmp4.zst >> tmpZst2 |
744 | zstd -d -f --filelist=tmpZst --filelist=tmpZst2 |
745 | test -f tmp1 |
746 | test -f tmp2 |
747 | test -f tmp3 |
748 | test -f tmp4 |
749 | |
750 | println "test : survive the list of files with too long filenames (--filelist=FILE)" |
751 | datagen -g5M > tmp_badList |
752 | zstd -qq -f --filelist=tmp_badList && die "should have failed : file name length is too long" # printing very long text garbage on console will cause CI failure |
753 | |
754 | println "test : survive a list of files which is text garbage (--filelist=FILE)" |
755 | datagen > tmp_badList |
756 | zstd -qq -f --filelist=tmp_badList && die "should have failed : list is text garbage" # printing very long text garbage on console will cause CI failure |
757 | |
758 | println "test : survive a list of files which is binary garbage (--filelist=FILE)" |
759 | datagen -P0 -g1M > tmp_badList |
760 | zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console |
761 | |
762 | println "test : try to overflow internal list of files (--filelist=FILE)" |
763 | touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 |
764 | ls tmp* > tmpList |
765 | zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list |
766 | rm -rf tmp* |
767 | |
768 | println "\n===> --[no-]content-size tests" |
769 | |
770 | datagen > tmp_contentsize |
771 | zstd -f tmp_contentsize |
772 | zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" |
773 | zstd -f --no-content-size tmp_contentsize |
774 | zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die |
775 | zstd -f --content-size tmp_contentsize |
776 | zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" |
777 | zstd -f --content-size --no-content-size tmp_contentsize |
778 | zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die |
779 | rm -rf tmp* |
780 | |
781 | println "test : show-default-cparams regular" |
782 | datagen > tmp |
783 | zstd --show-default-cparams -f tmp |
784 | zstd --show-default-cparams -d tmp.zst && die "error: can't use --show-default-cparams in decompression mode" |
785 | rm -rf tmp* |
786 | |
787 | println "test : show-default-cparams recursive" |
788 | mkdir tmp_files |
789 | datagen -g15000 > tmp_files/tmp1 |
790 | datagen -g129000 > tmp_files/tmp2 |
791 | datagen -g257000 > tmp_files/tmp3 |
792 | zstd --show-default-cparams -f -r tmp_files |
793 | rm -rf tmp* |
794 | |
795 | println "test : show compression parameters in verbose mode" |
796 | datagen > tmp |
797 | zstd -vv tmp 2>&1 | \ |
798 | grep -q -E -- "--zstd=wlog=[[:digit:]]+,clog=[[:digit:]]+,hlog=[[:digit:]]+,\ |
799 | slog=[[:digit:]]+,mml=[[:digit:]]+,tlen=[[:digit:]]+,strat=[[:digit:]]+" |
800 | rm -rf tmp* |
801 | |
802 | println "\n===> Advanced compression parameters " |
803 | println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" |
804 | println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" |
805 | println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" |
806 | println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist |
807 | test ! -f tmp.zst # tmp.zst should not be created |
808 | roundTripTest -g512K |
809 | roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" |
810 | roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" |
811 | roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" |
812 | roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" |
813 | roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" |
814 | roundTripTest -g64K "19 --zstd=strat=9" # btultra2 |
815 | |
816 | |
817 | println "\n===> Pass-Through mode " |
818 | println "Hello world 1!" | zstd -df |
819 | println "Hello world 2!" | zstd -dcf |
820 | println "Hello world 3!" > tmp1 |
821 | zstd -dcf tmp1 |
822 | println "" | zstd -df > tmp1 |
823 | println "" > tmp2 |
824 | $DIFF -q tmp1 tmp2 |
825 | println "1" | zstd -df > tmp1 |
826 | println "1" > tmp2 |
827 | $DIFF -q tmp1 tmp2 |
828 | println "12" | zstd -df > tmp1 |
829 | println "12" > tmp2 |
830 | $DIFF -q tmp1 tmp2 |
831 | rm -rf tmp* |
832 | |
833 | |
834 | println "\n===> frame concatenation " |
835 | println "hello " > hello.tmp |
836 | println "world!" > world.tmp |
837 | cat hello.tmp world.tmp > helloworld.tmp |
838 | zstd -c hello.tmp > hello.zst |
839 | zstd -c world.tmp > world.zst |
840 | zstd -c hello.tmp world.tmp > helloworld.zst |
841 | zstd -dc helloworld.zst > result.tmp |
842 | $DIFF helloworld.tmp result.tmp |
843 | cat hello.zst world.zst > helloworld.zst |
844 | zstd -dc helloworld.zst > result.tmp |
845 | cat result.tmp |
846 | $DIFF helloworld.tmp result.tmp |
847 | println "frame concatenation without checksum" |
848 | zstd -c hello.tmp > hello.zst --no-check |
849 | zstd -c world.tmp > world.zst --no-check |
850 | cat hello.zst world.zst > helloworld.zstd |
851 | zstd -dc helloworld.zst > result.tmp |
852 | $DIFF helloworld.tmp result.tmp |
853 | println "testing zstdcat symlink" |
854 | ln -sf "$ZSTD_BIN" zstdcat |
855 | $EXE_PREFIX ./zstdcat helloworld.zst > result.tmp |
856 | $DIFF helloworld.tmp result.tmp |
857 | ln -s helloworld.zst helloworld.link.zst |
858 | $EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp |
859 | $DIFF helloworld.tmp result.tmp |
860 | rm -f zstdcat |
861 | rm -f result.tmp |
862 | println "testing zcat symlink" |
863 | ln -sf "$ZSTD_BIN" zcat |
864 | $EXE_PREFIX ./zcat helloworld.zst > result.tmp |
865 | $DIFF helloworld.tmp result.tmp |
866 | $EXE_PREFIX ./zcat helloworld.link.zst > result.tmp |
867 | $DIFF helloworld.tmp result.tmp |
868 | rm -f zcat |
869 | rm -f ./*.tmp ./*.zstd |
870 | println "frame concatenation tests completed" |
871 | |
872 | |
873 | if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] && [ "$UNAME" != "AIX" ]; then |
874 | println "\n**** flush write error test **** " |
875 | |
876 | println "println foo | zstd > /dev/full" |
877 | println foo | zstd > /dev/full && die "write error not detected!" |
878 | println "println foo | zstd | zstd -d > /dev/full" |
879 | println foo | zstd | zstd -d > /dev/full && die "write error not detected!" |
880 | |
881 | fi |
882 | |
883 | |
884 | if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then |
885 | |
886 | println "\n===> symbolic link test " |
887 | |
888 | rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst |
889 | println "hello world" > hello.tmp |
890 | ln -s hello.tmp world.tmp |
891 | ln -s hello.tmp world2.tmp |
892 | zstd world.tmp hello.tmp || true |
893 | test -f hello.tmp.zst # regular file should have been compressed! |
894 | test ! -f world.tmp.zst # symbolic link should not have been compressed! |
895 | zstd world.tmp || true |
896 | test ! -f world.tmp.zst # symbolic link should not have been compressed! |
897 | zstd world.tmp world2.tmp || true |
898 | test ! -f world.tmp.zst # symbolic link should not have been compressed! |
899 | test ! -f world2.tmp.zst # symbolic link should not have been compressed! |
900 | zstd world.tmp hello.tmp -f |
901 | test -f world.tmp.zst # symbolic link should have been compressed with --force |
902 | rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst |
903 | |
904 | fi |
905 | |
906 | |
907 | println "\n===> test sparse file support " |
908 | |
909 | datagen -g5M -P100 > tmpSparse |
910 | zstd tmpSparse -c | zstd -dv -o tmpSparseRegen |
911 | $DIFF -s tmpSparse tmpSparseRegen |
912 | zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse |
913 | $DIFF -s tmpSparse tmpOutSparse |
914 | zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse |
915 | $DIFF -s tmpSparse tmpOutNoSparse |
916 | ls -ls tmpSparse* # look at file size and block size on disk |
917 | datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) |
918 | datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd |
919 | ls -ls tmpSparseOdd # look at file size and block size on disk |
920 | println "\n Sparse Compatibility with Console :" |
921 | println "Hello World 1 !" | zstd | zstd -d -c |
922 | println "Hello World 2 !" | zstd | zstd -d | cat |
923 | println "\n Sparse Compatibility with Append :" |
924 | datagen -P100 -g1M > tmpSparse1M |
925 | cat tmpSparse1M tmpSparse1M > tmpSparse2M |
926 | zstd -v -f tmpSparse1M -o tmpSparseCompressed |
927 | zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated |
928 | zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated |
929 | ls -ls tmpSparse* # look at file size and block size on disk |
930 | $DIFF tmpSparse2M tmpSparseRegenerated |
931 | rm -f tmpSparse* |
932 | |
933 | |
934 | println "\n===> stream-size mode" |
935 | |
936 | datagen -g11000 > tmp |
937 | println "test : basic file compression vs sized streaming compression" |
938 | file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) |
939 | stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c) |
940 | if [ "$stream_size" -gt "$file_size" ]; then |
941 | die "hinted compression larger than expected" |
942 | fi |
943 | println "test : sized streaming compression and decompression" |
944 | cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000 |
945 | zstd -df tmp.zst -o tmp_decompress |
946 | cmp tmp tmp_decompress || die "difference between original and decompressed file" |
947 | println "test : incorrect stream size" |
948 | cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" |
949 | |
950 | println "\n===> zstd zero weight dict test " |
951 | rm -f tmp* |
952 | cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input |
953 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input |
954 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp |
955 | $DIFF tmp_decomp tmp_input |
956 | rm -rf tmp* |
957 | |
958 | println "\n===> zstd (valid) zero weight dict test " |
959 | rm -f tmp* |
960 | # 0 has a non-zero weight in the dictionary |
961 | echo "0000000000000000000000000" > tmp_input |
962 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input |
963 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp |
964 | $DIFF tmp_decomp tmp_input |
965 | rm -rf tmp* |
966 | |
967 | println "\n===> size-hint mode" |
968 | |
969 | datagen -g11000 > tmp |
970 | datagen -g11000 > tmp2 |
971 | datagen > tmpDict |
972 | println "test : basic file compression vs hinted streaming compression" |
973 | file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) |
974 | stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c) |
975 | if [ "$stream_size" -ge "$file_size" ]; then |
976 | die "hinted compression larger than expected" |
977 | fi |
978 | println "test : hinted streaming compression and decompression" |
979 | cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000 |
980 | zstd -df tmp.zst -o tmp_decompress |
981 | cmp tmp tmp_decompress || die "difference between original and decompressed file" |
982 | println "test : hinted streaming compression with dictionary" |
983 | cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict |
984 | println "test : multiple file compression with hints and dictionary" |
985 | zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2 |
986 | zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp |
987 | zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2 |
988 | cmp tmp.zst tmp1_.zst || die "first file's output differs" |
989 | cmp tmp2.zst tmp2_.zst || die "second file's output differs" |
990 | println "test : incorrect hinted stream sizes" |
991 | cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high |
992 | cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low |
993 | cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high |
994 | cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low |
995 | println "test : allows and interprets K,KB,KiB,M,MB and MiB suffix" |
996 | cat tmp | zstd -14 -f --size-hint=11K | zstd -t |
997 | cat tmp | zstd -14 -f --size-hint=11KB | zstd -t |
998 | cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t |
999 | cat tmp | zstd -14 -f --size-hint=1M | zstd -t |
1000 | cat tmp | zstd -14 -f --size-hint=1MB | zstd -t |
1001 | cat tmp | zstd -14 -f --size-hint=1MiB | zstd -t |
1002 | |
1003 | |
1004 | println "\n===> dictionary tests " |
1005 | println "- Test high/low compressibility corpus training" |
1006 | datagen -g12M -P90 > tmpCorpusHighCompress |
1007 | datagen -g12M -P5 > tmpCorpusLowCompress |
1008 | zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress |
1009 | zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress |
1010 | rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress |
1011 | println "- Test with raw dict (content only) " |
1012 | datagen > tmpDict |
1013 | datagen -g1M | $MD5SUM > tmp1 |
1014 | datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2 |
1015 | $DIFF -q tmp1 tmp2 |
1016 | println "- Create first dictionary " |
1017 | TESTFILE="$PRGDIR"/zstdcli.c |
1018 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1019 | cp "$TESTFILE" tmp |
1020 | println "- Test dictionary compression with tmpDict as an input file and dictionary" |
1021 | zstd -f tmpDict -D tmpDict && die "compression error not detected!" |
1022 | println "- Dictionary compression roundtrip" |
1023 | zstd -f tmp -D tmpDict |
1024 | zstd -d tmp.zst -D tmpDict -fo result |
1025 | $DIFF "$TESTFILE" result |
1026 | println "- Dictionary compression with hlog < clog" |
1027 | zstd -6f tmp -D tmpDict --zstd=clog=25,hlog=23 |
1028 | println "- Dictionary compression with btlazy2 strategy" |
1029 | zstd -f tmp -D tmpDict --zstd=strategy=6 |
1030 | zstd -d tmp.zst -D tmpDict -fo result |
1031 | $DIFF "$TESTFILE" result |
1032 | if [ -e /proc/self/fd/0 ]; then |
1033 | println "- Test rejecting irregular dictionary file" |
1034 | cat tmpDict | zstd -f tmp -D /proc/self/fd/0 && die "Piped dictionary should fail!" |
1035 | cat tmpDict | zstd -d tmp.zst -D /proc/self/fd/0 -f && die "Piped dictionary should fail!" |
1036 | fi |
1037 | if [ -n "$hasMT" ] |
1038 | then |
1039 | println "- Test dictionary compression with multithreading " |
1040 | datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict # fails with v1.3.2 |
1041 | fi |
1042 | println "- Create second (different) dictionary " |
1043 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
1044 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
1045 | println "- Create dictionary with short dictID" |
1046 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
1047 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
1048 | println "- Create dictionary with wrong dictID parameter order (must fail)" |
1049 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " |
1050 | println "- Create dictionary with size limit" |
1051 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v |
1052 | println "- Create dictionary with small size limit" |
1053 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v |
1054 | println "- Create dictionary with wrong parameter order (must fail)" |
1055 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " |
1056 | println "- Compress without dictID" |
1057 | zstd -f tmp -D tmpDict1 --no-dictID |
1058 | zstd -d tmp.zst -D tmpDict -fo result |
1059 | $DIFF "$TESTFILE" result |
1060 | println "- Compress multiple files with dictionary" |
1061 | rm -rf dirTestDict |
1062 | mkdir dirTestDict |
1063 | cp "$TESTDIR"/*.c dirTestDict |
1064 | cp "$PRGDIR"/*.c dirTestDict |
1065 | cp "$PRGDIR"/*.h dirTestDict |
1066 | $MD5SUM dirTestDict/* > tmph1 |
1067 | zstd -f --rm dirTestDict/* -D tmpDictC |
1068 | zstd -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default |
1069 | case "$UNAME" in |
1070 | Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 |
1071 | *) $MD5SUM -c tmph1 ;; |
1072 | esac |
1073 | rm -rf dirTestDict |
1074 | println "- dictionary builder on bogus input" |
1075 | println "Hello World" > tmp |
1076 | zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" |
1077 | datagen -P0 -g10M > tmp |
1078 | zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" |
1079 | println "- Test -o before --train" |
1080 | rm -f tmpDict dictionary |
1081 | zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c |
1082 | test -f tmpDict |
1083 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c |
1084 | test -f dictionary |
1085 | if [ -n "$hasMT" ] |
1086 | then |
1087 | println "- Create dictionary with multithreading enabled" |
1088 | zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1089 | fi |
1090 | rm -f tmp* dictionary |
1091 | |
1092 | println "- Test --memory for dictionary compression" |
1093 | datagen -g12M -P90 > tmpCorpusHighCompress |
1094 | zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)" |
1095 | zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr |
1096 | cat zstTrainWithMemLimitStdErr | grep "setting manual memory limit for dictionary training data at 5 MB" |
1097 | cat zstTrainWithMemLimitStdErr | grep "Training samples set too large (12 MB); training on 5 MB only..." |
1098 | rm zstTrainWithMemLimitStdErr |
1099 | |
1100 | println "\n===> fastCover dictionary builder : advanced options " |
1101 | TESTFILE="$PRGDIR"/zstdcli.c |
1102 | datagen > tmpDict |
1103 | println "- Create first dictionary" |
1104 | zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1105 | cp "$TESTFILE" tmp |
1106 | zstd -f tmp -D tmpDict |
1107 | zstd -d tmp.zst -D tmpDict -fo result |
1108 | $DIFF "$TESTFILE" result |
1109 | println "- Create second (different) dictionary" |
1110 | zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
1111 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
1112 | zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file" |
1113 | println "- Create dictionary with short dictID" |
1114 | zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
1115 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
1116 | println "- Create dictionaries with shrink-dict flag enabled" |
1117 | zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict |
1118 | zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 |
1119 | zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 |
1120 | zstd --train-fastcover=shrink=5,steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict3 |
1121 | println "- Create dictionary with size limit" |
1122 | zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K |
1123 | println "- Create dictionary using all samples for both training and testing" |
1124 | zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1125 | println "- Create dictionary using f=16" |
1126 | zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1127 | zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" |
1128 | println "- Create dictionary using accel=2" |
1129 | zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1130 | println "- Create dictionary using accel=10" |
1131 | zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1132 | println "- Create dictionary with multithreading" |
1133 | zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1134 | println "- Test -o before --train-fastcover" |
1135 | rm -f tmpDict dictionary |
1136 | zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c |
1137 | test -f tmpDict |
1138 | zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c |
1139 | test -f dictionary |
1140 | rm -f tmp* dictionary |
1141 | |
1142 | |
1143 | println "\n===> legacy dictionary builder " |
1144 | |
1145 | TESTFILE="$PRGDIR"/zstdcli.c |
1146 | datagen > tmpDict |
1147 | println "- Create first dictionary" |
1148 | zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1149 | cp "$TESTFILE" tmp |
1150 | zstd -f tmp -D tmpDict |
1151 | zstd -d tmp.zst -D tmpDict -fo result |
1152 | $DIFF "$TESTFILE" result |
1153 | zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)" |
1154 | println "- Create second (different) dictionary" |
1155 | zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
1156 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
1157 | println "- Create dictionary with short dictID" |
1158 | zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
1159 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
1160 | println "- Create dictionary with size limit" |
1161 | zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K |
1162 | println "- Test -o before --train-legacy" |
1163 | rm -f tmpDict dictionary |
1164 | zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c |
1165 | test -f tmpDict |
1166 | zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c |
1167 | test -f dictionary |
1168 | rm -f tmp* dictionary |
1169 | |
1170 | |
1171 | println "\n===> integrity tests " |
1172 | |
1173 | println "test one file (tmp1.zst) " |
1174 | datagen > tmp1 |
1175 | zstd tmp1 |
1176 | zstd -t tmp1.zst |
1177 | zstd --test tmp1.zst |
1178 | println "test multiple files (*.zst) " |
1179 | zstd -t ./*.zst |
1180 | println "test bad files (*) " |
1181 | zstd -t ./* && die "bad files not detected !" |
1182 | zstd -t tmp1 && die "bad file not detected !" |
1183 | cp tmp1 tmp2.zst |
1184 | zstd -t tmp2.zst && die "bad file not detected !" |
1185 | datagen -g0 > tmp3 |
1186 | zstd -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad |
1187 | println "test --rm and --test combined " |
1188 | zstd -t --rm tmp1.zst |
1189 | test -f tmp1.zst # check file is still present |
1190 | cp tmp1.zst tmp2.zst |
1191 | zstd -t tmp1.zst tmp2.zst --rm |
1192 | test -f tmp1.zst # check file is still present |
1193 | test -f tmp2.zst # check file is still present |
1194 | split -b16384 tmp1.zst tmpSplit. |
1195 | zstd -t tmpSplit.* && die "bad file not detected !" |
1196 | datagen | zstd -c | zstd -t |
1197 | |
1198 | |
1199 | println "\n===> golden files tests " |
1200 | |
1201 | zstd -t -r "$TESTDIR/golden-decompression" |
1202 | zstd -c -r "$TESTDIR/golden-compression" | zstd -t |
1203 | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http" -c | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" -t |
1204 | |
1205 | |
1206 | println "\n===> benchmark mode tests " |
1207 | |
1208 | println "bench one file" |
1209 | datagen > tmp1 |
1210 | zstd -bi0 tmp1 |
1211 | println "bench multiple levels" |
1212 | zstd -i0b0e3 tmp1 |
1213 | println "bench negative level" |
1214 | zstd -bi0 --fast tmp1 |
1215 | println "with recursive and quiet modes" |
1216 | zstd -rqi0b1e2 tmp1 |
1217 | println "benchmark decompression only" |
1218 | zstd -f tmp1 |
1219 | zstd -b -d -i0 tmp1.zst |
1220 | println "benchmark can fail - decompression on invalid data" |
1221 | zstd -b -d -i0 tmp1 && die "invalid .zst data => benchmark should have failed" |
1222 | |
1223 | GZIPMODE=1 |
1224 | zstd --format=gzip -V || GZIPMODE=0 |
1225 | if [ $GZIPMODE -eq 1 ]; then |
1226 | println "benchmark mode is only compatible with zstd" |
1227 | zstd --format=gzip -b tmp1 && die "-b should be incompatible with gzip format!" |
1228 | fi |
1229 | |
1230 | println "\n===> zstd compatibility tests " |
1231 | |
1232 | datagen > tmp |
1233 | rm -f tmp.zst |
1234 | zstd --format=zstd -f tmp |
1235 | test -f tmp.zst |
1236 | |
1237 | |
1238 | println "\n===> gzip compatibility tests " |
1239 | |
1240 | GZIPMODE=1 |
1241 | zstd --format=gzip -V || GZIPMODE=0 |
1242 | if [ $GZIPMODE -eq 1 ]; then |
1243 | println "gzip support detected" |
1244 | GZIPEXE=1 |
1245 | gzip -V || GZIPEXE=0 |
1246 | if [ $GZIPEXE -eq 1 ]; then |
1247 | datagen > tmp |
1248 | zstd --format=gzip -f tmp |
1249 | gzip -t -v tmp.gz |
1250 | gzip -f tmp |
1251 | zstd -d -f -v tmp.gz |
1252 | rm -f tmp* |
1253 | else |
1254 | println "gzip binary not detected" |
1255 | fi |
1256 | else |
1257 | println "gzip mode not supported" |
1258 | fi |
1259 | |
1260 | |
1261 | println "\n===> gzip frame tests " |
1262 | |
1263 | if [ $GZIPMODE -eq 1 ]; then |
1264 | datagen > tmp |
1265 | zstd -f --format=gzip tmp |
1266 | zstd -f tmp |
1267 | cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp |
1268 | truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
1269 | rm -f tmp* |
1270 | else |
1271 | println "gzip mode not supported" |
1272 | fi |
1273 | |
1274 | if [ $GZIPMODE -eq 1 ]; then |
1275 | datagen > tmp |
1276 | rm -f tmp.zst |
1277 | zstd --format=gzip --format=zstd -f tmp |
1278 | test -f tmp.zst |
1279 | fi |
1280 | |
1281 | println "\n===> xz compatibility tests " |
1282 | |
1283 | LZMAMODE=1 |
1284 | zstd --format=xz -V || LZMAMODE=0 |
1285 | if [ $LZMAMODE -eq 1 ]; then |
1286 | println "xz support detected" |
1287 | XZEXE=1 |
1288 | xz -Q -V && lzma -Q -V || XZEXE=0 |
1289 | if [ $XZEXE -eq 1 ]; then |
1290 | println "Testing zstd xz and lzma support" |
1291 | datagen > tmp |
1292 | zstd --format=lzma -f tmp |
1293 | zstd --format=xz -f tmp |
1294 | xz -Q -t -v tmp.xz |
1295 | xz -Q -t -v tmp.lzma |
1296 | xz -Q -f -k tmp |
1297 | lzma -Q -f -k --lzma1 tmp |
1298 | zstd -d -f -v tmp.xz |
1299 | zstd -d -f -v tmp.lzma |
1300 | rm -f tmp* |
1301 | println "Creating symlinks" |
1302 | ln -s "$ZSTD_BIN" ./xz |
1303 | ln -s "$ZSTD_BIN" ./unxz |
1304 | ln -s "$ZSTD_BIN" ./lzma |
1305 | ln -s "$ZSTD_BIN" ./unlzma |
1306 | println "Testing xz and lzma symlinks" |
1307 | datagen > tmp |
1308 | ./xz tmp |
1309 | xz -Q -d tmp.xz |
1310 | ./lzma tmp |
1311 | lzma -Q -d tmp.lzma |
1312 | println "Testing unxz and unlzma symlinks" |
1313 | xz -Q tmp |
1314 | ./xz -d tmp.xz |
1315 | lzma -Q tmp |
1316 | ./lzma -d tmp.lzma |
1317 | rm -f xz unxz lzma unlzma |
1318 | rm -f tmp* |
1319 | else |
1320 | println "xz binary not detected" |
1321 | fi |
1322 | else |
1323 | println "xz mode not supported" |
1324 | fi |
1325 | |
1326 | |
1327 | println "\n===> xz frame tests " |
1328 | |
1329 | if [ $LZMAMODE -eq 1 ]; then |
1330 | datagen > tmp |
1331 | zstd -f --format=xz tmp |
1332 | zstd -f --format=lzma tmp |
1333 | zstd -f tmp |
1334 | cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp |
1335 | truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
1336 | truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
1337 | rm -f tmp* |
1338 | else |
1339 | println "xz mode not supported" |
1340 | fi |
1341 | |
1342 | println "\n===> lz4 compatibility tests " |
1343 | |
1344 | LZ4MODE=1 |
1345 | zstd --format=lz4 -V || LZ4MODE=0 |
1346 | if [ $LZ4MODE -eq 1 ]; then |
1347 | println "lz4 support detected" |
1348 | LZ4EXE=1 |
1349 | lz4 -V || LZ4EXE=0 |
1350 | if [ $LZ4EXE -eq 1 ]; then |
1351 | datagen > tmp |
1352 | zstd --format=lz4 -f tmp |
1353 | lz4 -t -v tmp.lz4 |
1354 | lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout |
1355 | zstd -d -f -v tmp.lz4 |
1356 | rm -f tmp* |
1357 | else |
1358 | println "lz4 binary not detected" |
1359 | fi |
1360 | else |
1361 | println "lz4 mode not supported" |
1362 | fi |
1363 | |
1364 | |
1365 | if [ $LZ4MODE -eq 1 ]; then |
1366 | println "\n===> lz4 frame tests " |
1367 | datagen > tmp |
1368 | zstd -f --format=lz4 tmp |
1369 | zstd -f tmp |
1370 | cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp |
1371 | truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
1372 | rm -f tmp* |
1373 | else |
1374 | println "\nlz4 mode not supported" |
1375 | fi |
1376 | |
1377 | |
1378 | println "\n===> suffix list test" |
1379 | |
1380 | ! zstd -d tmp.abc 2> tmplg |
1381 | |
1382 | if [ $GZIPMODE -ne 1 ]; then |
1383 | grep ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" |
1384 | fi |
1385 | |
1386 | if [ $LZMAMODE -ne 1 ]; then |
1387 | grep ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" |
1388 | grep ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" |
1389 | fi |
1390 | |
1391 | if [ $LZ4MODE -ne 1 ]; then |
1392 | grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" |
1393 | fi |
1394 | |
1395 | touch tmp1 |
1396 | zstd tmp1 -o tmp1.zstd |
1397 | zstd -d -f tmp1.zstd # support .zstd suffix even though it's not the default suffix |
1398 | |
1399 | println "\n===> tar extension tests " |
1400 | |
1401 | rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd |
1402 | |
1403 | datagen > tmp |
1404 | tar -cf tmp.tar tmp |
1405 | zstd tmp.tar -o tmp.tzst |
1406 | rm -f tmp.tar |
1407 | zstd -d tmp.tzst |
1408 | [ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" |
1409 | rm -f tmp.tar tmp.tzst |
1410 | |
1411 | if [ $GZIPMODE -eq 1 ]; then |
1412 | tar -f - -c tmp | gzip > tmp.tgz |
1413 | zstd -d tmp.tgz |
1414 | [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!" |
1415 | rm -f tmp.tar tmp.tgz |
1416 | fi |
1417 | |
1418 | if [ $LZMAMODE -eq 1 ]; then |
1419 | tar -f - -c tmp | zstd --format=xz > tmp.txz |
1420 | zstd -d tmp.txz |
1421 | [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" |
1422 | rm -f tmp.tar tmp.txz |
1423 | fi |
1424 | |
1425 | if [ $LZ4MODE -eq 1 ]; then |
1426 | tar -f - -c tmp | zstd --format=lz4 > tmp.tlz4 |
1427 | zstd -d tmp.tlz4 |
1428 | [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" |
1429 | rm -f tmp.tar tmp.tlz4 |
1430 | fi |
1431 | |
1432 | touch tmp.t tmp.tz tmp.tzs |
1433 | ! zstd -d tmp.t |
1434 | ! zstd -d tmp.tz |
1435 | ! zstd -d tmp.tzs |
1436 | |
1437 | |
1438 | println "\n===> zstd round-trip tests " |
1439 | |
1440 | roundTripTest |
1441 | roundTripTest -g15K # TableID==3 |
1442 | roundTripTest -g127K # TableID==2 |
1443 | roundTripTest -g255K # TableID==1 |
1444 | roundTripTest -g522K # TableID==0 |
1445 | roundTripTest -g519K 6 # greedy, hash chain |
1446 | roundTripTest -g517K 16 # btlazy2 |
1447 | roundTripTest -g516K 19 # btopt |
1448 | |
1449 | fileRoundTripTest -g500K |
1450 | |
1451 | println "\n===> zstd long distance matching round-trip tests " |
1452 | roundTripTest -g0 "2 --single-thread --long" |
1453 | roundTripTest -g1000K "1 --single-thread --long" |
1454 | roundTripTest -g517K "6 --single-thread --long" |
1455 | roundTripTest -g516K "16 --single-thread --long" |
1456 | roundTripTest -g518K "19 --single-thread --long" |
1457 | roundTripTest -g2M "22 --single-thread --ultra --long" |
1458 | fileRoundTripTest -g5M "3 --single-thread --long" |
1459 | |
1460 | |
1461 | roundTripTest -g96K "5 --single-thread" |
1462 | if [ -n "$hasMT" ] |
1463 | then |
1464 | println "\n===> zstdmt round-trip tests " |
1465 | roundTripTest -g4M "1 -T0" |
1466 | roundTripTest -g4M "1 -T0 --auto-threads=physical" |
1467 | roundTripTest -g4M "1 -T0 --auto-threads=logical" |
1468 | roundTripTest -g8M "3 -T2" |
1469 | roundTripTest -g8M "19 --long" |
1470 | roundTripTest -g8000K "2 --threads=2" |
1471 | fileRoundTripTest -g4M "19 -T2 -B1M" |
1472 | |
1473 | println "\n===> zstdmt long distance matching round-trip tests " |
1474 | roundTripTest -g8M "3 --long=24 -T2" |
1475 | |
1476 | println "\n===> zstdmt environment variable tests " |
1477 | echo "multifoo" >> mt_tmp |
1478 | ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting |
1479 | ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting |
1480 | ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting |
1481 | ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting |
1482 | ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting |
1483 | ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting |
1484 | ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= |
1485 | ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage |
1486 | ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread |
1487 | # temporary envvar changes in the above tests would actually persist in macos /bin/sh |
1488 | unset ZSTD_NBTHREADS |
1489 | rm -f mt_tmp* |
1490 | |
1491 | println "\n===> ovLog tests " |
1492 | datagen -g2MB > tmp |
1493 | refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c) |
1494 | ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) |
1495 | ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) |
1496 | if [ "$refSize" -eq "$ov9Size" ]; then |
1497 | echo ov9Size should be different from refSize |
1498 | exit 1 |
1499 | fi |
1500 | if [ "$refSize" -eq "$ov1Size" ]; then |
1501 | echo ov1Size should be different from refSize |
1502 | exit 1 |
1503 | fi |
1504 | if [ "$ov9Size" -ge "$ov1Size" ]; then |
1505 | echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size" |
1506 | exit 1 |
1507 | fi |
1508 | |
1509 | else |
1510 | println "\n===> no multithreading, skipping zstdmt tests " |
1511 | fi |
1512 | |
1513 | rm -f tmp* |
1514 | |
1515 | println "\n===> zstd --list/-l single frame tests " |
1516 | datagen > tmp1 |
1517 | datagen > tmp2 |
1518 | datagen > tmp3 |
1519 | zstd tmp* |
1520 | zstd -l ./*.zst |
1521 | zstd -lv ./*.zst | grep "Decompressed Size:" # check that decompressed size is present in header |
1522 | zstd --list ./*.zst |
1523 | zstd --list -v ./*.zst |
1524 | |
1525 | println "\n===> zstd --list/-l multiple frame tests " |
1526 | cat tmp1.zst tmp2.zst > tmp12.zst |
1527 | cat tmp12.zst tmp3.zst > tmp123.zst |
1528 | zstd -l ./*.zst |
1529 | zstd -lv ./*.zst |
1530 | |
1531 | println "\n===> zstd --list/-l error detection tests " |
1532 | zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file" |
1533 | zstd --list tmp* && die "-l must fail on non-zstd file" |
1534 | zstd -lv tmp1* && die "-l must fail on non-zstd file" |
1535 | zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" |
1536 | |
1537 | println "test : detect truncated compressed file " |
1538 | TEST_DATA_FILE=truncatable-input.txt |
1539 | FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst |
1540 | TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst |
1541 | datagen -g50000 > $TEST_DATA_FILE |
1542 | zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE |
1543 | dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE |
1544 | zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" |
1545 | |
1546 | rm -f $TEST_DATA_FILE |
1547 | rm -f $FULL_COMPRESSED_FILE |
1548 | rm -f $TRUNCATED_COMPRESSED_FILE |
1549 | |
1550 | println "\n===> zstd --list/-l errors when presented with stdin / no files" |
1551 | zstd -l && die "-l must fail on empty list of files" |
1552 | zstd -l - && die "-l does not work on stdin" |
1553 | zstd -l < tmp1.zst && die "-l does not work on stdin" |
1554 | zstd -l - < tmp1.zst && die "-l does not work on stdin" |
1555 | zstd -l - tmp1.zst && die "-l does not work on stdin" |
1556 | zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin" |
1557 | zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty |
1558 | |
1559 | println "\n===> zstd --list/-l test with null files " |
1560 | datagen -g0 > tmp5 |
1561 | zstd tmp5 |
1562 | zstd -l tmp5.zst |
1563 | zstd -l tmp5* && die "-l must fail on non-zstd file" |
1564 | zstd -lv tmp5.zst | grep "Decompressed Size: 0 B (0 B)" # check that 0 size is present in header |
1565 | zstd -lv tmp5* && die "-l must fail on non-zstd file" |
1566 | |
1567 | println "\n===> zstd --list/-l test with no content size field " |
1568 | datagen -g513K | zstd > tmp6.zst |
1569 | zstd -l tmp6.zst |
1570 | zstd -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" |
1571 | |
1572 | println "\n===> zstd --list/-l test with no checksum " |
1573 | zstd -f --no-check tmp1 |
1574 | zstd -l tmp1.zst |
1575 | zstd -lv tmp1.zst |
1576 | |
1577 | println "\n===> zstd trace tests " |
1578 | zstd -f --trace tmp.trace tmp1 |
1579 | zstd -f --trace tmp.trace tmp1 tmp2 tmp3 |
1580 | zstd -f --trace tmp.trace tmp1 tmp2 tmp3 -o /dev/null |
1581 | zstd -f --trace tmp.trace tmp1 tmp2 tmp3 --single-thread |
1582 | zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null |
1583 | zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null --single-thread |
1584 | zstd --trace tmp.trace -t tmp1.zst |
1585 | zstd --trace tmp.trace -t tmp1.zst tmp2.zst |
1586 | zstd -f --trace tmp.trace -d tmp1.zst |
1587 | zstd -f --trace tmp.trace -d tmp1.zst tmp2.zst tmp3.zst |
1588 | zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1 |
1589 | zstd -b1e10i0 --trace tmp.trace tmp1 |
1590 | zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3 |
1591 | |
1592 | rm -f tmp* |
1593 | |
1594 | |
1595 | println "\n===> zstd long distance matching tests " |
1596 | roundTripTest -g0 " --single-thread --long" |
1597 | roundTripTest -g9M "2 --single-thread --long" |
1598 | # Test parameter parsing |
1599 | roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB" |
1600 | roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB" |
1601 | roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB" |
1602 | roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" |
1603 | |
1604 | |
1605 | println "\n===> zstd long distance matching with optimal parser compressed size tests " |
1606 | optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) |
1607 | longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) |
1608 | optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) |
1609 | longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) |
1610 | optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) |
1611 | longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) |
1612 | if [ "$longCSize16" -gt "$optCSize16" ]; then |
1613 | echo using --long on compression level 16 should not cause compressed size regression |
1614 | exit 1 |
1615 | elif [ "$longCSize19" -gt "$optCSize19" ]; then |
1616 | echo using --long on compression level 19 should not cause compressed size regression |
1617 | exit 1 |
1618 | elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then |
1619 | echo using --long on compression level 19 with wLog=23 should not cause compressed size regression |
1620 | exit 1 |
1621 | fi |
1622 | |
1623 | println "\n===> zstd asyncio tests " |
1624 | |
1625 | addFrame() { |
1626 | datagen -g2M -s$2 >> tmp_uncompressed |
1627 | datagen -g2M -s$2 | zstd -1 --format=$1 >> tmp_compressed.zst |
1628 | } |
1629 | |
1630 | addTwoFrames() { |
1631 | addFrame $1 1 |
1632 | addFrame $1 2 |
1633 | } |
1634 | |
1635 | testAsyncIO() { |
1636 | roundTripTest -g2M "3 --asyncio --format=$1" |
1637 | roundTripTest -g2M "3 --no-asyncio --format=$1" |
1638 | } |
1639 | |
1640 | rm -f tmp_compressed tmp_uncompressed |
1641 | testAsyncIO zstd |
1642 | addTwoFrames zstd |
1643 | if [ $GZIPMODE -eq 1 ]; then |
1644 | testAsyncIO gzip |
1645 | addTwoFrames gzip |
1646 | fi |
1647 | if [ $LZMAMODE -eq 1 ]; then |
1648 | testAsyncIO lzma |
1649 | addTwoFrames lzma |
1650 | fi |
1651 | if [ $LZ4MODE -eq 1 ]; then |
1652 | testAsyncIO lz4 |
1653 | addTwoFrames lz4 |
1654 | fi |
1655 | cat tmp_uncompressed | $MD5SUM > tmp2 |
1656 | zstd -d tmp_compressed.zst --asyncio -c | $MD5SUM > tmp1 |
1657 | $DIFF -q tmp1 tmp2 |
1658 | rm tmp1 |
1659 | zstd -d tmp_compressed.zst --no-asyncio -c | $MD5SUM > tmp1 |
1660 | $DIFF -q tmp1 tmp2 |
1661 | |
1662 | if [ "$1" != "--test-large-data" ]; then |
1663 | println "Skipping large data tests" |
1664 | exit 0 |
1665 | fi |
1666 | |
1667 | |
1668 | ############################################################################# |
1669 | |
1670 | |
1671 | if [ -n "$hasMT" ] |
1672 | then |
1673 | println "\n===> adaptive mode " |
1674 | roundTripTest -g270000000 " --adapt" |
1675 | roundTripTest -g27000000 " --adapt=min=1,max=4" |
1676 | roundTripTest -g27000000 " --adapt=min=-2,max=-1" |
1677 | println "===> test: --adapt must fail on incoherent bounds " |
1678 | datagen > tmp |
1679 | zstd --adapt= tmp && die "invalid compression parameter" |
1680 | zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" |
1681 | |
1682 | println "\n===> rsyncable mode " |
1683 | roundTripTest -g10M " --rsyncable" |
1684 | roundTripTest -g10M " --rsyncable -B100K" |
1685 | println "===> test: --rsyncable must fail with --single-thread" |
1686 | zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" |
1687 | fi |
1688 | |
1689 | println "\n===> patch-from=origin tests" |
1690 | datagen -g1000 -P50 > tmp_dict |
1691 | datagen -g1000 -P10 > tmp_patch |
1692 | zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff |
1693 | zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon |
1694 | $DIFF -s tmp_patch_recon tmp_patch |
1695 | |
1696 | println "\n===> alternate syntax: patch-from origin" |
1697 | zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff |
1698 | zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon |
1699 | $DIFF -s tmp_patch_recon tmp_patch |
1700 | rm -rf tmp_* |
1701 | |
1702 | println "\n===> patch-from recursive tests" |
1703 | mkdir tmp_dir |
1704 | datagen > tmp_dir/tmp1 |
1705 | datagen > tmp_dir/tmp2 |
1706 | datagen > tmp_dict |
1707 | zstd --patch-from=tmp_dict -r tmp_dir && die |
1708 | rm -rf tmp* |
1709 | |
1710 | println "\n===> patch-from long mode trigger larger file test" |
1711 | datagen -g5000000 > tmp_dict |
1712 | datagen -g5000000 > tmp_patch |
1713 | zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered" |
1714 | rm -rf tmp* |
1715 | |
1716 | println "\n===> patch-from very large dictionary and file test" |
1717 | datagen -g550000000 -P0 > tmp_dict |
1718 | datagen -g100000000 -P1 > tmp_patch |
1719 | zstd --long=30 -1f --patch-from tmp_dict tmp_patch |
1720 | zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon |
1721 | $DIFF -s tmp_patch_recon tmp_patch |
1722 | rm -rf tmp* |
1723 | |
1724 | println "\n===> patch-from --stream-size test" |
1725 | datagen -g1000 -P50 > tmp_dict |
1726 | datagen -g1000 -P10 > tmp_patch |
1727 | cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die |
1728 | cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff |
1729 | rm -rf tmp* |
1730 | |
1731 | println "\n===> large files tests " |
1732 | |
1733 | roundTripTest -g270000000 1 |
1734 | roundTripTest -g250000000 2 |
1735 | roundTripTest -g230000000 3 |
1736 | |
1737 | roundTripTest -g140000000 -P60 4 |
1738 | roundTripTest -g130000000 -P62 5 |
1739 | roundTripTest -g120000000 -P65 6 |
1740 | |
1741 | roundTripTest -g70000000 -P70 7 |
1742 | roundTripTest -g60000000 -P71 8 |
1743 | roundTripTest -g50000000 -P73 9 |
1744 | |
1745 | roundTripTest -g35000000 -P75 10 |
1746 | roundTripTest -g30000000 -P76 11 |
1747 | roundTripTest -g25000000 -P78 12 |
1748 | |
1749 | roundTripTest -g18000013 -P80 13 |
1750 | roundTripTest -g18000014 -P80 14 |
1751 | roundTripTest -g18000015 -P81 15 |
1752 | roundTripTest -g18000016 -P84 16 |
1753 | roundTripTest -g18000017 -P88 17 |
1754 | roundTripTest -g18000018 -P94 18 |
1755 | roundTripTest -g18000019 -P96 19 |
1756 | |
1757 | roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25" |
1758 | roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an overflow rescale |
1759 | |
1760 | fileRoundTripTest -g4193M -P99 1 |
1761 | |
1762 | |
1763 | println "\n===> zstd long, long distance matching round-trip tests " |
1764 | roundTripTest -g270000000 "1 --single-thread --long" |
1765 | roundTripTest -g130000000 -P60 "5 --single-thread --long" |
1766 | roundTripTest -g35000000 -P70 "8 --single-thread --long" |
1767 | roundTripTest -g18000001 -P80 "18 --single-thread --long" |
1768 | # Test large window logs |
1769 | roundTripTest -g700M -P50 "1 --single-thread --long=29" |
1770 | roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" |
1771 | |
1772 | |
1773 | if [ -n "$hasMT" ] |
1774 | then |
1775 | println "\n===> zstdmt long round-trip tests " |
1776 | roundTripTest -g80000000 -P99 "19 -T2" " " |
1777 | roundTripTest -g5000000000 -P99 "1 -T2" " " |
1778 | roundTripTest -g500000000 -P97 "1 -T999" " " |
1779 | fileRoundTripTest -g4103M -P98 " -T0" " " |
1780 | roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " |
1781 | # Exposes the bug in https://github.com/facebook/zstd/pull/1678 |
1782 | # This test fails on 4 different travis builds at the time of writing |
1783 | # because it needs to allocate 8 GB of memory. |
1784 | # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000" |
1785 | else |
1786 | println "\n**** no multithreading, skipping zstdmt tests **** " |
1787 | fi |
1788 | |
1789 | |
1790 | println "\n===> cover dictionary builder : advanced options " |
1791 | |
1792 | TESTFILE="$PRGDIR"/zstdcli.c |
1793 | datagen > tmpDict |
1794 | println "- Create first dictionary" |
1795 | zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1796 | cp "$TESTFILE" tmp |
1797 | zstd -f tmp -D tmpDict |
1798 | zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the same time" |
1799 | zstd -d tmp.zst -D tmpDict -fo result |
1800 | $DIFF "$TESTFILE" result |
1801 | zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" |
1802 | println "- Create second (different) dictionary" |
1803 | zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
1804 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
1805 | println "- Create dictionary using shrink-dict flag" |
1806 | zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict |
1807 | zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 |
1808 | zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 |
1809 | zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3 |
1810 | println "- Create dictionary with short dictID" |
1811 | zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
1812 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
1813 | println "- Create dictionary with size limit" |
1814 | zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K |
1815 | println "- Compare size of dictionary from 90% training samples with 80% training samples" |
1816 | zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1817 | zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1818 | println "- Create dictionary using all samples for both training and testing" |
1819 | zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1820 | println "- Test -o before --train-cover" |
1821 | rm -f tmpDict dictionary |
1822 | zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c |
1823 | test -f tmpDict |
1824 | zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c |
1825 | test -f dictionary |
1826 | rm -f tmp* dictionary |
1827 | |
1828 | rm -f tmp* |