648db22b |
1 | #!/bin/sh |
2 | |
f535537f |
3 | set -e # exit immediately on error |
4 | # set -x # print commands before execution (debug) |
648db22b |
5 | |
6 | unset ZSTD_CLEVEL |
7 | unset ZSTD_NBTHREADS |
8 | |
9 | |
10 | die() { |
11 | println "$@" 1>&2 |
12 | exit 1 |
13 | } |
14 | |
15 | datagen() { |
16 | "$DATAGEN_BIN" "$@" |
17 | } |
18 | |
19 | zstd() { |
f535537f |
20 | if [ -z "$EXE_PREFIX" ]; then |
648db22b |
21 | "$ZSTD_BIN" "$@" |
22 | else |
f535537f |
23 | "$EXE_PREFIX" "$ZSTD_BIN" "$@" |
648db22b |
24 | fi |
25 | } |
26 | |
27 | sudoZstd() { |
f535537f |
28 | if [ -z "$EXE_PREFIX" ]; then |
648db22b |
29 | sudo "$ZSTD_BIN" "$@" |
30 | else |
f535537f |
31 | sudo "$EXE_PREFIX" "$ZSTD_BIN" "$@" |
648db22b |
32 | fi |
33 | } |
34 | |
35 | roundTripTest() { |
36 | if [ -n "$3" ]; then |
37 | cLevel="$3" |
38 | proba="$2" |
39 | else |
40 | cLevel="$2" |
41 | proba="" |
42 | fi |
43 | if [ -n "$4" ]; then |
44 | dLevel="$4" |
45 | else |
46 | dLevel="$cLevel" |
47 | fi |
48 | |
49 | rm -f tmp1 tmp2 |
50 | println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel" |
51 | datagen $1 $proba | $MD5SUM > tmp1 |
52 | datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 |
53 | $DIFF -q tmp1 tmp2 |
54 | } |
55 | |
56 | fileRoundTripTest() { |
57 | if [ -n "$3" ]; then |
58 | local_c="$3" |
59 | local_p="$2" |
60 | else |
61 | local_c="$2" |
62 | local_p="" |
63 | fi |
64 | if [ -n "$4" ]; then |
65 | local_d="$4" |
66 | else |
67 | local_d="$local_c" |
68 | fi |
69 | |
70 | rm -f tmp.zst tmp.md5.1 tmp.md5.2 |
71 | println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d" |
72 | datagen $1 $local_p > tmp |
73 | < tmp $MD5SUM > tmp.md5.1 |
74 | zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 |
75 | $DIFF -q tmp.md5.1 tmp.md5.2 |
76 | } |
77 | |
78 | truncateLastByte() { |
79 | dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" |
80 | } |
81 | |
82 | println() { |
83 | printf '%b\n' "${*}" |
84 | } |
85 | |
86 | if [ -z "${size}" ]; then |
87 | size= |
88 | else |
89 | size=${size} |
90 | fi |
91 | |
92 | SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) |
93 | PRGDIR="$SCRIPT_DIR/../programs" |
94 | TESTDIR="$SCRIPT_DIR/../tests" |
f535537f |
95 | UNAME=${UNAME:-$(uname)} |
96 | GREP=${GREP:-grep} |
97 | |
98 | case "$UNAME" in |
99 | SunOS) DIFF=${DIFF:-gdiff} ;; |
100 | *) DIFF=${DIFF:-diff} ;; |
101 | esac |
648db22b |
102 | |
103 | detectedTerminal=false |
104 | if [ -t 0 ] && [ -t 1 ] |
105 | then |
106 | detectedTerminal=true |
107 | fi |
108 | isTerminal=${isTerminal:-$detectedTerminal} |
109 | |
110 | isWindows=false |
111 | INTOVOID="/dev/null" |
112 | case "$UNAME" in |
113 | GNU) DEVDEVICE="/dev/random" ;; |
114 | *) DEVDEVICE="/dev/zero" ;; |
115 | esac |
116 | case "$OS" in |
117 | Windows*) |
118 | isWindows=true |
119 | INTOVOID="NUL" |
120 | DEVDEVICE="NUL" |
121 | ;; |
122 | esac |
123 | |
124 | case "$UNAME" in |
125 | Darwin) MD5SUM="md5 -r" ;; |
126 | FreeBSD) MD5SUM="gmd5sum" ;; |
127 | NetBSD) MD5SUM="md5 -n" ;; |
128 | OpenBSD) MD5SUM="md5" ;; |
129 | *) MD5SUM="md5sum" ;; |
130 | esac |
131 | |
132 | MTIME="stat -c %Y" |
133 | case "$UNAME" in |
134 | Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; |
135 | esac |
136 | |
137 | assertSameMTime() { |
138 | MT1=$($MTIME "$1") |
139 | MT2=$($MTIME "$2") |
140 | echo MTIME $MT1 $MT2 |
141 | [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)" |
142 | } |
143 | |
144 | GET_PERMS="stat -c %a" |
145 | case "$UNAME" in |
146 | Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; |
147 | esac |
148 | |
149 | assertFilePermissions() { |
150 | STAT1=$($GET_PERMS "$1") |
151 | STAT2=$2 |
152 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" |
153 | } |
154 | |
155 | assertSamePermissions() { |
156 | STAT1=$($GET_PERMS "$1") |
157 | STAT2=$($GET_PERMS "$2") |
158 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" |
159 | } |
160 | |
648db22b |
161 | |
162 | # check if ZSTD_BIN is defined. if not, use the default value |
163 | if [ -z "${ZSTD_BIN}" ]; then |
164 | println "\nZSTD_BIN is not set. Using the default value..." |
165 | ZSTD_BIN="$PRGDIR/zstd" |
166 | fi |
167 | |
168 | # check if DATAGEN_BIN is defined. if not, use the default value |
169 | if [ -z "${DATAGEN_BIN}" ]; then |
170 | println "\nDATAGEN_BIN is not set. Using the default value..." |
171 | DATAGEN_BIN="$TESTDIR/datagen" |
172 | fi |
173 | |
174 | # Why was this line here ? Generates a strange ZSTD_BIN when EXE_PREFIX is non empty |
175 | # ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN" |
176 | |
177 | # assertions |
178 | [ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd following the instructions in README.md" |
179 | [ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. " |
180 | println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" |
181 | |
f535537f |
182 | if echo hello | zstd -v -T2 2>&1 > $INTOVOID | $GREP -q 'multi-threading is disabled' |
648db22b |
183 | then |
184 | hasMT="" |
185 | else |
186 | hasMT="true" |
187 | fi |
188 | |
189 | |
190 | zstd -vvV |
191 | |
192 | println "\n===> simple tests " |
193 | |
194 | datagen > tmp |
195 | zstd -h |
196 | zstd -H |
197 | zstd -V |
198 | println "test : basic compression " |
199 | zstd -f tmp # trivial compression case, creates tmp.zst |
200 | zstd -f -z tmp |
201 | zstd -f -k tmp |
202 | zstd -f -C tmp |
203 | println "test : basic decompression" |
204 | zstd -df tmp.zst # trivial decompression case (overwrites tmp) |
205 | println "test : too large compression level => auto-fix" |
206 | zstd -99 -f tmp # too large compression level, automatic sized down |
207 | zstd -5000000000 -f tmp && die "too large numeric value : must fail" |
208 | println "test : --fast aka negative compression levels" |
209 | zstd --fast -f tmp # == -1 |
210 | zstd --fast=3 -f tmp # == -3 |
211 | zstd --fast=200000 -f tmp # too low compression level, automatic fixed |
212 | zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail" |
213 | zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" |
214 | println "test : too large numeric argument" |
215 | zstd --fast=9999999999 -f tmp && die "should have refused numeric value" |
216 | println "test : set compression level with environment variable ZSTD_CLEVEL" |
217 | |
218 | ZSTD_CLEVEL=12 zstd -f tmp # positive compression level |
219 | ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level |
220 | ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign |
221 | ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting |
222 | ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting |
223 | ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting |
224 | ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting |
225 | ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting |
226 | ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting |
227 | println "test : override ZSTD_CLEVEL with command line option" |
228 | ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option |
229 | |
230 | # temporary envvar changes in the above tests would actually persist in macos /bin/sh |
231 | unset ZSTD_CLEVEL |
232 | |
233 | |
234 | println "test : compress to stdout" |
235 | zstd tmp -c > tmpCompressed |
236 | zstd tmp --stdout > tmpCompressed # long command format |
f535537f |
237 | |
238 | println "test : compress to named file (-o)" |
648db22b |
239 | rm -f tmpCompressed |
240 | zstd tmp -o tmpCompressed |
241 | test -f tmpCompressed # file must be created |
f535537f |
242 | |
648db22b |
243 | println "test : force write, correct order" |
244 | zstd tmp -fo tmpCompressed |
f535537f |
245 | |
246 | println "test : -c + -o : last one wins" |
247 | rm -f tmpOut |
248 | zstd tmp -c > tmpCompressed -o tmpOut |
249 | test -f tmpOut # file must be created |
250 | rm -f tmpCompressed |
251 | zstd tmp -o tmpOut -c > tmpCompressed |
252 | test -f tmpCompressed # file must be created |
253 | |
648db22b |
254 | println "test : forgotten argument" |
255 | cp tmp tmp2 |
256 | zstd tmp2 -fo && die "-o must be followed by filename " |
257 | println "test : implied stdout when input is stdin" |
258 | println bob | zstd | zstd -d |
259 | if [ "$isTerminal" = true ]; then |
260 | println "test : compressed data to terminal" |
261 | println bob | zstd && die "should have refused : compressed data to terminal" |
262 | println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" |
263 | zstd -d > $INTOVOID && die "should have refused : compressed data from terminal" |
264 | fi |
265 | println "test : null-length file roundtrip" |
266 | println -n '' | zstd - --stdout | zstd -d --stdout |
267 | println "test : ensure small file doesn't add 3-bytes null block" |
268 | datagen -g1 > tmp1 |
f535537f |
269 | zstd tmp1 -c | wc -c | $GREP "14" |
270 | zstd < tmp1 | wc -c | $GREP "14" |
648db22b |
271 | println "test : decompress file with wrong suffix (must fail)" |
272 | zstd -d tmpCompressed && die "wrong suffix error not detected!" |
273 | zstd -df tmp && die "should have refused : wrong extension" |
274 | println "test : decompress into stdout" |
275 | zstd -d tmpCompressed -c > tmpResult # decompression using stdout |
276 | zstd --decompress tmpCompressed -c > tmpResult |
277 | zstd --decompress tmpCompressed --stdout > tmpResult |
278 | println "test : decompress from stdin into stdout" |
279 | zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout |
280 | zstd -dc - < tmp.zst > $INTOVOID |
281 | zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used |
282 | zstd -d - < tmp.zst > $INTOVOID |
283 | println "test : impose memory limitation (must fail)" |
284 | datagen -g500K > tmplimit |
285 | zstd -f tmplimit |
286 | zstd -d -f tmplimit.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" |
287 | zstd -d -f tmplimit.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command |
288 | zstd -d -f tmplimit.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command |
289 | zstd -d -f tmplimit.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command |
290 | rm -f tmplimit tmplimit.zst |
291 | println "test : overwrite protection" |
292 | zstd -q tmp && die "overwrite check failed!" |
293 | println "test : force overwrite" |
294 | zstd -q -f tmp |
295 | zstd -q --force tmp |
296 | println "test : overwrite readonly file" |
297 | rm -f tmpro tmpro.zst |
298 | println foo > tmpro.zst |
299 | println foo > tmpro |
300 | chmod 400 tmpro.zst |
301 | zstd -q tmpro && die "should have refused to overwrite read-only file" |
302 | zstd -q -f tmpro |
303 | println "test: --no-progress flag" |
304 | zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress |
305 | zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress |
306 | println "test: --progress flag" |
f535537f |
307 | zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' |
308 | zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' |
309 | zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' |
648db22b |
310 | rm -f tmpro tmpro.zst |
311 | println "test: overwrite input file (must fail)" |
312 | zstd tmp -fo tmp && die "zstd compression overwrote the input file" |
313 | zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" |
314 | println "test: detect that input file does not exist" |
315 | zstd nothere && die "zstd hasn't detected that input file does not exist" |
316 | println "test: --[no-]compress-literals" |
317 | zstd tmp -c --no-compress-literals -1 | zstd -t |
318 | zstd tmp -c --no-compress-literals --fast=1 | zstd -t |
319 | zstd tmp -c --no-compress-literals -19 | zstd -t |
320 | zstd tmp -c --compress-literals -1 | zstd -t |
321 | zstd tmp -c --compress-literals --fast=1 | zstd -t |
322 | zstd tmp -c --compress-literals -19 | zstd -t |
323 | zstd -b --fast=1 -i0e1 tmp --compress-literals |
324 | zstd -b --fast=1 -i0e1 tmp --no-compress-literals |
325 | println "test: --no-check for decompression" |
326 | zstd -f tmp -o tmp_corrupt.zst --check |
327 | zstd -f tmp -o tmp.zst --no-check |
328 | printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp |
329 | zstd -d -f tmp_corrupt.zst --no-check |
330 | zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides |
331 | zstd -d -f tmp.zst --no-check |
332 | |
333 | if [ "$isWindows" = false ] && [ "$UNAME" != "AIX" ]; then |
334 | if [ -n "$(which readelf)" ]; then |
335 | println "test: check if binary has executable stack (#2963)" |
f535537f |
336 | readelf -lW "$ZSTD_BIN" | $GREP 'GNU_STACK .* RW ' || die "zstd binary has executable stack!" |
648db22b |
337 | fi |
338 | fi |
339 | |
f535537f |
340 | println "\n===> multiple_thread test " |
341 | |
342 | datagen > tmp |
343 | println "test : single-thread " |
344 | zstd --fast --single-thread tmp -o tmpMT0 |
345 | println "test : one worker thread (default)" |
346 | zstd --fast -T1 tmp -o tmpMT1 |
347 | println "test : two worker threads " |
348 | zstd --fast -T2 tmp -o tmpMT2 |
349 | println "test : 16-thread " |
350 | zstd --fast -T16 tmp -o tmpMT3 |
351 | println "test : 127-thread " |
352 | zstd --fast -T127 tmp -o tmpMT4 |
353 | println "test : 128-thread " |
354 | zstd --fast -T128 tmp -o tmpMT5 |
355 | println "test : max allowed numeric value is 4294967295 " |
356 | zstd --fast -4294967295 tmp -o tmpMT6 |
357 | println "test : numeric value overflows 32-bit unsigned int " |
358 | zstd --fast -4294967296 tmp -o tmptest9 && die "max allowed numeric value is 4294967295" |
359 | |
360 | datagen > tmp |
361 | println "test : basic compression " |
362 | zstd -f tmp # trivial compression case, creates tmp.zst |
363 | println "test : basic decompression" |
364 | zstd -d -f -T1 tmp.zst |
365 | println "note : decompression does not support -T mode, but execution support" |
366 | rm -rf tmpMT* |
367 | |
368 | println "\n===> --fast_argument test " |
369 | datagen > tmp |
370 | println "test : basic compression " |
371 | zstd -f tmp # trivial compression case, creates tmp.zst |
372 | println "test: --fast=1" |
373 | zstd --fast=1 -f tmp |
374 | println "test: --fast=99" |
375 | zstd --fast=99 -f tmp |
376 | println "test: Invalid value -- negative number" |
377 | zstd --fast=-1 -f tmp && die "error: Invalid value -- negative number" |
378 | println "test: Invalid value -- zero" |
379 | zstd --fast=0 -f tmp && die "error: Invalid value -- 0 number" |
380 | println "test: max allowed numeric argument of --fast is 4294967295" |
381 | zstd --fast=4294967295 -f tmp |
382 | println "test: numeric value overflows 32-bit unsigned int " |
383 | zstd --fast=4294967296 -f tmp && die "max allowed argument of --fast is 4294967295" |
384 | |
648db22b |
385 | println "\n===> --exclude-compressed flag" |
386 | rm -rf precompressedFilterTestDir |
387 | mkdir -p precompressedFilterTestDir |
388 | datagen $size > precompressedFilterTestDir/input.5 |
389 | datagen $size > precompressedFilterTestDir/input.6 |
390 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
391 | datagen $size > precompressedFilterTestDir/input.7 |
392 | datagen $size > precompressedFilterTestDir/input.8 |
393 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
394 | test ! -f precompressedFilterTestDir/input.5.zst.zst |
395 | test ! -f precompressedFilterTestDir/input.6.zst.zst |
396 | file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst` |
397 | file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst` |
398 | if [ $file2timestamp -ge $file1timestamp ]; then |
399 | println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again." |
400 | else |
401 | println "Test is not successful" |
402 | fi |
403 | # File Extension check. |
404 | datagen $size > precompressedFilterTestDir/input.zstbar |
405 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
406 | # zstd should compress input.zstbar |
407 | test -f precompressedFilterTestDir/input.zstbar.zst |
408 | # Check without the --exclude-compressed flag |
409 | zstd --long --rm -r precompressedFilterTestDir |
410 | # Files should get compressed again without the --exclude-compressed flag. |
411 | test -f precompressedFilterTestDir/input.5.zst.zst |
412 | test -f precompressedFilterTestDir/input.6.zst.zst |
f535537f |
413 | |
414 | # Test some other compressed file extensions |
415 | datagen $size > precompressedFilterTestDir/input.flac |
416 | datagen $size > precompressedFilterTestDir/input.mov |
417 | datagen $size > precompressedFilterTestDir/input.mp3 |
418 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
419 | test ! -f precompressedFilterTestDir/input.flac.zst |
420 | test ! -f precompressedFilterTestDir/input.mov.zst |
421 | test ! -f precompressedFilterTestDir/input.mp3.zst |
422 | zstd --long --rm -r precompressedFilterTestDir |
423 | test -f precompressedFilterTestDir/input.flac.zst |
424 | test -f precompressedFilterTestDir/input.mov.zst |
425 | test -f precompressedFilterTestDir/input.mp3.zst |
648db22b |
426 | rm -rf precompressedFilterTestDir |
427 | println "Test completed" |
428 | |
429 | |
430 | |
431 | println "\n===> warning prompts should not occur if stdin is an input" |
432 | println "y" > tmpPrompt |
433 | println "hello world" >> tmpPrompt |
434 | zstd tmpPrompt -f |
435 | zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" |
436 | zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f |
437 | zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated |
438 | $DIFF tmpPromptRegenerated tmpPrompt # the first 'y' character should not be swallowed |
439 | |
440 | echo 'yes' | zstd tmpPrompt -v -o tmpPrompt.zst # accept piped "y" input to force overwrite when using files |
441 | echo 'yes' | zstd < tmpPrompt -v -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" |
442 | zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove" |
443 | |
444 | println "Test completed" |
445 | |
446 | |
447 | println "\n===> recursive mode test " |
448 | # combination of -r with empty list of input file |
449 | zstd -c -r < tmp > tmp.zst |
450 | |
451 | # combination of -r with empty folder |
452 | mkdir -p tmpEmptyDir |
453 | zstd -r tmpEmptyDir |
454 | rm -rf tmpEmptyDir |
455 | |
456 | |
457 | println "\n===> file removal" |
458 | zstd -f --rm tmp |
459 | test ! -f tmp # tmp should no longer be present |
460 | zstd -f -d --rm tmp.zst |
461 | test ! -f tmp.zst # tmp.zst should no longer be present |
462 | println "test: --rm is disabled when output is stdout" |
463 | test -f tmp |
464 | zstd --rm tmp -c > $INTOVOID |
465 | test -f tmp # tmp shall still be there |
f535537f |
466 | zstd --rm tmp --stdout > $INTOVOID |
467 | test -f tmp # tmp shall still be there |
648db22b |
468 | zstd -f --rm tmp -c > $INTOVOID |
469 | test -f tmp # tmp shall still be there |
470 | zstd -f tmp -c > $INTOVOID --rm |
471 | test -f tmp # tmp shall still be there |
472 | println "test: --rm is disabled when multiple inputs are concatenated into a single output" |
473 | cp tmp tmp2 |
474 | zstd --rm tmp tmp2 -c > $INTOVOID |
475 | test -f tmp |
476 | test -f tmp2 |
477 | rm -f tmp3.zst |
478 | echo 'y' | zstd -v tmp tmp2 -o tmp3.zst --rm # prompt for confirmation |
479 | test -f tmp |
480 | test -f tmp2 |
481 | zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt |
482 | test -f tmp |
483 | test -f tmp2 |
484 | zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate" |
f535537f |
485 | println "test: --rm is active with -o when single input" |
486 | rm -f tmp2.zst |
487 | zstd --rm tmp2 -o tmp2.zst |
488 | test -f tmp2.zst |
489 | test ! -f tmp2 |
490 | println "test: -c followed by -o => -o wins, so --rm remains active" # (#3719) |
491 | rm tmp2.zst |
492 | cp tmp tmp2 |
493 | zstd --rm tmp2 -c > $INTOVOID -o tmp2.zst |
494 | test ! -f tmp2 |
495 | println "test: -o followed by -c => -c wins, so --rm is disabled" # (#3719) |
496 | rm tmp3.zst |
497 | cp tmp tmp2 |
498 | zstd -v --rm tmp2 -o tmp2.zst -c > tmp3.zst |
499 | test -f tmp2 |
500 | test -f tmp3.zst |
648db22b |
501 | println "test : should quietly not remove non-regular file" |
502 | println hello > tmp |
503 | zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" |
f535537f |
504 | $GREP "Refusing to remove non-regular file" tmplog && die |
648db22b |
505 | rm -f tmplog |
f535537f |
506 | zstd tmp -f -o "$INTOVOID" 2>&1 | $GREP "Refusing to remove non-regular file" && die |
648db22b |
507 | println "test : --rm on stdin" |
508 | println a | zstd --rm > $INTOVOID # --rm should remain silent |
509 | rm -f tmp |
510 | zstd -f tmp && die "tmp not present : should have failed" |
511 | test ! -f tmp.zst # tmp.zst should not be created |
512 | println "test : -d -f do not delete destination when source is not present" |
513 | touch tmp # create destination file |
514 | zstd -d -f tmp.zst && die "attempt to decompress a non existing file" |
515 | test -f tmp # destination file should still be present |
516 | println "test : -f do not delete destination when source is not present" |
517 | rm -f tmp # erase source file |
518 | touch tmp.zst # create destination file |
519 | zstd -f tmp && die "attempt to compress a non existing file" |
520 | test -f tmp.zst # destination file should still be present |
521 | rm -rf tmp* # may also erase tmp* directory from previous failed run |
522 | |
523 | |
524 | println "\n===> decompression only tests " |
525 | # the following test verifies that the decoder is compatible with RLE as first block |
526 | # older versions of zstd cli are not able to decode such corner case. |
527 | # As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions. |
528 | dd bs=1048576 count=1 if=/dev/zero of=tmp |
529 | zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" |
530 | $DIFF -s tmp1 tmp |
531 | |
532 | touch tmp_empty |
533 | zstd -d -o tmp2 "$TESTDIR/golden-decompression/empty-block.zst" |
534 | $DIFF -s tmp2 tmp_empty |
f535537f |
535 | |
536 | zstd -t "$TESTDIR/golden-decompression/zeroSeq_2B.zst" |
537 | |
538 | zstd -t "$TESTDIR/golden-decompression-errors/zeroSeq_extraneous.zst" && die "invalid Sequences section should have been detected" |
539 | |
648db22b |
540 | rm -f tmp* |
541 | |
542 | println "\n===> compress multiple files" |
543 | println hello > tmp1 |
544 | println world > tmp2 |
545 | zstd tmp1 tmp2 -o "$INTOVOID" -f |
546 | zstd tmp1 tmp2 -c | zstd -t |
547 | echo 'y' | zstd -v tmp1 tmp2 -o tmp.zst |
548 | test ! -f tmp1.zst |
549 | test ! -f tmp2.zst |
550 | zstd tmp1 tmp2 |
551 | zstd -t tmp1.zst tmp2.zst |
552 | zstd -dc tmp1.zst tmp2.zst |
553 | zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f |
554 | echo 'y' | zstd -v -d tmp1.zst tmp2.zst -o tmp |
555 | touch tmpexists |
556 | zstd tmp1 tmp2 -f -o tmpexists |
557 | zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite" |
558 | println gooder > tmp_rm1 |
559 | println boi > tmp_rm2 |
560 | println worldly > tmp_rm3 |
561 | echo 'y' | zstd -v tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst |
562 | test -f tmp_rm1 |
563 | test -f tmp_rm2 |
564 | cp tmp_rm3.zst tmp_rm4.zst |
565 | echo 'Y' | zstd -v -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm |
566 | test -f tmp_rm3.zst |
567 | test -f tmp_rm4.zst |
568 | println gooder > tmpexists1 |
569 | zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID |
570 | # Bug: PR #972 |
571 | if [ "$?" -eq 139 ]; then |
572 | die "should not have segfaulted" |
573 | fi |
574 | test -f tmpexists1 |
575 | test -f tmpexists |
576 | println "\n===> multiple files and shell completion " |
577 | datagen -s1 > tmp1 2> $INTOVOID |
578 | datagen -s2 -g100K > tmp2 2> $INTOVOID |
579 | datagen -s3 -g1M > tmp3 2> $INTOVOID |
580 | println "compress tmp* : " |
581 | zstd -f tmp* |
582 | test -f tmp1.zst |
583 | test -f tmp2.zst |
584 | test -f tmp3.zst |
585 | rm -f tmp1 tmp2 tmp3 |
586 | println "decompress tmp* : " |
587 | zstd -df ./*.zst |
588 | test -f tmp1 |
589 | test -f tmp2 |
590 | test -f tmp3 |
591 | println "compress tmp* into stdout > tmpall : " |
592 | zstd -c tmp1 tmp2 tmp3 > tmpall |
593 | test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) |
594 | println "decompress tmpall* into stdout > tmpdec : " |
595 | cp tmpall tmpall2 |
596 | zstd -dc tmpall* > tmpdec |
597 | test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) |
598 | println "compress multiple files including a missing one (notHere) : " |
599 | zstd -f tmp1 notHere tmp2 && die "missing file not detected!" |
600 | rm -f tmp* |
601 | |
602 | |
603 | if [ "$isWindows" = false ] ; then |
604 | println "\n===> zstd fifo named pipe test " |
605 | echo "Hello World!" > tmp_original |
606 | mkfifo tmp_named_pipe |
607 | # note : fifo test doesn't work in combination with `dd` or `cat` |
608 | echo "Hello World!" > tmp_named_pipe & |
609 | zstd tmp_named_pipe -o tmp_compressed |
610 | zstd -d -o tmp_decompressed tmp_compressed |
611 | $DIFF -s tmp_original tmp_decompressed |
612 | rm -rf tmp* |
613 | fi |
614 | |
615 | println "\n===> zstd created file permissions tests" |
616 | if [ "$isWindows" = false ] ; then |
617 | rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove |
618 | |
619 | ORIGINAL_UMASK=$(umask) |
620 | umask 0000 |
621 | |
622 | datagen > tmp1 |
623 | datagen > tmp2 |
624 | assertFilePermissions tmp1 666 |
625 | assertFilePermissions tmp2 666 |
626 | |
627 | println "test : copy 666 permissions in file -> file compression " |
628 | zstd -f tmp1 -o tmp1.zst |
629 | assertSamePermissions tmp1 tmp1.zst |
630 | println "test : copy 666 permissions in file -> file decompression " |
631 | zstd -f -d tmp1.zst -o tmp1.out |
632 | assertSamePermissions tmp1.zst tmp1.out |
633 | |
634 | rm -f tmp1.zst tmp1.out |
635 | |
636 | println "test : copy 400 permissions in file -> file compression (write to a read-only file) " |
637 | chmod 0400 tmp1 |
638 | assertFilePermissions tmp1 400 |
639 | zstd -f tmp1 -o tmp1.zst |
640 | assertSamePermissions tmp1 tmp1.zst |
641 | println "test : copy 400 permissions in file -> file decompression (write to a read-only file) " |
642 | zstd -f -d tmp1.zst -o tmp1 |
643 | assertSamePermissions tmp1.zst tmp1 |
644 | |
645 | rm -f tmp1.zst tmp1.out |
646 | |
647 | println "test : check created permissions from stdin input in compression " |
648 | zstd -f -o tmp1.zst < tmp1 |
649 | assertFilePermissions tmp1.zst 666 |
650 | println "test : check created permissions from stdin input in decompression " |
651 | zstd -f -d -o tmp1.out < tmp1.zst |
652 | assertFilePermissions tmp1.out 666 |
653 | |
654 | rm -f tmp1.zst tmp1.out |
655 | |
656 | println "test : check created permissions from multiple inputs in compression " |
657 | zstd -f tmp1 tmp2 -o tmp1.zst |
658 | assertFilePermissions tmp1.zst 666 |
659 | println "test : check created permissions from multiple inputs in decompression " |
660 | cp tmp1.zst tmp2.zst |
661 | zstd -f -d tmp1.zst tmp2.zst -o tmp1.out |
662 | assertFilePermissions tmp1.out 666 |
663 | |
664 | rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out |
665 | |
666 | println "test : check permissions on pre-existing output file in compression " |
667 | chmod 0600 tmp1 |
668 | touch tmp1.zst |
669 | chmod 0400 tmp1.zst |
670 | zstd -f tmp1 -o tmp1.zst |
671 | assertFilePermissions tmp1.zst 600 |
672 | println "test : check permissions on pre-existing output file in decompression " |
673 | chmod 0400 tmp1.zst |
674 | touch tmp1.out |
675 | chmod 0200 tmp1.out |
676 | zstd -f -d tmp1.zst -o tmp1.out |
677 | assertFilePermissions tmp1.out 400 |
678 | |
679 | umask 0666 |
680 | chmod 0666 tmp1 tmp2 |
681 | |
682 | rm -f tmp1.zst tmp1.out |
683 | |
684 | println "test : respect umask when compressing from stdin input " |
685 | zstd -f -o tmp1.zst < tmp1 |
686 | assertFilePermissions tmp1.zst 0 |
687 | println "test : respect umask when decompressing from stdin input " |
688 | chmod 0666 tmp1.zst |
689 | zstd -f -d -o tmp1.out < tmp1.zst |
690 | assertFilePermissions tmp1.out 0 |
691 | |
692 | rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out |
693 | umask $ORIGINAL_UMASK |
694 | fi |
695 | |
696 | if [ -n "$DEVNULLRIGHTS" ] ; then |
697 | # these tests requires sudo rights, which is uncommon. |
698 | # they are only triggered if DEVNULLRIGHTS macro is defined. |
699 | println "\n===> checking /dev/null permissions are unaltered " |
700 | datagen > tmp |
701 | sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions |
702 | sudoZstd tmp -c > $INTOVOID |
703 | zstd tmp -f -o tmp.zst |
704 | sudoZstd -d tmp.zst -c > $INTOVOID |
705 | sudoZstd -d tmp.zst -o $INTOVOID |
f535537f |
706 | ls -las $INTOVOID | $GREP "rw-rw-rw-" |
648db22b |
707 | fi |
708 | |
709 | if [ -n "$READFROMBLOCKDEVICE" ] ; then |
710 | # This creates a temporary block device, which is only possible on unix-y |
711 | # systems, is somewhat invasive, and requires sudo. For these reasons, you |
712 | # have to specifically ask for this test. |
713 | println "\n===> checking that zstd can read from a block device" |
714 | datagen -g65536 > tmp.img |
715 | sudo losetup -fP tmp.img |
f535537f |
716 | LOOP_DEV=$(losetup -a | $GREP 'tmp\.img' | cut -f1 -d:) |
648db22b |
717 | [ -z "$LOOP_DEV" ] && die "failed to get loopback device" |
718 | sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f" |
719 | sudoZstd -f $LOOP_DEV -c > tmp.img.zst |
720 | zstd -d tmp.img.zst -o tmp.img.copy |
721 | sudo losetup -d $LOOP_DEV |
722 | $DIFF -s tmp.img tmp.img.copy || die "round trip failed" |
723 | rm -f tmp.img tmp.img.zst tmp.img.copy |
724 | fi |
725 | |
726 | println "\n===> zstd created file timestamp tests" |
727 | datagen > tmp |
728 | touch -m -t 200001010000.00 tmp |
729 | println "test : copy mtime in file -> file compression " |
730 | zstd -f tmp -o tmp.zst |
731 | assertSameMTime tmp tmp.zst |
732 | println "test : copy mtime in file -> file decompression " |
733 | zstd -f -d tmp.zst -o tmp.out |
734 | assertSameMTime tmp.zst tmp.out |
735 | rm -f tmp |
736 | |
737 | println "\n===> compress multiple files into an output directory, --output-dir-flat" |
738 | println henlo > tmp1 |
739 | mkdir tmpInputTestDir |
740 | mkdir tmpInputTestDir/we |
741 | mkdir tmpInputTestDir/we/must |
742 | mkdir tmpInputTestDir/we/must/go |
743 | mkdir tmpInputTestDir/we/must/go/deeper |
744 | println cool > tmpInputTestDir/we/must/go/deeper/tmp2 |
745 | mkdir tmpOutDir |
746 | zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir |
747 | test -f tmpOutDir/tmp1.zst |
748 | test -f tmpOutDir/tmp2.zst |
749 | println "test : decompress multiple files into an output directory, --output-dir-flat" |
750 | mkdir tmpOutDirDecomp |
751 | zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp |
752 | test -f tmpOutDirDecomp/tmp2 |
753 | test -f tmpOutDirDecomp/tmp1 |
754 | rm -f tmpOutDirDecomp/* |
755 | zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp |
756 | test -f tmpOutDirDecomp/tmp2 |
757 | test -f tmpOutDirDecomp/tmp1 |
758 | rm -rf tmp* |
759 | |
760 | if [ "$isWindows" = false ] ; then |
761 | println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror" |
762 | println "test --output-dir-mirror" > tmp1 |
763 | mkdir -p tmpInputTestDir/we/.../..must/go/deeper.. |
764 | println cool > tmpInputTestDir/we/.../..must/go/deeper../tmp2 |
765 | zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir |
766 | test -f tmpOutDir/tmp1.zst |
767 | test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst |
768 | |
769 | println "test: compress input dir will be ignored if it has '..'" |
770 | zstd -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" |
771 | zstd -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input cannot contain '..'" |
772 | zstd -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" |
773 | test ! -d non-exist |
774 | |
775 | println "test: compress input dir should succeed with benign uses of '..'" |
776 | zstd -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout |
777 | test -d tmpout |
778 | |
779 | println "test : decompress multiple files into an output directory, --output-dir-mirror" |
780 | zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp |
781 | test -f tmpOutDirDecomp/tmpOutDir/tmp1 |
782 | test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2 |
783 | |
784 | println "test: decompress input dir will be ignored if it has '..'" |
785 | zstd -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "input cannot contain '..'" |
786 | test ! -d non-exist |
787 | |
788 | rm -rf tmp* |
789 | fi |
790 | |
791 | |
792 | println "test : compress multiple files reading them from a file, --filelist=FILE" |
793 | println "Hello world!, file1" > tmp1 |
794 | println "Hello world!, file2" > tmp2 |
795 | println tmp1 > tmp_fileList |
796 | println tmp2 >> tmp_fileList |
797 | zstd -f --filelist=tmp_fileList |
798 | test -f tmp2.zst |
799 | test -f tmp1.zst |
800 | |
801 | println "test : alternate syntax: --filelist FILE" |
802 | zstd -f --filelist tmp_fileList |
803 | test -f tmp2.zst |
804 | test -f tmp1.zst |
805 | |
806 | println "test : reading file list from a symlink, --filelist=FILE" |
807 | rm -f *.zst |
808 | ln -s tmp_fileList tmp_symLink |
809 | zstd -f --filelist=tmp_symLink |
810 | test -f tmp2.zst |
811 | test -f tmp1.zst |
812 | |
813 | println "test : compress multiple files reading them from multiple files, --filelist=FILE" |
814 | rm -f *.zst |
815 | println "Hello world!, file3" > tmp3 |
816 | println "Hello world!, file4" > tmp4 |
817 | println tmp3 > tmp_fileList2 |
818 | println tmp4 >> tmp_fileList2 |
819 | zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2 |
820 | test -f tmp1.zst |
821 | test -f tmp2.zst |
822 | test -f tmp3.zst |
823 | test -f tmp4.zst |
824 | |
825 | println "test : decompress multiple files reading them from a file, --filelist=FILE" |
826 | rm -f tmp1 tmp2 |
827 | println tmp1.zst > tmpZst |
828 | println tmp2.zst >> tmpZst |
829 | zstd -d -f --filelist=tmpZst |
830 | test -f tmp1 |
831 | test -f tmp2 |
832 | |
833 | println "test : decompress multiple files reading them from multiple files, --filelist=FILE" |
834 | rm -f tmp1 tmp2 tmp3 tmp4 |
835 | println tmp3.zst > tmpZst2 |
836 | println tmp4.zst >> tmpZst2 |
837 | zstd -d -f --filelist=tmpZst --filelist=tmpZst2 |
838 | test -f tmp1 |
839 | test -f tmp2 |
840 | test -f tmp3 |
841 | test -f tmp4 |
842 | |
843 | println "test : survive the list of files with too long filenames (--filelist=FILE)" |
844 | datagen -g5M > tmp_badList |
845 | zstd -qq -f --filelist=tmp_badList && die "should have failed : file name length is too long" # printing very long text garbage on console will cause CI failure |
846 | |
847 | println "test : survive a list of files which is text garbage (--filelist=FILE)" |
848 | datagen > tmp_badList |
849 | zstd -qq -f --filelist=tmp_badList && die "should have failed : list is text garbage" # printing very long text garbage on console will cause CI failure |
850 | |
851 | println "test : survive a list of files which is binary garbage (--filelist=FILE)" |
852 | datagen -P0 -g1M > tmp_badList |
853 | zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console |
854 | |
855 | println "test : try to overflow internal list of files (--filelist=FILE)" |
856 | touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 |
857 | ls tmp* > tmpList |
858 | zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list |
859 | rm -rf tmp* |
860 | |
861 | println "\n===> --[no-]content-size tests" |
862 | |
863 | datagen > tmp_contentsize |
864 | zstd -f tmp_contentsize |
f535537f |
865 | zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" |
648db22b |
866 | zstd -f --no-content-size tmp_contentsize |
f535537f |
867 | zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die |
648db22b |
868 | zstd -f --content-size tmp_contentsize |
f535537f |
869 | zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" |
648db22b |
870 | zstd -f --content-size --no-content-size tmp_contentsize |
f535537f |
871 | zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die |
648db22b |
872 | rm -rf tmp* |
873 | |
874 | println "test : show-default-cparams regular" |
875 | datagen > tmp |
876 | zstd --show-default-cparams -f tmp |
877 | zstd --show-default-cparams -d tmp.zst && die "error: can't use --show-default-cparams in decompression mode" |
878 | rm -rf tmp* |
879 | |
880 | println "test : show-default-cparams recursive" |
881 | mkdir tmp_files |
882 | datagen -g15000 > tmp_files/tmp1 |
883 | datagen -g129000 > tmp_files/tmp2 |
884 | datagen -g257000 > tmp_files/tmp3 |
885 | zstd --show-default-cparams -f -r tmp_files |
886 | rm -rf tmp* |
887 | |
888 | println "test : show compression parameters in verbose mode" |
889 | datagen > tmp |
890 | zstd -vv tmp 2>&1 | \ |
f535537f |
891 | $GREP -q -- "--zstd=wlog=[0-9]*,clog=[0-9]*,hlog=[0-9]*,slog=[0-9]*,mml=[0-9]*,tlen=[0-9]*,strat=[0-9]*" |
648db22b |
892 | rm -rf tmp* |
893 | |
894 | println "\n===> Advanced compression parameters " |
895 | println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" |
896 | println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" |
897 | println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" |
898 | println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist |
899 | test ! -f tmp.zst # tmp.zst should not be created |
900 | roundTripTest -g512K |
901 | roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" |
902 | roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" |
903 | roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" |
904 | roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" |
905 | roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" |
906 | roundTripTest -g64K "19 --zstd=strat=9" # btultra2 |
907 | |
908 | |
909 | println "\n===> Pass-Through mode " |
910 | println "Hello world 1!" | zstd -df |
911 | println "Hello world 2!" | zstd -dcf |
912 | println "Hello world 3!" > tmp1 |
913 | zstd -dcf tmp1 |
914 | println "" | zstd -df > tmp1 |
915 | println "" > tmp2 |
916 | $DIFF -q tmp1 tmp2 |
917 | println "1" | zstd -df > tmp1 |
918 | println "1" > tmp2 |
919 | $DIFF -q tmp1 tmp2 |
920 | println "12" | zstd -df > tmp1 |
921 | println "12" > tmp2 |
922 | $DIFF -q tmp1 tmp2 |
923 | rm -rf tmp* |
924 | |
925 | |
926 | println "\n===> frame concatenation " |
927 | println "hello " > hello.tmp |
928 | println "world!" > world.tmp |
929 | cat hello.tmp world.tmp > helloworld.tmp |
930 | zstd -c hello.tmp > hello.zst |
931 | zstd -c world.tmp > world.zst |
932 | zstd -c hello.tmp world.tmp > helloworld.zst |
933 | zstd -dc helloworld.zst > result.tmp |
934 | $DIFF helloworld.tmp result.tmp |
935 | cat hello.zst world.zst > helloworld.zst |
936 | zstd -dc helloworld.zst > result.tmp |
937 | cat result.tmp |
938 | $DIFF helloworld.tmp result.tmp |
939 | println "frame concatenation without checksum" |
940 | zstd -c hello.tmp > hello.zst --no-check |
941 | zstd -c world.tmp > world.zst --no-check |
942 | cat hello.zst world.zst > helloworld.zstd |
943 | zstd -dc helloworld.zst > result.tmp |
944 | $DIFF helloworld.tmp result.tmp |
945 | println "testing zstdcat symlink" |
946 | ln -sf "$ZSTD_BIN" zstdcat |
947 | $EXE_PREFIX ./zstdcat helloworld.zst > result.tmp |
948 | $DIFF helloworld.tmp result.tmp |
949 | ln -s helloworld.zst helloworld.link.zst |
950 | $EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp |
951 | $DIFF helloworld.tmp result.tmp |
952 | rm -f zstdcat |
953 | rm -f result.tmp |
954 | println "testing zcat symlink" |
955 | ln -sf "$ZSTD_BIN" zcat |
956 | $EXE_PREFIX ./zcat helloworld.zst > result.tmp |
957 | $DIFF helloworld.tmp result.tmp |
958 | $EXE_PREFIX ./zcat helloworld.link.zst > result.tmp |
959 | $DIFF helloworld.tmp result.tmp |
960 | rm -f zcat |
961 | rm -f ./*.tmp ./*.zstd |
962 | println "frame concatenation tests completed" |
963 | |
964 | |
965 | if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] && [ "$UNAME" != "AIX" ]; then |
966 | println "\n**** flush write error test **** " |
967 | |
968 | println "println foo | zstd > /dev/full" |
969 | println foo | zstd > /dev/full && die "write error not detected!" |
970 | println "println foo | zstd | zstd -d > /dev/full" |
971 | println foo | zstd | zstd -d > /dev/full && die "write error not detected!" |
972 | |
973 | fi |
974 | |
975 | |
976 | if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then |
977 | |
978 | println "\n===> symbolic link test " |
979 | |
980 | rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst |
981 | println "hello world" > hello.tmp |
982 | ln -s hello.tmp world.tmp |
983 | ln -s hello.tmp world2.tmp |
984 | zstd world.tmp hello.tmp || true |
985 | test -f hello.tmp.zst # regular file should have been compressed! |
986 | test ! -f world.tmp.zst # symbolic link should not have been compressed! |
987 | zstd world.tmp || true |
988 | test ! -f world.tmp.zst # symbolic link should not have been compressed! |
989 | zstd world.tmp world2.tmp || true |
990 | test ! -f world.tmp.zst # symbolic link should not have been compressed! |
991 | test ! -f world2.tmp.zst # symbolic link should not have been compressed! |
992 | zstd world.tmp hello.tmp -f |
993 | test -f world.tmp.zst # symbolic link should have been compressed with --force |
994 | rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst |
995 | |
996 | fi |
997 | |
998 | |
999 | println "\n===> test sparse file support " |
1000 | |
1001 | datagen -g5M -P100 > tmpSparse |
1002 | zstd tmpSparse -c | zstd -dv -o tmpSparseRegen |
1003 | $DIFF -s tmpSparse tmpSparseRegen |
1004 | zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse |
1005 | $DIFF -s tmpSparse tmpOutSparse |
1006 | zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse |
1007 | $DIFF -s tmpSparse tmpOutNoSparse |
1008 | ls -ls tmpSparse* # look at file size and block size on disk |
1009 | datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) |
1010 | datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd |
1011 | ls -ls tmpSparseOdd # look at file size and block size on disk |
1012 | println "\n Sparse Compatibility with Console :" |
1013 | println "Hello World 1 !" | zstd | zstd -d -c |
1014 | println "Hello World 2 !" | zstd | zstd -d | cat |
1015 | println "\n Sparse Compatibility with Append :" |
1016 | datagen -P100 -g1M > tmpSparse1M |
1017 | cat tmpSparse1M tmpSparse1M > tmpSparse2M |
1018 | zstd -v -f tmpSparse1M -o tmpSparseCompressed |
1019 | zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated |
1020 | zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated |
1021 | ls -ls tmpSparse* # look at file size and block size on disk |
1022 | $DIFF tmpSparse2M tmpSparseRegenerated |
1023 | rm -f tmpSparse* |
1024 | |
1025 | |
1026 | println "\n===> stream-size mode" |
1027 | |
1028 | datagen -g11000 > tmp |
1029 | println "test : basic file compression vs sized streaming compression" |
1030 | file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) |
1031 | stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c) |
1032 | if [ "$stream_size" -gt "$file_size" ]; then |
1033 | die "hinted compression larger than expected" |
1034 | fi |
1035 | println "test : sized streaming compression and decompression" |
1036 | cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000 |
1037 | zstd -df tmp.zst -o tmp_decompress |
1038 | cmp tmp tmp_decompress || die "difference between original and decompressed file" |
1039 | println "test : incorrect stream size" |
1040 | cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" |
1041 | |
1042 | println "\n===> zstd zero weight dict test " |
1043 | rm -f tmp* |
1044 | cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input |
1045 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input |
1046 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp |
1047 | $DIFF tmp_decomp tmp_input |
1048 | rm -rf tmp* |
1049 | |
1050 | println "\n===> zstd (valid) zero weight dict test " |
1051 | rm -f tmp* |
1052 | # 0 has a non-zero weight in the dictionary |
1053 | echo "0000000000000000000000000" > tmp_input |
1054 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input |
1055 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp |
1056 | $DIFF tmp_decomp tmp_input |
1057 | rm -rf tmp* |
1058 | |
1059 | println "\n===> size-hint mode" |
1060 | |
1061 | datagen -g11000 > tmp |
1062 | datagen -g11000 > tmp2 |
1063 | datagen > tmpDict |
1064 | println "test : basic file compression vs hinted streaming compression" |
1065 | file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) |
1066 | stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c) |
1067 | if [ "$stream_size" -ge "$file_size" ]; then |
1068 | die "hinted compression larger than expected" |
1069 | fi |
1070 | println "test : hinted streaming compression and decompression" |
1071 | cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000 |
1072 | zstd -df tmp.zst -o tmp_decompress |
1073 | cmp tmp tmp_decompress || die "difference between original and decompressed file" |
1074 | println "test : hinted streaming compression with dictionary" |
1075 | cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict |
1076 | println "test : multiple file compression with hints and dictionary" |
1077 | zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2 |
1078 | zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp |
1079 | zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2 |
1080 | cmp tmp.zst tmp1_.zst || die "first file's output differs" |
1081 | cmp tmp2.zst tmp2_.zst || die "second file's output differs" |
1082 | println "test : incorrect hinted stream sizes" |
1083 | cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high |
1084 | cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low |
1085 | cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high |
1086 | cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low |
1087 | println "test : allows and interprets K,KB,KiB,M,MB and MiB suffix" |
1088 | cat tmp | zstd -14 -f --size-hint=11K | zstd -t |
1089 | cat tmp | zstd -14 -f --size-hint=11KB | zstd -t |
1090 | cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t |
1091 | cat tmp | zstd -14 -f --size-hint=1M | zstd -t |
1092 | cat tmp | zstd -14 -f --size-hint=1MB | zstd -t |
1093 | cat tmp | zstd -14 -f --size-hint=1MiB | zstd -t |
1094 | |
1095 | |
1096 | println "\n===> dictionary tests " |
1097 | println "- Test high/low compressibility corpus training" |
1098 | datagen -g12M -P90 > tmpCorpusHighCompress |
1099 | datagen -g12M -P5 > tmpCorpusLowCompress |
1100 | zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress |
1101 | zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress |
1102 | rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress |
1103 | println "- Test with raw dict (content only) " |
1104 | datagen > tmpDict |
1105 | datagen -g1M | $MD5SUM > tmp1 |
1106 | datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2 |
1107 | $DIFF -q tmp1 tmp2 |
1108 | println "- Create first dictionary " |
1109 | TESTFILE="$PRGDIR"/zstdcli.c |
1110 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1111 | cp "$TESTFILE" tmp |
1112 | println "- Test dictionary compression with tmpDict as an input file and dictionary" |
1113 | zstd -f tmpDict -D tmpDict && die "compression error not detected!" |
1114 | println "- Dictionary compression roundtrip" |
1115 | zstd -f tmp -D tmpDict |
1116 | zstd -d tmp.zst -D tmpDict -fo result |
1117 | $DIFF "$TESTFILE" result |
1118 | println "- Dictionary compression with hlog < clog" |
1119 | zstd -6f tmp -D tmpDict --zstd=clog=25,hlog=23 |
1120 | println "- Dictionary compression with btlazy2 strategy" |
1121 | zstd -f tmp -D tmpDict --zstd=strategy=6 |
1122 | zstd -d tmp.zst -D tmpDict -fo result |
1123 | $DIFF "$TESTFILE" result |
1124 | if [ -e /proc/self/fd/0 ]; then |
1125 | println "- Test rejecting irregular dictionary file" |
1126 | cat tmpDict | zstd -f tmp -D /proc/self/fd/0 && die "Piped dictionary should fail!" |
1127 | cat tmpDict | zstd -d tmp.zst -D /proc/self/fd/0 -f && die "Piped dictionary should fail!" |
1128 | fi |
1129 | if [ -n "$hasMT" ] |
1130 | then |
1131 | println "- Test dictionary compression with multithreading " |
1132 | datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict # fails with v1.3.2 |
1133 | fi |
1134 | println "- Create second (different) dictionary " |
1135 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
1136 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
1137 | println "- Create dictionary with short dictID" |
1138 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
1139 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
1140 | println "- Create dictionary with wrong dictID parameter order (must fail)" |
1141 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " |
1142 | println "- Create dictionary with size limit" |
1143 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v |
1144 | println "- Create dictionary with small size limit" |
1145 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v |
1146 | println "- Create dictionary with wrong parameter order (must fail)" |
1147 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " |
1148 | println "- Compress without dictID" |
1149 | zstd -f tmp -D tmpDict1 --no-dictID |
1150 | zstd -d tmp.zst -D tmpDict -fo result |
1151 | $DIFF "$TESTFILE" result |
1152 | println "- Compress multiple files with dictionary" |
1153 | rm -rf dirTestDict |
1154 | mkdir dirTestDict |
1155 | cp "$TESTDIR"/*.c dirTestDict |
1156 | cp "$PRGDIR"/*.c dirTestDict |
1157 | cp "$PRGDIR"/*.h dirTestDict |
1158 | $MD5SUM dirTestDict/* > tmph1 |
1159 | zstd -f --rm dirTestDict/* -D tmpDictC |
1160 | zstd -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default |
1161 | case "$UNAME" in |
1162 | Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 |
1163 | *) $MD5SUM -c tmph1 ;; |
1164 | esac |
1165 | rm -rf dirTestDict |
1166 | println "- dictionary builder on bogus input" |
1167 | println "Hello World" > tmp |
1168 | zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" |
1169 | datagen -P0 -g10M > tmp |
1170 | zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" |
1171 | println "- Test -o before --train" |
1172 | rm -f tmpDict dictionary |
1173 | zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c |
1174 | test -f tmpDict |
1175 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c |
1176 | test -f dictionary |
1177 | if [ -n "$hasMT" ] |
1178 | then |
1179 | println "- Create dictionary with multithreading enabled" |
1180 | zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1181 | fi |
1182 | rm -f tmp* dictionary |
1183 | |
1184 | println "- Test --memory for dictionary compression" |
1185 | datagen -g12M -P90 > tmpCorpusHighCompress |
1186 | zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)" |
1187 | zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr |
f535537f |
1188 | cat zstTrainWithMemLimitStdErr | $GREP "setting manual memory limit for dictionary training data at 5 MB" |
1189 | cat zstTrainWithMemLimitStdErr | $GREP "Training samples set too large (12 MB); training on 5 MB only..." |
648db22b |
1190 | rm zstTrainWithMemLimitStdErr |
1191 | |
1192 | println "\n===> fastCover dictionary builder : advanced options " |
1193 | TESTFILE="$PRGDIR"/zstdcli.c |
1194 | datagen > tmpDict |
1195 | println "- Create first dictionary" |
1196 | zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1197 | cp "$TESTFILE" tmp |
1198 | zstd -f tmp -D tmpDict |
1199 | zstd -d tmp.zst -D tmpDict -fo result |
1200 | $DIFF "$TESTFILE" result |
1201 | println "- Create second (different) dictionary" |
1202 | zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
1203 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
1204 | zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file" |
1205 | println "- Create dictionary with short dictID" |
1206 | zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
1207 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
1208 | println "- Create dictionaries with shrink-dict flag enabled" |
1209 | zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict |
1210 | zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 |
1211 | zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 |
1212 | zstd --train-fastcover=shrink=5,steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict3 |
1213 | println "- Create dictionary with size limit" |
1214 | zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K |
1215 | println "- Create dictionary using all samples for both training and testing" |
1216 | zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1217 | println "- Create dictionary using f=16" |
1218 | zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1219 | zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" |
1220 | println "- Create dictionary using accel=2" |
1221 | zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1222 | println "- Create dictionary using accel=10" |
1223 | zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1224 | println "- Create dictionary with multithreading" |
1225 | zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1226 | println "- Test -o before --train-fastcover" |
1227 | rm -f tmpDict dictionary |
1228 | zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c |
1229 | test -f tmpDict |
1230 | zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c |
1231 | test -f dictionary |
1232 | rm -f tmp* dictionary |
1233 | |
1234 | |
1235 | println "\n===> legacy dictionary builder " |
1236 | |
1237 | TESTFILE="$PRGDIR"/zstdcli.c |
1238 | datagen > tmpDict |
1239 | println "- Create first dictionary" |
1240 | zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1241 | cp "$TESTFILE" tmp |
1242 | zstd -f tmp -D tmpDict |
1243 | zstd -d tmp.zst -D tmpDict -fo result |
1244 | $DIFF "$TESTFILE" result |
1245 | zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)" |
1246 | println "- Create second (different) dictionary" |
1247 | zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
1248 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
1249 | println "- Create dictionary with short dictID" |
1250 | zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
1251 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
1252 | println "- Create dictionary with size limit" |
1253 | zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K |
1254 | println "- Test -o before --train-legacy" |
1255 | rm -f tmpDict dictionary |
1256 | zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c |
1257 | test -f tmpDict |
1258 | zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c |
1259 | test -f dictionary |
1260 | rm -f tmp* dictionary |
1261 | |
1262 | |
1263 | println "\n===> integrity tests " |
1264 | |
1265 | println "test one file (tmp1.zst) " |
1266 | datagen > tmp1 |
1267 | zstd tmp1 |
1268 | zstd -t tmp1.zst |
1269 | zstd --test tmp1.zst |
1270 | println "test multiple files (*.zst) " |
1271 | zstd -t ./*.zst |
1272 | println "test bad files (*) " |
1273 | zstd -t ./* && die "bad files not detected !" |
1274 | zstd -t tmp1 && die "bad file not detected !" |
1275 | cp tmp1 tmp2.zst |
1276 | zstd -t tmp2.zst && die "bad file not detected !" |
1277 | datagen -g0 > tmp3 |
1278 | zstd -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad |
1279 | println "test --rm and --test combined " |
1280 | zstd -t --rm tmp1.zst |
1281 | test -f tmp1.zst # check file is still present |
1282 | cp tmp1.zst tmp2.zst |
1283 | zstd -t tmp1.zst tmp2.zst --rm |
1284 | test -f tmp1.zst # check file is still present |
1285 | test -f tmp2.zst # check file is still present |
1286 | split -b16384 tmp1.zst tmpSplit. |
1287 | zstd -t tmpSplit.* && die "bad file not detected !" |
1288 | datagen | zstd -c | zstd -t |
1289 | |
1290 | |
1291 | println "\n===> golden files tests " |
1292 | |
1293 | zstd -t -r "$TESTDIR/golden-decompression" |
1294 | zstd -c -r "$TESTDIR/golden-compression" | zstd -t |
1295 | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http" -c | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" -t |
1296 | |
1297 | |
1298 | println "\n===> benchmark mode tests " |
1299 | |
1300 | println "bench one file" |
1301 | datagen > tmp1 |
1302 | zstd -bi0 tmp1 |
1303 | println "bench multiple levels" |
1304 | zstd -i0b0e3 tmp1 |
1305 | println "bench negative level" |
1306 | zstd -bi0 --fast tmp1 |
1307 | println "with recursive and quiet modes" |
1308 | zstd -rqi0b1e2 tmp1 |
1309 | println "benchmark decompression only" |
1310 | zstd -f tmp1 |
1311 | zstd -b -d -i0 tmp1.zst |
1312 | println "benchmark can fail - decompression on invalid data" |
1313 | zstd -b -d -i0 tmp1 && die "invalid .zst data => benchmark should have failed" |
1314 | |
1315 | GZIPMODE=1 |
1316 | zstd --format=gzip -V || GZIPMODE=0 |
1317 | if [ $GZIPMODE -eq 1 ]; then |
1318 | println "benchmark mode is only compatible with zstd" |
1319 | zstd --format=gzip -b tmp1 && die "-b should be incompatible with gzip format!" |
1320 | fi |
1321 | |
1322 | println "\n===> zstd compatibility tests " |
1323 | |
1324 | datagen > tmp |
1325 | rm -f tmp.zst |
1326 | zstd --format=zstd -f tmp |
1327 | test -f tmp.zst |
1328 | |
1329 | |
1330 | println "\n===> gzip compatibility tests " |
1331 | |
1332 | GZIPMODE=1 |
1333 | zstd --format=gzip -V || GZIPMODE=0 |
1334 | if [ $GZIPMODE -eq 1 ]; then |
1335 | println "gzip support detected" |
1336 | GZIPEXE=1 |
1337 | gzip -V || GZIPEXE=0 |
1338 | if [ $GZIPEXE -eq 1 ]; then |
1339 | datagen > tmp |
1340 | zstd --format=gzip -f tmp |
1341 | gzip -t -v tmp.gz |
1342 | gzip -f tmp |
1343 | zstd -d -f -v tmp.gz |
1344 | rm -f tmp* |
1345 | else |
1346 | println "gzip binary not detected" |
1347 | fi |
1348 | else |
1349 | println "gzip mode not supported" |
1350 | fi |
1351 | |
1352 | |
1353 | println "\n===> gzip frame tests " |
1354 | |
1355 | if [ $GZIPMODE -eq 1 ]; then |
1356 | datagen > tmp |
1357 | zstd -f --format=gzip tmp |
1358 | zstd -f tmp |
1359 | cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp |
1360 | truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
1361 | rm -f tmp* |
1362 | else |
1363 | println "gzip mode not supported" |
1364 | fi |
1365 | |
1366 | if [ $GZIPMODE -eq 1 ]; then |
1367 | datagen > tmp |
1368 | rm -f tmp.zst |
1369 | zstd --format=gzip --format=zstd -f tmp |
1370 | test -f tmp.zst |
1371 | fi |
1372 | |
1373 | println "\n===> xz compatibility tests " |
1374 | |
1375 | LZMAMODE=1 |
1376 | zstd --format=xz -V || LZMAMODE=0 |
1377 | if [ $LZMAMODE -eq 1 ]; then |
1378 | println "xz support detected" |
1379 | XZEXE=1 |
1380 | xz -Q -V && lzma -Q -V || XZEXE=0 |
1381 | if [ $XZEXE -eq 1 ]; then |
1382 | println "Testing zstd xz and lzma support" |
1383 | datagen > tmp |
1384 | zstd --format=lzma -f tmp |
1385 | zstd --format=xz -f tmp |
1386 | xz -Q -t -v tmp.xz |
1387 | xz -Q -t -v tmp.lzma |
1388 | xz -Q -f -k tmp |
1389 | lzma -Q -f -k --lzma1 tmp |
1390 | zstd -d -f -v tmp.xz |
1391 | zstd -d -f -v tmp.lzma |
1392 | rm -f tmp* |
1393 | println "Creating symlinks" |
1394 | ln -s "$ZSTD_BIN" ./xz |
1395 | ln -s "$ZSTD_BIN" ./unxz |
1396 | ln -s "$ZSTD_BIN" ./lzma |
1397 | ln -s "$ZSTD_BIN" ./unlzma |
1398 | println "Testing xz and lzma symlinks" |
1399 | datagen > tmp |
1400 | ./xz tmp |
1401 | xz -Q -d tmp.xz |
1402 | ./lzma tmp |
1403 | lzma -Q -d tmp.lzma |
1404 | println "Testing unxz and unlzma symlinks" |
1405 | xz -Q tmp |
1406 | ./xz -d tmp.xz |
1407 | lzma -Q tmp |
1408 | ./lzma -d tmp.lzma |
1409 | rm -f xz unxz lzma unlzma |
1410 | rm -f tmp* |
1411 | else |
1412 | println "xz binary not detected" |
1413 | fi |
1414 | else |
1415 | println "xz mode not supported" |
1416 | fi |
1417 | |
1418 | |
1419 | println "\n===> xz frame tests " |
1420 | |
1421 | if [ $LZMAMODE -eq 1 ]; then |
1422 | datagen > tmp |
1423 | zstd -f --format=xz tmp |
1424 | zstd -f --format=lzma tmp |
1425 | zstd -f tmp |
1426 | cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp |
1427 | truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
1428 | truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
1429 | rm -f tmp* |
1430 | else |
1431 | println "xz mode not supported" |
1432 | fi |
1433 | |
1434 | println "\n===> lz4 compatibility tests " |
1435 | |
1436 | LZ4MODE=1 |
1437 | zstd --format=lz4 -V || LZ4MODE=0 |
1438 | if [ $LZ4MODE -eq 1 ]; then |
1439 | println "lz4 support detected" |
1440 | LZ4EXE=1 |
1441 | lz4 -V || LZ4EXE=0 |
1442 | if [ $LZ4EXE -eq 1 ]; then |
1443 | datagen > tmp |
1444 | zstd --format=lz4 -f tmp |
1445 | lz4 -t -v tmp.lz4 |
1446 | lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout |
1447 | zstd -d -f -v tmp.lz4 |
1448 | rm -f tmp* |
1449 | else |
1450 | println "lz4 binary not detected" |
1451 | fi |
1452 | else |
1453 | println "lz4 mode not supported" |
1454 | fi |
1455 | |
1456 | |
1457 | if [ $LZ4MODE -eq 1 ]; then |
1458 | println "\n===> lz4 frame tests " |
1459 | datagen > tmp |
1460 | zstd -f --format=lz4 tmp |
1461 | zstd -f tmp |
1462 | cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp |
1463 | truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
1464 | rm -f tmp* |
1465 | else |
1466 | println "\nlz4 mode not supported" |
1467 | fi |
1468 | |
1469 | |
1470 | println "\n===> suffix list test" |
1471 | |
1472 | ! zstd -d tmp.abc 2> tmplg |
1473 | |
1474 | if [ $GZIPMODE -ne 1 ]; then |
f535537f |
1475 | $GREP ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" |
648db22b |
1476 | fi |
1477 | |
1478 | if [ $LZMAMODE -ne 1 ]; then |
f535537f |
1479 | $GREP ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" |
1480 | $GREP ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" |
648db22b |
1481 | fi |
1482 | |
1483 | if [ $LZ4MODE -ne 1 ]; then |
f535537f |
1484 | $GREP ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" |
648db22b |
1485 | fi |
1486 | |
1487 | touch tmp1 |
1488 | zstd tmp1 -o tmp1.zstd |
1489 | zstd -d -f tmp1.zstd # support .zstd suffix even though it's not the default suffix |
1490 | |
1491 | println "\n===> tar extension tests " |
1492 | |
1493 | rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd |
1494 | |
1495 | datagen > tmp |
1496 | tar -cf tmp.tar tmp |
1497 | zstd tmp.tar -o tmp.tzst |
1498 | rm -f tmp.tar |
1499 | zstd -d tmp.tzst |
1500 | [ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" |
1501 | rm -f tmp.tar tmp.tzst |
1502 | |
1503 | if [ $GZIPMODE -eq 1 ]; then |
1504 | tar -f - -c tmp | gzip > tmp.tgz |
1505 | zstd -d tmp.tgz |
1506 | [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!" |
1507 | rm -f tmp.tar tmp.tgz |
1508 | fi |
1509 | |
1510 | if [ $LZMAMODE -eq 1 ]; then |
1511 | tar -f - -c tmp | zstd --format=xz > tmp.txz |
1512 | zstd -d tmp.txz |
1513 | [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" |
1514 | rm -f tmp.tar tmp.txz |
1515 | fi |
1516 | |
1517 | if [ $LZ4MODE -eq 1 ]; then |
1518 | tar -f - -c tmp | zstd --format=lz4 > tmp.tlz4 |
1519 | zstd -d tmp.tlz4 |
1520 | [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" |
1521 | rm -f tmp.tar tmp.tlz4 |
1522 | fi |
1523 | |
1524 | touch tmp.t tmp.tz tmp.tzs |
1525 | ! zstd -d tmp.t |
1526 | ! zstd -d tmp.tz |
1527 | ! zstd -d tmp.tzs |
1528 | |
1529 | |
1530 | println "\n===> zstd round-trip tests " |
1531 | |
1532 | roundTripTest |
1533 | roundTripTest -g15K # TableID==3 |
1534 | roundTripTest -g127K # TableID==2 |
1535 | roundTripTest -g255K # TableID==1 |
1536 | roundTripTest -g522K # TableID==0 |
1537 | roundTripTest -g519K 6 # greedy, hash chain |
1538 | roundTripTest -g517K 16 # btlazy2 |
1539 | roundTripTest -g516K 19 # btopt |
1540 | |
1541 | fileRoundTripTest -g500K |
1542 | |
1543 | println "\n===> zstd long distance matching round-trip tests " |
1544 | roundTripTest -g0 "2 --single-thread --long" |
1545 | roundTripTest -g1000K "1 --single-thread --long" |
1546 | roundTripTest -g517K "6 --single-thread --long" |
1547 | roundTripTest -g516K "16 --single-thread --long" |
1548 | roundTripTest -g518K "19 --single-thread --long" |
1549 | roundTripTest -g2M "22 --single-thread --ultra --long" |
1550 | fileRoundTripTest -g5M "3 --single-thread --long" |
1551 | |
1552 | |
1553 | roundTripTest -g96K "5 --single-thread" |
1554 | if [ -n "$hasMT" ] |
1555 | then |
1556 | println "\n===> zstdmt round-trip tests " |
1557 | roundTripTest -g4M "1 -T0" |
1558 | roundTripTest -g4M "1 -T0 --auto-threads=physical" |
1559 | roundTripTest -g4M "1 -T0 --auto-threads=logical" |
1560 | roundTripTest -g8M "3 -T2" |
1561 | roundTripTest -g8M "19 --long" |
1562 | roundTripTest -g8000K "2 --threads=2" |
1563 | fileRoundTripTest -g4M "19 -T2 -B1M" |
1564 | |
1565 | println "\n===> zstdmt long distance matching round-trip tests " |
1566 | roundTripTest -g8M "3 --long=24 -T2" |
1567 | |
1568 | println "\n===> zstdmt environment variable tests " |
1569 | echo "multifoo" >> mt_tmp |
1570 | ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting |
1571 | ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting |
1572 | ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting |
1573 | ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting |
1574 | ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting |
1575 | ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting |
1576 | ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= |
1577 | ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage |
1578 | ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread |
1579 | # temporary envvar changes in the above tests would actually persist in macos /bin/sh |
1580 | unset ZSTD_NBTHREADS |
1581 | rm -f mt_tmp* |
1582 | |
1583 | println "\n===> ovLog tests " |
1584 | datagen -g2MB > tmp |
1585 | refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c) |
1586 | ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) |
1587 | ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) |
1588 | if [ "$refSize" -eq "$ov9Size" ]; then |
1589 | echo ov9Size should be different from refSize |
1590 | exit 1 |
1591 | fi |
1592 | if [ "$refSize" -eq "$ov1Size" ]; then |
1593 | echo ov1Size should be different from refSize |
1594 | exit 1 |
1595 | fi |
1596 | if [ "$ov9Size" -ge "$ov1Size" ]; then |
1597 | echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size" |
1598 | exit 1 |
1599 | fi |
1600 | |
1601 | else |
1602 | println "\n===> no multithreading, skipping zstdmt tests " |
1603 | fi |
1604 | |
1605 | rm -f tmp* |
1606 | |
1607 | println "\n===> zstd --list/-l single frame tests " |
1608 | datagen > tmp1 |
1609 | datagen > tmp2 |
1610 | datagen > tmp3 |
1611 | zstd tmp* |
1612 | zstd -l ./*.zst |
f535537f |
1613 | zstd -lv ./*.zst | $GREP "Decompressed Size:" # check that decompressed size is present in header |
648db22b |
1614 | zstd --list ./*.zst |
1615 | zstd --list -v ./*.zst |
1616 | |
1617 | println "\n===> zstd --list/-l multiple frame tests " |
1618 | cat tmp1.zst tmp2.zst > tmp12.zst |
1619 | cat tmp12.zst tmp3.zst > tmp123.zst |
1620 | zstd -l ./*.zst |
1621 | zstd -lv ./*.zst |
1622 | |
1623 | println "\n===> zstd --list/-l error detection tests " |
1624 | zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file" |
1625 | zstd --list tmp* && die "-l must fail on non-zstd file" |
1626 | zstd -lv tmp1* && die "-l must fail on non-zstd file" |
1627 | zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" |
1628 | |
1629 | println "test : detect truncated compressed file " |
1630 | TEST_DATA_FILE=truncatable-input.txt |
1631 | FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst |
1632 | TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst |
1633 | datagen -g50000 > $TEST_DATA_FILE |
1634 | zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE |
1635 | dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE |
1636 | zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" |
1637 | |
1638 | rm -f $TEST_DATA_FILE |
1639 | rm -f $FULL_COMPRESSED_FILE |
1640 | rm -f $TRUNCATED_COMPRESSED_FILE |
1641 | |
1642 | println "\n===> zstd --list/-l errors when presented with stdin / no files" |
1643 | zstd -l && die "-l must fail on empty list of files" |
1644 | zstd -l - && die "-l does not work on stdin" |
1645 | zstd -l < tmp1.zst && die "-l does not work on stdin" |
1646 | zstd -l - < tmp1.zst && die "-l does not work on stdin" |
1647 | zstd -l - tmp1.zst && die "-l does not work on stdin" |
1648 | zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin" |
1649 | zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty |
1650 | |
1651 | println "\n===> zstd --list/-l test with null files " |
1652 | datagen -g0 > tmp5 |
1653 | zstd tmp5 |
1654 | zstd -l tmp5.zst |
1655 | zstd -l tmp5* && die "-l must fail on non-zstd file" |
f535537f |
1656 | zstd -lv tmp5.zst | $GREP "Decompressed Size: 0 B (0 B)" # check that 0 size is present in header |
648db22b |
1657 | zstd -lv tmp5* && die "-l must fail on non-zstd file" |
1658 | |
1659 | println "\n===> zstd --list/-l test with no content size field " |
1660 | datagen -g513K | zstd > tmp6.zst |
1661 | zstd -l tmp6.zst |
f535537f |
1662 | zstd -lv tmp6.zst | $GREP "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" |
648db22b |
1663 | |
1664 | println "\n===> zstd --list/-l test with no checksum " |
1665 | zstd -f --no-check tmp1 |
1666 | zstd -l tmp1.zst |
1667 | zstd -lv tmp1.zst |
1668 | |
1669 | println "\n===> zstd trace tests " |
1670 | zstd -f --trace tmp.trace tmp1 |
1671 | zstd -f --trace tmp.trace tmp1 tmp2 tmp3 |
1672 | zstd -f --trace tmp.trace tmp1 tmp2 tmp3 -o /dev/null |
1673 | zstd -f --trace tmp.trace tmp1 tmp2 tmp3 --single-thread |
1674 | zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null |
1675 | zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null --single-thread |
1676 | zstd --trace tmp.trace -t tmp1.zst |
1677 | zstd --trace tmp.trace -t tmp1.zst tmp2.zst |
1678 | zstd -f --trace tmp.trace -d tmp1.zst |
1679 | zstd -f --trace tmp.trace -d tmp1.zst tmp2.zst tmp3.zst |
1680 | zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1 |
1681 | zstd -b1e10i0 --trace tmp.trace tmp1 |
1682 | zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3 |
1683 | |
1684 | rm -f tmp* |
1685 | |
1686 | |
1687 | println "\n===> zstd long distance matching tests " |
1688 | roundTripTest -g0 " --single-thread --long" |
1689 | roundTripTest -g9M "2 --single-thread --long" |
1690 | # Test parameter parsing |
1691 | roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB" |
1692 | roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB" |
1693 | roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB" |
1694 | roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" |
1695 | |
1696 | |
f535537f |
1697 | if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -ne "1" ]; then |
1698 | println "\n===> zstd long distance matching with optimal parser compressed size tests " |
1699 | optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) |
1700 | longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) |
1701 | optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) |
1702 | longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) |
1703 | optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) |
1704 | longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) |
1705 | if [ "$longCSize16" -gt "$optCSize16" ]; then |
1706 | echo using --long on compression level 16 should not cause compressed size regression |
1707 | exit 1 |
1708 | elif [ "$longCSize19" -gt "$optCSize19" ]; then |
1709 | echo using --long on compression level 19 should not cause compressed size regression |
1710 | exit 1 |
1711 | elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then |
1712 | echo using --long on compression level 19 with wLog=23 should not cause compressed size regression |
1713 | exit 1 |
1714 | fi |
648db22b |
1715 | fi |
1716 | |
1717 | println "\n===> zstd asyncio tests " |
1718 | |
1719 | addFrame() { |
1720 | datagen -g2M -s$2 >> tmp_uncompressed |
1721 | datagen -g2M -s$2 | zstd -1 --format=$1 >> tmp_compressed.zst |
1722 | } |
1723 | |
1724 | addTwoFrames() { |
1725 | addFrame $1 1 |
1726 | addFrame $1 2 |
1727 | } |
1728 | |
1729 | testAsyncIO() { |
1730 | roundTripTest -g2M "3 --asyncio --format=$1" |
1731 | roundTripTest -g2M "3 --no-asyncio --format=$1" |
1732 | } |
1733 | |
1734 | rm -f tmp_compressed tmp_uncompressed |
1735 | testAsyncIO zstd |
1736 | addTwoFrames zstd |
1737 | if [ $GZIPMODE -eq 1 ]; then |
1738 | testAsyncIO gzip |
1739 | addTwoFrames gzip |
1740 | fi |
1741 | if [ $LZMAMODE -eq 1 ]; then |
1742 | testAsyncIO lzma |
1743 | addTwoFrames lzma |
1744 | fi |
1745 | if [ $LZ4MODE -eq 1 ]; then |
1746 | testAsyncIO lz4 |
1747 | addTwoFrames lz4 |
1748 | fi |
1749 | cat tmp_uncompressed | $MD5SUM > tmp2 |
1750 | zstd -d tmp_compressed.zst --asyncio -c | $MD5SUM > tmp1 |
1751 | $DIFF -q tmp1 tmp2 |
1752 | rm tmp1 |
1753 | zstd -d tmp_compressed.zst --no-asyncio -c | $MD5SUM > tmp1 |
1754 | $DIFF -q tmp1 tmp2 |
1755 | |
1756 | if [ "$1" != "--test-large-data" ]; then |
1757 | println "Skipping large data tests" |
1758 | exit 0 |
1759 | fi |
1760 | |
1761 | |
1762 | ############################################################################# |
1763 | |
1764 | |
1765 | if [ -n "$hasMT" ] |
1766 | then |
1767 | println "\n===> adaptive mode " |
1768 | roundTripTest -g270000000 " --adapt" |
1769 | roundTripTest -g27000000 " --adapt=min=1,max=4" |
1770 | roundTripTest -g27000000 " --adapt=min=-2,max=-1" |
1771 | println "===> test: --adapt must fail on incoherent bounds " |
1772 | datagen > tmp |
1773 | zstd --adapt= tmp && die "invalid compression parameter" |
1774 | zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" |
1775 | |
1776 | println "\n===> rsyncable mode " |
1777 | roundTripTest -g10M " --rsyncable" |
1778 | roundTripTest -g10M " --rsyncable -B100K" |
1779 | println "===> test: --rsyncable must fail with --single-thread" |
1780 | zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" |
1781 | fi |
1782 | |
1783 | println "\n===> patch-from=origin tests" |
1784 | datagen -g1000 -P50 > tmp_dict |
1785 | datagen -g1000 -P10 > tmp_patch |
1786 | zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff |
1787 | zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon |
1788 | $DIFF -s tmp_patch_recon tmp_patch |
1789 | |
1790 | println "\n===> alternate syntax: patch-from origin" |
1791 | zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff |
1792 | zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon |
1793 | $DIFF -s tmp_patch_recon tmp_patch |
1794 | rm -rf tmp_* |
1795 | |
1796 | println "\n===> patch-from recursive tests" |
1797 | mkdir tmp_dir |
1798 | datagen > tmp_dir/tmp1 |
1799 | datagen > tmp_dir/tmp2 |
1800 | datagen > tmp_dict |
1801 | zstd --patch-from=tmp_dict -r tmp_dir && die |
1802 | rm -rf tmp* |
1803 | |
1804 | println "\n===> patch-from long mode trigger larger file test" |
f535537f |
1805 | if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -eq "1" ]; then |
1806 | # if binary tree strategies are excluded, the threshold is different |
1807 | datagen -g10000000 > tmp_dict |
1808 | datagen -g10000000 > tmp_patch |
1809 | else |
1810 | datagen -g5000000 > tmp_dict |
1811 | datagen -g5000000 > tmp_patch |
1812 | fi |
1813 | zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | $GREP "long mode automatically triggered" |
648db22b |
1814 | rm -rf tmp* |
1815 | |
1816 | println "\n===> patch-from very large dictionary and file test" |
1817 | datagen -g550000000 -P0 > tmp_dict |
1818 | datagen -g100000000 -P1 > tmp_patch |
1819 | zstd --long=30 -1f --patch-from tmp_dict tmp_patch |
1820 | zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon |
1821 | $DIFF -s tmp_patch_recon tmp_patch |
1822 | rm -rf tmp* |
1823 | |
1824 | println "\n===> patch-from --stream-size test" |
1825 | datagen -g1000 -P50 > tmp_dict |
1826 | datagen -g1000 -P10 > tmp_patch |
1827 | cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die |
1828 | cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff |
1829 | rm -rf tmp* |
1830 | |
1831 | println "\n===> large files tests " |
1832 | |
1833 | roundTripTest -g270000000 1 |
1834 | roundTripTest -g250000000 2 |
1835 | roundTripTest -g230000000 3 |
1836 | |
1837 | roundTripTest -g140000000 -P60 4 |
1838 | roundTripTest -g130000000 -P62 5 |
1839 | roundTripTest -g120000000 -P65 6 |
1840 | |
1841 | roundTripTest -g70000000 -P70 7 |
1842 | roundTripTest -g60000000 -P71 8 |
1843 | roundTripTest -g50000000 -P73 9 |
1844 | |
1845 | roundTripTest -g35000000 -P75 10 |
1846 | roundTripTest -g30000000 -P76 11 |
1847 | roundTripTest -g25000000 -P78 12 |
1848 | |
1849 | roundTripTest -g18000013 -P80 13 |
1850 | roundTripTest -g18000014 -P80 14 |
1851 | roundTripTest -g18000015 -P81 15 |
1852 | roundTripTest -g18000016 -P84 16 |
1853 | roundTripTest -g18000017 -P88 17 |
1854 | roundTripTest -g18000018 -P94 18 |
1855 | roundTripTest -g18000019 -P96 19 |
1856 | |
1857 | roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25" |
1858 | roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an overflow rescale |
1859 | |
1860 | fileRoundTripTest -g4193M -P99 1 |
1861 | |
1862 | |
1863 | println "\n===> zstd long, long distance matching round-trip tests " |
1864 | roundTripTest -g270000000 "1 --single-thread --long" |
1865 | roundTripTest -g130000000 -P60 "5 --single-thread --long" |
1866 | roundTripTest -g35000000 -P70 "8 --single-thread --long" |
1867 | roundTripTest -g18000001 -P80 "18 --single-thread --long" |
1868 | # Test large window logs |
1869 | roundTripTest -g700M -P50 "1 --single-thread --long=29" |
1870 | roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" |
1871 | |
1872 | |
1873 | if [ -n "$hasMT" ] |
1874 | then |
1875 | println "\n===> zstdmt long round-trip tests " |
1876 | roundTripTest -g80000000 -P99 "19 -T2" " " |
1877 | roundTripTest -g5000000000 -P99 "1 -T2" " " |
1878 | roundTripTest -g500000000 -P97 "1 -T999" " " |
1879 | fileRoundTripTest -g4103M -P98 " -T0" " " |
1880 | roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " |
1881 | # Exposes the bug in https://github.com/facebook/zstd/pull/1678 |
1882 | # This test fails on 4 different travis builds at the time of writing |
1883 | # because it needs to allocate 8 GB of memory. |
1884 | # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000" |
1885 | else |
1886 | println "\n**** no multithreading, skipping zstdmt tests **** " |
1887 | fi |
1888 | |
1889 | |
1890 | println "\n===> cover dictionary builder : advanced options " |
1891 | |
1892 | TESTFILE="$PRGDIR"/zstdcli.c |
1893 | datagen > tmpDict |
1894 | println "- Create first dictionary" |
1895 | zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
1896 | cp "$TESTFILE" tmp |
1897 | zstd -f tmp -D tmpDict |
1898 | zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the same time" |
1899 | zstd -d tmp.zst -D tmpDict -fo result |
1900 | $DIFF "$TESTFILE" result |
1901 | zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" |
1902 | println "- Create second (different) dictionary" |
1903 | zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
1904 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
1905 | println "- Create dictionary using shrink-dict flag" |
1906 | zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict |
1907 | zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 |
1908 | zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 |
1909 | zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3 |
1910 | println "- Create dictionary with short dictID" |
1911 | zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
1912 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
1913 | println "- Create dictionary with size limit" |
1914 | zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K |
1915 | println "- Compare size of dictionary from 90% training samples with 80% training samples" |
1916 | zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1917 | zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1918 | println "- Create dictionary using all samples for both training and testing" |
1919 | zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
1920 | println "- Test -o before --train-cover" |
1921 | rm -f tmpDict dictionary |
1922 | zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c |
1923 | test -f tmpDict |
1924 | zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c |
1925 | test -f dictionary |
1926 | rm -f tmp* dictionary |
1927 | |
1928 | rm -f tmp* |