| 1 | #!/bin/sh |
| 2 | |
| 3 | set -e # exit immediately on error |
| 4 | # set -x # print commands before execution (debug) |
| 5 | |
| 6 | unset ZSTD_CLEVEL |
| 7 | unset ZSTD_NBTHREADS |
| 8 | |
| 9 | |
| 10 | die() { |
| 11 | println "$@" 1>&2 |
| 12 | exit 1 |
| 13 | } |
| 14 | |
| 15 | datagen() { |
| 16 | "$DATAGEN_BIN" "$@" |
| 17 | } |
| 18 | |
| 19 | zstd() { |
| 20 | if [ -z "$EXE_PREFIX" ]; then |
| 21 | "$ZSTD_BIN" "$@" |
| 22 | else |
| 23 | "$EXE_PREFIX" "$ZSTD_BIN" "$@" |
| 24 | fi |
| 25 | } |
| 26 | |
| 27 | sudoZstd() { |
| 28 | if [ -z "$EXE_PREFIX" ]; then |
| 29 | sudo "$ZSTD_BIN" "$@" |
| 30 | else |
| 31 | sudo "$EXE_PREFIX" "$ZSTD_BIN" "$@" |
| 32 | fi |
| 33 | } |
| 34 | |
| 35 | roundTripTest() { |
| 36 | if [ -n "$3" ]; then |
| 37 | cLevel="$3" |
| 38 | proba="$2" |
| 39 | else |
| 40 | cLevel="$2" |
| 41 | proba="" |
| 42 | fi |
| 43 | if [ -n "$4" ]; then |
| 44 | dLevel="$4" |
| 45 | else |
| 46 | dLevel="$cLevel" |
| 47 | fi |
| 48 | |
| 49 | rm -f tmp1 tmp2 |
| 50 | println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel" |
| 51 | datagen $1 $proba | $MD5SUM > tmp1 |
| 52 | datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 |
| 53 | $DIFF -q tmp1 tmp2 |
| 54 | } |
| 55 | |
| 56 | fileRoundTripTest() { |
| 57 | if [ -n "$3" ]; then |
| 58 | local_c="$3" |
| 59 | local_p="$2" |
| 60 | else |
| 61 | local_c="$2" |
| 62 | local_p="" |
| 63 | fi |
| 64 | if [ -n "$4" ]; then |
| 65 | local_d="$4" |
| 66 | else |
| 67 | local_d="$local_c" |
| 68 | fi |
| 69 | |
| 70 | rm -f tmp.zst tmp.md5.1 tmp.md5.2 |
| 71 | println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d" |
| 72 | datagen $1 $local_p > tmp |
| 73 | < tmp $MD5SUM > tmp.md5.1 |
| 74 | zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 |
| 75 | $DIFF -q tmp.md5.1 tmp.md5.2 |
| 76 | } |
| 77 | |
| 78 | truncateLastByte() { |
| 79 | dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" |
| 80 | } |
| 81 | |
| 82 | println() { |
| 83 | printf '%b\n' "${*}" |
| 84 | } |
| 85 | |
| 86 | if [ -z "${size}" ]; then |
| 87 | size= |
| 88 | else |
| 89 | size=${size} |
| 90 | fi |
| 91 | |
| 92 | SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) |
| 93 | PRGDIR="$SCRIPT_DIR/../programs" |
| 94 | TESTDIR="$SCRIPT_DIR/../tests" |
| 95 | UNAME=${UNAME:-$(uname)} |
| 96 | GREP=${GREP:-grep} |
| 97 | |
| 98 | case "$UNAME" in |
| 99 | SunOS) DIFF=${DIFF:-gdiff} ;; |
| 100 | *) DIFF=${DIFF:-diff} ;; |
| 101 | esac |
| 102 | |
| 103 | detectedTerminal=false |
| 104 | if [ -t 0 ] && [ -t 1 ] |
| 105 | then |
| 106 | detectedTerminal=true |
| 107 | fi |
| 108 | isTerminal=${isTerminal:-$detectedTerminal} |
| 109 | |
| 110 | isWindows=false |
| 111 | INTOVOID="/dev/null" |
| 112 | case "$UNAME" in |
| 113 | GNU) DEVDEVICE="/dev/random" ;; |
| 114 | *) DEVDEVICE="/dev/zero" ;; |
| 115 | esac |
| 116 | case "$OS" in |
| 117 | Windows*) |
| 118 | isWindows=true |
| 119 | INTOVOID="NUL" |
| 120 | DEVDEVICE="NUL" |
| 121 | ;; |
| 122 | esac |
| 123 | |
| 124 | case "$UNAME" in |
| 125 | Darwin) MD5SUM="md5 -r" ;; |
| 126 | FreeBSD) MD5SUM="gmd5sum" ;; |
| 127 | NetBSD) MD5SUM="md5 -n" ;; |
| 128 | OpenBSD) MD5SUM="md5" ;; |
| 129 | *) MD5SUM="md5sum" ;; |
| 130 | esac |
| 131 | |
| 132 | MTIME="stat -c %Y" |
| 133 | case "$UNAME" in |
| 134 | Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;; |
| 135 | esac |
| 136 | |
| 137 | assertSameMTime() { |
| 138 | MT1=$($MTIME "$1") |
| 139 | MT2=$($MTIME "$2") |
| 140 | echo MTIME $MT1 $MT2 |
| 141 | [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)" |
| 142 | } |
| 143 | |
| 144 | GET_PERMS="stat -c %a" |
| 145 | case "$UNAME" in |
| 146 | Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;; |
| 147 | esac |
| 148 | |
| 149 | assertFilePermissions() { |
| 150 | STAT1=$($GET_PERMS "$1") |
| 151 | STAT2=$2 |
| 152 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)" |
| 153 | } |
| 154 | |
| 155 | assertSamePermissions() { |
| 156 | STAT1=$($GET_PERMS "$1") |
| 157 | STAT2=$($GET_PERMS "$2") |
| 158 | [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)" |
| 159 | } |
| 160 | |
| 161 | |
| 162 | # check if ZSTD_BIN is defined. if not, use the default value |
| 163 | if [ -z "${ZSTD_BIN}" ]; then |
| 164 | println "\nZSTD_BIN is not set. Using the default value..." |
| 165 | ZSTD_BIN="$PRGDIR/zstd" |
| 166 | fi |
| 167 | |
| 168 | # check if DATAGEN_BIN is defined. if not, use the default value |
| 169 | if [ -z "${DATAGEN_BIN}" ]; then |
| 170 | println "\nDATAGEN_BIN is not set. Using the default value..." |
| 171 | DATAGEN_BIN="$TESTDIR/datagen" |
| 172 | fi |
| 173 | |
| 174 | # Why was this line here ? Generates a strange ZSTD_BIN when EXE_PREFIX is non empty |
| 175 | # ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN" |
| 176 | |
| 177 | # assertions |
| 178 | [ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd following the instructions in README.md" |
| 179 | [ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. " |
| 180 | println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" |
| 181 | |
| 182 | if echo hello | zstd -v -T2 2>&1 > $INTOVOID | $GREP -q 'multi-threading is disabled' |
| 183 | then |
| 184 | hasMT="" |
| 185 | else |
| 186 | hasMT="true" |
| 187 | fi |
| 188 | |
| 189 | |
| 190 | zstd -vvV |
| 191 | |
| 192 | println "\n===> simple tests " |
| 193 | |
| 194 | datagen > tmp |
| 195 | zstd -h |
| 196 | zstd -H |
| 197 | zstd -V |
| 198 | println "test : basic compression " |
| 199 | zstd -f tmp # trivial compression case, creates tmp.zst |
| 200 | zstd -f -z tmp |
| 201 | zstd -f -k tmp |
| 202 | zstd -f -C tmp |
| 203 | println "test : basic decompression" |
| 204 | zstd -df tmp.zst # trivial decompression case (overwrites tmp) |
| 205 | println "test : too large compression level => auto-fix" |
| 206 | zstd -99 -f tmp # too large compression level, automatic sized down |
| 207 | zstd -5000000000 -f tmp && die "too large numeric value : must fail" |
| 208 | println "test : --fast aka negative compression levels" |
| 209 | zstd --fast -f tmp # == -1 |
| 210 | zstd --fast=3 -f tmp # == -3 |
| 211 | zstd --fast=200000 -f tmp # too low compression level, automatic fixed |
| 212 | zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail" |
| 213 | zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" |
| 214 | println "test : too large numeric argument" |
| 215 | zstd --fast=9999999999 -f tmp && die "should have refused numeric value" |
| 216 | println "test : set compression level with environment variable ZSTD_CLEVEL" |
| 217 | |
| 218 | ZSTD_CLEVEL=12 zstd -f tmp # positive compression level |
| 219 | ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level |
| 220 | ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign |
| 221 | ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting |
| 222 | ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting |
| 223 | ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting |
| 224 | ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting |
| 225 | ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting |
| 226 | ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting |
| 227 | println "test : override ZSTD_CLEVEL with command line option" |
| 228 | ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option |
| 229 | |
| 230 | # temporary envvar changes in the above tests would actually persist in macos /bin/sh |
| 231 | unset ZSTD_CLEVEL |
| 232 | |
| 233 | |
| 234 | println "test : compress to stdout" |
| 235 | zstd tmp -c > tmpCompressed |
| 236 | zstd tmp --stdout > tmpCompressed # long command format |
| 237 | |
| 238 | println "test : compress to named file (-o)" |
| 239 | rm -f tmpCompressed |
| 240 | zstd tmp -o tmpCompressed |
| 241 | test -f tmpCompressed # file must be created |
| 242 | |
| 243 | println "test : force write, correct order" |
| 244 | zstd tmp -fo tmpCompressed |
| 245 | |
| 246 | println "test : -c + -o : last one wins" |
| 247 | rm -f tmpOut |
| 248 | zstd tmp -c > tmpCompressed -o tmpOut |
| 249 | test -f tmpOut # file must be created |
| 250 | rm -f tmpCompressed |
| 251 | zstd tmp -o tmpOut -c > tmpCompressed |
| 252 | test -f tmpCompressed # file must be created |
| 253 | |
| 254 | println "test : forgotten argument" |
| 255 | cp tmp tmp2 |
| 256 | zstd tmp2 -fo && die "-o must be followed by filename " |
| 257 | println "test : implied stdout when input is stdin" |
| 258 | println bob | zstd | zstd -d |
| 259 | if [ "$isTerminal" = true ]; then |
| 260 | println "test : compressed data to terminal" |
| 261 | println bob | zstd && die "should have refused : compressed data to terminal" |
| 262 | println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" |
| 263 | zstd -d > $INTOVOID && die "should have refused : compressed data from terminal" |
| 264 | fi |
| 265 | println "test : null-length file roundtrip" |
| 266 | println -n '' | zstd - --stdout | zstd -d --stdout |
| 267 | println "test : ensure small file doesn't add 3-bytes null block" |
| 268 | datagen -g1 > tmp1 |
| 269 | zstd tmp1 -c | wc -c | $GREP "14" |
| 270 | zstd < tmp1 | wc -c | $GREP "14" |
| 271 | println "test : decompress file with wrong suffix (must fail)" |
| 272 | zstd -d tmpCompressed && die "wrong suffix error not detected!" |
| 273 | zstd -df tmp && die "should have refused : wrong extension" |
| 274 | println "test : decompress into stdout" |
| 275 | zstd -d tmpCompressed -c > tmpResult # decompression using stdout |
| 276 | zstd --decompress tmpCompressed -c > tmpResult |
| 277 | zstd --decompress tmpCompressed --stdout > tmpResult |
| 278 | println "test : decompress from stdin into stdout" |
| 279 | zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout |
| 280 | zstd -dc - < tmp.zst > $INTOVOID |
| 281 | zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used |
| 282 | zstd -d - < tmp.zst > $INTOVOID |
| 283 | println "test : impose memory limitation (must fail)" |
| 284 | datagen -g500K > tmplimit |
| 285 | zstd -f tmplimit |
| 286 | zstd -d -f tmplimit.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" |
| 287 | zstd -d -f tmplimit.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command |
| 288 | zstd -d -f tmplimit.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command |
| 289 | zstd -d -f tmplimit.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command |
| 290 | rm -f tmplimit tmplimit.zst |
| 291 | println "test : overwrite protection" |
| 292 | zstd -q tmp && die "overwrite check failed!" |
| 293 | println "test : force overwrite" |
| 294 | zstd -q -f tmp |
| 295 | zstd -q --force tmp |
| 296 | println "test : overwrite readonly file" |
| 297 | rm -f tmpro tmpro.zst |
| 298 | println foo > tmpro.zst |
| 299 | println foo > tmpro |
| 300 | chmod 400 tmpro.zst |
| 301 | zstd -q tmpro && die "should have refused to overwrite read-only file" |
| 302 | zstd -q -f tmpro |
| 303 | println "test: --no-progress flag" |
| 304 | zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress |
| 305 | zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress |
| 306 | println "test: --progress flag" |
| 307 | zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' |
| 308 | zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' |
| 309 | zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | $GREP '[A-Za-z0-9._ ]*: [0-9]* bytes' |
| 310 | rm -f tmpro tmpro.zst |
| 311 | println "test: overwrite input file (must fail)" |
| 312 | zstd tmp -fo tmp && die "zstd compression overwrote the input file" |
| 313 | zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" |
| 314 | println "test: detect that input file does not exist" |
| 315 | zstd nothere && die "zstd hasn't detected that input file does not exist" |
| 316 | println "test: --[no-]compress-literals" |
| 317 | zstd tmp -c --no-compress-literals -1 | zstd -t |
| 318 | zstd tmp -c --no-compress-literals --fast=1 | zstd -t |
| 319 | zstd tmp -c --no-compress-literals -19 | zstd -t |
| 320 | zstd tmp -c --compress-literals -1 | zstd -t |
| 321 | zstd tmp -c --compress-literals --fast=1 | zstd -t |
| 322 | zstd tmp -c --compress-literals -19 | zstd -t |
| 323 | zstd -b --fast=1 -i0e1 tmp --compress-literals |
| 324 | zstd -b --fast=1 -i0e1 tmp --no-compress-literals |
| 325 | println "test: --no-check for decompression" |
| 326 | zstd -f tmp -o tmp_corrupt.zst --check |
| 327 | zstd -f tmp -o tmp.zst --no-check |
| 328 | printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp |
| 329 | zstd -d -f tmp_corrupt.zst --no-check |
| 330 | zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides |
| 331 | zstd -d -f tmp.zst --no-check |
| 332 | |
| 333 | if [ "$isWindows" = false ] && [ "$UNAME" != "AIX" ]; then |
| 334 | if [ -n "$(which readelf)" ]; then |
| 335 | println "test: check if binary has executable stack (#2963)" |
| 336 | readelf -lW "$ZSTD_BIN" | $GREP 'GNU_STACK .* RW ' || die "zstd binary has executable stack!" |
| 337 | fi |
| 338 | fi |
| 339 | |
| 340 | println "\n===> multiple_thread test " |
| 341 | |
| 342 | datagen > tmp |
| 343 | println "test : single-thread " |
| 344 | zstd --fast --single-thread tmp -o tmpMT0 |
| 345 | println "test : one worker thread (default)" |
| 346 | zstd --fast -T1 tmp -o tmpMT1 |
| 347 | println "test : two worker threads " |
| 348 | zstd --fast -T2 tmp -o tmpMT2 |
| 349 | println "test : 16-thread " |
| 350 | zstd --fast -T16 tmp -o tmpMT3 |
| 351 | println "test : 127-thread " |
| 352 | zstd --fast -T127 tmp -o tmpMT4 |
| 353 | println "test : 128-thread " |
| 354 | zstd --fast -T128 tmp -o tmpMT5 |
| 355 | println "test : max allowed numeric value is 4294967295 " |
| 356 | zstd --fast -4294967295 tmp -o tmpMT6 |
| 357 | println "test : numeric value overflows 32-bit unsigned int " |
| 358 | zstd --fast -4294967296 tmp -o tmptest9 && die "max allowed numeric value is 4294967295" |
| 359 | |
| 360 | datagen > tmp |
| 361 | println "test : basic compression " |
| 362 | zstd -f tmp # trivial compression case, creates tmp.zst |
| 363 | println "test : basic decompression" |
| 364 | zstd -d -f -T1 tmp.zst |
| 365 | println "note : decompression does not support -T mode, but execution support" |
| 366 | rm -rf tmpMT* |
| 367 | |
| 368 | println "\n===> --fast_argument test " |
| 369 | datagen > tmp |
| 370 | println "test : basic compression " |
| 371 | zstd -f tmp # trivial compression case, creates tmp.zst |
| 372 | println "test: --fast=1" |
| 373 | zstd --fast=1 -f tmp |
| 374 | println "test: --fast=99" |
| 375 | zstd --fast=99 -f tmp |
| 376 | println "test: Invalid value -- negative number" |
| 377 | zstd --fast=-1 -f tmp && die "error: Invalid value -- negative number" |
| 378 | println "test: Invalid value -- zero" |
| 379 | zstd --fast=0 -f tmp && die "error: Invalid value -- 0 number" |
| 380 | println "test: max allowed numeric argument of --fast is 4294967295" |
| 381 | zstd --fast=4294967295 -f tmp |
| 382 | println "test: numeric value overflows 32-bit unsigned int " |
| 383 | zstd --fast=4294967296 -f tmp && die "max allowed argument of --fast is 4294967295" |
| 384 | |
| 385 | println "\n===> --exclude-compressed flag" |
| 386 | rm -rf precompressedFilterTestDir |
| 387 | mkdir -p precompressedFilterTestDir |
| 388 | datagen $size > precompressedFilterTestDir/input.5 |
| 389 | datagen $size > precompressedFilterTestDir/input.6 |
| 390 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
| 391 | datagen $size > precompressedFilterTestDir/input.7 |
| 392 | datagen $size > precompressedFilterTestDir/input.8 |
| 393 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
| 394 | test ! -f precompressedFilterTestDir/input.5.zst.zst |
| 395 | test ! -f precompressedFilterTestDir/input.6.zst.zst |
| 396 | file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst` |
| 397 | file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst` |
| 398 | if [ $file2timestamp -ge $file1timestamp ]; then |
| 399 | println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again." |
| 400 | else |
| 401 | println "Test is not successful" |
| 402 | fi |
| 403 | # File Extension check. |
| 404 | datagen $size > precompressedFilterTestDir/input.zstbar |
| 405 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
| 406 | # zstd should compress input.zstbar |
| 407 | test -f precompressedFilterTestDir/input.zstbar.zst |
| 408 | # Check without the --exclude-compressed flag |
| 409 | zstd --long --rm -r precompressedFilterTestDir |
| 410 | # Files should get compressed again without the --exclude-compressed flag. |
| 411 | test -f precompressedFilterTestDir/input.5.zst.zst |
| 412 | test -f precompressedFilterTestDir/input.6.zst.zst |
| 413 | |
| 414 | # Test some other compressed file extensions |
| 415 | datagen $size > precompressedFilterTestDir/input.flac |
| 416 | datagen $size > precompressedFilterTestDir/input.mov |
| 417 | datagen $size > precompressedFilterTestDir/input.mp3 |
| 418 | zstd --exclude-compressed --long --rm -r precompressedFilterTestDir |
| 419 | test ! -f precompressedFilterTestDir/input.flac.zst |
| 420 | test ! -f precompressedFilterTestDir/input.mov.zst |
| 421 | test ! -f precompressedFilterTestDir/input.mp3.zst |
| 422 | zstd --long --rm -r precompressedFilterTestDir |
| 423 | test -f precompressedFilterTestDir/input.flac.zst |
| 424 | test -f precompressedFilterTestDir/input.mov.zst |
| 425 | test -f precompressedFilterTestDir/input.mp3.zst |
| 426 | rm -rf precompressedFilterTestDir |
| 427 | println "Test completed" |
| 428 | |
| 429 | |
| 430 | |
| 431 | println "\n===> warning prompts should not occur if stdin is an input" |
| 432 | println "y" > tmpPrompt |
| 433 | println "hello world" >> tmpPrompt |
| 434 | zstd tmpPrompt -f |
| 435 | zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" |
| 436 | zstd < tmpPrompt -o tmpPrompt.zst -f # should successfully overwrite with -f |
| 437 | zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated |
| 438 | $DIFF tmpPromptRegenerated tmpPrompt # the first 'y' character should not be swallowed |
| 439 | |
| 440 | echo 'yes' | zstd tmpPrompt -v -o tmpPrompt.zst # accept piped "y" input to force overwrite when using files |
| 441 | echo 'yes' | zstd < tmpPrompt -v -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite" |
| 442 | zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove" |
| 443 | |
| 444 | println "Test completed" |
| 445 | |
| 446 | |
| 447 | println "\n===> recursive mode test " |
| 448 | # combination of -r with empty list of input file |
| 449 | zstd -c -r < tmp > tmp.zst |
| 450 | |
| 451 | # combination of -r with empty folder |
| 452 | mkdir -p tmpEmptyDir |
| 453 | zstd -r tmpEmptyDir |
| 454 | rm -rf tmpEmptyDir |
| 455 | |
| 456 | |
| 457 | println "\n===> file removal" |
| 458 | zstd -f --rm tmp |
| 459 | test ! -f tmp # tmp should no longer be present |
| 460 | zstd -f -d --rm tmp.zst |
| 461 | test ! -f tmp.zst # tmp.zst should no longer be present |
| 462 | println "test: --rm is disabled when output is stdout" |
| 463 | test -f tmp |
| 464 | zstd --rm tmp -c > $INTOVOID |
| 465 | test -f tmp # tmp shall still be there |
| 466 | zstd --rm tmp --stdout > $INTOVOID |
| 467 | test -f tmp # tmp shall still be there |
| 468 | zstd -f --rm tmp -c > $INTOVOID |
| 469 | test -f tmp # tmp shall still be there |
| 470 | zstd -f tmp -c > $INTOVOID --rm |
| 471 | test -f tmp # tmp shall still be there |
| 472 | println "test: --rm is disabled when multiple inputs are concatenated into a single output" |
| 473 | cp tmp tmp2 |
| 474 | zstd --rm tmp tmp2 -c > $INTOVOID |
| 475 | test -f tmp |
| 476 | test -f tmp2 |
| 477 | rm -f tmp3.zst |
| 478 | echo 'y' | zstd -v tmp tmp2 -o tmp3.zst --rm # prompt for confirmation |
| 479 | test -f tmp |
| 480 | test -f tmp2 |
| 481 | zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt |
| 482 | test -f tmp |
| 483 | test -f tmp2 |
| 484 | zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate" |
| 485 | println "test: --rm is active with -o when single input" |
| 486 | rm -f tmp2.zst |
| 487 | zstd --rm tmp2 -o tmp2.zst |
| 488 | test -f tmp2.zst |
| 489 | test ! -f tmp2 |
| 490 | println "test: -c followed by -o => -o wins, so --rm remains active" # (#3719) |
| 491 | rm tmp2.zst |
| 492 | cp tmp tmp2 |
| 493 | zstd --rm tmp2 -c > $INTOVOID -o tmp2.zst |
| 494 | test ! -f tmp2 |
| 495 | println "test: -o followed by -c => -c wins, so --rm is disabled" # (#3719) |
| 496 | rm tmp3.zst |
| 497 | cp tmp tmp2 |
| 498 | zstd -v --rm tmp2 -o tmp2.zst -c > tmp3.zst |
| 499 | test -f tmp2 |
| 500 | test -f tmp3.zst |
| 501 | println "test : should quietly not remove non-regular file" |
| 502 | println hello > tmp |
| 503 | zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" |
| 504 | $GREP "Refusing to remove non-regular file" tmplog && die |
| 505 | rm -f tmplog |
| 506 | zstd tmp -f -o "$INTOVOID" 2>&1 | $GREP "Refusing to remove non-regular file" && die |
| 507 | println "test : --rm on stdin" |
| 508 | println a | zstd --rm > $INTOVOID # --rm should remain silent |
| 509 | rm -f tmp |
| 510 | zstd -f tmp && die "tmp not present : should have failed" |
| 511 | test ! -f tmp.zst # tmp.zst should not be created |
| 512 | println "test : -d -f do not delete destination when source is not present" |
| 513 | touch tmp # create destination file |
| 514 | zstd -d -f tmp.zst && die "attempt to decompress a non existing file" |
| 515 | test -f tmp # destination file should still be present |
| 516 | println "test : -f do not delete destination when source is not present" |
| 517 | rm -f tmp # erase source file |
| 518 | touch tmp.zst # create destination file |
| 519 | zstd -f tmp && die "attempt to compress a non existing file" |
| 520 | test -f tmp.zst # destination file should still be present |
| 521 | rm -rf tmp* # may also erase tmp* directory from previous failed run |
| 522 | |
| 523 | |
| 524 | println "\n===> decompression only tests " |
| 525 | # the following test verifies that the decoder is compatible with RLE as first block |
| 526 | # older versions of zstd cli are not able to decode such corner case. |
| 527 | # As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions. |
| 528 | dd bs=1048576 count=1 if=/dev/zero of=tmp |
| 529 | zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" |
| 530 | $DIFF -s tmp1 tmp |
| 531 | |
| 532 | touch tmp_empty |
| 533 | zstd -d -o tmp2 "$TESTDIR/golden-decompression/empty-block.zst" |
| 534 | $DIFF -s tmp2 tmp_empty |
| 535 | |
| 536 | zstd -t "$TESTDIR/golden-decompression/zeroSeq_2B.zst" |
| 537 | |
| 538 | zstd -t "$TESTDIR/golden-decompression-errors/zeroSeq_extraneous.zst" && die "invalid Sequences section should have been detected" |
| 539 | |
| 540 | rm -f tmp* |
| 541 | |
| 542 | println "\n===> compress multiple files" |
| 543 | println hello > tmp1 |
| 544 | println world > tmp2 |
| 545 | zstd tmp1 tmp2 -o "$INTOVOID" -f |
| 546 | zstd tmp1 tmp2 -c | zstd -t |
| 547 | echo 'y' | zstd -v tmp1 tmp2 -o tmp.zst |
| 548 | test ! -f tmp1.zst |
| 549 | test ! -f tmp2.zst |
| 550 | zstd tmp1 tmp2 |
| 551 | zstd -t tmp1.zst tmp2.zst |
| 552 | zstd -dc tmp1.zst tmp2.zst |
| 553 | zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f |
| 554 | echo 'y' | zstd -v -d tmp1.zst tmp2.zst -o tmp |
| 555 | touch tmpexists |
| 556 | zstd tmp1 tmp2 -f -o tmpexists |
| 557 | zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite" |
| 558 | println gooder > tmp_rm1 |
| 559 | println boi > tmp_rm2 |
| 560 | println worldly > tmp_rm3 |
| 561 | echo 'y' | zstd -v tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst |
| 562 | test -f tmp_rm1 |
| 563 | test -f tmp_rm2 |
| 564 | cp tmp_rm3.zst tmp_rm4.zst |
| 565 | echo 'Y' | zstd -v -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm |
| 566 | test -f tmp_rm3.zst |
| 567 | test -f tmp_rm4.zst |
| 568 | println gooder > tmpexists1 |
| 569 | zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID |
| 570 | # Bug: PR #972 |
| 571 | if [ "$?" -eq 139 ]; then |
| 572 | die "should not have segfaulted" |
| 573 | fi |
| 574 | test -f tmpexists1 |
| 575 | test -f tmpexists |
| 576 | println "\n===> multiple files and shell completion " |
| 577 | datagen -s1 > tmp1 2> $INTOVOID |
| 578 | datagen -s2 -g100K > tmp2 2> $INTOVOID |
| 579 | datagen -s3 -g1M > tmp3 2> $INTOVOID |
| 580 | println "compress tmp* : " |
| 581 | zstd -f tmp* |
| 582 | test -f tmp1.zst |
| 583 | test -f tmp2.zst |
| 584 | test -f tmp3.zst |
| 585 | rm -f tmp1 tmp2 tmp3 |
| 586 | println "decompress tmp* : " |
| 587 | zstd -df ./*.zst |
| 588 | test -f tmp1 |
| 589 | test -f tmp2 |
| 590 | test -f tmp3 |
| 591 | println "compress tmp* into stdout > tmpall : " |
| 592 | zstd -c tmp1 tmp2 tmp3 > tmpall |
| 593 | test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) |
| 594 | println "decompress tmpall* into stdout > tmpdec : " |
| 595 | cp tmpall tmpall2 |
| 596 | zstd -dc tmpall* > tmpdec |
| 597 | test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) |
| 598 | println "compress multiple files including a missing one (notHere) : " |
| 599 | zstd -f tmp1 notHere tmp2 && die "missing file not detected!" |
| 600 | rm -f tmp* |
| 601 | |
| 602 | |
| 603 | if [ "$isWindows" = false ] ; then |
| 604 | println "\n===> zstd fifo named pipe test " |
| 605 | echo "Hello World!" > tmp_original |
| 606 | mkfifo tmp_named_pipe |
| 607 | # note : fifo test doesn't work in combination with `dd` or `cat` |
| 608 | echo "Hello World!" > tmp_named_pipe & |
| 609 | zstd tmp_named_pipe -o tmp_compressed |
| 610 | zstd -d -o tmp_decompressed tmp_compressed |
| 611 | $DIFF -s tmp_original tmp_decompressed |
| 612 | rm -rf tmp* |
| 613 | fi |
| 614 | |
| 615 | println "\n===> zstd created file permissions tests" |
| 616 | if [ "$isWindows" = false ] ; then |
| 617 | rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove |
| 618 | |
| 619 | ORIGINAL_UMASK=$(umask) |
| 620 | umask 0000 |
| 621 | |
| 622 | datagen > tmp1 |
| 623 | datagen > tmp2 |
| 624 | assertFilePermissions tmp1 666 |
| 625 | assertFilePermissions tmp2 666 |
| 626 | |
| 627 | println "test : copy 666 permissions in file -> file compression " |
| 628 | zstd -f tmp1 -o tmp1.zst |
| 629 | assertSamePermissions tmp1 tmp1.zst |
| 630 | println "test : copy 666 permissions in file -> file decompression " |
| 631 | zstd -f -d tmp1.zst -o tmp1.out |
| 632 | assertSamePermissions tmp1.zst tmp1.out |
| 633 | |
| 634 | rm -f tmp1.zst tmp1.out |
| 635 | |
| 636 | println "test : copy 400 permissions in file -> file compression (write to a read-only file) " |
| 637 | chmod 0400 tmp1 |
| 638 | assertFilePermissions tmp1 400 |
| 639 | zstd -f tmp1 -o tmp1.zst |
| 640 | assertSamePermissions tmp1 tmp1.zst |
| 641 | println "test : copy 400 permissions in file -> file decompression (write to a read-only file) " |
| 642 | zstd -f -d tmp1.zst -o tmp1 |
| 643 | assertSamePermissions tmp1.zst tmp1 |
| 644 | |
| 645 | rm -f tmp1.zst tmp1.out |
| 646 | |
| 647 | println "test : check created permissions from stdin input in compression " |
| 648 | zstd -f -o tmp1.zst < tmp1 |
| 649 | assertFilePermissions tmp1.zst 666 |
| 650 | println "test : check created permissions from stdin input in decompression " |
| 651 | zstd -f -d -o tmp1.out < tmp1.zst |
| 652 | assertFilePermissions tmp1.out 666 |
| 653 | |
| 654 | rm -f tmp1.zst tmp1.out |
| 655 | |
| 656 | println "test : check created permissions from multiple inputs in compression " |
| 657 | zstd -f tmp1 tmp2 -o tmp1.zst |
| 658 | assertFilePermissions tmp1.zst 666 |
| 659 | println "test : check created permissions from multiple inputs in decompression " |
| 660 | cp tmp1.zst tmp2.zst |
| 661 | zstd -f -d tmp1.zst tmp2.zst -o tmp1.out |
| 662 | assertFilePermissions tmp1.out 666 |
| 663 | |
| 664 | rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out |
| 665 | |
| 666 | println "test : check permissions on pre-existing output file in compression " |
| 667 | chmod 0600 tmp1 |
| 668 | touch tmp1.zst |
| 669 | chmod 0400 tmp1.zst |
| 670 | zstd -f tmp1 -o tmp1.zst |
| 671 | assertFilePermissions tmp1.zst 600 |
| 672 | println "test : check permissions on pre-existing output file in decompression " |
| 673 | chmod 0400 tmp1.zst |
| 674 | touch tmp1.out |
| 675 | chmod 0200 tmp1.out |
| 676 | zstd -f -d tmp1.zst -o tmp1.out |
| 677 | assertFilePermissions tmp1.out 400 |
| 678 | |
| 679 | umask 0666 |
| 680 | chmod 0666 tmp1 tmp2 |
| 681 | |
| 682 | rm -f tmp1.zst tmp1.out |
| 683 | |
| 684 | println "test : respect umask when compressing from stdin input " |
| 685 | zstd -f -o tmp1.zst < tmp1 |
| 686 | assertFilePermissions tmp1.zst 0 |
| 687 | println "test : respect umask when decompressing from stdin input " |
| 688 | chmod 0666 tmp1.zst |
| 689 | zstd -f -d -o tmp1.out < tmp1.zst |
| 690 | assertFilePermissions tmp1.out 0 |
| 691 | |
| 692 | rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out |
| 693 | umask $ORIGINAL_UMASK |
| 694 | fi |
| 695 | |
| 696 | if [ -n "$DEVNULLRIGHTS" ] ; then |
| 697 | # these tests requires sudo rights, which is uncommon. |
| 698 | # they are only triggered if DEVNULLRIGHTS macro is defined. |
| 699 | println "\n===> checking /dev/null permissions are unaltered " |
| 700 | datagen > tmp |
| 701 | sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions |
| 702 | sudoZstd tmp -c > $INTOVOID |
| 703 | zstd tmp -f -o tmp.zst |
| 704 | sudoZstd -d tmp.zst -c > $INTOVOID |
| 705 | sudoZstd -d tmp.zst -o $INTOVOID |
| 706 | ls -las $INTOVOID | $GREP "rw-rw-rw-" |
| 707 | fi |
| 708 | |
| 709 | if [ -n "$READFROMBLOCKDEVICE" ] ; then |
| 710 | # This creates a temporary block device, which is only possible on unix-y |
| 711 | # systems, is somewhat invasive, and requires sudo. For these reasons, you |
| 712 | # have to specifically ask for this test. |
| 713 | println "\n===> checking that zstd can read from a block device" |
| 714 | datagen -g65536 > tmp.img |
| 715 | sudo losetup -fP tmp.img |
| 716 | LOOP_DEV=$(losetup -a | $GREP 'tmp\.img' | cut -f1 -d:) |
| 717 | [ -z "$LOOP_DEV" ] && die "failed to get loopback device" |
| 718 | sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f" |
| 719 | sudoZstd -f $LOOP_DEV -c > tmp.img.zst |
| 720 | zstd -d tmp.img.zst -o tmp.img.copy |
| 721 | sudo losetup -d $LOOP_DEV |
| 722 | $DIFF -s tmp.img tmp.img.copy || die "round trip failed" |
| 723 | rm -f tmp.img tmp.img.zst tmp.img.copy |
| 724 | fi |
| 725 | |
| 726 | println "\n===> zstd created file timestamp tests" |
| 727 | datagen > tmp |
| 728 | touch -m -t 200001010000.00 tmp |
| 729 | println "test : copy mtime in file -> file compression " |
| 730 | zstd -f tmp -o tmp.zst |
| 731 | assertSameMTime tmp tmp.zst |
| 732 | println "test : copy mtime in file -> file decompression " |
| 733 | zstd -f -d tmp.zst -o tmp.out |
| 734 | assertSameMTime tmp.zst tmp.out |
| 735 | rm -f tmp |
| 736 | |
| 737 | println "\n===> compress multiple files into an output directory, --output-dir-flat" |
| 738 | println henlo > tmp1 |
| 739 | mkdir tmpInputTestDir |
| 740 | mkdir tmpInputTestDir/we |
| 741 | mkdir tmpInputTestDir/we/must |
| 742 | mkdir tmpInputTestDir/we/must/go |
| 743 | mkdir tmpInputTestDir/we/must/go/deeper |
| 744 | println cool > tmpInputTestDir/we/must/go/deeper/tmp2 |
| 745 | mkdir tmpOutDir |
| 746 | zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir |
| 747 | test -f tmpOutDir/tmp1.zst |
| 748 | test -f tmpOutDir/tmp2.zst |
| 749 | println "test : decompress multiple files into an output directory, --output-dir-flat" |
| 750 | mkdir tmpOutDirDecomp |
| 751 | zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp |
| 752 | test -f tmpOutDirDecomp/tmp2 |
| 753 | test -f tmpOutDirDecomp/tmp1 |
| 754 | rm -f tmpOutDirDecomp/* |
| 755 | zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp |
| 756 | test -f tmpOutDirDecomp/tmp2 |
| 757 | test -f tmpOutDirDecomp/tmp1 |
| 758 | rm -rf tmp* |
| 759 | |
| 760 | if [ "$isWindows" = false ] ; then |
| 761 | println "\n===> compress multiple files into an output directory and mirror input folder, --output-dir-mirror" |
| 762 | println "test --output-dir-mirror" > tmp1 |
| 763 | mkdir -p tmpInputTestDir/we/.../..must/go/deeper.. |
| 764 | println cool > tmpInputTestDir/we/.../..must/go/deeper../tmp2 |
| 765 | zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir |
| 766 | test -f tmpOutDir/tmp1.zst |
| 767 | test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst |
| 768 | |
| 769 | println "test: compress input dir will be ignored if it has '..'" |
| 770 | zstd -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" |
| 771 | zstd -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input cannot contain '..'" |
| 772 | zstd -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'" |
| 773 | test ! -d non-exist |
| 774 | |
| 775 | println "test: compress input dir should succeed with benign uses of '..'" |
| 776 | zstd -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout |
| 777 | test -d tmpout |
| 778 | |
| 779 | println "test : decompress multiple files into an output directory, --output-dir-mirror" |
| 780 | zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp |
| 781 | test -f tmpOutDirDecomp/tmpOutDir/tmp1 |
| 782 | test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2 |
| 783 | |
| 784 | println "test: decompress input dir will be ignored if it has '..'" |
| 785 | zstd -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "input cannot contain '..'" |
| 786 | test ! -d non-exist |
| 787 | |
| 788 | rm -rf tmp* |
| 789 | fi |
| 790 | |
| 791 | |
| 792 | println "test : compress multiple files reading them from a file, --filelist=FILE" |
| 793 | println "Hello world!, file1" > tmp1 |
| 794 | println "Hello world!, file2" > tmp2 |
| 795 | println tmp1 > tmp_fileList |
| 796 | println tmp2 >> tmp_fileList |
| 797 | zstd -f --filelist=tmp_fileList |
| 798 | test -f tmp2.zst |
| 799 | test -f tmp1.zst |
| 800 | |
| 801 | println "test : alternate syntax: --filelist FILE" |
| 802 | zstd -f --filelist tmp_fileList |
| 803 | test -f tmp2.zst |
| 804 | test -f tmp1.zst |
| 805 | |
| 806 | println "test : reading file list from a symlink, --filelist=FILE" |
| 807 | rm -f *.zst |
| 808 | ln -s tmp_fileList tmp_symLink |
| 809 | zstd -f --filelist=tmp_symLink |
| 810 | test -f tmp2.zst |
| 811 | test -f tmp1.zst |
| 812 | |
| 813 | println "test : compress multiple files reading them from multiple files, --filelist=FILE" |
| 814 | rm -f *.zst |
| 815 | println "Hello world!, file3" > tmp3 |
| 816 | println "Hello world!, file4" > tmp4 |
| 817 | println tmp3 > tmp_fileList2 |
| 818 | println tmp4 >> tmp_fileList2 |
| 819 | zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2 |
| 820 | test -f tmp1.zst |
| 821 | test -f tmp2.zst |
| 822 | test -f tmp3.zst |
| 823 | test -f tmp4.zst |
| 824 | |
| 825 | println "test : decompress multiple files reading them from a file, --filelist=FILE" |
| 826 | rm -f tmp1 tmp2 |
| 827 | println tmp1.zst > tmpZst |
| 828 | println tmp2.zst >> tmpZst |
| 829 | zstd -d -f --filelist=tmpZst |
| 830 | test -f tmp1 |
| 831 | test -f tmp2 |
| 832 | |
| 833 | println "test : decompress multiple files reading them from multiple files, --filelist=FILE" |
| 834 | rm -f tmp1 tmp2 tmp3 tmp4 |
| 835 | println tmp3.zst > tmpZst2 |
| 836 | println tmp4.zst >> tmpZst2 |
| 837 | zstd -d -f --filelist=tmpZst --filelist=tmpZst2 |
| 838 | test -f tmp1 |
| 839 | test -f tmp2 |
| 840 | test -f tmp3 |
| 841 | test -f tmp4 |
| 842 | |
| 843 | println "test : survive the list of files with too long filenames (--filelist=FILE)" |
| 844 | datagen -g5M > tmp_badList |
| 845 | zstd -qq -f --filelist=tmp_badList && die "should have failed : file name length is too long" # printing very long text garbage on console will cause CI failure |
| 846 | |
| 847 | println "test : survive a list of files which is text garbage (--filelist=FILE)" |
| 848 | datagen > tmp_badList |
| 849 | zstd -qq -f --filelist=tmp_badList && die "should have failed : list is text garbage" # printing very long text garbage on console will cause CI failure |
| 850 | |
| 851 | println "test : survive a list of files which is binary garbage (--filelist=FILE)" |
| 852 | datagen -P0 -g1M > tmp_badList |
| 853 | zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console |
| 854 | |
| 855 | println "test : try to overflow internal list of files (--filelist=FILE)" |
| 856 | touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 |
| 857 | ls tmp* > tmpList |
| 858 | zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list |
| 859 | rm -rf tmp* |
| 860 | |
| 861 | println "\n===> --[no-]content-size tests" |
| 862 | |
| 863 | datagen > tmp_contentsize |
| 864 | zstd -f tmp_contentsize |
| 865 | zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" |
| 866 | zstd -f --no-content-size tmp_contentsize |
| 867 | zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die |
| 868 | zstd -f --content-size tmp_contentsize |
| 869 | zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" |
| 870 | zstd -f --content-size --no-content-size tmp_contentsize |
| 871 | zstd -lv tmp_contentsize.zst | $GREP "Decompressed Size:" && die |
| 872 | rm -rf tmp* |
| 873 | |
| 874 | println "test : show-default-cparams regular" |
| 875 | datagen > tmp |
| 876 | zstd --show-default-cparams -f tmp |
| 877 | zstd --show-default-cparams -d tmp.zst && die "error: can't use --show-default-cparams in decompression mode" |
| 878 | rm -rf tmp* |
| 879 | |
| 880 | println "test : show-default-cparams recursive" |
| 881 | mkdir tmp_files |
| 882 | datagen -g15000 > tmp_files/tmp1 |
| 883 | datagen -g129000 > tmp_files/tmp2 |
| 884 | datagen -g257000 > tmp_files/tmp3 |
| 885 | zstd --show-default-cparams -f -r tmp_files |
| 886 | rm -rf tmp* |
| 887 | |
| 888 | println "test : show compression parameters in verbose mode" |
| 889 | datagen > tmp |
| 890 | zstd -vv tmp 2>&1 | \ |
| 891 | $GREP -q -- "--zstd=wlog=[0-9]*,clog=[0-9]*,hlog=[0-9]*,slog=[0-9]*,mml=[0-9]*,tlen=[0-9]*,strat=[0-9]*" |
| 892 | rm -rf tmp* |
| 893 | |
| 894 | println "\n===> Advanced compression parameters " |
| 895 | println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" |
| 896 | println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" |
| 897 | println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" |
| 898 | println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist |
| 899 | test ! -f tmp.zst # tmp.zst should not be created |
| 900 | roundTripTest -g512K |
| 901 | roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" |
| 902 | roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" |
| 903 | roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" |
| 904 | roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" |
| 905 | roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" |
| 906 | roundTripTest -g64K "19 --zstd=strat=9" # btultra2 |
| 907 | |
| 908 | |
| 909 | println "\n===> Pass-Through mode " |
| 910 | println "Hello world 1!" | zstd -df |
| 911 | println "Hello world 2!" | zstd -dcf |
| 912 | println "Hello world 3!" > tmp1 |
| 913 | zstd -dcf tmp1 |
| 914 | println "" | zstd -df > tmp1 |
| 915 | println "" > tmp2 |
| 916 | $DIFF -q tmp1 tmp2 |
| 917 | println "1" | zstd -df > tmp1 |
| 918 | println "1" > tmp2 |
| 919 | $DIFF -q tmp1 tmp2 |
| 920 | println "12" | zstd -df > tmp1 |
| 921 | println "12" > tmp2 |
| 922 | $DIFF -q tmp1 tmp2 |
| 923 | rm -rf tmp* |
| 924 | |
| 925 | |
| 926 | println "\n===> frame concatenation " |
| 927 | println "hello " > hello.tmp |
| 928 | println "world!" > world.tmp |
| 929 | cat hello.tmp world.tmp > helloworld.tmp |
| 930 | zstd -c hello.tmp > hello.zst |
| 931 | zstd -c world.tmp > world.zst |
| 932 | zstd -c hello.tmp world.tmp > helloworld.zst |
| 933 | zstd -dc helloworld.zst > result.tmp |
| 934 | $DIFF helloworld.tmp result.tmp |
| 935 | cat hello.zst world.zst > helloworld.zst |
| 936 | zstd -dc helloworld.zst > result.tmp |
| 937 | cat result.tmp |
| 938 | $DIFF helloworld.tmp result.tmp |
| 939 | println "frame concatenation without checksum" |
| 940 | zstd -c hello.tmp > hello.zst --no-check |
| 941 | zstd -c world.tmp > world.zst --no-check |
| 942 | cat hello.zst world.zst > helloworld.zstd |
| 943 | zstd -dc helloworld.zst > result.tmp |
| 944 | $DIFF helloworld.tmp result.tmp |
| 945 | println "testing zstdcat symlink" |
| 946 | ln -sf "$ZSTD_BIN" zstdcat |
| 947 | $EXE_PREFIX ./zstdcat helloworld.zst > result.tmp |
| 948 | $DIFF helloworld.tmp result.tmp |
| 949 | ln -s helloworld.zst helloworld.link.zst |
| 950 | $EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp |
| 951 | $DIFF helloworld.tmp result.tmp |
| 952 | rm -f zstdcat |
| 953 | rm -f result.tmp |
| 954 | println "testing zcat symlink" |
| 955 | ln -sf "$ZSTD_BIN" zcat |
| 956 | $EXE_PREFIX ./zcat helloworld.zst > result.tmp |
| 957 | $DIFF helloworld.tmp result.tmp |
| 958 | $EXE_PREFIX ./zcat helloworld.link.zst > result.tmp |
| 959 | $DIFF helloworld.tmp result.tmp |
| 960 | rm -f zcat |
| 961 | rm -f ./*.tmp ./*.zstd |
| 962 | println "frame concatenation tests completed" |
| 963 | |
| 964 | |
| 965 | if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] && [ "$UNAME" != "AIX" ]; then |
| 966 | println "\n**** flush write error test **** " |
| 967 | |
| 968 | println "println foo | zstd > /dev/full" |
| 969 | println foo | zstd > /dev/full && die "write error not detected!" |
| 970 | println "println foo | zstd | zstd -d > /dev/full" |
| 971 | println foo | zstd | zstd -d > /dev/full && die "write error not detected!" |
| 972 | |
| 973 | fi |
| 974 | |
| 975 | |
| 976 | if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then |
| 977 | |
| 978 | println "\n===> symbolic link test " |
| 979 | |
| 980 | rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst |
| 981 | println "hello world" > hello.tmp |
| 982 | ln -s hello.tmp world.tmp |
| 983 | ln -s hello.tmp world2.tmp |
| 984 | zstd world.tmp hello.tmp || true |
| 985 | test -f hello.tmp.zst # regular file should have been compressed! |
| 986 | test ! -f world.tmp.zst # symbolic link should not have been compressed! |
| 987 | zstd world.tmp || true |
| 988 | test ! -f world.tmp.zst # symbolic link should not have been compressed! |
| 989 | zstd world.tmp world2.tmp || true |
| 990 | test ! -f world.tmp.zst # symbolic link should not have been compressed! |
| 991 | test ! -f world2.tmp.zst # symbolic link should not have been compressed! |
| 992 | zstd world.tmp hello.tmp -f |
| 993 | test -f world.tmp.zst # symbolic link should have been compressed with --force |
| 994 | rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst |
| 995 | |
| 996 | fi |
| 997 | |
| 998 | |
| 999 | println "\n===> test sparse file support " |
| 1000 | |
| 1001 | datagen -g5M -P100 > tmpSparse |
| 1002 | zstd tmpSparse -c | zstd -dv -o tmpSparseRegen |
| 1003 | $DIFF -s tmpSparse tmpSparseRegen |
| 1004 | zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse |
| 1005 | $DIFF -s tmpSparse tmpOutSparse |
| 1006 | zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse |
| 1007 | $DIFF -s tmpSparse tmpOutNoSparse |
| 1008 | ls -ls tmpSparse* # look at file size and block size on disk |
| 1009 | datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) |
| 1010 | datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd |
| 1011 | ls -ls tmpSparseOdd # look at file size and block size on disk |
| 1012 | println "\n Sparse Compatibility with Console :" |
| 1013 | println "Hello World 1 !" | zstd | zstd -d -c |
| 1014 | println "Hello World 2 !" | zstd | zstd -d | cat |
| 1015 | println "\n Sparse Compatibility with Append :" |
| 1016 | datagen -P100 -g1M > tmpSparse1M |
| 1017 | cat tmpSparse1M tmpSparse1M > tmpSparse2M |
| 1018 | zstd -v -f tmpSparse1M -o tmpSparseCompressed |
| 1019 | zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated |
| 1020 | zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated |
| 1021 | ls -ls tmpSparse* # look at file size and block size on disk |
| 1022 | $DIFF tmpSparse2M tmpSparseRegenerated |
| 1023 | rm -f tmpSparse* |
| 1024 | |
| 1025 | |
| 1026 | println "\n===> stream-size mode" |
| 1027 | |
| 1028 | datagen -g11000 > tmp |
| 1029 | println "test : basic file compression vs sized streaming compression" |
| 1030 | file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) |
| 1031 | stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c) |
| 1032 | if [ "$stream_size" -gt "$file_size" ]; then |
| 1033 | die "hinted compression larger than expected" |
| 1034 | fi |
| 1035 | println "test : sized streaming compression and decompression" |
| 1036 | cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000 |
| 1037 | zstd -df tmp.zst -o tmp_decompress |
| 1038 | cmp tmp tmp_decompress || die "difference between original and decompressed file" |
| 1039 | println "test : incorrect stream size" |
| 1040 | cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" |
| 1041 | |
| 1042 | println "\n===> zstd zero weight dict test " |
| 1043 | rm -f tmp* |
| 1044 | cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input |
| 1045 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input |
| 1046 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp |
| 1047 | $DIFF tmp_decomp tmp_input |
| 1048 | rm -rf tmp* |
| 1049 | |
| 1050 | println "\n===> zstd (valid) zero weight dict test " |
| 1051 | rm -f tmp* |
| 1052 | # 0 has a non-zero weight in the dictionary |
| 1053 | echo "0000000000000000000000000" > tmp_input |
| 1054 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input |
| 1055 | zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp |
| 1056 | $DIFF tmp_decomp tmp_input |
| 1057 | rm -rf tmp* |
| 1058 | |
| 1059 | println "\n===> size-hint mode" |
| 1060 | |
| 1061 | datagen -g11000 > tmp |
| 1062 | datagen -g11000 > tmp2 |
| 1063 | datagen > tmpDict |
| 1064 | println "test : basic file compression vs hinted streaming compression" |
| 1065 | file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) |
| 1066 | stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c) |
| 1067 | if [ "$stream_size" -ge "$file_size" ]; then |
| 1068 | die "hinted compression larger than expected" |
| 1069 | fi |
| 1070 | println "test : hinted streaming compression and decompression" |
| 1071 | cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000 |
| 1072 | zstd -df tmp.zst -o tmp_decompress |
| 1073 | cmp tmp tmp_decompress || die "difference between original and decompressed file" |
| 1074 | println "test : hinted streaming compression with dictionary" |
| 1075 | cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict |
| 1076 | println "test : multiple file compression with hints and dictionary" |
| 1077 | zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2 |
| 1078 | zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp |
| 1079 | zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2 |
| 1080 | cmp tmp.zst tmp1_.zst || die "first file's output differs" |
| 1081 | cmp tmp2.zst tmp2_.zst || die "second file's output differs" |
| 1082 | println "test : incorrect hinted stream sizes" |
| 1083 | cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high |
| 1084 | cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low |
| 1085 | cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high |
| 1086 | cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low |
| 1087 | println "test : allows and interprets K,KB,KiB,M,MB and MiB suffix" |
| 1088 | cat tmp | zstd -14 -f --size-hint=11K | zstd -t |
| 1089 | cat tmp | zstd -14 -f --size-hint=11KB | zstd -t |
| 1090 | cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t |
| 1091 | cat tmp | zstd -14 -f --size-hint=1M | zstd -t |
| 1092 | cat tmp | zstd -14 -f --size-hint=1MB | zstd -t |
| 1093 | cat tmp | zstd -14 -f --size-hint=1MiB | zstd -t |
| 1094 | |
| 1095 | |
| 1096 | println "\n===> dictionary tests " |
| 1097 | println "- Test high/low compressibility corpus training" |
| 1098 | datagen -g12M -P90 > tmpCorpusHighCompress |
| 1099 | datagen -g12M -P5 > tmpCorpusLowCompress |
| 1100 | zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress |
| 1101 | zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress |
| 1102 | rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress |
| 1103 | println "- Test with raw dict (content only) " |
| 1104 | datagen > tmpDict |
| 1105 | datagen -g1M | $MD5SUM > tmp1 |
| 1106 | datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2 |
| 1107 | $DIFF -q tmp1 tmp2 |
| 1108 | println "- Create first dictionary " |
| 1109 | TESTFILE="$PRGDIR"/zstdcli.c |
| 1110 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
| 1111 | cp "$TESTFILE" tmp |
| 1112 | println "- Test dictionary compression with tmpDict as an input file and dictionary" |
| 1113 | zstd -f tmpDict -D tmpDict && die "compression error not detected!" |
| 1114 | println "- Dictionary compression roundtrip" |
| 1115 | zstd -f tmp -D tmpDict |
| 1116 | zstd -d tmp.zst -D tmpDict -fo result |
| 1117 | $DIFF "$TESTFILE" result |
| 1118 | println "- Dictionary compression with hlog < clog" |
| 1119 | zstd -6f tmp -D tmpDict --zstd=clog=25,hlog=23 |
| 1120 | println "- Dictionary compression with btlazy2 strategy" |
| 1121 | zstd -f tmp -D tmpDict --zstd=strategy=6 |
| 1122 | zstd -d tmp.zst -D tmpDict -fo result |
| 1123 | $DIFF "$TESTFILE" result |
| 1124 | if [ -e /proc/self/fd/0 ]; then |
| 1125 | println "- Test rejecting irregular dictionary file" |
| 1126 | cat tmpDict | zstd -f tmp -D /proc/self/fd/0 && die "Piped dictionary should fail!" |
| 1127 | cat tmpDict | zstd -d tmp.zst -D /proc/self/fd/0 -f && die "Piped dictionary should fail!" |
| 1128 | fi |
| 1129 | if [ -n "$hasMT" ] |
| 1130 | then |
| 1131 | println "- Test dictionary compression with multithreading " |
| 1132 | datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict # fails with v1.3.2 |
| 1133 | fi |
| 1134 | println "- Create second (different) dictionary " |
| 1135 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
| 1136 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
| 1137 | println "- Create dictionary with short dictID" |
| 1138 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
| 1139 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
| 1140 | println "- Create dictionary with wrong dictID parameter order (must fail)" |
| 1141 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " |
| 1142 | println "- Create dictionary with size limit" |
| 1143 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v |
| 1144 | println "- Create dictionary with small size limit" |
| 1145 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v |
| 1146 | println "- Create dictionary with wrong parameter order (must fail)" |
| 1147 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " |
| 1148 | println "- Compress without dictID" |
| 1149 | zstd -f tmp -D tmpDict1 --no-dictID |
| 1150 | zstd -d tmp.zst -D tmpDict -fo result |
| 1151 | $DIFF "$TESTFILE" result |
| 1152 | println "- Compress multiple files with dictionary" |
| 1153 | rm -rf dirTestDict |
| 1154 | mkdir dirTestDict |
| 1155 | cp "$TESTDIR"/*.c dirTestDict |
| 1156 | cp "$PRGDIR"/*.c dirTestDict |
| 1157 | cp "$PRGDIR"/*.h dirTestDict |
| 1158 | $MD5SUM dirTestDict/* > tmph1 |
| 1159 | zstd -f --rm dirTestDict/* -D tmpDictC |
| 1160 | zstd -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default |
| 1161 | case "$UNAME" in |
| 1162 | Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 |
| 1163 | *) $MD5SUM -c tmph1 ;; |
| 1164 | esac |
| 1165 | rm -rf dirTestDict |
| 1166 | println "- dictionary builder on bogus input" |
| 1167 | println "Hello World" > tmp |
| 1168 | zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" |
| 1169 | datagen -P0 -g10M > tmp |
| 1170 | zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" |
| 1171 | println "- Test -o before --train" |
| 1172 | rm -f tmpDict dictionary |
| 1173 | zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1174 | test -f tmpDict |
| 1175 | zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1176 | test -f dictionary |
| 1177 | if [ -n "$hasMT" ] |
| 1178 | then |
| 1179 | println "- Create dictionary with multithreading enabled" |
| 1180 | zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
| 1181 | fi |
| 1182 | rm -f tmp* dictionary |
| 1183 | |
| 1184 | println "- Test --memory for dictionary compression" |
| 1185 | datagen -g12M -P90 > tmpCorpusHighCompress |
| 1186 | zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)" |
| 1187 | zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr |
| 1188 | cat zstTrainWithMemLimitStdErr | $GREP "setting manual memory limit for dictionary training data at 5 MB" |
| 1189 | cat zstTrainWithMemLimitStdErr | $GREP "Training samples set too large (12 MB); training on 5 MB only..." |
| 1190 | rm zstTrainWithMemLimitStdErr |
| 1191 | |
| 1192 | println "\n===> fastCover dictionary builder : advanced options " |
| 1193 | TESTFILE="$PRGDIR"/zstdcli.c |
| 1194 | datagen > tmpDict |
| 1195 | println "- Create first dictionary" |
| 1196 | zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
| 1197 | cp "$TESTFILE" tmp |
| 1198 | zstd -f tmp -D tmpDict |
| 1199 | zstd -d tmp.zst -D tmpDict -fo result |
| 1200 | $DIFF "$TESTFILE" result |
| 1201 | println "- Create second (different) dictionary" |
| 1202 | zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
| 1203 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
| 1204 | zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file" |
| 1205 | println "- Create dictionary with short dictID" |
| 1206 | zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
| 1207 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
| 1208 | println "- Create dictionaries with shrink-dict flag enabled" |
| 1209 | zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict |
| 1210 | zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 |
| 1211 | zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 |
| 1212 | zstd --train-fastcover=shrink=5,steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict3 |
| 1213 | println "- Create dictionary with size limit" |
| 1214 | zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K |
| 1215 | println "- Create dictionary using all samples for both training and testing" |
| 1216 | zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1217 | println "- Create dictionary using f=16" |
| 1218 | zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1219 | zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" |
| 1220 | println "- Create dictionary using accel=2" |
| 1221 | zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1222 | println "- Create dictionary using accel=10" |
| 1223 | zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1224 | println "- Create dictionary with multithreading" |
| 1225 | zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1226 | println "- Test -o before --train-fastcover" |
| 1227 | rm -f tmpDict dictionary |
| 1228 | zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1229 | test -f tmpDict |
| 1230 | zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1231 | test -f dictionary |
| 1232 | rm -f tmp* dictionary |
| 1233 | |
| 1234 | |
| 1235 | println "\n===> legacy dictionary builder " |
| 1236 | |
| 1237 | TESTFILE="$PRGDIR"/zstdcli.c |
| 1238 | datagen > tmpDict |
| 1239 | println "- Create first dictionary" |
| 1240 | zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
| 1241 | cp "$TESTFILE" tmp |
| 1242 | zstd -f tmp -D tmpDict |
| 1243 | zstd -d tmp.zst -D tmpDict -fo result |
| 1244 | $DIFF "$TESTFILE" result |
| 1245 | zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)" |
| 1246 | println "- Create second (different) dictionary" |
| 1247 | zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
| 1248 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
| 1249 | println "- Create dictionary with short dictID" |
| 1250 | zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
| 1251 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
| 1252 | println "- Create dictionary with size limit" |
| 1253 | zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K |
| 1254 | println "- Test -o before --train-legacy" |
| 1255 | rm -f tmpDict dictionary |
| 1256 | zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1257 | test -f tmpDict |
| 1258 | zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1259 | test -f dictionary |
| 1260 | rm -f tmp* dictionary |
| 1261 | |
| 1262 | |
| 1263 | println "\n===> integrity tests " |
| 1264 | |
| 1265 | println "test one file (tmp1.zst) " |
| 1266 | datagen > tmp1 |
| 1267 | zstd tmp1 |
| 1268 | zstd -t tmp1.zst |
| 1269 | zstd --test tmp1.zst |
| 1270 | println "test multiple files (*.zst) " |
| 1271 | zstd -t ./*.zst |
| 1272 | println "test bad files (*) " |
| 1273 | zstd -t ./* && die "bad files not detected !" |
| 1274 | zstd -t tmp1 && die "bad file not detected !" |
| 1275 | cp tmp1 tmp2.zst |
| 1276 | zstd -t tmp2.zst && die "bad file not detected !" |
| 1277 | datagen -g0 > tmp3 |
| 1278 | zstd -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad |
| 1279 | println "test --rm and --test combined " |
| 1280 | zstd -t --rm tmp1.zst |
| 1281 | test -f tmp1.zst # check file is still present |
| 1282 | cp tmp1.zst tmp2.zst |
| 1283 | zstd -t tmp1.zst tmp2.zst --rm |
| 1284 | test -f tmp1.zst # check file is still present |
| 1285 | test -f tmp2.zst # check file is still present |
| 1286 | split -b16384 tmp1.zst tmpSplit. |
| 1287 | zstd -t tmpSplit.* && die "bad file not detected !" |
| 1288 | datagen | zstd -c | zstd -t |
| 1289 | |
| 1290 | |
| 1291 | println "\n===> golden files tests " |
| 1292 | |
| 1293 | zstd -t -r "$TESTDIR/golden-decompression" |
| 1294 | zstd -c -r "$TESTDIR/golden-compression" | zstd -t |
| 1295 | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http" -c | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" -t |
| 1296 | |
| 1297 | |
| 1298 | println "\n===> benchmark mode tests " |
| 1299 | |
| 1300 | println "bench one file" |
| 1301 | datagen > tmp1 |
| 1302 | zstd -bi0 tmp1 |
| 1303 | println "bench multiple levels" |
| 1304 | zstd -i0b0e3 tmp1 |
| 1305 | println "bench negative level" |
| 1306 | zstd -bi0 --fast tmp1 |
| 1307 | println "with recursive and quiet modes" |
| 1308 | zstd -rqi0b1e2 tmp1 |
| 1309 | println "benchmark decompression only" |
| 1310 | zstd -f tmp1 |
| 1311 | zstd -b -d -i0 tmp1.zst |
| 1312 | println "benchmark can fail - decompression on invalid data" |
| 1313 | zstd -b -d -i0 tmp1 && die "invalid .zst data => benchmark should have failed" |
| 1314 | |
| 1315 | GZIPMODE=1 |
| 1316 | zstd --format=gzip -V || GZIPMODE=0 |
| 1317 | if [ $GZIPMODE -eq 1 ]; then |
| 1318 | println "benchmark mode is only compatible with zstd" |
| 1319 | zstd --format=gzip -b tmp1 && die "-b should be incompatible with gzip format!" |
| 1320 | fi |
| 1321 | |
| 1322 | println "\n===> zstd compatibility tests " |
| 1323 | |
| 1324 | datagen > tmp |
| 1325 | rm -f tmp.zst |
| 1326 | zstd --format=zstd -f tmp |
| 1327 | test -f tmp.zst |
| 1328 | |
| 1329 | |
| 1330 | println "\n===> gzip compatibility tests " |
| 1331 | |
| 1332 | GZIPMODE=1 |
| 1333 | zstd --format=gzip -V || GZIPMODE=0 |
| 1334 | if [ $GZIPMODE -eq 1 ]; then |
| 1335 | println "gzip support detected" |
| 1336 | GZIPEXE=1 |
| 1337 | gzip -V || GZIPEXE=0 |
| 1338 | if [ $GZIPEXE -eq 1 ]; then |
| 1339 | datagen > tmp |
| 1340 | zstd --format=gzip -f tmp |
| 1341 | gzip -t -v tmp.gz |
| 1342 | gzip -f tmp |
| 1343 | zstd -d -f -v tmp.gz |
| 1344 | rm -f tmp* |
| 1345 | else |
| 1346 | println "gzip binary not detected" |
| 1347 | fi |
| 1348 | else |
| 1349 | println "gzip mode not supported" |
| 1350 | fi |
| 1351 | |
| 1352 | |
| 1353 | println "\n===> gzip frame tests " |
| 1354 | |
| 1355 | if [ $GZIPMODE -eq 1 ]; then |
| 1356 | datagen > tmp |
| 1357 | zstd -f --format=gzip tmp |
| 1358 | zstd -f tmp |
| 1359 | cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp |
| 1360 | truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
| 1361 | rm -f tmp* |
| 1362 | else |
| 1363 | println "gzip mode not supported" |
| 1364 | fi |
| 1365 | |
| 1366 | if [ $GZIPMODE -eq 1 ]; then |
| 1367 | datagen > tmp |
| 1368 | rm -f tmp.zst |
| 1369 | zstd --format=gzip --format=zstd -f tmp |
| 1370 | test -f tmp.zst |
| 1371 | fi |
| 1372 | |
| 1373 | println "\n===> xz compatibility tests " |
| 1374 | |
| 1375 | LZMAMODE=1 |
| 1376 | zstd --format=xz -V || LZMAMODE=0 |
| 1377 | if [ $LZMAMODE -eq 1 ]; then |
| 1378 | println "xz support detected" |
| 1379 | XZEXE=1 |
| 1380 | xz -Q -V && lzma -Q -V || XZEXE=0 |
| 1381 | if [ $XZEXE -eq 1 ]; then |
| 1382 | println "Testing zstd xz and lzma support" |
| 1383 | datagen > tmp |
| 1384 | zstd --format=lzma -f tmp |
| 1385 | zstd --format=xz -f tmp |
| 1386 | xz -Q -t -v tmp.xz |
| 1387 | xz -Q -t -v tmp.lzma |
| 1388 | xz -Q -f -k tmp |
| 1389 | lzma -Q -f -k --lzma1 tmp |
| 1390 | zstd -d -f -v tmp.xz |
| 1391 | zstd -d -f -v tmp.lzma |
| 1392 | rm -f tmp* |
| 1393 | println "Creating symlinks" |
| 1394 | ln -s "$ZSTD_BIN" ./xz |
| 1395 | ln -s "$ZSTD_BIN" ./unxz |
| 1396 | ln -s "$ZSTD_BIN" ./lzma |
| 1397 | ln -s "$ZSTD_BIN" ./unlzma |
| 1398 | println "Testing xz and lzma symlinks" |
| 1399 | datagen > tmp |
| 1400 | ./xz tmp |
| 1401 | xz -Q -d tmp.xz |
| 1402 | ./lzma tmp |
| 1403 | lzma -Q -d tmp.lzma |
| 1404 | println "Testing unxz and unlzma symlinks" |
| 1405 | xz -Q tmp |
| 1406 | ./xz -d tmp.xz |
| 1407 | lzma -Q tmp |
| 1408 | ./lzma -d tmp.lzma |
| 1409 | rm -f xz unxz lzma unlzma |
| 1410 | rm -f tmp* |
| 1411 | else |
| 1412 | println "xz binary not detected" |
| 1413 | fi |
| 1414 | else |
| 1415 | println "xz mode not supported" |
| 1416 | fi |
| 1417 | |
| 1418 | |
| 1419 | println "\n===> xz frame tests " |
| 1420 | |
| 1421 | if [ $LZMAMODE -eq 1 ]; then |
| 1422 | datagen > tmp |
| 1423 | zstd -f --format=xz tmp |
| 1424 | zstd -f --format=lzma tmp |
| 1425 | zstd -f tmp |
| 1426 | cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp |
| 1427 | truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
| 1428 | truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
| 1429 | rm -f tmp* |
| 1430 | else |
| 1431 | println "xz mode not supported" |
| 1432 | fi |
| 1433 | |
| 1434 | println "\n===> lz4 compatibility tests " |
| 1435 | |
| 1436 | LZ4MODE=1 |
| 1437 | zstd --format=lz4 -V || LZ4MODE=0 |
| 1438 | if [ $LZ4MODE -eq 1 ]; then |
| 1439 | println "lz4 support detected" |
| 1440 | LZ4EXE=1 |
| 1441 | lz4 -V || LZ4EXE=0 |
| 1442 | if [ $LZ4EXE -eq 1 ]; then |
| 1443 | datagen > tmp |
| 1444 | zstd --format=lz4 -f tmp |
| 1445 | lz4 -t -v tmp.lz4 |
| 1446 | lz4 -f -m tmp # ensure result is sent into tmp.lz4, not stdout |
| 1447 | zstd -d -f -v tmp.lz4 |
| 1448 | rm -f tmp* |
| 1449 | else |
| 1450 | println "lz4 binary not detected" |
| 1451 | fi |
| 1452 | else |
| 1453 | println "lz4 mode not supported" |
| 1454 | fi |
| 1455 | |
| 1456 | |
| 1457 | if [ $LZ4MODE -eq 1 ]; then |
| 1458 | println "\n===> lz4 frame tests " |
| 1459 | datagen > tmp |
| 1460 | zstd -f --format=lz4 tmp |
| 1461 | zstd -f tmp |
| 1462 | cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp |
| 1463 | truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" |
| 1464 | rm -f tmp* |
| 1465 | else |
| 1466 | println "\nlz4 mode not supported" |
| 1467 | fi |
| 1468 | |
| 1469 | |
| 1470 | println "\n===> suffix list test" |
| 1471 | |
| 1472 | ! zstd -d tmp.abc 2> tmplg |
| 1473 | |
| 1474 | if [ $GZIPMODE -ne 1 ]; then |
| 1475 | $GREP ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" |
| 1476 | fi |
| 1477 | |
| 1478 | if [ $LZMAMODE -ne 1 ]; then |
| 1479 | $GREP ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" |
| 1480 | $GREP ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" |
| 1481 | fi |
| 1482 | |
| 1483 | if [ $LZ4MODE -ne 1 ]; then |
| 1484 | $GREP ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" |
| 1485 | fi |
| 1486 | |
| 1487 | touch tmp1 |
| 1488 | zstd tmp1 -o tmp1.zstd |
| 1489 | zstd -d -f tmp1.zstd # support .zstd suffix even though it's not the default suffix |
| 1490 | |
| 1491 | println "\n===> tar extension tests " |
| 1492 | |
| 1493 | rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd |
| 1494 | |
| 1495 | datagen > tmp |
| 1496 | tar -cf tmp.tar tmp |
| 1497 | zstd tmp.tar -o tmp.tzst |
| 1498 | rm -f tmp.tar |
| 1499 | zstd -d tmp.tzst |
| 1500 | [ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" |
| 1501 | rm -f tmp.tar tmp.tzst |
| 1502 | |
| 1503 | if [ $GZIPMODE -eq 1 ]; then |
| 1504 | tar -f - -c tmp | gzip > tmp.tgz |
| 1505 | zstd -d tmp.tgz |
| 1506 | [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!" |
| 1507 | rm -f tmp.tar tmp.tgz |
| 1508 | fi |
| 1509 | |
| 1510 | if [ $LZMAMODE -eq 1 ]; then |
| 1511 | tar -f - -c tmp | zstd --format=xz > tmp.txz |
| 1512 | zstd -d tmp.txz |
| 1513 | [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" |
| 1514 | rm -f tmp.tar tmp.txz |
| 1515 | fi |
| 1516 | |
| 1517 | if [ $LZ4MODE -eq 1 ]; then |
| 1518 | tar -f - -c tmp | zstd --format=lz4 > tmp.tlz4 |
| 1519 | zstd -d tmp.tlz4 |
| 1520 | [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" |
| 1521 | rm -f tmp.tar tmp.tlz4 |
| 1522 | fi |
| 1523 | |
| 1524 | touch tmp.t tmp.tz tmp.tzs |
| 1525 | ! zstd -d tmp.t |
| 1526 | ! zstd -d tmp.tz |
| 1527 | ! zstd -d tmp.tzs |
| 1528 | |
| 1529 | |
| 1530 | println "\n===> zstd round-trip tests " |
| 1531 | |
| 1532 | roundTripTest |
| 1533 | roundTripTest -g15K # TableID==3 |
| 1534 | roundTripTest -g127K # TableID==2 |
| 1535 | roundTripTest -g255K # TableID==1 |
| 1536 | roundTripTest -g522K # TableID==0 |
| 1537 | roundTripTest -g519K 6 # greedy, hash chain |
| 1538 | roundTripTest -g517K 16 # btlazy2 |
| 1539 | roundTripTest -g516K 19 # btopt |
| 1540 | |
| 1541 | fileRoundTripTest -g500K |
| 1542 | |
| 1543 | println "\n===> zstd long distance matching round-trip tests " |
| 1544 | roundTripTest -g0 "2 --single-thread --long" |
| 1545 | roundTripTest -g1000K "1 --single-thread --long" |
| 1546 | roundTripTest -g517K "6 --single-thread --long" |
| 1547 | roundTripTest -g516K "16 --single-thread --long" |
| 1548 | roundTripTest -g518K "19 --single-thread --long" |
| 1549 | roundTripTest -g2M "22 --single-thread --ultra --long" |
| 1550 | fileRoundTripTest -g5M "3 --single-thread --long" |
| 1551 | |
| 1552 | |
| 1553 | roundTripTest -g96K "5 --single-thread" |
| 1554 | if [ -n "$hasMT" ] |
| 1555 | then |
| 1556 | println "\n===> zstdmt round-trip tests " |
| 1557 | roundTripTest -g4M "1 -T0" |
| 1558 | roundTripTest -g4M "1 -T0 --auto-threads=physical" |
| 1559 | roundTripTest -g4M "1 -T0 --auto-threads=logical" |
| 1560 | roundTripTest -g8M "3 -T2" |
| 1561 | roundTripTest -g8M "19 --long" |
| 1562 | roundTripTest -g8000K "2 --threads=2" |
| 1563 | fileRoundTripTest -g4M "19 -T2 -B1M" |
| 1564 | |
| 1565 | println "\n===> zstdmt long distance matching round-trip tests " |
| 1566 | roundTripTest -g8M "3 --long=24 -T2" |
| 1567 | |
| 1568 | println "\n===> zstdmt environment variable tests " |
| 1569 | echo "multifoo" >> mt_tmp |
| 1570 | ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting |
| 1571 | ZSTD_NBTHREADS='' zstd -f mt_tmp # empty env var, warn and revert to default setting |
| 1572 | ZSTD_NBTHREADS=- zstd -f mt_tmp # malformed env var, warn and revert to default setting |
| 1573 | ZSTD_NBTHREADS=a zstd -f mt_tmp # malformed env var, warn and revert to default setting |
| 1574 | ZSTD_NBTHREADS=+a zstd -f mt_tmp # malformed env var, warn and revert to default setting |
| 1575 | ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting |
| 1576 | ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting= |
| 1577 | ZSTD_NBTHREADS=2 zstd -f mt_tmp # correct usage |
| 1578 | ZSTD_NBTHREADS=1 zstd -f mt_tmp # correct usage: single thread |
| 1579 | # temporary envvar changes in the above tests would actually persist in macos /bin/sh |
| 1580 | unset ZSTD_NBTHREADS |
| 1581 | rm -f mt_tmp* |
| 1582 | |
| 1583 | println "\n===> ovLog tests " |
| 1584 | datagen -g2MB > tmp |
| 1585 | refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c) |
| 1586 | ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) |
| 1587 | ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) |
| 1588 | if [ "$refSize" -eq "$ov9Size" ]; then |
| 1589 | echo ov9Size should be different from refSize |
| 1590 | exit 1 |
| 1591 | fi |
| 1592 | if [ "$refSize" -eq "$ov1Size" ]; then |
| 1593 | echo ov1Size should be different from refSize |
| 1594 | exit 1 |
| 1595 | fi |
| 1596 | if [ "$ov9Size" -ge "$ov1Size" ]; then |
| 1597 | echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size" |
| 1598 | exit 1 |
| 1599 | fi |
| 1600 | |
| 1601 | else |
| 1602 | println "\n===> no multithreading, skipping zstdmt tests " |
| 1603 | fi |
| 1604 | |
| 1605 | rm -f tmp* |
| 1606 | |
| 1607 | println "\n===> zstd --list/-l single frame tests " |
| 1608 | datagen > tmp1 |
| 1609 | datagen > tmp2 |
| 1610 | datagen > tmp3 |
| 1611 | zstd tmp* |
| 1612 | zstd -l ./*.zst |
| 1613 | zstd -lv ./*.zst | $GREP "Decompressed Size:" # check that decompressed size is present in header |
| 1614 | zstd --list ./*.zst |
| 1615 | zstd --list -v ./*.zst |
| 1616 | |
| 1617 | println "\n===> zstd --list/-l multiple frame tests " |
| 1618 | cat tmp1.zst tmp2.zst > tmp12.zst |
| 1619 | cat tmp12.zst tmp3.zst > tmp123.zst |
| 1620 | zstd -l ./*.zst |
| 1621 | zstd -lv ./*.zst |
| 1622 | |
| 1623 | println "\n===> zstd --list/-l error detection tests " |
| 1624 | zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file" |
| 1625 | zstd --list tmp* && die "-l must fail on non-zstd file" |
| 1626 | zstd -lv tmp1* && die "-l must fail on non-zstd file" |
| 1627 | zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" |
| 1628 | |
| 1629 | println "test : detect truncated compressed file " |
| 1630 | TEST_DATA_FILE=truncatable-input.txt |
| 1631 | FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst |
| 1632 | TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst |
| 1633 | datagen -g50000 > $TEST_DATA_FILE |
| 1634 | zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE |
| 1635 | dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE |
| 1636 | zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" |
| 1637 | |
| 1638 | rm -f $TEST_DATA_FILE |
| 1639 | rm -f $FULL_COMPRESSED_FILE |
| 1640 | rm -f $TRUNCATED_COMPRESSED_FILE |
| 1641 | |
| 1642 | println "\n===> zstd --list/-l errors when presented with stdin / no files" |
| 1643 | zstd -l && die "-l must fail on empty list of files" |
| 1644 | zstd -l - && die "-l does not work on stdin" |
| 1645 | zstd -l < tmp1.zst && die "-l does not work on stdin" |
| 1646 | zstd -l - < tmp1.zst && die "-l does not work on stdin" |
| 1647 | zstd -l - tmp1.zst && die "-l does not work on stdin" |
| 1648 | zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin" |
| 1649 | zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty |
| 1650 | |
| 1651 | println "\n===> zstd --list/-l test with null files " |
| 1652 | datagen -g0 > tmp5 |
| 1653 | zstd tmp5 |
| 1654 | zstd -l tmp5.zst |
| 1655 | zstd -l tmp5* && die "-l must fail on non-zstd file" |
| 1656 | zstd -lv tmp5.zst | $GREP "Decompressed Size: 0 B (0 B)" # check that 0 size is present in header |
| 1657 | zstd -lv tmp5* && die "-l must fail on non-zstd file" |
| 1658 | |
| 1659 | println "\n===> zstd --list/-l test with no content size field " |
| 1660 | datagen -g513K | zstd > tmp6.zst |
| 1661 | zstd -l tmp6.zst |
| 1662 | zstd -lv tmp6.zst | $GREP "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" |
| 1663 | |
| 1664 | println "\n===> zstd --list/-l test with no checksum " |
| 1665 | zstd -f --no-check tmp1 |
| 1666 | zstd -l tmp1.zst |
| 1667 | zstd -lv tmp1.zst |
| 1668 | |
| 1669 | println "\n===> zstd trace tests " |
| 1670 | zstd -f --trace tmp.trace tmp1 |
| 1671 | zstd -f --trace tmp.trace tmp1 tmp2 tmp3 |
| 1672 | zstd -f --trace tmp.trace tmp1 tmp2 tmp3 -o /dev/null |
| 1673 | zstd -f --trace tmp.trace tmp1 tmp2 tmp3 --single-thread |
| 1674 | zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null |
| 1675 | zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null --single-thread |
| 1676 | zstd --trace tmp.trace -t tmp1.zst |
| 1677 | zstd --trace tmp.trace -t tmp1.zst tmp2.zst |
| 1678 | zstd -f --trace tmp.trace -d tmp1.zst |
| 1679 | zstd -f --trace tmp.trace -d tmp1.zst tmp2.zst tmp3.zst |
| 1680 | zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1 |
| 1681 | zstd -b1e10i0 --trace tmp.trace tmp1 |
| 1682 | zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3 |
| 1683 | |
| 1684 | rm -f tmp* |
| 1685 | |
| 1686 | |
| 1687 | println "\n===> zstd long distance matching tests " |
| 1688 | roundTripTest -g0 " --single-thread --long" |
| 1689 | roundTripTest -g9M "2 --single-thread --long" |
| 1690 | # Test parameter parsing |
| 1691 | roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB" |
| 1692 | roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB" |
| 1693 | roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB" |
| 1694 | roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" |
| 1695 | |
| 1696 | |
| 1697 | if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -ne "1" ]; then |
| 1698 | println "\n===> zstd long distance matching with optimal parser compressed size tests " |
| 1699 | optCSize16=$(datagen -g511K | zstd -16 -c | wc -c) |
| 1700 | longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c) |
| 1701 | optCSize19=$(datagen -g2M | zstd -19 -c | wc -c) |
| 1702 | longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c) |
| 1703 | optCSize19wlog23=$(datagen -g2M | zstd -19 -c --zstd=wlog=23 | wc -c) |
| 1704 | longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c) |
| 1705 | if [ "$longCSize16" -gt "$optCSize16" ]; then |
| 1706 | echo using --long on compression level 16 should not cause compressed size regression |
| 1707 | exit 1 |
| 1708 | elif [ "$longCSize19" -gt "$optCSize19" ]; then |
| 1709 | echo using --long on compression level 19 should not cause compressed size regression |
| 1710 | exit 1 |
| 1711 | elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then |
| 1712 | echo using --long on compression level 19 with wLog=23 should not cause compressed size regression |
| 1713 | exit 1 |
| 1714 | fi |
| 1715 | fi |
| 1716 | |
| 1717 | println "\n===> zstd asyncio tests " |
| 1718 | |
| 1719 | addFrame() { |
| 1720 | datagen -g2M -s$2 >> tmp_uncompressed |
| 1721 | datagen -g2M -s$2 | zstd -1 --format=$1 >> tmp_compressed.zst |
| 1722 | } |
| 1723 | |
| 1724 | addTwoFrames() { |
| 1725 | addFrame $1 1 |
| 1726 | addFrame $1 2 |
| 1727 | } |
| 1728 | |
| 1729 | testAsyncIO() { |
| 1730 | roundTripTest -g2M "3 --asyncio --format=$1" |
| 1731 | roundTripTest -g2M "3 --no-asyncio --format=$1" |
| 1732 | } |
| 1733 | |
| 1734 | rm -f tmp_compressed tmp_uncompressed |
| 1735 | testAsyncIO zstd |
| 1736 | addTwoFrames zstd |
| 1737 | if [ $GZIPMODE -eq 1 ]; then |
| 1738 | testAsyncIO gzip |
| 1739 | addTwoFrames gzip |
| 1740 | fi |
| 1741 | if [ $LZMAMODE -eq 1 ]; then |
| 1742 | testAsyncIO lzma |
| 1743 | addTwoFrames lzma |
| 1744 | fi |
| 1745 | if [ $LZ4MODE -eq 1 ]; then |
| 1746 | testAsyncIO lz4 |
| 1747 | addTwoFrames lz4 |
| 1748 | fi |
| 1749 | cat tmp_uncompressed | $MD5SUM > tmp2 |
| 1750 | zstd -d tmp_compressed.zst --asyncio -c | $MD5SUM > tmp1 |
| 1751 | $DIFF -q tmp1 tmp2 |
| 1752 | rm tmp1 |
| 1753 | zstd -d tmp_compressed.zst --no-asyncio -c | $MD5SUM > tmp1 |
| 1754 | $DIFF -q tmp1 tmp2 |
| 1755 | |
| 1756 | if [ "$1" != "--test-large-data" ]; then |
| 1757 | println "Skipping large data tests" |
| 1758 | exit 0 |
| 1759 | fi |
| 1760 | |
| 1761 | |
| 1762 | ############################################################################# |
| 1763 | |
| 1764 | |
| 1765 | if [ -n "$hasMT" ] |
| 1766 | then |
| 1767 | println "\n===> adaptive mode " |
| 1768 | roundTripTest -g270000000 " --adapt" |
| 1769 | roundTripTest -g27000000 " --adapt=min=1,max=4" |
| 1770 | roundTripTest -g27000000 " --adapt=min=-2,max=-1" |
| 1771 | println "===> test: --adapt must fail on incoherent bounds " |
| 1772 | datagen > tmp |
| 1773 | zstd --adapt= tmp && die "invalid compression parameter" |
| 1774 | zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" |
| 1775 | |
| 1776 | println "\n===> rsyncable mode " |
| 1777 | roundTripTest -g10M " --rsyncable" |
| 1778 | roundTripTest -g10M " --rsyncable -B100K" |
| 1779 | println "===> test: --rsyncable must fail with --single-thread" |
| 1780 | zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" |
| 1781 | fi |
| 1782 | |
| 1783 | println "\n===> patch-from=origin tests" |
| 1784 | datagen -g1000 -P50 > tmp_dict |
| 1785 | datagen -g1000 -P10 > tmp_patch |
| 1786 | zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff |
| 1787 | zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon |
| 1788 | $DIFF -s tmp_patch_recon tmp_patch |
| 1789 | |
| 1790 | println "\n===> alternate syntax: patch-from origin" |
| 1791 | zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff |
| 1792 | zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon |
| 1793 | $DIFF -s tmp_patch_recon tmp_patch |
| 1794 | rm -rf tmp_* |
| 1795 | |
| 1796 | println "\n===> patch-from recursive tests" |
| 1797 | mkdir tmp_dir |
| 1798 | datagen > tmp_dir/tmp1 |
| 1799 | datagen > tmp_dir/tmp2 |
| 1800 | datagen > tmp_dict |
| 1801 | zstd --patch-from=tmp_dict -r tmp_dir && die |
| 1802 | rm -rf tmp* |
| 1803 | |
| 1804 | println "\n===> patch-from long mode trigger larger file test" |
| 1805 | if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -eq "1" ]; then |
| 1806 | # if binary tree strategies are excluded, the threshold is different |
| 1807 | datagen -g10000000 > tmp_dict |
| 1808 | datagen -g10000000 > tmp_patch |
| 1809 | else |
| 1810 | datagen -g5000000 > tmp_dict |
| 1811 | datagen -g5000000 > tmp_patch |
| 1812 | fi |
| 1813 | zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | $GREP "long mode automatically triggered" |
| 1814 | rm -rf tmp* |
| 1815 | |
| 1816 | println "\n===> patch-from very large dictionary and file test" |
| 1817 | datagen -g550000000 -P0 > tmp_dict |
| 1818 | datagen -g100000000 -P1 > tmp_patch |
| 1819 | zstd --long=30 -1f --patch-from tmp_dict tmp_patch |
| 1820 | zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon |
| 1821 | $DIFF -s tmp_patch_recon tmp_patch |
| 1822 | rm -rf tmp* |
| 1823 | |
| 1824 | println "\n===> patch-from --stream-size test" |
| 1825 | datagen -g1000 -P50 > tmp_dict |
| 1826 | datagen -g1000 -P10 > tmp_patch |
| 1827 | cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die |
| 1828 | cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff |
| 1829 | rm -rf tmp* |
| 1830 | |
| 1831 | println "\n===> large files tests " |
| 1832 | |
| 1833 | roundTripTest -g270000000 1 |
| 1834 | roundTripTest -g250000000 2 |
| 1835 | roundTripTest -g230000000 3 |
| 1836 | |
| 1837 | roundTripTest -g140000000 -P60 4 |
| 1838 | roundTripTest -g130000000 -P62 5 |
| 1839 | roundTripTest -g120000000 -P65 6 |
| 1840 | |
| 1841 | roundTripTest -g70000000 -P70 7 |
| 1842 | roundTripTest -g60000000 -P71 8 |
| 1843 | roundTripTest -g50000000 -P73 9 |
| 1844 | |
| 1845 | roundTripTest -g35000000 -P75 10 |
| 1846 | roundTripTest -g30000000 -P76 11 |
| 1847 | roundTripTest -g25000000 -P78 12 |
| 1848 | |
| 1849 | roundTripTest -g18000013 -P80 13 |
| 1850 | roundTripTest -g18000014 -P80 14 |
| 1851 | roundTripTest -g18000015 -P81 15 |
| 1852 | roundTripTest -g18000016 -P84 16 |
| 1853 | roundTripTest -g18000017 -P88 17 |
| 1854 | roundTripTest -g18000018 -P94 18 |
| 1855 | roundTripTest -g18000019 -P96 19 |
| 1856 | |
| 1857 | roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25" |
| 1858 | roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an overflow rescale |
| 1859 | |
| 1860 | fileRoundTripTest -g4193M -P99 1 |
| 1861 | |
| 1862 | |
| 1863 | println "\n===> zstd long, long distance matching round-trip tests " |
| 1864 | roundTripTest -g270000000 "1 --single-thread --long" |
| 1865 | roundTripTest -g130000000 -P60 "5 --single-thread --long" |
| 1866 | roundTripTest -g35000000 -P70 "8 --single-thread --long" |
| 1867 | roundTripTest -g18000001 -P80 "18 --single-thread --long" |
| 1868 | # Test large window logs |
| 1869 | roundTripTest -g700M -P50 "1 --single-thread --long=29" |
| 1870 | roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" |
| 1871 | |
| 1872 | |
| 1873 | if [ -n "$hasMT" ] |
| 1874 | then |
| 1875 | println "\n===> zstdmt long round-trip tests " |
| 1876 | roundTripTest -g80000000 -P99 "19 -T2" " " |
| 1877 | roundTripTest -g5000000000 -P99 "1 -T2" " " |
| 1878 | roundTripTest -g500000000 -P97 "1 -T999" " " |
| 1879 | fileRoundTripTest -g4103M -P98 " -T0" " " |
| 1880 | roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " |
| 1881 | # Exposes the bug in https://github.com/facebook/zstd/pull/1678 |
| 1882 | # This test fails on 4 different travis builds at the time of writing |
| 1883 | # because it needs to allocate 8 GB of memory. |
| 1884 | # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000" |
| 1885 | else |
| 1886 | println "\n**** no multithreading, skipping zstdmt tests **** " |
| 1887 | fi |
| 1888 | |
| 1889 | |
| 1890 | println "\n===> cover dictionary builder : advanced options " |
| 1891 | |
| 1892 | TESTFILE="$PRGDIR"/zstdcli.c |
| 1893 | datagen > tmpDict |
| 1894 | println "- Create first dictionary" |
| 1895 | zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict |
| 1896 | cp "$TESTFILE" tmp |
| 1897 | zstd -f tmp -D tmpDict |
| 1898 | zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the same time" |
| 1899 | zstd -d tmp.zst -D tmpDict -fo result |
| 1900 | $DIFF "$TESTFILE" result |
| 1901 | zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" |
| 1902 | println "- Create second (different) dictionary" |
| 1903 | zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC |
| 1904 | zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" |
| 1905 | println "- Create dictionary using shrink-dict flag" |
| 1906 | zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict |
| 1907 | zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 |
| 1908 | zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 |
| 1909 | zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3 |
| 1910 | println "- Create dictionary with short dictID" |
| 1911 | zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 |
| 1912 | cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" |
| 1913 | println "- Create dictionary with size limit" |
| 1914 | zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K |
| 1915 | println "- Compare size of dictionary from 90% training samples with 80% training samples" |
| 1916 | zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1917 | zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1918 | println "- Create dictionary using all samples for both training and testing" |
| 1919 | zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1920 | println "- Test -o before --train-cover" |
| 1921 | rm -f tmpDict dictionary |
| 1922 | zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1923 | test -f tmpDict |
| 1924 | zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c |
| 1925 | test -f dictionary |
| 1926 | rm -f tmp* dictionary |
| 1927 | |
| 1928 | rm -f tmp* |