#!/bin/sh -e

# Tool to bundle multiple C/C++ source files, inlining any includes.
#
# TODO: ROOTS, FOUND, etc., as arrays (since they fail on paths with spaces)
#
# Author: Carl Woffenden, Numfum GmbH (this script is released under a CC0 license/Public Domain)

# Common file roots
ROOTS="."

# -x option excluded includes
XINCS=""

# -k option includes to keep as include directives
KINCS=""

# Files previously visited
FOUND=""

# Optional destination file (empty string to write to stdout)
DESTN=""

# Whether the "#pragma once" directives should be written to the output
PONCE=0

# Prints the script usage then exits
usage() {
  echo "Usage: $0 [-r <path>] [-x <header>] [-k <header>] [-o <outfile>] infile"
  echo "  -r file root search path"
  echo "  -x file to completely exclude from inlining"
  echo "  -k file to exclude from inlining but keep the include directive"
  echo "  -p keep any '#pragma once' directives (removed by default)"
  echo "  -o output file (otherwise stdout)"
  echo "Example: $0 -r ../my/path - r ../other/path -o out.c in.c"
  exit 1
}

# Tests that the grep implementation works as expected (older OSX grep fails)
test_deps() {
  if ! echo '#include "foo"' | grep -Eq '^\s*#\s*include\s*".+"'; then
    echo "Aborting: the grep implementation fails to parse include lines"
    exit 1
  fi
  if ! echo '"foo.h"' | sed -E 's/"([^"]+)"/\1/' | grep -Eq '^foo\.h$'; then
    echo "Aborting: sed is unavailable or non-functional"
    exit 1
  fi
}

# Test if glob pattern $1 matches subject $2 (see fnmatch(3))
fnmatch() {
  case "$2" in
  $1)
    return 0
    ;;
  esac
  return 1
}

# Test if line $1 is local include directive
is_include_line() {
  fnmatch "*#*include*" "$1" || return 1
  printf "%s\n" "$1" | grep -Eq '^\s*#\s*include\s*".+"'
}

# Test if line $1 is pragma once directive
is_pragma_once_line() {
  fnmatch "*#*pragma*once*" "$1" || return 1
  printf "%s\n" "$1" | grep -Eq '^\s*#\s*pragma\s*once\s*'
}

# Tests if list $1 has item $2 (returning zero on a match)
# (originally used grep -Eq "(^|\s*)$2(\$|\s*))
readonly list_FS="$IFS"
list_has_item() {
  # Re: escaping glob pattern special characters in item string:
  #
  # bash (tested 3.2.57, 5.1.4), dash (tested 0.5.10.2), NetBSD /bin/sh
  # (tested 8.2), and Solaris /bin/sh (tested 11.4) require escaping
  # backslashes in a bracket expression despite POSIX specifying that
  # backslash loses significance in a bracket expression.
  #
  # Conversely, neither FreeBSD /bin/sh (tested 12.2) nor OpenBSD /bin/sh
  # (tested 7.1) obey backslash-escaping in case statement patterns even
  # outside bracket expressions, so escape special characters using bracket
  # expressions.
  #
  # Solaris /bin/sh (tested 11.4) requires vertical bar (|) to be escaped.
  #
  # All accommodations should behave as expected under strict POSIX semantics.
  if fnmatch "*[\\*?[|]*" "$2"; then
    set -- "$1" "$(printf '%s\n' "$2" | sed -e 's/[*?[|]/[&]/g; s/[\]/[\\&]/g')"
  fi
  for item_P in "*[$list_FS]$2[$list_FS]*" "*[$list_FS]$2" "$2[$list_FS]*" "$2"; do
    fnmatch "${item_P}" "$1" && return 0
  done
  return 1
}

# Adds a new line with the supplied arguments to $DESTN (or stdout)
write_line() {
  if [ -n "$DESTN" ]; then
    printf '%s\n' "$@" >> "$DESTN"
  else
    printf '%s\n' "$@"
  fi
}

log_line() {
  echo $@ >&2
}

# Find this file!
resolve_include() {
  local srcdir=$1
  local inc=$2
  for root in $srcdir $ROOTS; do
    if [ -f "$root/$inc" ]; then
      # Try to reduce the file path into a canonical form (so that multiple)
      # includes of the same file are successfully deduplicated, even if they
      # are expressed differently.
      local relpath="$(realpath --relative-to . "$root/$inc" 2>/dev/null)"
      if [ "$relpath" != "" ]; then # not all realpaths support --relative-to
        echo "$relpath"
        return 0
      fi
      local relpath="$(realpath "$root/$inc" 2>/dev/null)"
      if [ "$relpath" != "" ]; then # not all distros have realpath...
        echo "$relpath"
        return 0
      fi
      # Fallback on Python to reduce the path if the above fails.
      local relpath=$(python -c "import os,sys; print os.path.relpath(sys.argv[1])" "$root/$inc" 2>/dev/null)
      if [ "$relpath" != "" ]; then # not all distros have realpath...
        echo "$relpath"
        return 0
      fi
      # Worst case, fall back to just the root + relative include path. The
      # problem with this is that it is possible to emit multiple different
      # resolved paths to the same file, depending on exactly how its included.
      # Since the main loop below keeps a list of the resolved paths it's
      # already included, in order to avoid repeated includes, this failure to
      # produce a canonical/reduced path can lead to multiple inclusions of the
      # same file. But it seems like the resulting single file library still
      # works (hurray include guards!), so I guess it's ok.
      echo "$root/$inc"
      return 0
    fi
  done
  return 1
}

# Adds the contents of $1 with any of its includes inlined
add_file() {
  local file=$1
  if [ -n "$file" ]; then
    log_line "Processing: $file"
    # Get directory of the current so we can resolve relative includes
    local srcdir="$(dirname "$file")"
    # Read the file
    local line=
    while IFS= read -r line; do
      if is_include_line "$line"; then
        # We have an include directive so strip the (first) file
        local inc=$(echo "$line" | grep -Eo '".*"' | sed -E 's/"([^"]+)"/\1/' | head -1)
        local res_inc="$(resolve_include "$srcdir" "$inc")"
        if list_has_item "$XINCS" "$inc"; then
          # The file was excluded so error if the source attempts to use it
          write_line "#error Using excluded file: $inc (re-amalgamate source to fix)"
          log_line "Excluding: $inc"
        else
          if ! list_has_item "$FOUND" "$res_inc"; then
            # The file was not previously encountered
            FOUND="$FOUND $res_inc"
            if list_has_item "$KINCS" "$inc"; then
              # But the include was flagged to keep as included
              write_line "/**** *NOT* inlining $inc ****/"
              write_line "$line"
              log_line "Not Inlining: $inc"
            else
              # The file was neither excluded nor seen before so inline it
              write_line "/**** start inlining $inc ****/"
              add_file "$res_inc"
              write_line "/**** ended inlining $inc ****/"
            fi
          else
            write_line "/**** skipping file: $inc ****/"
          fi
        fi
      else
        # Skip any 'pragma once' directives, otherwise write the source line
        local write=$PONCE
        if [ $write -eq 0 ]; then
          if ! is_pragma_once_line "$line"; then
            write=1
          fi
        fi
        if [ $write -ne 0 ]; then
          write_line "$line"
        fi
      fi
    done < "$file"
  else
    write_line "#error Unable to find \"$1\""
    log_line "Error: Unable to find: \"$1\""
  fi
}

while getopts ":r:x:k:po:" opts; do
  case $opts in
  r)
    ROOTS="$ROOTS $OPTARG"
    ;;
  x)
    XINCS="$XINCS $OPTARG"
    ;;
  k)
    KINCS="$KINCS $OPTARG"
    ;;
  p)
    PONCE=1
    ;;
  o)
    DESTN="$OPTARG"
    ;;
  *)
    usage
    ;;
  esac
done
shift $((OPTIND-1))

if [ -n "$1" ]; then
  if [ -f "$1" ]; then
    if [ -n "$DESTN" ]; then
      printf "" > "$DESTN"
    fi
    test_deps
    log_line "Processing using the slower shell script; this might take a while"
    add_file "$1"
  else
    echo "Input file not found: \"$1\""
    exit 1
  fi
else
  usage
fi
exit 0


#!/bin/sh

# Where to find the sources
ZSTD_SRC_ROOT="../../lib"

# Amalgamate the sources
echo "Amalgamating files..."
# Using the faster Python script if we have 3.8 or higher
if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then
  ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
else
  ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstddeclib.c zstddeclib-in.c
fi
# Did combining work?
if [ $? -ne 0 ]; then
  echo "Combine script: FAILED"
  exit 1
fi
echo "Combine script: PASSED"


#!/bin/sh

# Where to find the sources
ZSTD_SRC_ROOT="../../lib"

# Amalgamate the sources
echo "Amalgamating files..."
# Using the faster Python script if we have 3.8 or higher
if python3 -c 'import sys; assert sys.version_info >= (3,8)' 2>/dev/null; then
  ./combine.py -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
else
  ./combine.sh -r "$ZSTD_SRC_ROOT" -x legacy/zstd_legacy.h -o zstd.c zstd-in.c
fi
# Did combining work?
if [ $? -ne 0 ]; then
  echo "Combine script: FAILED"
  exit 1
fi
echo "Combine script: PASSED"


#!/bin/sh

LIBVER_MAJOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
LIBVER_MINOR_SCRIPT=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
LIBVER_PATCH_SCRIPT=`sed -n '/define ZSTD_VERSION_RELEASE/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ../../lib/zstd.h`
LIBVER_SCRIPT=$LIBVER_MAJOR_SCRIPT.$LIBVER_MINOR_SCRIPT.$LIBVER_PATCH_SCRIPT

echo ZSTD_VERSION=$LIBVER_SCRIPT
./gen_html $LIBVER_SCRIPT ../../lib/zstd.h ./zstd_manual.html


# !/bin/sh
set -e

# Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
# The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and
# 16 GB of RAM and an SSD.

# silesia is a directory that can be downloaded from
# http://mattmahoney.net/dc/silesia.html
# ls -l silesia/
# total 203M
# -rwxr-xr-x 1 terrelln 9.8M Apr 12  2002 dickens
# -rwxr-xr-x 1 terrelln  49M May 31  2002 mozilla
# -rwxr-xr-x 1 terrelln 9.6M Mar 20  2003 mr
# -rwxr-xr-x 1 terrelln  32M Apr  2  2002 nci
# -rwxr-xr-x 1 terrelln 5.9M Jul  4  2002 ooffice
# -rwxr-xr-x 1 terrelln 9.7M Apr 11  2002 osdb
# -rwxr-xr-x 1 terrelln 6.4M Apr  2  2002 reymont
# -rwxr-xr-x 1 terrelln  21M Mar 25  2002 samba
# -rwxr-xr-x 1 terrelln 7.0M Mar 24  2002 sao
# -rwxr-xr-x 1 terrelln  40M Mar 25  2002 webster
# -rwxr-xr-x 1 terrelln 8.1M Apr  4  2002 x-ray
# -rwxr-xr-x 1 terrelln 5.1M Nov 30  2000 xml

# $HOME is on a ext4 filesystem
BENCHMARK_DIR="$HOME/silesia/"
N=10

# Normalize the environment
sudo umount /mnt/btrfs 2> /dev/null > /dev/null || true
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
sudo rm -rf /mnt/btrfs/*
sync
sudo umount /mnt/btrfs
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs

# Run the benchmark
echo "Compression"
time sh -c "for i in \$(seq $N); do sudo cp -r $BENCHMARK_DIR /mnt/btrfs/\$i; done; sync"

echo "Approximate compression ratio"
printf "%d / %d\n"                                                             \
  $(df /mnt/btrfs --output=used -B 1 | tail -n 1)                              \
  $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);

# Unmount and remount to avoid any caching
sudo umount /mnt/btrfs
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs

echo "Decompression"
time sudo tar -c /mnt/btrfs 2> /dev/null | wc -c > /dev/null

sudo rm -rf /mnt/btrfs/*
sudo umount /mnt/btrfs

# Run for each of -o compress-force={none, lzo, zlib, zstd} 5 times and take the
# min time and ratio.
# Ran zstd with compression levels {1, 3, 6, 9, 12, 15}.
# Original size: 2119415342 B (using du /mnt/btrfs)

# none
# compress: 4.205 s
# decompress: 3.090 s
# ratio: 0.99

# lzo
# compress: 5.328 s
# decompress: 4.793 s
# ratio: 1.66

# zlib
# compress: 32.588 s
# decompress: 8.791 s
# ratio : 2.58

# zstd 1
# compress: 8.147 s
# decompress: 5.527 s
# ratio : 2.57

# zstd 3
# compress: 12.207 s
# decompress: 5.195 s
# ratio : 2.71

# zstd 6
# compress: 30.253 s
# decompress: 5.324 s
# ratio : 2.87

# zstd 9
# compress: 49.659 s
# decompress: 5.220 s
# ratio : 2.92

# zstd 12
# compress: 99.245 s
# decompress: 5.193 s
# ratio : 2.93

# zstd 15
# compress: 196.997 s
# decompress: 5.992 s
# ratio : 3.01


# !/bin/sh
set -e

# Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
# The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and
# 16 GB of RAM and an SSD.

# silesia is a directory that can be downloaded from
# http://mattmahoney.net/dc/silesia.html
# ls -l silesia/
# total 203M
# -rwxr-xr-x 1 terrelln 9.8M Apr 12  2002 dickens
# -rwxr-xr-x 1 terrelln  49M May 31  2002 mozilla
# -rwxr-xr-x 1 terrelln 9.6M Mar 20  2003 mr
# -rwxr-xr-x 1 terrelln  32M Apr  2  2002 nci
# -rwxr-xr-x 1 terrelln 5.9M Jul  4  2002 ooffice
# -rwxr-xr-x 1 terrelln 9.7M Apr 11  2002 osdb
# -rwxr-xr-x 1 terrelln 6.4M Apr  2  2002 reymont
# -rwxr-xr-x 1 terrelln  21M Mar 25  2002 samba
# -rwxr-xr-x 1 terrelln 7.0M Mar 24  2002 sao
# -rwxr-xr-x 1 terrelln  40M Mar 25  2002 webster
# -rwxr-xr-x 1 terrelln 8.1M Apr  4  2002 x-ray
# -rwxr-xr-x 1 terrelln 5.1M Nov 30  2000 xml

# $HOME is on a ext4 filesystem
BENCHMARK_FILE="linux-4.11.6.tar"
BENCHMARK_DIR="$HOME/$BENCHMARK_FILE"

# Normalize the environment
sudo umount /mnt/btrfs 2> /dev/null > /dev/null || true
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs
sudo rm -rf /mnt/btrfs/*
sync
sudo umount /mnt/btrfs
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs

# Run the benchmark
echo "Copy"
time sh -c "sudo cp -r $BENCHMARK_DIR /mnt/btrfs/$BENCHMARK_FILE && sync"

echo "Approximate tarred compression ratio"
printf "%d / %d\n"                                                             \
  $(df /mnt/btrfs --output=used -B 1 | tail -n 1)                              \
  $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);

# Unmount and remount to avoid any caching
sudo umount /mnt/btrfs
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs

echo "Extract"
time sh -c "sudo tar -C /mnt/btrfs -xf /mnt/btrfs/$BENCHMARK_FILE && sync"

# Remove the tarball, leaving only the extracted data
sudo rm /mnt/btrfs/$BENCHMARK_FILE
# Unmount and remount to avoid any caching
sudo umount /mnt/btrfs
sudo mount -t btrfs $@ /dev/sda3 /mnt/btrfs

echo "Approximate extracted compression ratio"
printf "%d / %d\n"                                                             \
  $(df /mnt/btrfs --output=used -B 1 | tail -n 1)                              \
  $(sudo du /mnt/btrfs -b -d 0 | tr '\t' '\n' | head -n 1);

echo "Read"
time sudo tar -c /mnt/btrfs 2> /dev/null | wc -c > /dev/null

sudo rm -rf /mnt/btrfs/*
sudo umount /mnt/btrfs

# Run for each of -o compress-force={none, lzo, zlib, zstd} 5 times and take the
# min time and ratio.

# none
# copy: 0.981 s
# extract: 5.501 s
# read: 8.807 s
# tarball ratio: 0.97
# extracted ratio: 0.78

# lzo
# copy: 1.631 s
# extract: 8.458 s
# read: 8.585 s
# tarball ratio: 2.06
# extracted ratio: 1.38

# zlib
# copy: 7.750 s
# extract: 21.544 s
# read: 11.744 s
# tarball ratio : 3.40
# extracted ratio: 1.86

# zstd 1
# copy: 2.579 s
# extract: 11.479 s
# read: 9.389 s
# tarball ratio : 3.57
# extracted ratio: 1.85


# !/bin/sh
set -e

# Benchmarks run on a Ubuntu 14.04 VM with 2 cores and 4 GiB of RAM.
# The VM is running on a Macbook Pro with a 3.1 GHz Intel Core i7 processor and
# 16 GB of RAM and an SSD.

# $BENCHMARK_DIR is generated with the following commands, from the Ubuntu image
# ubuntu-16.10-desktop-amd64.iso.
# > mkdir mnt
# > sudo mount -o loop ubuntu-16.10-desktop-amd64.iso mnt
# > cp mnt/casper/filesystem.squashfs .
# > sudo unsquashfs filesystem.squashfs

# $HOME is on a ext4 filesystem
BENCHMARK_DIR="$HOME/squashfs-root/"
BENCHMARK_FS="$HOME/filesystem.squashfs"

# Normalize the environment
sudo rm -f $BENCHMARK_FS 2> /dev/null > /dev/null || true
sudo umount /mnt/squashfs 2> /dev/null > /dev/null || true

# Run the benchmark
echo "Compression"
echo "sudo mksquashfs $BENCHMARK_DIR $BENCHMARK_FS $@"
time sudo mksquashfs $BENCHMARK_DIR $BENCHMARK_FS $@ 2> /dev/null > /dev/null

echo "Approximate compression ratio"
printf "%d / %d\n"                                                             \
  $(sudo du -sx --block-size=1 $BENCHMARK_DIR | cut -f1)                       \
  $(sudo du -sx --block-size=1 $BENCHMARK_FS  | cut -f1);

# Mount the filesystem
sudo mount -t squashfs $BENCHMARK_FS /mnt/squashfs

echo "Decompression"
time sudo tar -c /mnt/squashfs 2> /dev/null | wc -c > /dev/null

sudo umount /mnt/squashfs


#!/usr/bin/env sh

set -e

SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
INCLUDE_DIR="$SCRIPT_DIR/../linux/include"
LIB_DIR="$SCRIPT_DIR/../linux/lib"

print() {
    printf '%b' "${*}"
}

println() {
    printf '%b\n' "${*}"
}

die() {
    println "$@" 1>&2
    exit 1
}

test_not_present() {
    print "Testing that '$1' is not present... "
    grep -r $1 "$INCLUDE_DIR" "$LIB_DIR" && die "Fail!"
    println "Okay"
}

println "This test checks that the macro removal process worked as expected"
println "If this test fails, then freestanding.py wasn't able to remove one of these"
println "macros from the source code completely. You'll either need to rewrite the check"
println "or improve freestanding.py."
println ""

test_not_present "ZSTD_NO_INTRINSICS"
test_not_present "ZSTD_NO_UNUSED_FUNCTIONS"
test_not_present "ZSTD_LEGACY_SUPPORT"
test_not_present "STATIC_BMI2"
test_not_present "ZSTD_DLL_EXPORT"
test_not_present "ZSTD_DLL_IMPORT"
test_not_present "__ICCARM__"
test_not_present "_MSC_VER"
test_not_present "_WIN32"
test_not_present "__linux__"


#!/bin/sh -e

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

ECHO=echo
RM="rm -f"
GREP="grep"
INTOVOID="/dev/null"

die() {
    $ECHO "$@" 1>&2
    exit 1
}

isPresent() {
    $GREP $@ tmplog || die "$@" "should be present"
}

mustBeAbsent() {
    $GREP $@ tmplog && die "$@ should not be there !!"
    $ECHO "$@ correctly not present"  # for some reason, this $ECHO must exist, otherwise mustBeAbsent() always fails (??)
}

# default compilation : all features enabled - no zbuff
$ECHO "testing default library compilation"
CFLAGS= make -C $DIR/../lib libzstd libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM tmplog

# Check that the exec-stack bit isn't set
readelf -lW $DIR/../lib/libzstd.so | $GREP "GNU_STACK" > tmplog
mustBeAbsent "RWE"
$RM $DIR/../lib/libzstd.a $DIR/../lib/libzstd.so* tmplog

# compression disabled => also disable zdict
$ECHO "testing with compression disabled"
ZSTD_LIB_COMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
mustBeAbsent "zstd_compress.o"
isPresent "zstd_decompress.o"
mustBeAbsent "zdict.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog

# decompression disabled => also disable legacy
$ECHO "testing with decompression disabled"
ZSTD_LIB_DECOMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
mustBeAbsent "zstd_decompress.o"
isPresent "zdict.o"
mustBeAbsent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog

# deprecated function disabled => only remove zbuff
$ECHO "testing with deprecated functions disabled"
ZSTD_LIB_DEPRECATED=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog

# deprecated function enabled => zbuff present
$ECHO "testing with deprecated functions enabled"
ZSTD_LIB_DEPRECATED=1 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
isPresent "zdict.o"
isPresent "zstd_v07.o"
isPresent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog

# dictionary builder disabled => only remove zdict
$ECHO "testing with dictionary builder disabled"
ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
isPresent "zstd_decompress.o"
mustBeAbsent "zdict.o"
isPresent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog

# both decompression and dictionary builder disabled => only compression remains
$ECHO "testing with both decompression and dictionary builder disabled (only compression remains)"
ZSTD_LIB_DECOMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID
nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog
isPresent "zstd_compress.o"
mustBeAbsent "zstd_decompress.o"
mustBeAbsent "zdict.o"
mustBeAbsent "zstd_v07.o"
mustBeAbsent "zbuff_compress.o"
$RM $DIR/../lib/libzstd.a tmplog


#!/bin/sh

set -e

unset ZSTD_CLEVEL
unset ZSTD_NBTHREADS

die() {
    println "$@" 1>&2
    exit 1
}

datagen() {
    "$DATAGEN_BIN" "$@"
}

zstd() {
    if [ -z "$EXEC_PREFIX" ]; then
        "$ZSTD_BIN" "$@"
    else
        "$EXEC_PREFIX" "$ZSTD_BIN" "$@"
    fi
}

sudoZstd() {
    if [ -z "$EXEC_PREFIX" ]; then
        sudo "$ZSTD_BIN" "$@"
    else
        sudo "$EXEC_PREFIX" "$ZSTD_BIN" "$@"
    fi
}

roundTripTest() {
    if [ -n "$3" ]; then
        cLevel="$3"
        proba="$2"
    else
        cLevel="$2"
        proba=""
    fi
    if [ -n "$4" ]; then
        dLevel="$4"
    else
        dLevel="$cLevel"
    fi

    rm -f tmp1 tmp2
    println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel"
    datagen $1 $proba | $MD5SUM > tmp1
    datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel  | $MD5SUM > tmp2
    $DIFF -q tmp1 tmp2
}

fileRoundTripTest() {
    if [ -n "$3" ]; then
        local_c="$3"
        local_p="$2"
    else
        local_c="$2"
        local_p=""
    fi
    if [ -n "$4" ]; then
        local_d="$4"
    else
        local_d="$local_c"
    fi

    rm -f tmp.zst tmp.md5.1 tmp.md5.2
    println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d"
    datagen $1 $local_p > tmp
    < tmp $MD5SUM > tmp.md5.1
    zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2
    $DIFF -q tmp.md5.1 tmp.md5.2
}

truncateLastByte() {
    dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1"
}

println() {
    printf '%b\n' "${*}"
}

if [ -z "${size}" ]; then
    size=
else
    size=${size}
fi

SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
PRGDIR="$SCRIPT_DIR/../programs"
TESTDIR="$SCRIPT_DIR/../tests"
UNAME=$(uname)

detectedTerminal=false
if [ -t 0 ] && [ -t 1 ]
then
    detectedTerminal=true
fi
isTerminal=${isTerminal:-$detectedTerminal}

isWindows=false
INTOVOID="/dev/null"
case "$UNAME" in
  GNU) DEVDEVICE="/dev/random" ;;
  *) DEVDEVICE="/dev/zero" ;;
esac
case "$OS" in
  Windows*)
    isWindows=true
    INTOVOID="NUL"
    DEVDEVICE="NUL"
    ;;
esac

case "$UNAME" in
  Darwin) MD5SUM="md5 -r" ;;
  FreeBSD) MD5SUM="gmd5sum" ;;
  NetBSD) MD5SUM="md5 -n" ;;
  OpenBSD) MD5SUM="md5" ;;
  *) MD5SUM="md5sum" ;;
esac

MTIME="stat -c %Y"
case "$UNAME" in
    Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
esac

assertSameMTime() {
    MT1=$($MTIME "$1")
    MT2=$($MTIME "$2")
    echo MTIME $MT1 $MT2
    [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)"
}

GET_PERMS="stat -c %a"
case "$UNAME" in
    Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;;
esac

assertFilePermissions() {
    STAT1=$($GET_PERMS "$1")
    STAT2=$2
    [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)"
}

assertSamePermissions() {
    STAT1=$($GET_PERMS "$1")
    STAT2=$($GET_PERMS "$2")
    [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)"
}

DIFF="diff"
case "$UNAME" in
  SunOS) DIFF="gdiff" ;;
esac

# check if ZSTD_BIN is defined. if not, use the default value
if [ -z "${ZSTD_BIN}" ]; then
  println "\nZSTD_BIN is not set. Using the default value..."
  ZSTD_BIN="$PRGDIR/zstd"
fi

# check if DATAGEN_BIN is defined. if not, use the default value
if [ -z "${DATAGEN_BIN}" ]; then
  println "\nDATAGEN_BIN is not set. Using the default value..."
  DATAGEN_BIN="$TESTDIR/datagen"
fi

# Why was this line here ? Generates a strange ZSTD_BIN when EXE_PREFIX is non empty
# ZSTD_BIN="$EXE_PREFIX$ZSTD_BIN"

# assertions
[ -n "$ZSTD_BIN" ] || die "zstd not found at $ZSTD_BIN! \n Please define ZSTD_BIN pointing to the zstd binary. You might also consider rebuilding zstd following the instructions in README.md"
[ -n "$DATAGEN_BIN" ] || die "datagen not found at $DATAGEN_BIN! \n Please define DATAGEN_BIN pointing to the datagen binary. You might also consider rebuilding zstd tests following the instructions in README.md. "
println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'"

if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled'
then
    hasMT=""
else
    hasMT="true"
fi

zstd -vvV

println "\n===>  simple tests "

datagen > tmp
zstd -h
zstd -H
zstd -V
println "test : basic compression "
zstd -f tmp                      # trivial compression case, creates tmp.zst
zstd -f -z tmp
zstd -f -k tmp
zstd -f -C tmp
println "test : basic decompression"
zstd -df tmp.zst                 # trivial decompression case (overwrites tmp)
println "test : too large compression level => auto-fix"
zstd -99 -f tmp  # too large compression level, automatic sized down
zstd -5000000000 -f tmp && die "too large numeric value : must fail"
println "test : --fast aka negative compression levels"
zstd --fast -f tmp  # == -1
zstd --fast=3 -f tmp  # == -3
zstd --fast=200000 -f tmp  # too low compression level, automatic fixed
zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail"
zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0"
println "test : too large numeric argument"
zstd --fast=9999999999 -f tmp  && die "should have refused numeric value"
println "test : set compression level with environment variable ZSTD_CLEVEL"

ZSTD_CLEVEL=12  zstd -f tmp # positive compression level
ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level
ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign
ZSTD_CLEVEL=''  zstd -f tmp # empty env var, warn and revert to default setting
ZSTD_CLEVEL=-   zstd -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=a   zstd -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=+a  zstd -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting
ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting
println "test : override ZSTD_CLEVEL with command line option"
ZSTD_CLEVEL=12  zstd --fast=3 -f tmp # overridden by command line option

# temporary envvar changes in the above tests would actually persist in macos /bin/sh
unset ZSTD_CLEVEL

println "test : compress to stdout"
zstd tmp -c > tmpCompressed
zstd tmp --stdout > tmpCompressed       # long command format
println "test : compress to named file"
rm -f tmpCompressed
zstd tmp -o tmpCompressed
test -f tmpCompressed   # file must be created
println "test : force write, correct order"
zstd tmp -fo tmpCompressed
println "test : forgotten argument"
cp tmp tmp2
zstd tmp2 -fo && die "-o must be followed by filename "
println "test : implied stdout when input is stdin"
println bob | zstd | zstd -d
if [ "$isTerminal" = true ]; then
println "test : compressed data to terminal"
println bob | zstd && die "should have refused : compressed data to terminal"
println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)"
zstd -d > $INTOVOID && die "should have refused : compressed data from terminal"
fi
println "test : null-length file roundtrip"
println -n '' | zstd - --stdout | zstd -d --stdout
println "test : ensure small file doesn't add 3-bytes null block"
datagen -g1 > tmp1
zstd tmp1 -c | wc -c | grep "14"
zstd < tmp1  | wc -c | grep "14"
println "test : decompress file with wrong suffix (must fail)"
zstd -d tmpCompressed && die "wrong suffix error not detected!"
zstd -df tmp && die "should have refused : wrong extension"
println "test : decompress into stdout"
zstd -d tmpCompressed -c > tmpResult    # decompression using stdout
zstd --decompress tmpCompressed -c > tmpResult
zstd --decompress tmpCompressed --stdout > tmpResult
println "test : decompress from stdin into stdout"
zstd -dc   < tmp.zst > $INTOVOID   # combine decompression, stdin & stdout
zstd -dc - < tmp.zst > $INTOVOID
zstd -d    < tmp.zst > $INTOVOID   # implicit stdout when stdin is used
zstd -d  - < tmp.zst > $INTOVOID
println "test : impose memory limitation (must fail)"
datagen -g500K > tmplimit
zstd -f tmplimit
zstd -d -f tmplimit.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed"
zstd -d -f tmplimit.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed"  # long command
zstd -d -f tmplimit.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed"  # long command
zstd -d -f tmplimit.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed"  # long command
rm -f tmplimit tmplimit.zst
println "test : overwrite protection"
zstd -q tmp && die "overwrite check failed!"
println "test : force overwrite"
zstd -q -f tmp
zstd -q --force tmp
println "test : overwrite readonly file"
rm -f tmpro tmpro.zst
println foo > tmpro.zst
println foo > tmpro
chmod 400 tmpro.zst
zstd -q tmpro && die "should have refused to overwrite read-only file"
zstd -q -f tmpro
println "test: --no-progress flag"
zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress
zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress
println "test: --progress flag"
zstd tmpro -c | zstd -d -f -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes"
zstd tmpro -c | zstd -d -f -q -o "$INTOVOID" --progress 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes"
zstd tmpro -c | zstd -d -f -v -o "$INTOVOID" 2>&1 | grep -E "[A-Za-z0-9._ ]+: [0-9]+ bytes"
rm -f tmpro tmpro.zst
println "test: overwrite input file (must fail)"
zstd tmp -fo tmp && die "zstd compression overwrote the input file"
zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file"
println "test: detect that input file does not exist"
zstd nothere && die "zstd hasn't detected that input file does not exist"
println "test: --[no-]compress-literals"
zstd tmp -c --no-compress-literals -1       | zstd -t
zstd tmp -c --no-compress-literals --fast=1 | zstd -t
zstd tmp -c --no-compress-literals -19      | zstd -t
zstd tmp -c --compress-literals    -1       | zstd -t
zstd tmp -c --compress-literals    --fast=1 | zstd -t
zstd tmp -c --compress-literals    -19      | zstd -t
zstd -b --fast=1 -i0e1 tmp --compress-literals
zstd -b --fast=1 -i0e1 tmp --no-compress-literals
println "test: --no-check for decompression"
zstd -f tmp -o tmp_corrupt.zst --check
zstd -f tmp -o tmp.zst --no-check
printf '\xDE\xAD\xBE\xEF' | dd of=tmp_corrupt.zst bs=1 seek=$(($(wc -c < "tmp_corrupt.zst") - 4)) count=4 conv=notrunc # corrupt checksum in tmp
zstd -d -f tmp_corrupt.zst --no-check
zstd -d -f tmp_corrupt.zst --check --no-check # final flag overrides
zstd -d -f tmp.zst --no-check

if [ "$isWindows" = false ] && [ "$UNAME" != "AIX" ]; then
  if [ -n "$(which readelf)" ]; then
    println "test: check if binary has executable stack (#2963)"
    readelf -lW "$ZSTD_BIN" | grep 'GNU_STACK .* RW ' || die "zstd binary has executable stack!"
  fi
fi

println "\n===>  --exclude-compressed flag"
rm -rf precompressedFilterTestDir
mkdir -p precompressedFilterTestDir
datagen $size > precompressedFilterTestDir/input.5
datagen $size > precompressedFilterTestDir/input.6
zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
datagen $size > precompressedFilterTestDir/input.7
datagen $size > precompressedFilterTestDir/input.8
zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
test ! -f precompressedFilterTestDir/input.5.zst.zst
test ! -f precompressedFilterTestDir/input.6.zst.zst
file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst`
file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst`
if [ $file2timestamp -ge $file1timestamp ]; then
  println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again."
else
  println "Test is not successful"
fi
# File Extension check.
datagen $size > precompressedFilterTestDir/input.zstbar
zstd --exclude-compressed --long --rm -r precompressedFilterTestDir
# zstd should compress input.zstbar
test -f precompressedFilterTestDir/input.zstbar.zst
# Check without the --exclude-compressed flag
zstd --long --rm -r precompressedFilterTestDir
# Files should get compressed again without the --exclude-compressed flag.
test -f precompressedFilterTestDir/input.5.zst.zst
test -f precompressedFilterTestDir/input.6.zst.zst
rm -rf precompressedFilterTestDir
println "Test completed"

println "\n===>  warning prompts should not occur if stdin is an input"
println "y" > tmpPrompt
println "hello world" >> tmpPrompt
zstd tmpPrompt -f
zstd < tmpPrompt -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite"
zstd < tmpPrompt -o tmpPrompt.zst -f    # should successfully overwrite with -f
zstd -q -d -f tmpPrompt.zst -o tmpPromptRegenerated
$DIFF tmpPromptRegenerated tmpPrompt    # the first 'y' character should not be swallowed

echo 'yes' | zstd tmpPrompt -v -o tmpPrompt.zst  # accept piped "y" input to force overwrite when using files
echo 'yes' | zstd < tmpPrompt -v -o tmpPrompt.zst && die "should have aborted immediately and failed to overwrite"
zstd tmpPrompt - < tmpPrompt -o tmpPromp.zst --rm && die "should have aborted immediately and failed to remove"

println "Test completed"

println "\n===>  recursive mode test "
# combination of -r with empty list of input file
zstd -c -r < tmp > tmp.zst

# combination of -r with empty folder
mkdir -p tmpEmptyDir
zstd -r tmpEmptyDir
rm -rf tmpEmptyDir

println "\n===>  file removal"
zstd -f --rm tmp
test ! -f tmp  # tmp should no longer be present
zstd -f -d --rm tmp.zst
test ! -f tmp.zst  # tmp.zst should no longer be present
println "test: --rm is disabled when output is stdout"
test -f tmp
zstd --rm tmp -c > $INTOVOID
test -f tmp # tmp shall still be there
zstd -f --rm tmp -c > $INTOVOID
test -f tmp # tmp shall still be there
zstd -f tmp -c > $INTOVOID --rm
test -f tmp # tmp shall still be there
println "test: --rm is disabled when multiple inputs are concatenated into a single output"
cp tmp tmp2
zstd --rm tmp tmp2 -c > $INTOVOID
test -f tmp
test -f tmp2
rm -f tmp3.zst
echo 'y' | zstd -v tmp tmp2 -o tmp3.zst --rm # prompt for confirmation
test -f tmp
test -f tmp2
zstd -f tmp tmp2 -o tmp3.zst --rm # just warns, no prompt
test -f tmp
test -f tmp2
zstd -q tmp tmp2 -o tmp3.zst --rm && die "should refuse to concatenate"

println "test : should quietly not remove non-regular file"
println hello > tmp
zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID"
grep "Refusing to remove non-regular file" tmplog && die
rm -f tmplog
zstd tmp -f -o "$INTOVOID" 2>&1 | grep "Refusing to remove non-regular file" && die
println "test : --rm on stdin"
println a | zstd --rm > $INTOVOID   # --rm should remain silent
rm -f tmp
zstd -f tmp && die "tmp not present : should have failed"
test ! -f tmp.zst  # tmp.zst should not be created
println "test : -d -f do not delete destination when source is not present"
touch tmp    # create destination file
zstd -d -f tmp.zst && die "attempt to decompress a non existing file"
test -f tmp  # destination file should still be present
println "test : -f do not delete destination when source is not present"
rm -f tmp         # erase source file
touch tmp.zst  # create destination file
zstd -f tmp && die "attempt to compress a non existing file"
test -f tmp.zst  # destination file should still be present
rm -rf tmp*  # may also erase tmp* directory from previous failed run

println "\n===>  decompression only tests "
# the following test verifies that the decoder is compatible with RLE as first block
# older versions of zstd cli are not able to decode such corner case.
# As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions.
dd bs=1048576 count=1 if=/dev/zero of=tmp
zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst"
$DIFF -s tmp1 tmp

touch tmp_empty
zstd -d -o tmp2 "$TESTDIR/golden-decompression/empty-block.zst"
$DIFF -s tmp2 tmp_empty

zstd -t "$TESTDIR/golden-decompression/zeroSeq_2B.zst"

zstd -t "$TESTDIR/golden-decompression-errors/zeroSeq_extraneous.zst" && die "invalid Sequences section should have been detected"

rm -f tmp*

println "\n===>  compress multiple files"
println hello > tmp1
println world > tmp2
zstd tmp1 tmp2 -o "$INTOVOID" -f
zstd tmp1 tmp2 -c | zstd -t
echo 'y' | zstd -v tmp1 tmp2 -o tmp.zst
test ! -f tmp1.zst
test ! -f tmp2.zst
zstd tmp1 tmp2
zstd -t tmp1.zst tmp2.zst
zstd -dc tmp1.zst tmp2.zst
zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f
echo 'y' | zstd -v -d tmp1.zst tmp2.zst -o tmp
touch tmpexists
zstd tmp1 tmp2 -f -o tmpexists
zstd tmp1 tmp2 -q -o tmpexists && die "should have refused to overwrite"
println gooder > tmp_rm1
println boi > tmp_rm2
println worldly > tmp_rm3
echo 'y' | zstd -v tmp_rm1 tmp_rm2 -v -o tmp_rm3.zst
test -f tmp_rm1
test -f tmp_rm2
cp tmp_rm3.zst tmp_rm4.zst
echo 'Y' | zstd -v -d tmp_rm3.zst tmp_rm4.zst -v -o tmp_rm_out --rm
test -f tmp_rm3.zst
test -f tmp_rm4.zst
println gooder > tmpexists1
zstd tmpexists1 tmpexists -c --rm -f > $INTOVOID
# Bug: PR #972
if [ "$?" -eq 139 ]; then
  die "should not have segfaulted"
fi
test -f tmpexists1
test -f tmpexists
println "\n===>  multiple files and shell completion "
datagen -s1        > tmp1 2> $INTOVOID
datagen -s2 -g100K > tmp2 2> $INTOVOID
datagen -s3 -g1M   > tmp3 2> $INTOVOID
println "compress tmp* : "
zstd -f tmp*
test -f tmp1.zst
test -f tmp2.zst
test -f tmp3.zst
rm -f tmp1 tmp2 tmp3
println "decompress tmp* : "
zstd -df ./*.zst
test -f tmp1
test -f tmp2
test -f tmp3
println "compress tmp* into stdout > tmpall : "
zstd -c tmp1 tmp2 tmp3 > tmpall
test -f tmpall  # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst)
println "decompress tmpall* into stdout > tmpdec : "
cp tmpall tmpall2
zstd -dc tmpall* > tmpdec
test -f tmpdec  # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3))
println "compress multiple files including a missing one (notHere) : "
zstd -f tmp1 notHere tmp2 && die "missing file not detected!"
rm -f tmp*

if [ "$isWindows" = false ] ; then
    println "\n===>  zstd fifo named pipe test "
    echo "Hello World!" > tmp_original
    mkfifo tmp_named_pipe
    # note : fifo test doesn't work in combination with `dd` or `cat`
    echo "Hello World!" > tmp_named_pipe &
    zstd tmp_named_pipe -o tmp_compressed
    zstd -d -o tmp_decompressed tmp_compressed
    $DIFF -s tmp_original tmp_decompressed
    rm -rf tmp*
fi

println "\n===>  zstd created file permissions tests"
if [ "$isWindows" = false ] ; then
    rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out # todo: remove

    ORIGINAL_UMASK=$(umask)
    umask 0000

    datagen > tmp1
    datagen > tmp2
    assertFilePermissions tmp1 666
    assertFilePermissions tmp2 666

    println "test : copy 666 permissions in file -> file compression "
    zstd -f tmp1 -o tmp1.zst
    assertSamePermissions tmp1 tmp1.zst
    println "test : copy 666 permissions in file -> file decompression "
    zstd -f -d tmp1.zst -o tmp1.out
    assertSamePermissions tmp1.zst tmp1.out

    rm -f tmp1.zst tmp1.out

    println "test : copy 400 permissions in file -> file compression (write to a read-only file) "
    chmod 0400 tmp1
    assertFilePermissions tmp1 400
    zstd -f tmp1 -o tmp1.zst
    assertSamePermissions tmp1 tmp1.zst
    println "test : copy 400 permissions in file -> file decompression (write to a read-only file) "
    zstd -f -d tmp1.zst -o tmp1
    assertSamePermissions tmp1.zst tmp1

    rm -f tmp1.zst tmp1.out

    println "test : check created permissions from stdin input in compression "
    zstd -f -o tmp1.zst < tmp1
    assertFilePermissions tmp1.zst 666
    println "test : check created permissions from stdin input in decompression "
    zstd -f -d -o tmp1.out < tmp1.zst
    assertFilePermissions tmp1.out 666

    rm -f tmp1.zst tmp1.out

    println "test : check created permissions from multiple inputs in compression "
    zstd -f tmp1 tmp2 -o tmp1.zst
    assertFilePermissions tmp1.zst 666
    println "test : check created permissions from multiple inputs in decompression "
    cp tmp1.zst tmp2.zst
    zstd -f -d tmp1.zst tmp2.zst -o tmp1.out
    assertFilePermissions tmp1.out 666

    rm -f tmp1.zst tmp2.zst tmp1.out tmp2.out

    println "test : check permissions on pre-existing output file in compression "
    chmod 0600 tmp1
    touch tmp1.zst
    chmod 0400 tmp1.zst
    zstd -f tmp1 -o tmp1.zst
    assertFilePermissions tmp1.zst 600
    println "test : check permissions on pre-existing output file in decompression "
    chmod 0400 tmp1.zst
    touch tmp1.out
    chmod 0200 tmp1.out
    zstd -f -d tmp1.zst -o tmp1.out
    assertFilePermissions tmp1.out 400

    umask 0666
    chmod 0666 tmp1 tmp2

    rm -f tmp1.zst tmp1.out

    println "test : respect umask when compressing from stdin input "
    zstd -f -o tmp1.zst < tmp1
    assertFilePermissions tmp1.zst 0
    println "test : respect umask when decompressing from stdin input "
    chmod 0666 tmp1.zst
    zstd -f -d -o tmp1.out < tmp1.zst
    assertFilePermissions tmp1.out 0

    rm -f tmp1 tmp2 tmp1.zst tmp2.zst tmp1.out tmp2.out
    umask $ORIGINAL_UMASK
fi

if [ -n "$DEVNULLRIGHTS" ] ; then
    # these tests requires sudo rights, which is uncommon.
    # they are only triggered if DEVNULLRIGHTS macro is defined.
    println "\n===> checking /dev/null permissions are unaltered "
    datagen > tmp
    sudoZstd tmp -o $INTOVOID   # sudo rights could modify /dev/null permissions
    sudoZstd tmp -c > $INTOVOID
    zstd tmp -f -o tmp.zst
    sudoZstd -d tmp.zst -c > $INTOVOID
    sudoZstd -d tmp.zst -o $INTOVOID
    ls -las $INTOVOID | grep "rw-rw-rw-"
fi

if [ -n "$READFROMBLOCKDEVICE" ] ; then
    # This creates a temporary block device, which is only possible on unix-y
    # systems, is somewhat invasive, and requires sudo. For these reasons, you
    # have to specifically ask for this test.
    println "\n===> checking that zstd can read from a block device"
    datagen -g65536 > tmp.img
    sudo losetup -fP tmp.img
    LOOP_DEV=$(losetup -a | grep 'tmp\.img' | cut -f1 -d:)
    [ -z "$LOOP_DEV" ] && die "failed to get loopback device"
    sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f"
    sudoZstd -f $LOOP_DEV -c > tmp.img.zst
    zstd -d tmp.img.zst -o tmp.img.copy
    sudo losetup -d $LOOP_DEV
    $DIFF -s tmp.img tmp.img.copy || die "round trip failed"
    rm -f tmp.img tmp.img.zst tmp.img.copy
fi

println "\n===>  zstd created file timestamp tests"
datagen > tmp
touch -m -t 200001010000.00 tmp
println "test : copy mtime in file -> file compression "
zstd -f tmp -o tmp.zst
assertSameMTime tmp tmp.zst
println "test : copy mtime in file -> file decompression "
zstd -f -d tmp.zst -o tmp.out
assertSameMTime tmp.zst tmp.out
rm -f tmp

println "\n===>  compress multiple files into an output directory, --output-dir-flat"
println henlo > tmp1
mkdir tmpInputTestDir
mkdir tmpInputTestDir/we
mkdir tmpInputTestDir/we/must
mkdir tmpInputTestDir/we/must/go
mkdir tmpInputTestDir/we/must/go/deeper
println cool > tmpInputTestDir/we/must/go/deeper/tmp2
mkdir tmpOutDir
zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir
test -f tmpOutDir/tmp1.zst
test -f tmpOutDir/tmp2.zst
println "test : decompress multiple files into an output directory, --output-dir-flat"
mkdir tmpOutDirDecomp
zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1
rm -f tmpOutDirDecomp/*
zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp
test -f tmpOutDirDecomp/tmp2
test -f tmpOutDirDecomp/tmp1
rm -rf tmp*

if [ "$isWindows" = false ] ; then
    println "\n===>  compress multiple files into an output directory and mirror input folder, --output-dir-mirror"
    println "test --output-dir-mirror" > tmp1
    mkdir -p tmpInputTestDir/we/.../..must/go/deeper..
    println cool > tmpInputTestDir/we/.../..must/go/deeper../tmp2
    zstd tmp1 -r tmpInputTestDir --output-dir-mirror tmpOutDir
    test -f tmpOutDir/tmp1.zst
    test -f tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2.zst

    println "test: compress input dir will be ignored if it has '..'"
    zstd  -r tmpInputTestDir/we/.../..must/../..mustgo/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'"
    zstd  -r tmpInputTestDir/we/.../..must/deeper../.. --output-dir-mirror non-exist && die "input cannot contain '..'"
    zstd  -r ../tests/tmpInputTestDir/we/.../..must/deeper.. --output-dir-mirror non-exist && die "input cannot contain '..'"
    test ! -d non-exist

    println "test: compress input dir should succeed with benign uses of '..'"
    zstd  -r tmpInputTestDir/we/.../..must/go/deeper.. --output-dir-mirror tmpout
    test -d tmpout

    println "test : decompress multiple files into an output directory, --output-dir-mirror"
    zstd tmpOutDir -r -d --output-dir-mirror tmpOutDirDecomp
    test -f tmpOutDirDecomp/tmpOutDir/tmp1
    test -f tmpOutDirDecomp/tmpOutDir/tmpInputTestDir/we/.../..must/go/deeper../tmp2

    println "test: decompress input dir will be ignored if it has '..'"
    zstd  -r tmpOutDir/tmpInputTestDir/we/.../..must/../..must --output-dir-mirror non-exist && die "input cannot contain '..'"
    test ! -d non-exist

    rm -rf tmp*
fi

println "test : compress multiple files reading them from a file, --filelist=FILE"
println "Hello world!, file1" > tmp1
println "Hello world!, file2" > tmp2
println tmp1 > tmp_fileList
println tmp2 >> tmp_fileList
zstd -f --filelist=tmp_fileList
test -f tmp2.zst
test -f tmp1.zst

println "test : alternate syntax: --filelist FILE"
zstd -f --filelist tmp_fileList
test -f tmp2.zst
test -f tmp1.zst

println "test : reading file list from a symlink, --filelist=FILE"
rm -f *.zst
ln -s tmp_fileList tmp_symLink
zstd -f --filelist=tmp_symLink
test -f tmp2.zst
test -f tmp1.zst

println "test : compress multiple files reading them from multiple files, --filelist=FILE"
rm -f *.zst
println "Hello world!, file3" > tmp3
println "Hello world!, file4" > tmp4
println tmp3 > tmp_fileList2
println tmp4 >> tmp_fileList2
zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2
test -f tmp1.zst
test -f tmp2.zst
test -f tmp3.zst
test -f tmp4.zst

println "test : decompress multiple files reading them from a file, --filelist=FILE"
rm -f tmp1 tmp2
println tmp1.zst > tmpZst
println tmp2.zst >> tmpZst
zstd -d -f --filelist=tmpZst
test -f tmp1
test -f tmp2

println "test : decompress multiple files reading them from multiple files, --filelist=FILE"
rm -f tmp1 tmp2 tmp3 tmp4
println tmp3.zst > tmpZst2
println tmp4.zst >> tmpZst2
zstd -d -f --filelist=tmpZst --filelist=tmpZst2
test -f tmp1
test -f tmp2
test -f tmp3
test -f tmp4

println "test : survive the list of files with too long filenames (--filelist=FILE)"
datagen -g5M > tmp_badList
zstd -qq -f --filelist=tmp_badList && die "should have failed : file name length is too long"  # printing very long text garbage on console will cause CI failure

println "test : survive a list of files which is text garbage (--filelist=FILE)"
datagen > tmp_badList
zstd -qq -f --filelist=tmp_badList && die "should have failed : list is text garbage"  # printing very long text garbage on console will cause CI failure

println "test : survive a list of files which is binary garbage (--filelist=FILE)"
datagen -P0 -g1M > tmp_badList
zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage"  # let's avoid printing binary garbage on console

println "test : try to overflow internal list of files (--filelist=FILE)"
touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6
ls tmp* > tmpList
zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3  # can trigger an overflow of internal file list
rm -rf tmp*

println "\n===> --[no-]content-size tests"

datagen > tmp_contentsize
zstd -f tmp_contentsize
zstd -lv tmp_contentsize.zst | grep "Decompressed Size:"
zstd -f --no-content-size tmp_contentsize
zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die
zstd -f --content-size tmp_contentsize
zstd -lv tmp_contentsize.zst | grep "Decompressed Size:"
zstd -f --content-size --no-content-size tmp_contentsize
zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die
rm -rf tmp*

println "test : show-default-cparams regular"
datagen > tmp
zstd --show-default-cparams -f tmp
zstd --show-default-cparams -d tmp.zst && die "error: can't use --show-default-cparams in decompression mode"
rm -rf tmp*

println "test : show-default-cparams recursive"
mkdir tmp_files
datagen -g15000 > tmp_files/tmp1
datagen -g129000 > tmp_files/tmp2
datagen -g257000 > tmp_files/tmp3
zstd --show-default-cparams -f -r tmp_files
rm -rf tmp*

println "test : show compression parameters in verbose mode"
datagen > tmp
zstd -vv tmp 2>&1 | \
grep -q -E -- "--zstd=wlog=[[:digit:]]+,clog=[[:digit:]]+,hlog=[[:digit:]]+,\
slog=[[:digit:]]+,mml=[[:digit:]]+,tlen=[[:digit:]]+,strat=[[:digit:]]+"
rm -rf tmp*

println "\n===>  Advanced compression parameters "
println "Hello world!" | zstd --zstd=windowLog=21,      - -o tmp.zst && die "wrong parameters not detected!"
println "Hello world!" | zstd --zstd=windowLo=21        - -o tmp.zst && die "wrong parameters not detected!"
println "Hello world!" | zstd --zstd=windowLog=21,slog  - -o tmp.zst && die "wrong parameters not detected!"
println "Hello world!" | zstd --zstd=strategy=10        - -o tmp.zst && die "parameter out of bound not detected!"  # > btultra2 : does not exist
test ! -f tmp.zst  # tmp.zst should not be created
roundTripTest -g512K
roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6"
roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6"
roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6"
roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7"
roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7"
roundTripTest -g64K  "19 --zstd=strat=9"   # btultra2

println "\n===>  Pass-Through mode "
println "Hello world 1!" | zstd -df
println "Hello world 2!" | zstd -dcf
println "Hello world 3!" > tmp1
zstd -dcf tmp1
println "" | zstd -df > tmp1
println "" > tmp2
$DIFF -q tmp1 tmp2
println "1" | zstd -df > tmp1
println "1" > tmp2
$DIFF -q tmp1 tmp2
println "12" | zstd -df > tmp1
println "12" > tmp2
$DIFF -q tmp1 tmp2
rm -rf tmp*

println "\n===>  frame concatenation "
println "hello " > hello.tmp
println "world!" > world.tmp
cat hello.tmp world.tmp > helloworld.tmp
zstd -c hello.tmp > hello.zst
zstd -c world.tmp > world.zst
zstd -c hello.tmp world.tmp > helloworld.zst
zstd -dc helloworld.zst > result.tmp
$DIFF helloworld.tmp result.tmp
cat hello.zst world.zst > helloworld.zst
zstd -dc helloworld.zst > result.tmp
cat result.tmp
$DIFF helloworld.tmp result.tmp
println "frame concatenation without checksum"
zstd -c hello.tmp > hello.zst --no-check
zstd -c world.tmp > world.zst --no-check
cat hello.zst world.zst > helloworld.zstd
zstd -dc helloworld.zst > result.tmp
$DIFF helloworld.tmp result.tmp
println "testing zstdcat symlink"
ln -sf "$ZSTD_BIN" zstdcat
$EXE_PREFIX ./zstdcat helloworld.zst > result.tmp
$DIFF helloworld.tmp result.tmp
ln -s helloworld.zst helloworld.link.zst
$EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp
$DIFF helloworld.tmp result.tmp
rm -f zstdcat
rm -f result.tmp
println "testing zcat symlink"
ln -sf "$ZSTD_BIN" zcat
$EXE_PREFIX ./zcat helloworld.zst > result.tmp
$DIFF helloworld.tmp result.tmp
$EXE_PREFIX ./zcat helloworld.link.zst > result.tmp
$DIFF helloworld.tmp result.tmp
rm -f zcat
rm -f ./*.tmp ./*.zstd
println "frame concatenation tests completed"

if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] && [ "$UNAME" != "AIX" ]; then
println "\n**** flush write error test **** "

println "println foo | zstd > /dev/full"
println foo | zstd > /dev/full && die "write error not detected!"
println "println foo | zstd | zstd -d > /dev/full"
println foo | zstd | zstd -d > /dev/full && die "write error not detected!"

fi

if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then

println "\n===>  symbolic link test "

rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst
println "hello world" > hello.tmp
ln -s hello.tmp world.tmp
ln -s hello.tmp world2.tmp
zstd world.tmp hello.tmp || true
test -f hello.tmp.zst  # regular file should have been compressed!
test ! -f world.tmp.zst  # symbolic link should not have been compressed!
zstd world.tmp || true
test ! -f world.tmp.zst  # symbolic link should not have been compressed!
zstd world.tmp world2.tmp || true
test ! -f world.tmp.zst  # symbolic link should not have been compressed!
test ! -f world2.tmp.zst  # symbolic link should not have been compressed!
zstd world.tmp hello.tmp -f
test -f world.tmp.zst  # symbolic link should have been compressed with --force
rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst

fi

println "\n===>  test sparse file support "

datagen -g5M  -P100 > tmpSparse
zstd tmpSparse -c | zstd -dv -o tmpSparseRegen
$DIFF -s tmpSparse tmpSparseRegen
zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse
$DIFF -s tmpSparse tmpOutSparse
zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse
$DIFF -s tmpSparse tmpOutNoSparse
ls -ls tmpSparse*  # look at file size and block size on disk
datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd   # Odd size file (to not finish on an exact nb of blocks)
datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd
ls -ls tmpSparseOdd  # look at file size and block size on disk
println "\n Sparse Compatibility with Console :"
println "Hello World 1 !" | zstd | zstd -d -c
println "Hello World 2 !" | zstd | zstd -d | cat
println "\n Sparse Compatibility with Append :"
datagen -P100 -g1M > tmpSparse1M
cat tmpSparse1M tmpSparse1M > tmpSparse2M
zstd -v -f tmpSparse1M -o tmpSparseCompressed
zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated
zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated
ls -ls tmpSparse*  # look at file size and block size on disk
$DIFF tmpSparse2M tmpSparseRegenerated
rm -f tmpSparse*

println "\n===>  stream-size mode"

datagen -g11000 > tmp
println "test : basic file compression vs sized streaming compression"
file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst)
stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c)
if [ "$stream_size" -gt "$file_size" ]; then
  die "hinted compression larger than expected"
fi
println "test : sized streaming compression and decompression"
cat tmp | zstd -14 -f tmp -o tmp.zst --stream-size=11000
zstd -df tmp.zst -o tmp_decompress
cmp tmp tmp_decompress || die "difference between original and decompressed file"
println "test : incorrect stream size"
cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size"

println "\n===>  zstd zero weight dict test "
rm -f tmp*
cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input
zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input
zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp
$DIFF tmp_decomp tmp_input
rm -rf tmp*

println "\n===>  zstd (valid) zero weight dict test "
rm -f tmp*
# 0 has a non-zero weight in the dictionary
echo "0000000000000000000000000" > tmp_input
zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input
zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp
$DIFF tmp_decomp tmp_input
rm -rf tmp*

println "\n===>  size-hint mode"

datagen -g11000 > tmp
datagen -g11000 > tmp2
datagen > tmpDict
println "test : basic file compression vs hinted streaming compression"
file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst)
stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c)
if [ "$stream_size" -ge "$file_size" ]; then
  die "hinted compression larger than expected"
fi
println "test : hinted streaming compression and decompression"
cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000
zstd -df tmp.zst -o tmp_decompress
cmp tmp tmp_decompress || die "difference between original and decompressed file"
println "test : hinted streaming compression with dictionary"
cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict
println "test : multiple file compression with hints and dictionary"
zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2
zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp
zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2
cmp tmp.zst tmp1_.zst || die "first file's output differs"
cmp tmp2.zst tmp2_.zst || die "second file's output differs"
println "test : incorrect hinted stream sizes"
cat tmp | zstd -14 -f --size-hint=11050 | zstd -t  # slightly too high
cat tmp | zstd -14 -f --size-hint=10950 | zstd -t  # slightly too low
cat tmp | zstd -14 -f --size-hint=22000 | zstd -t  # considerably too high
cat tmp | zstd -14 -f --size-hint=5500  | zstd -t  # considerably too low
println "test : allows and interprets K,KB,KiB,M,MB and MiB suffix"
cat tmp | zstd -14 -f --size-hint=11K | zstd -t
cat tmp | zstd -14 -f --size-hint=11KB | zstd -t
cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t
cat tmp | zstd -14 -f --size-hint=1M  | zstd -t
cat tmp | zstd -14 -f --size-hint=1MB  | zstd -t
cat tmp | zstd -14 -f --size-hint=1MiB  | zstd -t

println "\n===>  dictionary tests "
println "- Test high/low compressibility corpus training"
datagen -g12M -P90 > tmpCorpusHighCompress
datagen -g12M -P5 > tmpCorpusLowCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress
zstd --train -B2K tmpCorpusLowCompress -o tmpDictLowCompress
rm -f tmpCorpusHighCompress tmpCorpusLowCompress tmpDictHighCompress tmpDictLowCompress
println "- Test with raw dict (content only) "
datagen > tmpDict
datagen -g1M | $MD5SUM > tmp1
datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2
$DIFF -q tmp1 tmp2
println "- Create first dictionary "
TESTFILE="$PRGDIR"/zstdcli.c
zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp
println "- Test dictionary compression with tmpDict as an input file and dictionary"
zstd -f tmpDict -D tmpDict && die "compression error not detected!"
println "- Dictionary compression roundtrip"
zstd -f tmp -D tmpDict
zstd -d tmp.zst -D tmpDict -fo result
$DIFF "$TESTFILE" result
println "- Dictionary compression with hlog < clog"
zstd -6f tmp -D tmpDict --zstd=clog=25,hlog=23
println "- Dictionary compression with btlazy2 strategy"
zstd -f tmp -D tmpDict --zstd=strategy=6
zstd -d tmp.zst -D tmpDict -fo result
$DIFF "$TESTFILE" result
if [ -e /proc/self/fd/0 ]; then
    println "- Test rejecting irregular dictionary file"
    cat tmpDict | zstd -f tmp -D /proc/self/fd/0 && die "Piped dictionary should fail!"
    cat tmpDict | zstd -d tmp.zst -D /proc/self/fd/0 -f && die "Piped dictionary should fail!"
fi
if [ -n "$hasMT" ]
then
    println "- Test dictionary compression with multithreading "
    datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict   # fails with v1.3.2
fi
println "- Create second (different) dictionary "
zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
println "- Create dictionary with short dictID"
zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionary with wrong dictID parameter order (must fail)"
zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument "
println "- Create dictionary with size limit"
zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v
println "- Create dictionary with small size limit"
zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v
println "- Create dictionary with wrong parameter order (must fail)"
zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument "
println "- Compress without dictID"
zstd -f tmp -D tmpDict1 --no-dictID
zstd -d tmp.zst -D tmpDict -fo result
$DIFF "$TESTFILE" result
println "- Compress multiple files with dictionary"
rm -rf dirTestDict
mkdir dirTestDict
cp "$TESTDIR"/*.c dirTestDict
cp "$PRGDIR"/*.c dirTestDict
cp "$PRGDIR"/*.h dirTestDict
$MD5SUM dirTestDict/* > tmph1
zstd -f --rm dirTestDict/* -D tmpDictC
zstd -d --rm dirTestDict/*.zst -D tmpDictC  # note : use internal checksum by default
case "$UNAME" in
  Darwin) println "md5sum -c not supported on OS-X : test skipped" ;;  # not compatible with OS-X's md5
  *) $MD5SUM -c tmph1 ;;
esac
rm -rf dirTestDict
println "- dictionary builder on bogus input"
println "Hello World" > tmp
zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source"
datagen -P0 -g10M > tmp
zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise"
println "- Test -o before --train"
rm -f tmpDict dictionary
zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict
zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary
if [ -n "$hasMT" ]
then
  println "- Create dictionary with multithreading enabled"
  zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
fi
rm -f tmp* dictionary

println "- Test --memory for dictionary compression"
datagen -g12M -P90 > tmpCorpusHighCompress
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=10K && die "Dictionary training should fail : --memory too low (10K)"
zstd --train -B2K tmpCorpusHighCompress -o tmpDictHighCompress --memory=5MB 2> zstTrainWithMemLimitStdErr
cat zstTrainWithMemLimitStdErr | grep "setting manual memory limit for dictionary training data at 5 MB"
cat zstTrainWithMemLimitStdErr | grep "Training samples set too large (12 MB); training on 5 MB only..."
rm zstTrainWithMemLimitStdErr

println "\n===>  fastCover dictionary builder : advanced options "
TESTFILE="$PRGDIR"/zstdcli.c
datagen > tmpDict
println "- Create first dictionary"
zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp
zstd -f tmp -D tmpDict
zstd -d tmp.zst -D tmpDict -fo result
$DIFF "$TESTFILE" result
println "- Create second (different) dictionary"
zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file"
println "- Create dictionary with short dictID"
zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionaries with shrink-dict flag enabled"
zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict
zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1
zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2
zstd --train-fastcover=shrink=5,steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict3
println "- Create dictionary with size limit"
zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Create dictionary using all samples for both training and testing"
zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using f=16"
zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15"
println "- Create dictionary using accel=2"
zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using accel=10"
zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary with multithreading"
zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Test -o before --train-fastcover"
rm -f tmpDict dictionary
zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict
zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary
rm -f tmp* dictionary

println "\n===>  legacy dictionary builder "

TESTFILE="$PRGDIR"/zstdcli.c
datagen > tmpDict
println "- Create first dictionary"
zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp
zstd -f tmp -D tmpDict
zstd -d tmp.zst -D tmpDict -fo result
$DIFF "$TESTFILE" result
zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)"
println "- Create second (different) dictionary"
zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
println "- Create dictionary with short dictID"
zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionary with size limit"
zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Test -o before --train-legacy"
rm -f tmpDict dictionary
zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict
zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary
rm -f tmp* dictionary

println "\n===>  integrity tests "

println "test one file (tmp1.zst) "
datagen > tmp1
zstd tmp1
zstd -t tmp1.zst
zstd --test tmp1.zst
println "test multiple files (*.zst) "
zstd -t ./*.zst
println "test bad files (*) "
zstd -t ./* && die "bad files not detected !"
zstd -t tmp1 && die "bad file not detected !"
cp tmp1 tmp2.zst
zstd -t tmp2.zst && die "bad file not detected !"
datagen -g0 > tmp3
zstd -t tmp3 && die "bad file not detected !"   # detects 0-sized files as bad
println "test --rm and --test combined "
zstd -t --rm tmp1.zst
test -f tmp1.zst   # check file is still present
cp tmp1.zst tmp2.zst
zstd -t tmp1.zst tmp2.zst --rm
test -f tmp1.zst   # check file is still present
test -f tmp2.zst   # check file is still present
split -b16384 tmp1.zst tmpSplit.
zstd -t tmpSplit.* && die "bad file not detected !"
datagen | zstd -c | zstd -t

println "\n===>  golden files tests "

zstd -t -r "$TESTDIR/golden-decompression"
zstd -c -r "$TESTDIR/golden-compression" | zstd -t
zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" "$TESTDIR/golden-compression/http" -c | zstd -D "$TESTDIR/golden-dictionaries/http-dict-missing-symbols" -t

println "\n===>  benchmark mode tests "

println "bench one file"
datagen > tmp1
zstd -bi0 tmp1
println "bench multiple levels"
zstd -i0b0e3 tmp1
println "bench negative level"
zstd -bi0 --fast tmp1
println "with recursive and quiet modes"
zstd -rqi0b1e2 tmp1
println "benchmark decompression only"
zstd -f tmp1
zstd -b -d -i0 tmp1.zst
println "benchmark can fail - decompression on invalid data"
zstd -b -d -i0 tmp1 && die "invalid .zst data => benchmark should have failed"

GZIPMODE=1
zstd --format=gzip -V || GZIPMODE=0
if [ $GZIPMODE -eq 1 ]; then
    println "benchmark mode is only compatible with zstd"
    zstd --format=gzip -b tmp1 && die "-b should be incompatible with gzip format!"
fi

println "\n===>  zstd compatibility tests "

datagen > tmp
rm -f tmp.zst
zstd --format=zstd -f tmp
test -f tmp.zst

println "\n===>  gzip compatibility tests "

GZIPMODE=1
zstd --format=gzip -V || GZIPMODE=0
if [ $GZIPMODE -eq 1 ]; then
    println "gzip support detected"
    GZIPEXE=1
    gzip -V || GZIPEXE=0
    if [ $GZIPEXE -eq 1 ]; then
        datagen > tmp
        zstd --format=gzip -f tmp
        gzip -t -v tmp.gz
        gzip -f tmp
        zstd -d -f -v tmp.gz
        rm -f tmp*
    else
        println "gzip binary not detected"
    fi
else
    println "gzip mode not supported"
fi

println "\n===>  gzip frame tests "

if [ $GZIPMODE -eq 1 ]; then
    datagen > tmp
    zstd -f --format=gzip tmp
    zstd -f tmp
    cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp
    truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !"
    rm -f tmp*
else
    println "gzip mode not supported"
fi

if [ $GZIPMODE -eq 1 ]; then
    datagen > tmp
    rm -f tmp.zst
    zstd --format=gzip --format=zstd -f tmp
    test -f tmp.zst
fi

println "\n===>  xz compatibility tests "

LZMAMODE=1
zstd --format=xz -V || LZMAMODE=0
if [ $LZMAMODE -eq 1 ]; then
    println "xz support detected"
    XZEXE=1
    xz -Q -V && lzma -Q -V || XZEXE=0
    if [ $XZEXE -eq 1 ]; then
        println "Testing zstd xz and lzma support"
        datagen > tmp
        zstd --format=lzma -f tmp
        zstd --format=xz -f tmp
        xz -Q -t -v tmp.xz
        xz -Q -t -v tmp.lzma
        xz -Q -f -k tmp
        lzma -Q -f -k --lzma1 tmp
        zstd -d -f -v tmp.xz
        zstd -d -f -v tmp.lzma
        rm -f tmp*
        println "Creating symlinks"
        ln -s "$ZSTD_BIN" ./xz
        ln -s "$ZSTD_BIN" ./unxz
        ln -s "$ZSTD_BIN" ./lzma
        ln -s "$ZSTD_BIN" ./unlzma
        println "Testing xz and lzma symlinks"
        datagen > tmp
        ./xz tmp
        xz -Q -d tmp.xz
        ./lzma tmp
        lzma -Q -d tmp.lzma
        println "Testing unxz and unlzma symlinks"
        xz -Q tmp
        ./xz -d tmp.xz
        lzma -Q tmp
        ./lzma -d tmp.lzma
        rm -f xz unxz lzma unlzma
        rm -f tmp*
    else
        println "xz binary not detected"
    fi
else
    println "xz mode not supported"
fi

println "\n===>  xz frame tests "

if [ $LZMAMODE -eq 1 ]; then
    datagen > tmp
    zstd -f --format=xz tmp
    zstd -f --format=lzma tmp
    zstd -f tmp
    cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp
    truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !"
    truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !"
    rm -f tmp*
else
    println "xz mode not supported"
fi

println "\n===>  lz4 compatibility tests "

LZ4MODE=1
zstd --format=lz4 -V || LZ4MODE=0
if [ $LZ4MODE -eq 1 ]; then
    println "lz4 support detected"
    LZ4EXE=1
    lz4 -V || LZ4EXE=0
    if [ $LZ4EXE -eq 1 ]; then
        datagen > tmp
        zstd --format=lz4 -f tmp
        lz4 -t -v tmp.lz4
        lz4 -f -m tmp   # ensure result is sent into tmp.lz4, not stdout
        zstd -d -f -v tmp.lz4
        rm -f tmp*
    else
        println "lz4 binary not detected"
    fi
else
    println "lz4 mode not supported"
fi

if [ $LZ4MODE -eq 1 ]; then
    println "\n===>  lz4 frame tests "
    datagen > tmp
    zstd -f --format=lz4 tmp
    zstd -f tmp
    cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp
    truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !"
    rm -f tmp*
else
    println "\nlz4 mode not supported"
fi

println "\n===> suffix list test"

! zstd -d tmp.abc 2> tmplg

if [ $GZIPMODE -ne 1 ]; then
    grep ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed"
fi

if [ $LZMAMODE -ne 1 ]; then
    grep ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed"
    grep ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed"
fi

if [ $LZ4MODE -ne 1 ]; then
    grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed"
fi

touch tmp1
zstd tmp1 -o tmp1.zstd
zstd -d -f tmp1.zstd   # support .zstd suffix even though it's not the default suffix

println "\n===>  tar extension tests "

rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 tmp1.zstd

datagen > tmp
tar -cf tmp.tar tmp
zstd tmp.tar -o tmp.tzst
rm -f tmp.tar
zstd -d tmp.tzst
[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!"
rm -f tmp.tar tmp.tzst

if [ $GZIPMODE -eq 1 ]; then
    tar -f - -c tmp | gzip > tmp.tgz
    zstd -d tmp.tgz
    [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!"
    rm -f tmp.tar tmp.tgz
fi

if [ $LZMAMODE -eq 1 ]; then
    tar -f - -c tmp | zstd --format=xz > tmp.txz
    zstd -d tmp.txz
    [ -e tmp.tar ] || die ".txz failed to decompress to .tar!"
    rm -f tmp.tar tmp.txz
fi

if [ $LZ4MODE -eq 1 ]; then
    tar -f - -c tmp | zstd --format=lz4 > tmp.tlz4
    zstd -d tmp.tlz4
    [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!"
    rm -f tmp.tar tmp.tlz4
fi

touch tmp.t tmp.tz tmp.tzs
! zstd -d tmp.t
! zstd -d tmp.tz
! zstd -d tmp.tzs

println "\n===>  zstd round-trip tests "

roundTripTest
roundTripTest -g15K       # TableID==3
roundTripTest -g127K      # TableID==2
roundTripTest -g255K      # TableID==1
roundTripTest -g522K      # TableID==0
roundTripTest -g519K 6    # greedy, hash chain
roundTripTest -g517K 16   # btlazy2
roundTripTest -g516K 19   # btopt

fileRoundTripTest -g500K

println "\n===>  zstd long distance matching round-trip tests "
roundTripTest -g0 "2 --single-thread --long"
roundTripTest -g1000K "1 --single-thread --long"
roundTripTest -g517K "6 --single-thread --long"
roundTripTest -g516K "16 --single-thread --long"
roundTripTest -g518K "19 --single-thread --long"
roundTripTest -g2M "22 --single-thread --ultra --long"
fileRoundTripTest -g5M "3 --single-thread --long"

roundTripTest -g96K "5 --single-thread"
if [ -n "$hasMT" ]
then
    println "\n===>  zstdmt round-trip tests "
    roundTripTest -g4M "1 -T0"
    roundTripTest -g4M "1 -T0 --auto-threads=physical"
    roundTripTest -g4M "1 -T0 --auto-threads=logical"
    roundTripTest -g8M "3 -T2"
    roundTripTest -g8M "19 --long"
    roundTripTest -g8000K "2 --threads=2"
    fileRoundTripTest -g4M "19 -T2 -B1M"

    println "\n===>  zstdmt long distance matching round-trip tests "
    roundTripTest -g8M "3 --long=24 -T2"

    println "\n===>  zstdmt environment variable tests "
    echo "multifoo" >> mt_tmp
    ZSTD_NBTHREADS=-3 zstd -f mt_tmp # negative value, warn and revert to default setting
    ZSTD_NBTHREADS=''  zstd -f mt_tmp # empty env var, warn and revert to default setting
    ZSTD_NBTHREADS=-   zstd -f mt_tmp # malformed env var, warn and revert to default setting
    ZSTD_NBTHREADS=a   zstd -f mt_tmp # malformed env var, warn and revert to default setting
    ZSTD_NBTHREADS=+a  zstd -f mt_tmp # malformed env var, warn and revert to default setting
    ZSTD_NBTHREADS=3a7 zstd -f mt_tmp # malformed env var, warn and revert to default setting
    ZSTD_NBTHREADS=50000000000 zstd -f mt_tmp # numeric value too large, warn and revert to default setting=
    ZSTD_NBTHREADS=2  zstd -f mt_tmp # correct usage
    ZSTD_NBTHREADS=1  zstd -f mt_tmp # correct usage: single thread
    # temporary envvar changes in the above tests would actually persist in macos /bin/sh
    unset ZSTD_NBTHREADS
    rm -f mt_tmp*

    println "\n===>  ovLog tests "
    datagen -g2MB > tmp
    refSize=$(zstd tmp -6 -c --zstd=wlog=18         | wc -c)
    ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c)
    ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c)
    if [ "$refSize" -eq "$ov9Size" ]; then
        echo ov9Size should be different from refSize
        exit 1
    fi
    if [ "$refSize" -eq "$ov1Size" ]; then
        echo ov1Size should be different from refSize
        exit 1
    fi
    if [ "$ov9Size" -ge "$ov1Size" ]; then
        echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size"
        exit 1
    fi

else
    println "\n===>  no multithreading, skipping zstdmt tests "
fi

rm -f tmp*

println "\n===>  zstd --list/-l single frame tests "
datagen > tmp1
datagen > tmp2
datagen > tmp3
zstd tmp*
zstd -l ./*.zst
zstd -lv ./*.zst | grep "Decompressed Size:"  # check that decompressed size is present in header
zstd --list ./*.zst
zstd --list -v ./*.zst

println "\n===>  zstd --list/-l multiple frame tests "
cat tmp1.zst tmp2.zst > tmp12.zst
cat tmp12.zst tmp3.zst > tmp123.zst
zstd -l ./*.zst
zstd -lv ./*.zst

println "\n===>  zstd --list/-l error detection tests "
zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file"
zstd --list tmp* && die "-l must fail on non-zstd file"
zstd -lv tmp1* && die "-l must fail on non-zstd file"
zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file"

println "test : detect truncated compressed file "
TEST_DATA_FILE=truncatable-input.txt
FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst
TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst
datagen -g50000 > $TEST_DATA_FILE
zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE
dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE
zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file"

rm -f $TEST_DATA_FILE
rm -f $FULL_COMPRESSED_FILE
rm -f $TRUNCATED_COMPRESSED_FILE

println "\n===>  zstd --list/-l errors when presented with stdin / no files"
zstd -l && die "-l must fail on empty list of files"
zstd -l - && die "-l does not work on stdin"
zstd -l < tmp1.zst && die "-l does not work on stdin"
zstd -l - < tmp1.zst && die "-l does not work on stdin"
zstd -l - tmp1.zst && die "-l does not work on stdin"
zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin"
zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty

println "\n===>  zstd --list/-l test with null files "
datagen -g0 > tmp5
zstd tmp5
zstd -l tmp5.zst
zstd -l tmp5* && die "-l must fail on non-zstd file"
zstd -lv tmp5.zst | grep "Decompressed Size: 0 B (0 B)"  # check that 0 size is present in header
zstd -lv tmp5* && die "-l must fail on non-zstd file"

println "\n===>  zstd --list/-l test with no content size field "
datagen -g513K | zstd > tmp6.zst
zstd -l tmp6.zst
zstd -lv tmp6.zst | grep "Decompressed Size:"  && die "Field :Decompressed Size: should not be available in this compressed file"

println "\n===>   zstd --list/-l test with no checksum "
zstd -f --no-check tmp1
zstd -l tmp1.zst
zstd -lv tmp1.zst

println "\n===>  zstd trace tests "
zstd -f --trace tmp.trace tmp1
zstd -f --trace tmp.trace tmp1 tmp2 tmp3
zstd -f --trace tmp.trace tmp1 tmp2 tmp3 -o /dev/null
zstd -f --trace tmp.trace tmp1 tmp2 tmp3 --single-thread
zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null
zstd -f --trace tmp.trace -D tmp1 tmp2 tmp3 -o /dev/null --single-thread
zstd --trace tmp.trace -t tmp1.zst
zstd --trace tmp.trace -t tmp1.zst tmp2.zst
zstd -f --trace tmp.trace -d tmp1.zst
zstd -f --trace tmp.trace -d tmp1.zst tmp2.zst tmp3.zst
zstd -D tmp1 tmp2 -c | zstd --trace tmp.trace -t -D tmp1
zstd -b1e10i0 --trace tmp.trace tmp1
zstd -b1e10i0 --trace tmp.trace tmp1 tmp2 tmp3

rm -f tmp*

println "\n===>   zstd long distance matching tests "
roundTripTest -g0 " --single-thread --long"
roundTripTest -g9M "2 --single-thread --long"
# Test parameter parsing
roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB"
roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB"
roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB"
roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB"

if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -ne "1" ]; then
    println "\n===>  zstd long distance matching with optimal parser compressed size tests "
    optCSize16=$(datagen -g511K | zstd -16 -c | wc -c)
    longCSize16=$(datagen -g511K | zstd -16 --long -c | wc -c)
    optCSize19=$(datagen -g2M | zstd -19 -c | wc -c)
    longCSize19=$(datagen -g2M | zstd -19 --long -c | wc -c)
    optCSize19wlog23=$(datagen -g2M | zstd -19 -c  --zstd=wlog=23 | wc -c)
    longCSize19wlog23=$(datagen -g2M | zstd -19 -c --long=23 | wc -c)
    if [ "$longCSize16" -gt "$optCSize16" ]; then
        echo using --long on compression level 16 should not cause compressed size regression
        exit 1
    elif [ "$longCSize19" -gt "$optCSize19" ]; then
        echo using --long on compression level 19 should not cause compressed size regression
        exit 1
    elif [ "$longCSize19wlog23" -gt "$optCSize19wlog23" ]; then
        echo using --long on compression level 19 with wLog=23 should not cause compressed size regression
        exit 1
    fi
fi

println "\n===>  zstd asyncio tests "

addFrame() {
    datagen -g2M -s$2 >> tmp_uncompressed
    datagen -g2M -s$2 | zstd -1 --format=$1 >> tmp_compressed.zst
}

addTwoFrames() {
  addFrame $1 1
  addFrame $1 2
}

testAsyncIO() {
  roundTripTest -g2M "3 --asyncio --format=$1"
  roundTripTest -g2M "3 --no-asyncio --format=$1"
}

rm -f tmp_compressed tmp_uncompressed
testAsyncIO zstd
addTwoFrames zstd
if [ $GZIPMODE -eq 1 ]; then
  testAsyncIO gzip
  addTwoFrames gzip
fi
if [ $LZMAMODE -eq 1 ]; then
  testAsyncIO lzma
  addTwoFrames lzma
fi
if [ $LZ4MODE -eq 1 ]; then
  testAsyncIO lz4
  addTwoFrames lz4
fi
cat tmp_uncompressed | $MD5SUM > tmp2
zstd -d tmp_compressed.zst --asyncio -c | $MD5SUM > tmp1
$DIFF -q tmp1 tmp2
rm tmp1
zstd -d tmp_compressed.zst --no-asyncio -c | $MD5SUM > tmp1
$DIFF -q tmp1 tmp2

if [ "$1" != "--test-large-data" ]; then
    println "Skipping large data tests"
    exit 0
fi

#############################################################################

if [ -n "$hasMT" ]
then
    println "\n===>   adaptive mode "
    roundTripTest -g270000000 " --adapt"
    roundTripTest -g27000000 " --adapt=min=1,max=4"
    roundTripTest -g27000000 " --adapt=min=-2,max=-1"
    println "===>   test: --adapt must fail on incoherent bounds "
    datagen > tmp
    zstd --adapt= tmp && die "invalid compression parameter"
    zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds"

    println "\n===>   rsyncable mode "
    roundTripTest -g10M " --rsyncable"
    roundTripTest -g10M " --rsyncable -B100K"
    println "===>   test: --rsyncable must fail with --single-thread"
    zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread"
fi

println "\n===> patch-from=origin tests"
datagen -g1000 -P50 > tmp_dict
datagen -g1000 -P10 > tmp_patch
zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff
zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch

println "\n===> alternate syntax: patch-from origin"
zstd -f --patch-from tmp_dict tmp_patch -o tmp_patch_diff
zstd -df --patch-from tmp_dict tmp_patch_diff -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
rm -rf tmp_*

println "\n===> patch-from recursive tests"
mkdir tmp_dir
datagen > tmp_dir/tmp1
datagen > tmp_dir/tmp2
datagen > tmp_dict
zstd --patch-from=tmp_dict -r tmp_dir && die
rm -rf tmp*

println "\n===> patch-from long mode trigger larger file test"
if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -eq "1" ]; then
    # if binary tree strategies are excluded, the threshold is different
    datagen -g10000000 > tmp_dict
    datagen -g10000000 > tmp_patch
else
    datagen -g5000000 > tmp_dict
    datagen -g5000000 > tmp_patch
fi
zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automatically triggered"
rm -rf tmp*

println "\n===> patch-from very large dictionary and file test"
datagen -g550000000 -P0 > tmp_dict
datagen -g100000000 -P1 > tmp_patch
zstd --long=30 -1f --patch-from tmp_dict tmp_patch
zstd --long=30 -df --patch-from tmp_dict tmp_patch.zst -o tmp_patch_recon
$DIFF -s tmp_patch_recon tmp_patch
rm -rf tmp*

println "\n===> patch-from --stream-size test"
datagen -g1000 -P50 > tmp_dict
datagen -g1000 -P10 > tmp_patch
cat tmp_patch | zstd -f --patch-from=tmp_dict -c -o tmp_patch_diff && die
cat tmp_patch | zstd -f --patch-from=tmp_dict --stream-size=1000 -c -o tmp_patch_diff
rm -rf tmp*

println "\n===>   large files tests "

roundTripTest -g270000000 1
roundTripTest -g250000000 2
roundTripTest -g230000000 3

roundTripTest -g140000000 -P60 4
roundTripTest -g130000000 -P62 5
roundTripTest -g120000000 -P65 6

roundTripTest -g70000000 -P70 7
roundTripTest -g60000000 -P71 8
roundTripTest -g50000000 -P73 9

roundTripTest -g35000000 -P75 10
roundTripTest -g30000000 -P76 11
roundTripTest -g25000000 -P78 12

roundTripTest -g18000013 -P80 13
roundTripTest -g18000014 -P80 14
roundTripTest -g18000015 -P81 15
roundTripTest -g18000016 -P84 16
roundTripTest -g18000017 -P88 17
roundTripTest -g18000018 -P94 18
roundTripTest -g18000019 -P96 19

roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25"
roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25"   # ensure btlazy2 can survive an overflow rescale

fileRoundTripTest -g4193M -P99 1

println "\n===>   zstd long, long distance matching round-trip tests "
roundTripTest -g270000000 "1 --single-thread --long"
roundTripTest -g130000000 -P60 "5 --single-thread --long"
roundTripTest -g35000000 -P70 "8 --single-thread --long"
roundTripTest -g18000001 -P80  "18 --single-thread --long"
# Test large window logs
roundTripTest -g700M -P50 "1 --single-thread --long=29"
roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28"

if [ -n "$hasMT" ]
then
    println "\n===>   zstdmt long round-trip tests "
    roundTripTest -g80000000 -P99 "19 -T2" " "
    roundTripTest -g5000000000 -P99 "1 -T2" " "
    roundTripTest -g500000000 -P97 "1 -T999" " "
    fileRoundTripTest -g4103M -P98 " -T0" " "
    roundTripTest -g400000000 -P97 "1 --long=24 -T2" " "
    # Exposes the bug in https://github.com/facebook/zstd/pull/1678
    # This test fails on 4 different travis builds at the time of writing
    # because it needs to allocate 8 GB of memory.
    # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000"
else
    println "\n**** no multithreading, skipping zstdmt tests **** "
fi

println "\n===>  cover dictionary builder : advanced options "

TESTFILE="$PRGDIR"/zstdcli.c
datagen > tmpDict
println "- Create first dictionary"
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict
cp "$TESTFILE" tmp
zstd -f tmp -D tmpDict
zstd -f tmp -D tmpDict --patch-from=tmpDict && die "error: can't use -D and --patch-from=#at the same time"
zstd -d tmp.zst -D tmpDict -fo result
$DIFF "$TESTFILE" result
zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)"
println "- Create second (different) dictionary"
zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC
zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!"
println "- Create dictionary using shrink-dict flag"
zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict
zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1
zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2
zstd --train-cover=shrink=5,steps=256 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict3
println "- Create dictionary with short dictID"
zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1
cmp tmpDict tmpDict1 && die "dictionaries should have different ID !"
println "- Create dictionary with size limit"
zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K
println "- Compare size of dictionary from 90% training samples with 80% training samples"
zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Create dictionary using all samples for both training and testing"
zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c
println "- Test -o before --train-cover"
rm -f tmpDict dictionary
zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f tmpDict
zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c
test -f dictionary
rm -f tmp* dictionary

rm -f tmp*


#!/bin/sh
set -e
set -u
set -x

SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
PROG_DIR="$SCRIPT_DIR/../programs"

ZSTD="$PROG_DIR/zstd"
ZSTD_COMPRESS="$PROG_DIR/zstd-compress"
ZSTD_DECOMPRESS="$PROG_DIR/zstd-decompress"
ZSTD_NOLEGACY="$PROG_DIR/zstd-nolegacy"
ZSTD_DICTBUILDER="$PROG_DIR/zstd-dictBuilder"
ZSTD_FRUGAL="$PROG_DIR/zstd-frugal"
ZSTD_NOMT="$PROG_DIR/zstd-nomt"

println() {
    printf '%b\n' "${*}"
}

die() {
    println "$@" 1>&2
    exit 1
}

symbol_present() {
	(nm $1 || echo "symbol_present $@ failed") | grep $2
}

symbol_not_present() {
	symbol_present $@ && die "Binary '$1' mistakenly contains symbol '$2'" ||:
}

compress_not_present() {
	symbol_not_present "$1" ZSTD_compress
}

decompress_not_present() {
	symbol_not_present "$1" ZSTD_decompress
}

dict_not_present() {
	symbol_not_present "$1" ZDICT_
	symbol_not_present "$1" COVER_
}

cliextra_not_present() {
	symbol_not_present "$1" TRACE_
	symbol_not_present "$1" BMK_
}

legacy_not_present() {
	symbol_not_present "$1" ZSTDv0
}

test_help() {
	"$1" --help | grep -- "$2"
}

test_no_help() {
	test_help $@ && die "'$1' supports '$2' when it shouldn't" ||:
}

extras_not_present() {
	dict_not_present $@
	legacy_not_present $@
	cliextra_not_present $@
	test_no_help $@ "--train"
	test_no_help $@ "-b#"
}

test_compress() {
	echo "hello" | "$1" | "$ZSTD" -t
}

test_decompress() {
	echo "hello" | "$ZSTD" | "$1" -t
}

test_zstd() {
	test_compress $@
	test_decompress $@
}

extras_not_present "$ZSTD_FRUGAL"
extras_not_present "$ZSTD_COMPRESS"
extras_not_present "$ZSTD_DECOMPRESS"

compress_not_present "$ZSTD_DECOMPRESS"

decompress_not_present "$ZSTD_COMPRESS"
decompress_not_present "$ZSTD_DICTBUILDER"

cliextra_not_present "$ZSTD_DICTBUILDER"

legacy_not_present "$ZSTD_DICTBUILDER"
legacy_not_present "$ZSTD_NOLEGACY"

symbol_not_present "$ZSTD" ZSTDv01
symbol_not_present "$ZSTD" ZSTDv02
symbol_not_present "$ZSTD" ZSTDv03
symbol_not_present "$ZSTD" ZSTDv04

test_compress "$ZSTD_COMPRESS"
test_decompress "$ZSTD_DECOMPRESS"

test_zstd "$ZSTD_FRUGAL"
test_zstd "$ZSTD_NOLEGACY"

test_help "$ZSTD" '-b#'
test_help "$ZSTD" --train
test_help "$ZSTD_DICTBUILDER" --train

println "Success!"


#!/bin/sh

set -e

println "+ zstd -h"
zstd -h
println "+ zstd -H"
zstd -H
println "+ zstd --help"
zstd --help


#!/bin/sh

echo "some data" > file

println "+ zstd --memory=32LB file"
zstd --memory=32LB file && die "Should not allow bogus suffix"
println "+ zstd --memory=32LiB file"
zstd --memory=32LiB file && die "Should not allow bogus suffix"
println "+ zstd --memory=32A file"
zstd --memory=32A file && die "Should not allow bogus suffix"
println "+ zstd --memory=32r82347dn83 file"
zstd --memory=32r82347dn83 file && die "Should not allow bogus suffix"
println "+ zstd --memory=32asbdf file"
zstd --memory=32asbdf file && die "Should not allow bogus suffix"
println "+ zstd --memory=hello file"
zstd --memory=hello file && die "Should not allow non-numeric parameter"
println "+ zstd --memory=1 file"
zstd -q --memory=1 file && die "Should allow numeric parameter without suffix"
rm file.zst
println "+ zstd --memory=1K file"
zstd -q --memory=1K file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1KB file"
zstd -q --memory=1KB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1KiB file"
zstd -q --memory=1KiB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1M file"
zstd -q --memory=1M file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1MB file"
zstd -q --memory=1MB file && die "Should allow numeric parameter with expected suffix"
rm file.zst
println "+ zstd --memory=1MiB file"
zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
rm file.zst

rm file
exit 0


#!/bin/sh

println "+ zstd -r * --output-dir-mirror=\"\""
zstd -r * --output-dir-mirror="" && die "Should not allow empty output dir!"
println "+ zstd -r * --output-dir-flat=\"\""
zstd -r * --output-dir-flat="" && die "Should not allow empty output dir!"
exit 0


#!/bin/sh

set -e

zstd -V
zstd --version


#!/bin/sh

set -e

println "+ good path"
zstdgrep "1234" file file.zst
println "+ bad path"
zstdgrep "1234" bad.zst


#!/bin/sh

set -e

println "+ good path"
zstdless file.zst
println "+ pass parameters"
zstdless -N file.zst # This parameter does not produce line #s when piped, but still serves to test that the flag went to less and not zstd
println "+ bad path"
zstdless bad.zst >&2


#!/bin/sh

. "$COMMON/platform.sh"

zstd_supports_format()
{
	zstd -h | grep > $INTOVOID -- "--format=$1"
}

format_extension()
{
	if [ "$1" = "zstd" ]; then
		printf "zst"
	elif [ "$1" = "gzip" ]; then
		printf "gz"
	else
		printf "$1"
	fi
}


. "$COMMON/platform.sh"

MTIME="stat -c %Y"
case "$UNAME" in
    Darwin | FreeBSD | OpenBSD | NetBSD) MTIME="stat -f %m" ;;
esac

assertSameMTime() {
    MT1=$($MTIME "$1")
    MT2=$($MTIME "$2")
    echo MTIME $MT1 $MT2
    [ "$MT1" = "$MT2" ] || die "mtime on $1 doesn't match mtime on $2 ($MT1 != $MT2)"
}


. "$COMMON/platform.sh"

GET_PERMS="stat -c %a"
case "$UNAME" in
    Darwin | FreeBSD | OpenBSD | NetBSD) GET_PERMS="stat -f %Lp" ;;
esac

assertFilePermissions() {
    STAT1=$($GET_PERMS "$1")
    STAT2=$2
    [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match expected ($STAT1 != $STAT2)"
}

assertSamePermissions() {
    STAT1=$($GET_PERMS "$1")
    STAT2=$($GET_PERMS "$2")
    [ "$STAT1" = "$STAT2" ] || die "permissions on $1 don't match those on $2 ($STAT1 != $STAT2)"
}


#!/bin/sh

UNAME=$(uname)

isWindows=false
INTOVOID="/dev/null"
case "$UNAME" in
  GNU) DEVDEVICE="/dev/random" ;;
  *) DEVDEVICE="/dev/zero" ;;
esac
case "$OS" in
  Windows*)
    isWindows=true
    INTOVOID="NUL"
    DEVDEVICE="NUL"
    ;;
esac

case "$UNAME" in
  Darwin) MD5SUM="md5 -r" ;;
  FreeBSD) MD5SUM="gmd5sum" ;;
  NetBSD) MD5SUM="md5 -n" ;;
  OpenBSD) MD5SUM="md5" ;;
  *) MD5SUM="md5sum" ;;
esac

DIFF="diff"
case "$UNAME" in
  SunOS) DIFF="gdiff" ;;
esac

if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled'
then
    hasMT=""
else
    hasMT="true"
fi


#!/bin/sh

set -e

# Test --adapt
zstd -f file --adapt -c | zstd -t

datagen -g100M > file100M

# Pick parameters to force fast adaptation, even on slow systems
zstd --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"

# Adaption still happens with --no-progress
zstd --no-progress --adapt -vvvv -19 --zstd=wlog=10 file100M -o /dev/null 2>&1 | grep -q "faster speed , lighter compression"


#!/bin/sh

set -e

# Uncomment the set -v line for debugging
# set -v

# Test compression flags and check that they work
zstd file                 ; zstd -t file.zst
zstd -f file              ; zstd -t file.zst
zstd -f -z file           ; zstd -t file.zst
zstd -f -k file           ; zstd -t file.zst
zstd -f -C file           ; zstd -t file.zst
zstd -f --check file      ; zstd -t file.zst
zstd -f --no-check file   ; zstd -t file.zst
zstd -f -- file           ; zstd -t file.zst

# Test output file compression
zstd -o file-out.zst ; zstd -t file-out.zst
zstd -fo file-out.zst; zstd -t file-out.zst

# Test compression to stdout
zstd -c file       | zstd -t
zstd --stdout file | zstd -t
println bob | zstd | zstd -t

# Test keeping input file when compressing to stdout in gzip mode
if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
    $ZSTD_SYMLINK_DIR/gzip -c file       | zstd -t ; test -f file
    $ZSTD_SYMLINK_DIR/gzip --stdout file | zstd -t ; test -f file
fi

# Test --rm
cp file file-rm
zstd --rm file-rm; zstd -t file-rm.zst
test ! -f file-rm


#!/bin/sh

set -e

# Test --[no-]compress-literals
zstd file --no-compress-literals -1 -c       | zstd -t
zstd file --no-compress-literals -19 -c      | zstd -t
zstd file --no-compress-literals --fast=1 -c | zstd -t
zstd file --compress-literals -1 -c          | zstd -t
zstd file --compress-literals --fast=1 -c    | zstd -t


#!/bin/sh

. "$COMMON/format.sh"

set -e

# Test --format
zstd --format=zstd file -f
zstd -t file.zst
for format in "gzip" "lz4" "xz" "lzma"; do
	if zstd_supports_format $format; then
		zstd --format=$format file
		zstd -t file.$(format_extension $format)
		zstd -c --format=$format file | zstd -t --format=$format
	fi
done


#!/bin/sh

set -e

GOLDEN_DIR="$ZSTD_REPO_DIR/tests/golden-compression/"
cp -r "$GOLDEN_DIR" golden/

zstd -rf golden/ --output-dir-mirror golden-compressed/
zstd -r -t golden-compressed/

zstd --target-compressed-block-size=1024 -rf golden/ --output-dir-mirror golden-compressed/
zstd -r -t golden-compressed/

# PR #3517 block splitter corruption test
zstd -rf -19 --zstd=mml=7 golden/ --output-dir-mirror golden-compressed/
zstd -r -t golden-compressed/


#!/bin/sh

set -e

# Uncomment the set -v line for debugging
# set -v

# Test gzip specific compression option
if $(command -v $ZSTD_SYMLINK_DIR/gzip); then
    $ZSTD_SYMLINK_DIR/gzip --fast file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
    $ZSTD_SYMLINK_DIR/gzip --best file ; $ZSTD_SYMLINK_DIR/gzip -d file.gz

    # Test -n / --no-name: do not embed original filename in archive
    $ZSTD_SYMLINK_DIR/gzip -n file           ; grep -qv file file.gz  ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
    $ZSTD_SYMLINK_DIR/gzip --no-name file    ; grep -qv file file.gz  ; $ZSTD_SYMLINK_DIR/gzip -d file.gz
    $ZSTD_SYMLINK_DIR/gzip -c --no-name file | grep -qv file
fi