#!/usr/bin/env bash

# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2

# Copyright 2013-present Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# #     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o verbose
rm -rf /tmp/ops

function tt {
  bundle exec ./bin/taste-tester test -s localhost -y -c $1
}
(
  mkdir /tmp/ops
)
tt ./test/tt-auto.rb
if [ $? -ne 0 ]; then
  exit 0
fi


#!/usr/bin/env bash

# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2

# Copyright 2013-present Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e
set -o verbose
rm -rf /tmp/ops
GIT="git -c user.email='foo@bar.com' -c user.name='foobar'"

function tt {
  bundle exec ./bin/taste-tester test -s localhost -y -c $1 -v
}
(
  git init /tmp/ops
  cd /tmp/ops
  mkdir chef/cookbooks/cookbook1/recipes -p
  touch chef/cookbooks/cookbook1/recipes/default.rb
  echo "name 'cookbook1'" > chef/cookbooks/cookbook1/metadata.rb
  $GIT add . -A
  $GIT commit -m "Add cookbook1"
)
tt ./test/tt-git.rb
tt ./test/tt-auto.rb


#!/usr/bin/env bash

# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2

# Copyright 2013-present Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e
set -o verbose
rm -rf /tmp/ops
HG="hg --config ui.username=foo@bar.com"

function tt {
  bundle exec ./bin/taste-tester test -s localhost -y -c $1 -v
}
(
  mkdir /tmp/ops
  cd /tmp/ops
  hg init
  mkdir chef/cookbooks/cookbook1/recipes -p
  touch chef/cookbooks/cookbook1/recipes/default.rb
  echo "name 'cookbook1'" > chef/cookbooks/cookbook1/metadata.rb
  $HG add .
  $HG commit -m "Add cookbook1"
)
tt ./test/tt-hg.rb
tt ./test/tt-auto.rb


#!/usr/bin/env bash

# vim: syntax=ruby:expandtab:shiftwidth=2:softtabstop=2:tabstop=2

# Copyright 2013-present Facebook
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e
set -o verbose
./test/run-git.sh
./test/run-hg.sh
./test/run-broken.sh


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -euxo pipefail

release_version=${1?"Usage: $0 <release_version>"}

git checkout main
git pull

echo $release_version > testslide/version
sed -i -e "s/Version .*/Version $release_version/" \
    util/testslide-snippets/README.md \
    util/testslide-snippets/CHANGELOG.md
sed -i -e "s/\"version\":.*/\"version\": \"$release_version\",/" util/testslide-snippets/package.json
echo $release_version > pytest-testslide/testslide-version
git add testslide/version util/testslide-snippets
git add pytest-testslide/testslide-version
git commit -m "v$release_version"
git push origin main:release_v$release_version

git tag $release_version
git push origin $release_version


#! /bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.

# Add all the values from terraform/outputs.tf to the local env (call upcase with prefix HMA_)
# usage:
#    $ source scripts/set_tf_outputs_in_local_env.sh

[[ $_ != $0 ]] && echo "Error: this script needs to be 'source'd not run." && exit
for s in $(terraform -chdir=terraform output -json | jq -r "to_entries|map(\"HMA_\(.key|ascii_upcase)=\(.value.value|tostring)\")|.[]"); do
    export $s
done


#! /bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.

set -e

SCRIPT_NAME="$(basename \"$0\")"

if [[ -z "${DOCKER_TAG}" ]]; then
    echo "Please rerun with a DOCKER_TAG set in your local environment" >&2
    echo "e.g., DOCKER_TAG=my_prefix $0" >&2
    exit 1
fi

if [ -d "local-threatexchange" ]; then
    echo "################### WARNING ######################"
    echo "  Using a local copy of threatexchange from the   "
    echo "  local-threatexchange directory to build the "
    echo "  docker image."
    echo " Run './scripts/set_threatexchange_source pypi' if "
    echo " this is not what you want. "
    echo "##################################################"
fi

REPOSITORY_NAME="hma-lambda-dev"

ECS_PASSWORD="$(aws ecr get-login-password --region us-east-1)"
ECR_REPOSITORY_URI="$(aws ecr describe-repositories --repository-names hma-lambda-dev | jq -r '.repositories[0].repositoryUri')"

AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
echo "$ECS_PASSWORD" | docker login --username AWS --password-stdin "$ECR_REPOSITORY_URI"
docker build -f Dockerfile -t "$REPOSITORY_NAME" .
docker tag "$REPOSITORY_NAME" "${ECR_REPOSITORY_URI}:${DOCKER_TAG}"
docker push "${ECR_REPOSITORY_URI}:${DOCKER_TAG}"


#! /bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.

[[ ! -z "$TF_STATE_S3_BUCKET" ]] && s3_bucket="$TF_STATE_S3_BUCKET" || s3_bucket="threatexchange-tf-state"
[[ ! -z "$TF_STATE_S3_KEY" ]] && s3_key="$TF_STATE_S3_KEY" || s3_key="state\/hasher-matcher-actioner-$(whoami).tfstate"
[[ ! -z "$TF_STATE_AWS_REGION" ]] && aws_region="$TF_STATE_AWS_REGION" || aws_region="us-east-1"
[[ ! -z "$TF_STATE_LOCKING_TABLE" ]] && table_name="$TF_STATE_LOCKING_TABLE" || table_name="terraform-state-locking"

sed \
    -e "s/# TERRAFORM_STATE_S3_BUCKET/\"${s3_bucket}\"/" \
    -e "s/# TERRAFORM_STATE_KEY/\"${s3_key}\"/" \
    -e "s/# TERRAFORM_BUCKET_REGION/\"${aws_region}\"/" \
    -e "s/# TERRAFORM_STATE_LOCKING_TABLE/\"${table_name}\"/" \
    terraform/backend.tf.example


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.

set -e

make clean
phpize
./configure --enable-pdq
make
echo
echo REBUILT


#!/bin/sh
# Copyright (c) Meta Platforms, Inc. and affiliates.

for mp4 in ../sample-videos/*.mp4; do
  tmk=$(echo $(basename $mp4) | sed 's/mp4/tmk/')
  ../cpp/tmk-hash-video -i $mp4 -o $tmk -f /usr/local/bin/ffmpeg
done


#!/bin/bash

# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -eu

die() { echo "$1"; exit 1; }

while read filename; do \
  grep -q "^# Copyright (c) Meta Platforms, Inc. and affiliates." "$filename" ||
    die "Missing copyright in $filename"
done < <( git ls-tree -r --name-only HEAD | grep "\(.py\|\.sh\)$" )


#!/bin/bash -e
# vim:ts=2:sw=2:et:
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

cd "$(dirname "$0")"

set -x
PREFIX=${PREFIX:-/usr/local}
python3 build/fbcode_builder/getdeps.py build \
        --allow-system-packages \
        --src-dir=. \
        "--project-install-prefix=watchman:$PREFIX" \
        watchman
python3 build/fbcode_builder/getdeps.py fixup-dyn-deps \
        --allow-system-packages \
        --src-dir=. \
        "--project-install-prefix=watchman:$PREFIX" \
        --final-install-prefix "$PREFIX" \
        watchman built

find built -ls


#!/bin/bash -e
# vim:ts=2:sw=2:et:
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -x
python3 "$(dirname "$0")/build/fbcode_builder/getdeps.py" install-system-deps --recursive watchman


#!/bin/bash -e
# vim:ts=2:sw=2:et:
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -x
python3 "$(dirname "$0")/build/fbcode_builder/getdeps.py" test --allow-system-packages --no-testpilot watchman


#!/bin/bash -e
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

cd "$(dirname "$0")"

cd /__w/watchman/watchman

cd "$(git rev-parse --show-toplevel)"

# Why /usr/local? This .deb does not hold to any rigorous packaging
# standards (e.g. Debian), and is intended to replace existing ad hoc
# Watchman installations.
#
# Until we have a reason otherwise, let's remain compatible with
# legacy installation paths.
PREFIX="/usr/local"

# Generated by the prior invocation of ./autogen.sh
BUILT="/__w/watchman/watchman/built"

find "$BUILT"

set -x

# This is an implicit assertion that the `lib` directory is empty.
# We statically link dependencies for packaging purposes.
rmdir "$BUILT/lib"

# TODO: make a debuginfo package
strip "$BUILT/bin/watchman"
strip "$BUILT/bin/watchmanctl"

PACKAGE_VERSION=$("$BUILT/bin/watchman" --version)

PACKAGE_WORKDIR=$(mktemp -d)
trap 'rm -rf -- "$PACKAGE_WORKDIR"' EXIT

mkdir -p "$PACKAGE_WORKDIR$PREFIX"
cp -ar "$BUILT/bin" "$PACKAGE_WORKDIR$PREFIX/bin"

cp -ar /__w/watchman/watchman/watchman/build/package/watchman-deb/DEBIAN "$PACKAGE_WORKDIR"

python3 /__w/watchman/watchman/watchman/build/package/substcontrol.py "$PACKAGE_WORKDIR/DEBIAN/control" "$PACKAGE_VERSION" "$UBUNTU_VERSION"

mkdir -p /_debs

DEB_OUTPUT="/_debs/watchman.deb"

dpkg-deb -b "$PACKAGE_WORKDIR" "$DEB_OUTPUT"


#!/bin/bash -e
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

cd "$(dirname "$0")"

# buildkit truncates log output.
# https://stackoverflow.com/questions/65819424/is-there-a-way-to-increase-the-log-size-in-docker-when-building-a-container
export DOCKER_BUILDKIT=0

DB="docker build --progress plain"

UBUNTU_VERSIONS="18 20 22"

# There may be an opportunity to run some of these in parallel.
for uv in $UBUNTU_VERSIONS; do
    $DB --build-arg "UBUNTU_VERSION=$uv.04" -t "watchman-ubuntu-$uv-env" ubuntu-env

    $DB --build-arg "BASE_IMAGE=watchman-ubuntu-$uv-env" -t "watchman-ubuntu-$uv-build" watchman-build

    $DB --build-arg "BASE_IMAGE=watchman-ubuntu-$uv-build" -t "watchman-ubuntu-$uv-deb" watchman-deb

    mkdir -p "_debs/ubuntu-$uv"

    docker run --rm --mount "type=bind,source=$(pwd)/_debs,target=/_out" "watchman-ubuntu-$uv-deb" sh -c "cp /_debs/* /_out/ubuntu-$uv"
done


#!/bin/bash -e
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

cd "$(dirname "$0")"

cd /__w/watchman/watchman

cd "$(git rev-parse --show-toplevel)"

# Why /usr/local? This .deb does not hold to any rigorous packaging
# standards (e.g. Debian), and is intended to replace existing ad hoc
# Watchman installations.
#
# Until we have a reason otherwise, let's remain compatible with
# legacy installation paths.
PREFIX="/usr/local"

# Generated by the prior invocation of ./autogen.sh
BUILT="/__w/watchman/watchman/built"

find "$BUILT"

set -x

# This is an implicit assertion that the `lib` directory is empty.
# We statically link dependencies for packaging purposes.
rmdir "$BUILT/lib"

PACKAGE_VERSION=$("$BUILT/bin/watchman" --version)

PACKAGE_WORKDIR=$(mktemp -d)
trap 'rm -rf -- "$PACKAGE_WORKDIR"' EXIT

SPECS="$PACKAGE_WORKDIR/SPECS"

mkdir -p "$SPECS"

cp watchman/build/package/watchman.spec "$SPECS/watchman.spec"

rpmbuild \
    -bb \
    --define "dist .fc$FEDORA_VERSION" \
    --define "_rpmdir $PACKAGE_WORKDIR/RPMS" \
    --define "version $PACKAGE_VERSION" \
    --define "prefix $PREFIX/" \
    --define "image $BUILT" \
    "$SPECS/watchman.spec"

mkdir -p /_rpms
cp -a "$PACKAGE_WORKDIR"/RPMS/x86_64/*.rpm /_rpms

find /_rpms

for rpm in /_rpms/*.rpm; do
    echo "::set-output name=rpm_path::$rpm"
    echo "::set-output name=rpm_name::$(basename "$rpm")"
    break
done


#!/bin/bash -e
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

cd "$(dirname "$0")"
cd "$(git rev-parse --show-toplevel)"

# Why /usr/local? This .deb does not hold to any rigorous packaging
# standards (e.g. Debian), and is intended to replace existing ad hoc
# Watchman installations.
#
# Until we have a reason otherwise, let's remain compatible with
# legacy installation paths.
PREFIX="/usr/local"

set -x

if [[ $BUILT = "" ]]; then
    BUILT=$(mktemp -d)
    trap 'rm -rf -- "$BUILT"' EXIT

    sudo python3 build/fbcode_builder/getdeps.py install-system-deps \
         --recursive watchman

    python3 build/fbcode_builder/getdeps.py build \
            --allow-system-packages \
            --src-dir=. \
            "--project-install-prefix=watchman:$PREFIX" \
            watchman

    python3 build/fbcode_builder/getdeps.py fixup-dyn-deps \
            --allow-system-packages \
            --src-dir=. \
            "--project-install-prefix=watchman:$PREFIX" \
            --final-install-prefix "$PREFIX" \
            watchman \
            "$BUILT"
fi

# This is an implicit assertion that the `lib` directory is empty.
# We statically link dependencies for packaging purposes.
rmdir "$BUILT/lib"

PACKAGE_VERSION=$("$BUILT/bin/watchman" --version)

PACKAGE_WORKDIR=$(mktemp -d)
trap 'rm -rf -- "$PACKAGE_WORKDIR"' EXIT

mkdir -p "$PACKAGE_WORKDIR$PREFIX"
cp -ar "$BUILT/bin" "$PACKAGE_WORKDIR$PREFIX/bin"
cp -ar watchman/build/package/DEBIAN "$PACKAGE_WORKDIR"

sed -i "s/%VERSION%/$PACKAGE_VERSION/" "$PACKAGE_WORKDIR/DEBIAN/control"

mkdir -p /_debs

DEB_OUTPUT="/_debs/watchman_$PACKAGE_VERSION.deb"

# TODO: use dpkg-shlibdeps to automate generation of the runtime
# dependency list

dpkg-deb -b "$PACKAGE_WORKDIR" "$DEB_OUTPUT"


#!/bin/sh
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

#
# ===============================
# pywatchman pypi publishing tool
# ===============================
#
# this publishes the watchman python module to the pypi project
# index located at https://pypi.python.org/pypi/pywatchman. the
# metadata used for this process comes directly from setup.py.
#
# this requires write access to the pywatchman project on pypi
# as well as a ~/.pypirc of the following form:
#
#    [distutils]
#    index-servers = pypi
#
#    [pypi]
#    repository: https://pypi.python.org/pypi
#    username: <pypi_username>
#    password: <pypi_password>
#

python setup.py sdist bdist_wheel upload -r pypi


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# This is a dumb wrapper to satisfy the buck_sh_test rule
exec "$@"


#! /bin/sh
# Quick hack to figure out where is the space coming from

FILE=$1
if [ -z "$FILE" -o "$FILE" = "-h" ] ; then
  echo "Usage: $0 binary"
  exit 1
fi

TMPRESULT=`mktemp /tmp/nm_out.XXX`

RESULT=$TMPRESULT

echo "Working on $FILE, intermediate nm result in $RESULT"

nm -C --defined-only --print-size -l $FILE > $RESULT

echo "nm done... analyzing..."
#RESULT=/dev/shm/nm.out

gawk '
{sizeHex="0x" $2; size=strtonum(sizeHex); sum+=size}
/:[0-9]+$/ {
  if (size) {
    ssum += size;
    # not all lines of nm output do find a file with line number this finds
    # lines like    "xxxx/blah.cc:123" and returns "xxxx/blah"
    # for the xxx part it tries to keep it shorter by not allowing .
    # (nm sometimes outputs  path/./tmp/morepath)
    match($0, "([^ \t.]+)(\\.(h|c|cc|tcc|cpp|hpp|S|rl|ipp|y))?:[0-9]+$", arr);
    src=arr[1]
    # unknown extensions...
    if (length(src) < 5) print src, "  XXXX ", $0;
    # get rid of irrelevant stuff
    sub("/tmp/", "/", src)
    # somehow lots of ....fbcode...////////morestuf
    sub(".*///", "", src)
    # x/ or y/ ...
    sub("^./", "", src)
    # 20/....
    sub("^[0-9/]+", "", src)
    sizes[src] += size;
    match(src, "(.*)/[^/]*$", pkg);
    sizes[pkg[1] "/*"]+= size;
  }
}
END {
  print "Extracted file attribution for", ssum/1024/1024., "Mb out of",
     sum/1024/1024., "Mb :", int((1000*ssum/sum+.5))/10, "%"
  n = asorti(sizes, siter)
  for (s in sizes) {
    print s, sizes[s]
  }
}' $RESULT | sort -k 2 -n

rm $TMPRESULT


#!/bin/bash
find . -type f -a \( -name "*.h" -o -name "*.cpp" \) | xargs clang-format -i


#!/bin/sh
find . \( -name "*~" -o -name "*.orig" -o -name "*.rej" -o -name "#*#" \
    -o -name ".#*" \) -print0 | xargs -0 rm -v


#! /bin/sh

cat <<EOF
[{
    "alias": "wdt-opensource-linux",
    "command": "SandcastleUniversalCommand",
    "vcsType": "fbcode-fbsource",
    "capabilities": {
        "vcs": "fbcode-fbsource",
        "type": "lego-linux",
        "os": "ubuntu_16.04"
    },
    "priority": 3,
    "args": {
        "name": "wdt-opensource-linux",
        "oncall": "wdt",
        "timeout": 10800,
        "steps": [
            {
                "user": "facebook",
                "name": "Build Wdt OpenSource Linux",
                "shell": "wdt/build/lego-linux-build.sh",
                "required": true
            },
            {
                "user": "facebook",
                "name": "Run Wdt OpenSource Linux Tests",
                "shell": "cd wdt_build; CTEST_OUTPUT_ON_FAILURE=1 make test",
                "required": true
            }
        ]
    }
},
{
    "alias": "wdt-opensource-mac",
    "command": "SandcastleUniversalCommand",
    "vcsType": "fbcode-fbsource",
    "capabilities": {
        "vcs": "fbcode-fbsource",
        "type": "lego-mac",
    },
    "priority": 3,
    "args": {
        "name": "wdt-opensource-mac",
        "oncall": "wdt",
        "timeout": 10800,
        "steps": [
            {
                "user": "facebook",
                "name": "Build Wdt OpenSource Mac",
                "shell": "wdt/build/lego-mac-build.sh",
                "required": true
            },
            {
                "user": "facebook",
                "name": "Run Wdt OpenSource Mac Tests",
                "shell": "cd wdt_build; CTEST_OUTPUT_ON_FAILURE=1 make test",
                "required": true
            }
        ]
    }
}]
EOF


#! /bin/bash
# If using variables directly from the spec, the variables are replaced by the
# wrong value (from initial step)
echo "BOX_DIR=$BOX_DIR"
set -x
pwd
uname -a
mkdir wdt_build
cd wdt_build || exit 1
ls -l "$BOX_DIR"
cmake ../wdt -DFOLLY_SOURCE_DIR="$BOX_DIR" -DBUILD_TESTING=on && make -j


#! /bin/bash

# Remove the double-conversion part when Task#10922468 is done

set -x
pwd
uname -a
mkdir wdt_build
cd wdt_build || exit 1
cmake --version
export https_proxy=fwdproxy:8080
git clone https://github.com/floitsch/double-conversion.git
mkdir double-conversion-build
(cd double-conversion-build && \
 cmake -DBUILD_SHARED_LIBS=on \
       -DCMAKE_INSTALL_PREFIX="$HOME" \
       ../double-conversion; \
 make -j 4 && make install)

cmake ../wdt -DFOLLY_SOURCE_DIR="$(cd ..;pwd)" -DBUILD_TESTING=on \
      -DDOUBLECONV_INCLUDE_DIR="$HOME/include" \
      -DDOUBLECONV_LIBRARY="$HOME/lib/libdouble-conversion.dylib" && make -j


#! /bin/bash
# Script to setup the travis build env - changes here are likely needed
# in wdt_cont_build_setup*.sh too
set -x
set -e

date
uname -a
echo $HOSTNAME
mkdir $HOME/bin || true
export PATH=$HOME/bin:$PATH
export LD_LIBRARY_PATH=$HOME/lib:$LD_LIBRARY_PATH
export CMAKE_PREFIX_PATH=$HOME
openssl version
cd ..
git clone https://github.com/google/double-conversion.git
(mkdir double-conversion-build; cd double-conversion-build; cmake -DBUILD_SHARED_LIBS=on -DCMAKE_INSTALL_PREFIX=$HOME ../double-conversion; make -j 4 && make install)
git clone https://github.com/gflags/gflags.git
(mkdir gflags-build; cd gflags-build; cmake -DCMAKE_INSTALL_PREFIX=$HOME -DGFLAGS_NAMESPACE=google -DBUILD_SHARED_LIBS=on ../gflags && make -j 4 && make install)
git clone https://github.com/google/glog.git
( cd glog && ./autogen.sh && ./configure --with-gflags=$HOME --prefix=$HOME && make -j 4 && make install )
git clone https://github.com/facebook/folly.git
pwd ; ls -l
cd wdt
#set +e
set +x


#! /bin/bash
# Sets up MacOS build environment (for travis but can be used on other mac too)
set -x
#set -e

date
uname -a
echo $HOSTNAME
mkdir $HOME/bin || true
brew update
brew install openssl || true
cd ..
export PATH=/usr/local/opt/openssl/bin:$HOME/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/opt/openssl/lib:$HOME/lib:$LD_LIBRARY_PATH
export OPENSSL_ROOT_DIR=/usr/local/opt/openssl
export CMAKE_PREFIX_PATH=$HOME
openssl version -a
git clone https://github.com/google/double-conversion.git
(mkdir double-conversion-build; cd double-conversion-build; cmake -DBUILD_SHARED_LIBS=on -DCMAKE_INSTALL_PREFIX=$HOME ../double-conversion; make -j 4 && make install)
git clone https://github.com/gflags/gflags.git
(mkdir gflags-build; cd gflags-build; cmake -DCMAKE_INSTALL_PREFIX=$HOME -DGFLAGS_NAMESPACE=google -DBUILD_SHARED_LIBS=on ../gflags && make -j 4 && make install)
git clone https://github.com/google/glog.git
( cd glog && ./autogen.sh && ./configure --with-gflags=$HOME --prefix=$HOME && make -j 4 && make install )
git clone https://github.com/facebook/folly.git
pwd ; ls -l
cd wdt
#set +e
set +x


#! /bin/bash
# A lot of this is facebook specific but can give an idea of how
# to setup custom tests for WDT - Very similar to travis_linux.sh
# run this once and then _run will run a loop and email results
echo "if sending email later fails, you might need:"
echo "sudo sed -ie 's#^\(mailhub\).*$#\1=localhost#' /etc/ssmtp/ssmtp.conf"
echo "but that shouldn't be necessary anymore"
set -x
sudo yum install iproute-tc devtoolset-4-gcc-c++ automake autoconf boost-devel libtool
export PATH=/opt/rh/devtoolset-4/root/bin:$PATH
while sh -c "g++ --version | fgrep 4.8." ; do
  smcc add-services -e hhvm.oss $HOSTNAME:22
  date
  echo "you have an old g++ let see if chefctl can upgrade it"
  sleep 1
  sudo chefctl -i
done
echo congrats on
set -e
g++ --version
CDIR=/data/users/${USER}_wdt_contbuild
BINDIR=/data/users/${USER}_wdt_contbuild/bin
LIBDIR=/data/users/${USER}_wdt_contbuild/lib
sudo sh -c "mkdir -p $BINDIR || true"
sudo sh -c "mkdir -p $LIBDIR || true"
sudo chown -R $USER $CDIR
export PATH=$BINDIR:$PATH
export LD_LIBRARY_PATH=$LIBDIR:$LD_LIBRARY_PATH

cd $CDIR
hg-clone-fbsource --sparse .hgsparse-fbcode --simple

# open source part
export https_proxy=http://fwdproxy:8080
git clone https://cmake.org/cmake.git
cd cmake
./bootstrap --prefix=$CDIR --parallel=16
make -j 16
make install
cd ..

# should be same:
which cmake
ls -l $BINDIR/cmake

# similar as travis_linux.sh
git clone https://github.com/floitsch/double-conversion.git
(cd double-conversion; cmake -DBUILD_SHARED_LIBS=on -DCMAKE_INSTALL_PREFIX=$CDIR .; make -j 16 && make install)
git clone https://github.com/schuhschuh/gflags.git
(mkdir gflags/build; cd gflags/build; cmake -DCMAKE_INSTALL_PREFIX=$CDIR -DGFLAGS_NAMESPACE=google -DBUILD_SHARED_LIBS=on .. && make -j 16 && make install)
git clone https://github.com/google/glog.git
echo "if this fails with aclocal error; run autoreconf"
( cd glog && ./configure --with-gflags=$CDIR --prefix=$CDIR && autoreconf -vfi && make -j 16 && make install )
OPENSSL_VERSION=openssl-1.0.1u
wget https://www.openssl.org/source/$OPENSSL_VERSION.tar.gz
tar xfz $OPENSSL_VERSION.tar.gz
( cd $OPENSSL_VERSION ; ./config --prefix=$CDIR threads shared; make ; make install )
$CDIR/bin/openssl version
ldd $CDIR/bin/openssl
export OPENSSL_ROOT_DIR=$CDIR

mkdir cmake_wdt_build
cd cmake_wdt_build
cmake ../fbsource/fbcode/wdt -DFOLLY_SOURCE_DIR=$CDIR/fbsource/fbcode\
        -DBUILD_TESTING=1

time make -j 16

CTEST_OUTPUT_ON_FAILURE=1 time make test


#! /bin/sh
# Mac version of wdt_cont_build_setup.sh
set -x
set -e
CDIR=/data/users/${USER}_wdt_contbuild
sudo sh -c "mkdir -p $CDIR || true"
sudo chown $USER $CDIR
( echo p | svn list https://googlemock.googlecode.com ) || true
cd $CDIR
hg-clone-fbsource --sparse fbcode/wdt/.hgsparse --simple
mkdir cmake_wdt_build
cd cmake_wdt_build
echo "might need to get a new enough cmake if this fails:"
cmake ../fbsource/fbcode/wdt -DBUILD_TESTING=1 \
    -DFOLLY_SOURCE_DIR=$(cd ../fbsource/fbcode;pwd)
make -j 4
CTEST_OUTPUT_ON_FAILURE=1 make test


#! /bin/sh
# Very inneficient way to colorize log input
awk '/^I/ {system("tput setaf 2"); print $0} /^W/ {system("tput setaf 3"); print $0} /^E/ {system("tput setaf 1"); print $0} /^C/ {system("tput setaf 5"); print $0} /^[^IWEC]/ {system("tput sgr0"); print $0}'


#### TODO : make this work on  MacOS

IPTABLE_LOCK_FILE="/tmp/wdt.iptable.lock"

setTcOptions() {
  sudo tc qdisc add dev lo root netem delay 20ms 10ms duplicate 1% corrupt 0.1%
}

clearTcOptions() {
  sudo tc qdisc del dev lo root
}

runIptableCmd() {
  flock -x "$IPTABLE_LOCK_FILE" -c "$1"
}

blockSportByDropping() {
  UNBLOCK_CMD="sudo ip6tables -D INPUT -p tcp --sport $1 -j DROP"
  CMD="sudo ip6tables -A INPUT -p tcp --sport $1 -j DROP"
  runIptableCmd "$CMD"
}

blockDportByDropping() {
  UNBLOCK_CMD="sudo ip6tables -D INPUT -p tcp --dport $1 -j DROP"
  CMD="sudo ip6tables -A INPUT -p tcp --dport $1 -j DROP"
  runIptableCmd "$CMD"
}

blockSportByRejecting() {
  UNBLOCK_CMD="sudo ip6tables -D INPUT -p tcp --sport $1 -j REJECT"
  CMD="sudo ip6tables -A INPUT -p tcp --sport $1 -j REJECT"
  runIptableCmd "$CMD"
}

blockDportByRejecting() {
  UNBLOCK_CMD="sudo ip6tables -D INPUT -p tcp --dport $1 -j REJECT"
  CMD="sudo ip6tables -A INPUT -p tcp --dport $1 -j REJECT"
  runIptableCmd "$CMD"
}

undoLastIpTableChange() {
  if [ ! -z "$UNBLOCK_CMD" ]; then
    CMD="eval $UNBLOCK_CMD"
    runIptableCmd "$CMD"
    unset UNBLOCK_CMD
  fi
}

setBinaries() {
  if [ -z "$WDT_BINARY" ]; then
    # Inside fb, building with buck, look there:
    WDT_BINARY="./buck-out/gen/wdt/wdt"
    if [ ! -x "$WDT_BINARY" ]; then
      # Open source / Cmake / old fbmake path:
      WDT_BINARY="./_bin/wdt/wdt"
    fi
  fi
  if [ -z "$WDT_SENDER" ]; then
    WDT_SENDER=$WDT_BINARY
  fi
  if [ -z "$WDT_RECEIVER" ]; then
    WDT_RECEIVER=$WDT_BINARY
  fi
  if [ -z "$WDT_GEN_FILES" ]; then
    WDT_GEN_FILES="./buck-out/gen/wdt/bench/wdt_gen_files"
    if [ ! -x "$WDT_GEN_FILES" ]; then
      WDT_GEN_FILES="./_bin/wdt/bench/wdt_gen_files"
    fi
  fi
  if [ -z "$WDT_GEN_BIGRAMS" ]; then
    WDT_GEN_BIGRAMS="$(dirname "$0")/../bench/book1.bigrams"
  fi
  echo "Wdt binary: $WDT_BINARY, Sender: $WDT_SENDER, Receiver: $WDT_RECEIVER"
  echo "wdt_gen_files: $WDT_GEN_FILES - bigram data $WDT_GEN_BIGRAMS"
}

setDirectory() {
  if [ -z "$BASEDIR" ]; then
      BASEDIR="/dev/shm/wdtTest_$USER"
  fi
  mkdir -p "$BASEDIR"
  DIR=$(mktemp -d "$BASEDIR/XXXXX")
  SRC_DIR="$DIR/src"
  echo "Testing in $DIR - src dir $SRC_DIR"
  mkdir "$SRC_DIR"
}

simulateNetworkGlitchesByRejecting() {
  for ((i = 1; i <= ERROR_COUNT; i++))
  do
    sleep 0.3 # sleep for 300ms
    port=$((STARTING_PORT + RANDOM % threads))
    echo "blocking $port"
    if [ $((i % 2)) -eq 0 ]; then
      blockSportByRejecting $port
      sleep 0.7 # sleep for 700ms, read/write timeout is 500ms, so must sleep
                # more than that
      echo "unblocking $port"
      undoLastIpTableChange
    else
      blockDportByRejecting $port
      sleep 0.7 # sleep for 700ms, read/write timeout is 500ms, so must sleep
                # more than that
      echo "unblocking $port"
      undoLastIpTableChange
    fi
  done
}

simulateNetworkGlitchesByDropping() {
  for ((i = 1; i <= ERROR_COUNT; i++))
  do
    sleep 0.3 # sleep for 300ms
    port=$((STARTING_PORT + RANDOM % threads))
    echo "blocking $port"
    if [ $((i % 2)) -eq 0 ]; then
      blockSportByDropping $port
      sleep 0.7 # sleep for 700ms, read/write timeout is 500ms, so must sleep
                # more than that
      echo "unblocking $port"
      undoLastIpTableChange
    else
      blockDportByDropping $port
      sleep 0.7 # sleep for 700ms, read/write timeout is 500ms, so must sleep
                # more than that
      echo "unblocking $port"
      undoLastIpTableChange
    fi
  done
}

printServerLog() {
  echo "Server log($DIR/server${TEST_COUNT}.log):"
  cat $DIR/server${TEST_COUNT}.log
}

wdtExit() {
  undoLastIpTableChange
  if [ $1 -ne 0 ] ; then
      echo "Failing test $0 : Test#${TEST_COUNT} - Logs in $DIR"
  fi
  exit $1
}

checkLastCmdStatus() {
  LAST_STATUS=$?
  if [ $LAST_STATUS -ne 0 ] ; then
    echo "exiting abnormally with status $LAST_STATUS - aborting/failing test"
    printServerLog
    wdtExit $LAST_STATUS
  fi
}

checkLastCmdStatusExpectingFailure() {
  LAST_STATUS=$?
  if [ $LAST_STATUS -eq 0 ] ; then
    echo "expecting wdt failure, but transfer was successful, failing test"
    printServerLog
    wdtExit 1
  fi
}

waitForTransferEnd() {
  wait "$pidofreceiver"
  checkLastCmdStatus
  wait "$pidofsender"
  checkLastCmdStatus
}

waitForTransferEndWithoutCheckingStatus() {
  wait "$pidofreceiver"
  wait "$pidofsender"
}

waitForTransferEndExpectingFailure() {
  wait "$pidofreceiver"
  checkLastCmdStatusExpectingFailure
  wait "$pidofsender"
  checkLastCmdStatusExpectingFailure
}

killCurrentTransfer() {
  kill -9 "$pidofsender"
  kill -9 "$pidofreceiver"
  wait "$pidofsender"
  wait "$pidofreceiver"
}

generateRandomFiles() {
  if [ -z "$1" ]; then
    echo "generateRandomFile expects the directory to be passed as first \
      argument"
    wdtExit 1
  fi
  if [ -z "$2" ]; then
    echo "generateRandomFile expects base file size to be passed as second \
      argument"
    wdtExit 1
  fi
  mkdir -p "$1"
  fourx=$(echo "$2* 4" | bc -l)
  echo "Generating 4 files of $2 MBytes and 16 of $fourx Mbytes"
  for ((i = 0; i < 4; i++))
  do
    $WDT_GEN_FILES -stats_source="$WDT_GEN_BIGRAMS" \
        -directory="$1" -filename="sample${i}" -gen_size_mb="$2"
  done
  # we will generate 1G of random data. 16 files each of size 64mb.
  for ((i = 0; i < 16; i++))
  do
    $WDT_GEN_FILES -stats_source="$WDT_GEN_BIGRAMS" \
        -directory="$1" -filename="file${i}" -gen_size_mb="$fourx"
  done
}

removeDestination() {
  echo "Removing destination directory $DIR/dst${TEST_COUNT}"
  rm -rf "$DIR/dst${TEST_COUNT}"
}

verifyTransferAndCleanup() {
  if [ -z "$MD5SUM" ]; then
    MD5SUM=$(which md5sum)
    STATUS=$?
    if [ $STATUS -ne 0 ] ; then
      MD5SUM=$(which md5)
    fi
    echo "Found md5sum as $MD5SUM"
  fi

  stat $DIR/dst${TEST_COUNT}/.wdt.log 2> /dev/null
  if [ $? -eq 0 ]; then
    # transfer log present, verify for correctness
    echo "Verifying transfer log in $DIR/dst${TEST_COUNT}"
    $WDT_RECEIVER -directory $DIR/dst${TEST_COUNT} -parse_transfer_log
    checkLastCmdStatus
  fi

  if [ ! -f "$DIR/src.md5s" ]; then
    (cd "$DIR/src" ; ( find . -type f -print0 | xargs -0 "$MD5SUM" | sort ) \
      > ../src.md5s )
  fi

  if [ -d "$DIR/src1" ] && [ ! -f "$DIR/src1.md5s" ]; then
    (cd "$DIR/src1" ; ( find . -type f -print0 | xargs -0 "$MD5SUM" | sort ) \
      > ../src1.md5s )
  fi

  STATUS=0
  (cd "$DIR/dst${TEST_COUNT}" ; ( find . -type f -print0 | xargs -0 "$MD5SUM" \
    | sort | grep -v "\.wdt\.log\$") > "../dst${TEST_COUNT}.md5s" )
  echo "Verifying correctness for test $TEST_COUNT"
  echo "Should be no diff"
  USE_OTHER_SRC=$1
  if [ "$USE_OTHER_SRC" == "true" ]; then
    SRC_MD5=src1.md5s
  else
    SRC_MD5=src.md5s
  fi
  (cd "$DIR"; diff -u "$SRC_MD5" "dst${TEST_COUNT}.md5s")
  STATUS=$?
  # treating PROTOCOL_ERROR as errors
  grep "PROTOCOL_ERROR" "$DIR/server${TEST_COUNT}.log" > /dev/null
  if [ $? -eq 0 ]; then
    echo "server has PROTOCOL ERROR"
    STATUS=1
  fi
  grep "PROTOCOL_ERROR" "$DIR/client${TEST_COUNT}.log" > /dev/null
  if [ $? -eq 0 ]; then
    echo "client has PROTOCOL ERROR"
    STATUS=1
  fi

  if [ $STATUS -eq 0 ] ; then
    echo "Test $TEST_COUNT succeeded"
    removeDestination
  else
    printServerLog
    echo "Test $TEST_COUNT failed"
    wdtExit $STATUS
  fi
}

signalHandler() {
  echo "Caught signal, exiting..."
  wdtExit 1
}

extendWdtOptions() {
  WDTBIN_OPTS+=" $EXTRA_WDT_OPTIONS"
  WDTBIN_OPTS+=" $EXTRA_ENCRYPTION_CMD"
  if [ ! -z "$ENCRYPTION_TYPE" ]; then
    echo "encryption_type $ENCRYPTION_TYPE"
    WDTBIN_OPTS+=" -encryption_type=$ENCRYPTION_TYPE"
  fi
  if [ ! -z "$ENABLE_CHECKSUM" ]; then
    echo "enable_checksum $ENABLE_CHECKSUM"
    WDTBIN_OPTS+=" -enable_checksum=$ENABLE_CHECKSUM"
  fi
  WDTBIN_OPTS+=" -iv_change_interval_mb=32"
}

# without the following line, wdt piped to tee effectively has exit status of
# tee. source : http://petereisentraut.blogspot.com/2010/11/pipefail.html
set -o pipefail

trap signalHandler SIGINT SIGTERM
# Note this is not meant to be an example of a secure key, it's just for tests
# (thus the test_only in the option name) - the normal way is through the URL
# which generates a crypto. Add _secret for _secr in case of single digit pid.
TEST_ONLY_ENCRYPTION_KEY=$(echo not_a_key_$$_secret | fold -b -w 16 | head -n 1)
EXTRA_ENCRYPTION_CMD="-test_only_encryption_secret $TEST_ONLY_ENCRYPTION_KEY"


#! /bin/bash

source `dirname "$0"`/common_functions.sh
setBinaries

$WDT_RECEIVER -skip_writes -avg_mbytes_per_sec=10 -max_accept_retries=10
STATUS=$?
if [ $STATUS -ne 3 ]; then
  echo "Receiver should exit with status 3, but it exited with $STATUS"
  exit 1
fi
echo "Test passed"


#! /bin/bash

# Wrapper shell to ease use of WDT command line for remote cp
# Source of this file is fbcode/wdt/wcp.sh

# todo: make it work for multiple files and remote src/dir (full(er) scp syntax)
# and remove pkill once we have support for picking available ports
# (remote wdt will exit on its own after succesfull transfer)

usage="Usage:\n
`basename $0` [-p portbase] [-l loglevel] [-n] sourcefile
[user@]desthost:directory\n
  p : port base to use (8 sockets used by default from that port or 22356\n
  l : logging verbosity level to use\n
  n : no compression (gzip by default)\n
The wdt binary should be in the path on both source and destination\n
And wdt ports (22356-22364 unless specified with -p) should be available
on the destination"

# Defaults
MINLOGLEVEL=2
PORTBASE=22356
DOCOMPRESSION=1

while getopts "p:l:hn" opt
do case "$opt" in
        p)  PORTBASE="$OPTARG";;
        l)  MINLOGLEVEL="$OPTARG";;
        n)  DOCOMPRESSION=0;;
        h)  echo -e $usage; exit 1;;
        \?) echo -e $usage; exit 1;;
  esac
done
shift $((OPTIND-1))

if [ $# -ne 2 ] ; then
    echo -e $usage
    exit 1
fi

SRCPATH=$1
if [ ! -r $SRCPATH ] ; then
    echo "First argument ($SRCPATH) must be a readable file"
    exit 2
fi

DIR=`mktemp -d`
REMOTE=`echo $2|sed -e 's/^.*@//' -e 's/:.*//'`
SSHREMOTE=`echo $2|sed -e 's/:.*//'`
REMOTEDIR=`echo $2|sed -e 's/.*://'`
if [ -z "$REMOTEDIR" ] ; then
  REMOTEDIR=.
fi

FILENAME=`basename $SRCPATH`
SIZE=`stat -L -c %s $SRCPATH`
if [ $? -ne 0 ] ; then
  echo "Error stating $SRCPATH. aborting."
  exit 1
fi

TRANSFERID=$$

echo "Copying $FILENAME ($SRCPATH) to $REMOTE (using $SSHREMOTE in $REMOTEDIR)"

# Here we try to start asynchronously - the effect is things may fail on the
# server side and we won't know - also pkill may not be fast enough and
# data may still be sent to the previous process
echo "Starting destination side server"
if [ $DOCOMPRESSION -eq 1 ] ; then
    # Gzip compression:
    ssh -f $SSHREMOTE "pkill wdt; mkdir -p $REMOTEDIR; cd $REMOTEDIR; wdt --run_as_daemon=false --minloglevel=$MINLOGLEVEL --start_port $PORTBASE --transfer_id=$TRANSFERID && time gunzip $FILENAME.gz && echo 'Complete!';date; echo 'Dst checksum'; md5sum $FILENAME"
else
    # No compression:
    ssh -f $SSHREMOTE "pkill wdt; mkdir -p $REMOTEDIR; cd $REMOTEDIR; wdt --run_as_daemon=false --minloglevel=$MINLOGLEVEL --start_port $PORTBASE --transfer_id=$TRANSFERID; echo 'Complete!';date; echo 'Dst checksum'; md5sum $FILENAME"
fi
#REMOTEPID=$!

# Start counting time after ssh because ssh can ask for credentials/take a while
START_NANO=`date +%s%N`
START_MS=`expr $START_NANO / 1000000`
echo "Starting at `date` ($START_MS)"

if [ $DOCOMPRESSION -eq 1 ] ; then
  # Gzip
  echo "Compressing $SRCPATH ($SIZE)"
  time gzip -1 -c $SRCPATH > $DIR/$FILENAME.gz
  # LZ4
  #time lz4 -3 -z $SRCPATH - | (cd $DIR ; time split -b ${SPLIT}K - wdtTMP )
else
  # No compression
  ln -s $SRCPATH $DIR/$FILENAME
fi

date "+%s.%N"
time wdt --minloglevel=$MINLOGLEVEL --directory $DIR \
    --follow_symlinks --transfer_id=$TRANSFERID\
    --destination $REMOTE  --start_port $PORTBASE --ipv6=false # try both v4 and v6
STATUS=$?

END_NANO=`date +%s%N`
END_MS=`expr $END_NANO / 1000000`
DURATION=`expr $END_MS - $START_MS`
RATE=`expr 1000 \* $SIZE / 1024 / 1024 / $DURATION`

if [ $STATUS -eq 0 ] ; then
    echo "Succesfull transfer"
else
    echo "Failure (code $STATUS)"
fi

echo "All done client side! cleanup..."
rm -r $DIR &
#echo -n "e" | nc $REMOTE $PORTBASE

echo "Overall transfer @ $RATE Mbytes/sec ($DURATION ms for $SIZE uncompressed)"

echo "Source checksum:"
md5sum $SRCPATH

exit $STATUS


#! /bin/bash

# Modified version of wdt_max_send_test.sh to test crypto speed

echo "Run this sender performance test from fbcode/ directory"
echo "or give full path to wdt binary to use and staging directory"

set -o pipefail

if [ -z "$1" ]; then
  WDTBIN="_bin/wdt/wdt"
else
  WDTBIN="$1"
fi

if [ -z "$2" ]; then
    BASEDIR=/dev/shm/wdtTest_$USER
else
    BASEDIR=$2
fi

AWK=gawk

# TEST_COUNT is an environment variable. It is set up by the python benchmarking
# script.
if [ -z $TEST_COUNT ]; then
  TEST_COUNT=4
fi

# WDT_THROUGHPUT is an env var. Set it to overwrite the expected 16Gbyte/sec
if [ -z $WDT_THROUGHPUT ]; then
  WDT_THROUGHPUT=5000
fi

if [ -z "$HOSTNAME" ] ; then
    echo "HOSTNAME not set, will try with 'localhost'"
    HOSTNAME=localhost
else
    echo "Will self connect to HOSTNAME=$HOSTNAME"
fi

ENC_TYPE=aes128gcm
CRC=false

REMOTE=$HOSTNAME
SKIP_WRITES="true"

# Without throttling:
#WDTBIN_OPTS="-sleep_millis 1 -max_retries 3 -num_sockets 13"

#CPU normalization
NUM_CPU=`grep processor /proc/cpuinfo|wc -l`
# leave one cpu alone for the rest (or typically 2 because of /2)
NUM_THREADS=`echo $NUM_CPU/2|bc`
echo "Using $NUM_THREADS threads (on each sender, receiver) for $NUM_CPU CPUs"

# With, still gets almost same max (21G) with throttling set high enough
WDTBIN_OPTS="-sleep_millis 1 -max_retries 3 -num_ports $NUM_THREADS
-transfer_id=$$ -encryption_type=$ENC_TYPE --enable_checksum=$CRC
-exit_on_bad_flags=false"

WDTNAME=`basename $WDTBIN`
WDTCMD="$WDTBIN $WDTBIN_OPTS"

mkdir -p $BASEDIR
DIR=`mktemp -d --tmpdir=$BASEDIR`
echo "Testing $WDTBIN in $DIR"
if [ -z "$DIR" ]; then
  echo "Unable to create dir in $BASEDIR"
  exit 1
fi

pkill -x wdt
pkill -x wdt_fb
pkill -x $WDTNAME

mkdir $DIR/src

for size in 1k 64K 512K 1M 16M 256M 512M
do
    base=inp$size
    echo dd if=/dev/... of=$DIR/src/$base.1 bs=$size count=1
    dd if=/dev/zero of=$DIR/src/$base.1 bs=$size count=1
    for i in {2..32}
    do
        cp $DIR/src/$base.1 $DIR/src/$base.$i
    done
done
echo "Done with staging src test files"

#set -e

# wait for server to be up
#while [ `/bin/true | nc $REMOTE 22356; echo $?` -eq 1 ]
#do
#  echo "Server not up yet...`date`..."
#  sleep 0.5
#done
#echo "Server is up on $REMOTE 22356 - `date` - starting client run"

for ((i = 1; i <= TEST_COUNT; i++))
do
  echo "starting ${i}th run"
  TWO_PHASE_ARG=""
  # every other run will be two_phases
  [ $(($i % 2)) -eq 0 ] && TWO_PHASE_ARG="-two_phases"

  set -x
  time $WDTCMD -directory $DIR/dst$i -skip_writes=$SKIP_WRITES 2> \
    $DIR/server$i.log | time $WDTCMD -directory $DIR/src \
  $TWO_PHASE_ARG - |& tee $DIR/client$i.log
  set +x
  THROUGHPUT=`$AWK 'match($0, /.*Total sender throughput = ([0-9.]+)/, res) \
  {print res[1]} END {}' $DIR/client$i.log`
  echo "Server log:"
  cat $DIR/server$i.log
  echo "THROUGHPUT $THROUGHPUT"
  TRANSFER_TIME=`$AWK 'match($0, /.*Total sender time = ([0-9.]+)/, res) \
  {print res[1]} END {}' $DIR/client$i.log`
  echo "TRANSFER_TIME $TRANSFER_TIME"
done

MAXRATE=`$AWK 'BEGIN {max=0} match($0, /.*Total sender throughput = ([0-9.]+)/, res) {rate=res[1]; if (rate>max) max=rate} END {print int(max+.5)}' $DIR/client*.log`

ALLSPEEDS=`$AWK 'match($0, /.*Total sender throughput = ([0-9.]+)/, res) {printf("%s ", res[1])}' $DIR/client*.log`

echo "Deleting logs and staging in $DIR"
rm -rf $DIR

echo "Speed for all runs: $ALLSPEEDS"

# Normalize by CPU / number of threads
# (32 cores leaves 15 threads and can do ~20-22G, use 16 for margin of noise)
EXPECTED_SPEED=`echo $WDT_THROUGHPUT*$NUM_THREADS/16|bc`

echo "Best throughput for $WDTBIN"
if [ "$MAXRATE" -lt "$EXPECTED_SPEED" ]; then
    echo "Regression: $MAXRATE is too slow - before top"
    atop -l 1 1 | head -40
    echo "Regression: $MAXRATE is too slow for $NUM_CPU cpus ($EXPECTED_SPEED)"
    exit 1
else
    echo "Good rate $MAXRATE for $NUM_CPU cpus (threshold $EXPECTED_SPEED)"
    exit 0
fi

# all done


#! /bin/bash

source `dirname "$0"`/common_functions.sh

startNewTransfer() {
  if [ "$1" == "add" ] && [ $DELETE_EXTRA_FILES == "true" ]; then
    echo "Adding extra files for test ${TEST_COUNT}"
    fallocate -l 20M $SRC_DIR/extra_file${TEST_COUNT}_1
    fallocate -l 80M $SRC_DIR/extra_file${TEST_COUNT}_2
  fi
  if [ "$1" == "remove" ] && [ $DELETE_EXTRA_FILES == "true" ]; then
    echo "Removing extra files for test ${TEST_COUNT}"
    rm $SRC_DIR/extra_file${TEST_COUNT}_1 $SRC_DIR/extra_file${TEST_COUNT}_2
  fi
  $WDTBIN_SERVER -directory $DIR/dst${TEST_COUNT} -start_port=$STARTING_PORT \
  -transfer_id=$RECEIVER_ID -protocol_version=$RECEIVER_PROTOCOL_VERSION \
  -recovery_id=$RECOVERY_ID -disable_preallocation=$DISABLE_PREALLOCATION \
  >> $DIR/server${TEST_COUNT}.log 2>&1 &
  pidofreceiver=$!
  $WDTBIN_CLIENT -directory $SRC_DIR -destination $HOSTNAME \
  -start_port=$STARTING_PORT -block_size_mbytes=$BLOCK_SIZE_MBYTES \
  -transfer_id=$SENDER_ID -protocol_version=$SENDER_PROTOCOL_VERSION \
  -two_phases=$TWO_PHASES \
  |& tee -a $DIR/client${TEST_COUNT}.log &
  pidofsender=$!
}

usage="
The possible options to this script are
-s sender protocol version
-r receiver protocol version
-p start port
-d turns on file deletion
-c combination of options to run. Valid values are 1, 2, 3 and 4.
   1. pre-allocation and block-mode enabled, resumption done using transfer log
   2. pre-allocation disabled, block-mode enabled, resumption done using
      directory tree. This effectively disables resumption.
   3. pre-allocation and block-mode disabled, resumption done using directory
      tree
   4. pre-allocation enabled and block-mode disabled, resumption done using
      transfer log
"

#protocol versions, used to check version verification
#version 0 represents default version
SENDER_PROTOCOL_VERSION=0
RECEIVER_PROTOCOL_VERSION=0

DISABLE_PREALLOCATION=false
BLOCK_SIZE_MBYTES=10
RESUME_USING_DIR_TREE=false

RECOVERY_ID="foo"

STARTING_PORT=25000

DELETE_EXTRA_FILES=false

while getopts ":c:s:p:r:dh" opt; do
  case $opt in
    s) SENDER_PROTOCOL_VERSION="$OPTARG"
    ;;
    r) RECEIVER_PROTOCOL_VERSION="$OPTARG"
    ;;
    p) STARTING_PORT="$OPTARG"
    ;;
    d) DELETE_EXTRA_FILES=true
    ;;
    c)
      case $OPTARG in
        1) echo "pre-allocation and block-mode enabled, resumption using \
transfer log"
        # no need to change anything, settings are initialized for this case
        ;;
        2) echo "pre-allocation disabled, block-mode enabled, resumption done \
using directory tree, resumption is effectively disabled"
           DISABLE_PREALLOCATION=true
           RESUME_USING_DIR_TREE=true
        ;;
        3) echo "pre-allocation and block-mode disabled, resumption done \
using directory tree"
           BLOCK_SIZE_MBYTES=0
           DISABLE_PREALLOCATION=true
           RESUME_USING_DIR_TREE=true
        ;;
        4) echo "pre-allocation enabled, block-mode disabled, resumption done \
using transfer log"
        BLOCK_SIZE_MBYTES=0
        ;;
        *) echo "Invalid combination, valid values are 1, 2, 3 and 4"
           wdtExit 1
        ;;
      esac
    ;;
    h) echo "$usage"
        wdtExit
    ;;
    \?) echo "Invalid option -$OPTARG" >&2
    ;;
  esac
done

setBinaries

if [ -z "$TWO_PHASES" ]; then
  TWO_PHASES=false
fi

echo "sender protocol version $SENDER_PROTOCOL_VERSION, receiver protocol \
version $RECEIVER_PROTOCOL_VERSION"

if [ $RECEIVER_PROTOCOL_VERSION -ne 0 ] && [ $RECEIVER_PROTOCOL_VERSION -lt 23 \
] && [ -z $NO_ENCRYPT ]; then
  echo "Can not support encryption for receiver with version \
$RECEIVER_PROTOCOL_VERSION"
  # exit with 0 to make the test pass
  exit 0
fi

threads=4
ERROR_COUNT=5

#sender and receiver id, used to check transfer-id verification
SENDER_ID="123456"
RECEIVER_ID="123456"

WDTBIN_OPTS="-ipv6 -num_ports=$threads -full_reporting \
-max_mbytes_per_sec=25 -run_as_daemon=false \
-full_reporting -read_timeout_millis=500 -write_timeout_millis=500 \
-enable_download_resumption -treat_fewer_port_as_error \
-resume_using_dir_tree=$RESUME_USING_DIR_TREE -enable_perf_stat_collection \
-connect_timeout_millis 100 -delete_extra_files=$DELETE_EXTRA_FILES \
-exit_on_bad_flags=false"
extendWdtOptions
WDTBIN_CLIENT="$WDT_SENDER $WDTBIN_OPTS -buffer_size=2097152"
WDTBIN_SERVER="$WDT_RECEIVER $WDTBIN_OPTS -avg_mbytes_per_sec=80 "

BASEDIR=/dev/shm/wdtTest_$USER

mkdir -p $BASEDIR
DIR=`mktemp -d $BASEDIR/XXXXXX`
echo "Testing in $DIR"

#pkill -x wdt

SRC_DIR=$DIR/src

generateRandomFiles "$SRC_DIR" 4 # 4 Mb for smallest file

TEST_COUNT=0

echo "Testing that connection failure results in failed transfer"
# first create a deep directory structure
# this is done so that directory thread gets aborted before discovering any
# file
CURDIR=`pwd`
cd $DIR
for ((i = 0; i < 100; i++))
do
  mkdir d
  cd d
done
touch file
cd $CURDIR
# start the sender without starting receiver and set connect retries to 1
$WDT_SENDER -directory $DIR/d -destination $HOSTNAME -max_retries 1 \
-start_port $STARTING_PORT -num_ports $threads
checkLastCmdStatusExpectingFailure
TEST_COUNT=$((TEST_COUNT + 1))

echo "Download resumption test(1)"
startNewTransfer add
sleep 5
killCurrentTransfer
# rm a file to create an invalid log entry
rm -f $DIR/dst${TEST_COUNT}/file0
startNewTransfer remove
sleep 5
killCurrentTransfer
startNewTransfer
waitForTransferEnd
verifyTransferAndCleanup
TEST_COUNT=$((TEST_COUNT + 1))

echo "Download resumption test(2)"
startNewTransfer add
sleep 5
killCurrentTransfer
# change the file size in the receiver side
fallocate -l 70M $DIR/dst${TEST_COUNT}/file0
startNewTransfer remove
sleep 5
killCurrentTransfer
startNewTransfer
waitForTransferEnd
verifyTransferAndCleanup
TEST_COUNT=$((TEST_COUNT + 1))

echo "Download resumption test with log truncation"
startNewTransfer
sleep 5
killCurrentTransfer
LOG_FILE=$DIR/dst${TEST_COUNT}/.wdt.log
LOG_SIZE=`stat --printf="%s" $LOG_FILE`
NEW_LOG_SIZE=$(( LOG_SIZE / 2 ))
truncate -s $NEW_LOG_SIZE $LOG_FILE
startNewTransfer
waitForTransferEnd
verifyTransferAndCleanup
TEST_COUNT=$((TEST_COUNT + 1))

echo "Download resumption test with log corruption"
startNewTransfer
sleep 5
killCurrentTransfer
LOG_FILE=$DIR/dst${TEST_COUNT}/.wdt.log
# write some garbage at the end of the log
echo "abc" >> $LOG_FILE
startNewTransfer
sleep 2
killCurrentTransfer
startNewTransfer
waitForTransferEnd

# check whether we have .wdt.log.bug present or not
stat "${LOG_FILE}.bug"
if [ $? -ne 0 ]; then
  printServerLog
  echo "Transfer log corrupted, but not detected by wdt, failing test"
  wdtExit 1
fi
rm "${LOG_FILE}.bug"

verifyTransferAndCleanup
TEST_COUNT=$((TEST_COUNT + 1))

changePreallocationSettings() {
  if [ $DISABLE_PREALLOCATION == "false" ]; then
    DISABLE_PREALLOCATION="true"
  else
    DISABLE_PREALLOCATION="false"
  fi
}

if [ $RESUME_USING_DIR_TREE == "false" ]; then
  echo "Download resumption with pre-allocation settings changed in the middle"
  startNewTransfer
  sleep 5
  killCurrentTransfer
  # change pre-allocation settings
  changePreallocationSettings
  startNewTransfer
  sleep 5
  killCurrentTransfer
  # change it back
  changePreallocationSettings
  startNewTransfer
  waitForTransferEnd
  verifyTransferAndCleanup
  TEST_COUNT=$((TEST_COUNT + 1))
fi

echo "Download resumption with network error test"
startNewTransfer add
sleep 10
killCurrentTransfer
startNewTransfer remove
simulateNetworkGlitchesByDropping
waitForTransferEnd
verifyTransferAndCleanup
TEST_COUNT=$((TEST_COUNT + 1))

if [ $BLOCK_SIZE_MBYTES -gt 0 ]; then
  # try to change the block size and also disable block mode
  NEW_BLOCK_SIZES=(8 0)
else
  # try to enable block mode
  NEW_BLOCK_SIZES=(8)
fi

for NEW_BLOCK_SIZE in ${NEW_BLOCK_SIZES[*]}
do
  echo "Download resumption with network error and block size change test(1)"
  startNewTransfer
  simulateNetworkGlitchesByDropping
  killCurrentTransfer
  # change the block size for next transfer
  PRE_BLOCK_SIZE=$BLOCK_SIZE_MBYTES
  echo "Changing block size to $NEW_BLOCK_SIZE"
  BLOCK_SIZE_MBYTES=$NEW_BLOCK_SIZE
  startNewTransfer
  waitForTransferEnd
  verifyTransferAndCleanup
  echo "Resetting block size to $PRE_BLOCK_SIZE"
  BLOCK_SIZE_MBYTES=$PRE_BLOCK_SIZE
  TEST_COUNT=$((TEST_COUNT + 1))

  echo "Download resumption with network error and block size change test(2)"
  startNewTransfer
  simulateNetworkGlitchesByDropping
  killCurrentTransfer
  # change the block size for next transfer
  PRE_BLOCK_SIZE=$BLOCK_SIZE_MBYTES
  echo "Changing block size to $NEW_BLOCK_SIZE"
  BLOCK_SIZE_MBYTES=$NEW_BLOCK_SIZE
  startNewTransfer
  sleep 5
  killCurrentTransfer
  echo "Resetting block size to $PRE_BLOCK_SIZE"
  BLOCK_SIZE_MBYTES=$PRE_BLOCK_SIZE
  startNewTransfer
  waitForTransferEnd
  verifyTransferAndCleanup
  TEST_COUNT=$((TEST_COUNT + 1))
done

echo "Download resumption test for append-only file"
# truncate file0
cp $DIR/src/file0 $DIR/file0.bak
truncate -s 10M $DIR/src/file0
startNewTransfer
sleep 5
killCurrentTransfer
# restore file0
mv $DIR/file0.bak $DIR/src/file0
startNewTransfer
sleep 5
killCurrentTransfer
startNewTransfer
waitForTransferEnd
verifyTransferAndCleanup
TEST_COUNT=$((TEST_COUNT + 1))

if [ $RESUME_USING_DIR_TREE == "true" ] && [ $BLOCK_SIZE_MBYTES -eq 0 ]; then
  # combination 3
  echo "Test with download resumption disabled and block mode enabled \
for the sender in the middle"
  startNewTransfer
  sleep 2
  killCurrentTransfer
  PREV_WDT_CLIENT=$WDTBIN_CLIENT
  BLOCK_SIZE_MBYTES=1
  WDTBIN_CLIENT="$WDTBIN_CLIENT -enable_download_resumption=false"
  startNewTransfer
  simulateNetworkGlitchesByDropping
  killCurrentTransfer
  WDTBIN_CLIENT=$PREV_WDT_CLIENT
  BLOCK_SIZE_MBYTES=0
  startNewTransfer
  waitForTransferEnd
  verifyTransferAndCleanup
  TEST_COUNT=$((TEST_COUNT + 1))
fi

# abort set-up
ABORT_AFTER_SECONDS=5
ABORT_CHECK_INTERVAL_MILLIS=100
ABORT_AFTER_MILLIS=$((ABORT_AFTER_SECONDS * 1000))
EXPECTED_TRANSFER_DURATION_MILLIS=$((ABORT_AFTER_MILLIS + \
ABORT_CHECK_INTERVAL_MILLIS))
# add 800ms overhead. We need this because we can not control timeouts for disk
# writes
EXPECTED_TRANSFER_DURATION_MILLIS=$((EXPECTED_TRANSFER_DURATION_MILLIS + 800))

echo "Abort timing test(1) - Sender side abort"
WDTBIN_CLIENT_OLD=$WDTBIN_CLIENT
WDTBIN_CLIENT+=" -abort_check_interval_millis=$ABORT_CHECK_INTERVAL_MILLIS \
-abort_after_seconds=$ABORT_AFTER_SECONDS"
START_TIME_MILLIS=`date +%s%3N`
startNewTransfer
wait $pidofsender
END_TIME_MILLIS=`date +%s%3N`
wait $pidofreceiver
DURATION=$((END_TIME_MILLIS - START_TIME_MILLIS))
echo "Abort timing test, transfer duration ${DURATION} ms, expected duration \
${EXPECTED_TRANSFER_DURATION_MILLIS} ms."
if (( $DURATION > $EXPECTED_TRANSFER_DURATION_MILLIS \
  || $DURATION < $ABORT_AFTER_MILLIS )); then
  echo "Abort timing test failed, exiting"
  printServerLog
  wdtExit 1
fi
WDTBIN_CLIENT=$WDTBIN_CLIENT_OLD
removeDestination
TEST_COUNT=$((TEST_COUNT + 1))

echo "Abort timing test(2) - Receiver side abort"
WDTBIN_SERVER_OLD=$WDTBIN_SERVER
WDTBIN_SERVER+=" -abort_check_interval_millis=$ABORT_CHECK_INTERVAL_MILLIS \
-abort_after_seconds=$ABORT_AFTER_SECONDS"
# Block a port to the beginning
blockDportByDropping "$STARTING_PORT"
START_TIME_MILLIS=`date +%s%3N`
startNewTransfer
wait $pidofreceiver
END_TIME_MILLIS=`date +%s%3N`
wait $pidofsender
DURATION=$((END_TIME_MILLIS - START_TIME_MILLIS))
echo "Abort timing test, transfer duration ${DURATION} ms, expected duration \
${EXPECTED_TRANSFER_DURATION_MILLIS} ms."
undoLastIpTableChange
if (( $DURATION > $EXPECTED_TRANSFER_DURATION_MILLIS \
  || $DURATION < $ABORT_AFTER_MILLIS )); then
  echo "Abort timing test failed, exiting"
  printServerLog
  wdtExit 1
fi
WDTBIN_SERVER=$WDTBIN_SERVER_OLD
removeDestination
TEST_COUNT=$((TEST_COUNT + 1))

echo "Transfer-id mismatch test"
SENDER_ID_OLD=$SENDER_ID
SENDER_ID="abcdef"
startNewTransfer
waitForTransferEndExpectingFailure
SENDER_ID=$SENDER_ID_OLD
removeDestination
TEST_COUNT=$((TEST_COUNT + 1))

# create another src directory full of files with the same name and size as the
# actual src directory
mkdir -p $DIR/src1
cd $DIR/src1
for ((i = 0; i < 4; i++))
do
  fallocate -l 16M sample${i}
done
for ((i = 0; i < 16; i++))
do
  fallocate -l 64M file${i}
done
cd -

startAndKillIpv4Transfer() {
  PREV_WDT_CLIENT=$WDTBIN_CLIENT
  PREV_WDT_SERVER=$WDTBIN_SERVER
  WDTBIN_CLIENT+=" -ipv6=false -ipv4"
  WDTBIN_SERVER+=" -ipv6=false -ipv4"
  startNewTransfer
  sleep 5
  killCurrentTransfer
  # change it back to IPV6
  WDTBIN_CLIENT=$PREV_WDT_CLIENT
  WDTBIN_SERVER=$PREV_WDT_SERVER
}

echo "Test hostname mismatch(1)"
# start first transfer
startAndKillIpv4Transfer
# change src directory
SRC_DIR=$DIR/src1
startNewTransfer
waitForTransferEnd
SRC_DIR=$DIR/src
verifyTransferAndCleanup true
TEST_COUNT=$((TEST_COUNT + 1))

echo "Test hostname mismatch(2)"
# start first transfer
startAndKillIpv4Transfer
# change src directory
SRC_DIR=$DIR/src1
startNewTransfer
sleep 2
killCurrentTransfer
startNewTransfer
waitForTransferEnd
SRC_DIR=$DIR/src
verifyTransferAndCleanup true
TEST_COUNT=$((TEST_COUNT + 1))

echo "Test recovery-id mismatch"
startNewTransfer
sleep 5
killCurrentTransfer
# change recovery-id and src directory
RECOVERY_ID="bar"
SRC_DIR=$DIR/src1
startNewTransfer
sleep 2
killCurrentTransfer
startNewTransfer
waitForTransferEnd
SRC_DIR=$DIR/src
verifyTransferAndCleanup true
TEST_COUNT=$((TEST_COUNT + 1))

echo "Good run, deleting logs in $DIR"
rm -rf "$DIR"

wdtExit 0


#! /bin/bash

#
# Smaller/minimal version of wdt_e2e_simple_test.sh
#

source `dirname "$0"`/common_functions.sh
setBinaries

echo "Run from the cmake build dir (or ~/fbcode - or fbmake runtests)"

BASEDIR=/tmp/wdtTest_$USER
USE_ODIRECT=false
usage="
The possible options to this script are
-d base directory to use (defaults to $BASEDIR)
-o if the value is true, o_direct read is used
"

if [ "$1" == "-h" ]; then
  echo "$usage"
  exit 0
fi
while getopts ":d:h:o:" opt; do
  case $opt in
    d) BASEDIR="$OPTARG"
    ;;
    o)
    if [ "$OPTARG" == "true" ]; then
      echo "Testing with o_direct reads"
      USE_ODIRECT=true
    fi
    ;;
    h) echo "$usage"
       exit
    ;;
    \?) echo "Invalid option -$OPTARG" >&2
    ;;
  esac
done

# Set DO_VERIFY:
# to 1 : slow/expensive but checks correctness
# to 0 : fast for repeated benchmarking not for correctness
DO_VERIFY=1

if [ -z "$WDT_TEST_SYMLINKS" ] ; then
  WDT_TEST_SYMLINKS=1
fi
echo "WDT_TEST_SYMLINKS=$WDT_TEST_SYMLINKS"

# Verbose / to debug failure:
#WDTBIN="_bin/wdt/wdt -minloglevel 0 -v 99"
# Normal:
WDTBIN_OPTS="-minloglevel=0 -sleep_millis 1 -max_retries 999 -full_reporting "\
"-avg_mbytes_per_sec=3000 -max_mbytes_per_sec=3500 "\
"-num_ports=4 -throttler_log_time_millis=200"
extendWdtOptions
WDTBIN="$WDT_BINARY $WDTBIN_OPTS"
MD5SUM=`which md5sum`
STATUS=$?
if [ $STATUS -ne 0 ] ; then
  MD5SUM=`which md5`
fi
echo "Found md5sum as $MD5SUM"

if [ -z "$HOSTNAME" ] ; then
    echo "HOSTNAME not set, will try with 'localhost'"
    HOSTNAME=localhost
else
    echo "Will self connect to HOSTNAME=$HOSTNAME"
fi

mkdir -p $BASEDIR
DIR=`mktemp -d $BASEDIR/XXXXXX`
echo "Testing in $DIR"

#pkill -x wdt

mkdir $DIR/src
mkdir $DIR/extsrc

#cp -R wdt folly /usr/bin /usr/lib /usr/lib64 /usr/libexec /usr/share $DIR/src
#cp -R wdt folly /usr/bin /usr/lib /usr/lib64 /usr/libexec $DIR/src
#cp -R wdt $DIR/src

#for size in 1k 64K 512K 1M 16M 256M 512M 1G
#for size in 512K 1M 16M 256M 512M 1G
# In megabytes
for size in .0009765625 .0625 0.5 1 20
do
    base=inp$size
    for i in {1..8}
    do
        $WDT_GEN_FILES -stats_source="$WDT_GEN_BIGRAMS" \
                 -directory="$DIR/src" -filename="$base.$i" -gen_size_mb="$size"
    done
done
echo "done with setup"

if [ $WDT_TEST_SYMLINKS -eq 1 ]; then
  # test symlink issues
  (cd $DIR/src ; touch a; ln -s doesntexist badlink; dd if=/dev/zero of=c bs=1024 count=1; mkdir d; ln -s ../d d/e; ln -s ../c d/a)
  (cd $DIR/extsrc; mkdir TestDir; mkdir TestDir/test; cd TestDir; echo "Text1" >> file1; cd test; echo "Text2" >> file1; ln -s $DIR/extsrc/TestDir; cp -R $DIR/extsrc/TestDir $DIR/src)
fi

CMD="$WDTBIN -minloglevel=0 -directory $DIR/dst 2> $DIR/server.log | \
    $WDTBIN -directory $DIR/src -odirect_reads=$USE_ODIRECT - 2>&1 | \
    tee $DIR/client1.log"
echo "First transfer: $CMD"
eval $CMD
STATUS=$?
# TODO check for $? / crash... though diff will indirectly find that case

if [ $WDT_TEST_SYMLINKS -eq 1 ]; then
  CMD="$WDTBIN -minloglevel=0 -directory $DIR/dst_symlinks 2>> $DIR/server.log \
   | $WDTBIN -follow_symlinks -directory $DIR/src \
    -odirect_reads=$USE_ODIRECT - 2>&1 | tee $DIR/client2.log"
  echo "Second transfer: $CMD"
  eval $CMD
  # TODO check for $? / crash... though diff will indirectly find that case
fi

if [ $DO_VERIFY -eq 1 ] ; then
    echo "Verifying for run without follow_symlinks"
    echo "Checking for difference `date`"

    NUM_FILES=`(cd $DIR/dst && ( find . -type f | wc -l))`
    echo "Transfered `du -ks $DIR/dst` kbytes across $NUM_FILES files"

    (cd $DIR/src ; ( find . -type f -print0 | xargs -0 $MD5SUM | sort ) \
        > ../src.md5s )
    (cd $DIR/dst ; ( find . -type f -print0 | xargs -0 $MD5SUM | sort ) \
        > ../dst.md5s )

    echo "Should be no diff"
    (cd $DIR; diff -u src.md5s dst.md5s)
    STATUS=$?

  if [ $WDT_TEST_SYMLINKS -eq 1 ]; then
    echo "Verifying for run with follow_symlinks"
    echo "Checking for difference `date`"

    NUM_FILES=`(cd $DIR/dst_symlinks && ( find . -type f | wc -l))`
    echo "Transfered `du -ks $DIR/dst_symlinks` kbytes across $NUM_FILES files"

    (cd $DIR/src ; ( find -L . -type f -print0 | xargs -0 $MD5SUM | sort ) \
        > ../src_symlinks.md5s )
    (cd $DIR/dst_symlinks ; ( find . -type f -print0 | xargs -0 $MD5SUM \
        | sort ) > ../dst_symlinks.md5s )

    echo "Should be no diff"
    (cd $DIR; diff -u src_symlinks.md5s dst_symlinks.md5s)
    SYMLINK_STATUS=$?
    if [ $STATUS -eq 0 ] ; then
      STATUS=$SYMLINK_STATUS
    fi
  fi
else
    echo "Skipping independant verification"
fi

echo "Server logs:"
cat $DIR/server.log

if [ $STATUS -eq 0 ] ; then
  echo "Good run, deleting logs in $DIR"
  find $DIR -type d | xargs chmod 755 # cp -r can make some unreadable dir
  rm -rf $DIR
else
  echo "Bad run ($STATUS) - keeping full logs and partial transfer in $DIR"
fi

exit $STATUS


#! /bin/bash

# Multiple benchmarks - set DO_VERIFY=0 and
#
# wdt/wdt_e2e_test.sh |& awk '/All data.*Total throughput/ \
# {print $30} /^(real|user|sys)/ {print $0}'

# 100 loops:
#
# for i in {1..100} ;do date; wdt/wdt_e2e_test.sh \
# |& awk '/All data.*Total throughput/ {print int($30+.5)}' \
# >> ~/wdt_perfRes100; done &
#
# $ histogram -offset 7000 -divider 100 -percentile1 25 -percentile2 50 < ~/wdt_perfRes100
# # count,avg,min,max,stddev,100,8433.43,6506,9561,690.083
# # range, mid point, percentile, count
# < 7100 , 7050 , 3, 3
# >= 7100 < 7200 , 7150 , 7, 4
# >= 7200 < 7300 , 7250 , 10, 3
# >= 7300 < 7400 , 7350 , 10, 0
# >= 7400 < 7500 , 7450 , 11, 1
# >= 7500 < 7600 , 7550 , 12, 1
# >= 7600 < 7700 , 7650 , 14, 2
# >= 7700 < 7800 , 7750 , 15, 1
# >= 7800 < 7900 , 7850 , 15, 0
# >= 7900 < 8000 , 7950 , 19, 4
# >= 8000 < 8100 , 8050 , 27, 8
# >= 8100 < 8200 , 8150 , 35, 8
# >= 8200 < 8400 , 8300 , 43, 8
# >= 8400 < 8600 , 8500 , 59, 16
# >= 8600 < 8800 , 8700 , 68, 9
# >= 8800 < 9000 , 8900 , 80, 12
# >= 9000 < 9500 , 9250 , 97, 17
# >= 9500 < 10000 , 9750 , 100, 3
# # target 25.0%,8075.0
# # target 50.0%,8487.5
usage="
The possible options to this script are
-t #threads
-s specify 1 to save client logs to local dir
-a Average transfer rate to achieve
-p Peak Rate for Token Bucket
-d Delay (seconds) to introduce at receiver
"

echo "Run from ~/fbcode - or fbmake runtests"

source `dirname "$0"`/common_functions.sh
setBinaries

# Set DO_VERIFY:
# to 1 : slow/expensive but checks correctness
# to 0 : fast for repeated benchmarking not for correctness
DO_VERIFY=1
# echo e | $NC was used to stop daemon servers
#NC="nc -4" # ipv4 only

# Verbose:
#WDTBIN="_bin/wdt/wdt -minloglevel 0"
# Fastest:
BS=`expr 256 \* 1024`
threads=8
# set a default value for avg rate to test throttler
avg_rate=3000
max_rate=-1
keeplog=0
delay=0
if [ "$1" == "-h" ]; then
  echo "$usage"
  exit 0
fi
while getopts ":t:a:p:s:d:h:" opt; do
  case $opt in
    t) threads="$OPTARG"
    ;;
    s) keeplog="$OPTARG"
    ;;
    a) avg_rate="$OPTARG"
    ;;
    p) max_rate="$OPTARG"
    ;;
    d) delay="$OPTARG"
    ;;
    h) echo "$usage"
       exit
    ;;
    \?) echo "Invalid option -$OPTARG" >&2
    ;;
  esac
done
printf "(Sockets,Average rate, Max_rate, Save local?, Delay)=%s,%s,%s,%s,%s\n" "$threads" "$avg_rate" "$max_rate" "$keeplog" "$delay"
#WDTBIN_OPTS="-buffer_size=$BS -num_sockets=8 -minloglevel 2 -sleep_ms 1 -max_retries 999"
WDTBIN_OPTS="-minloglevel=0 -sleep_millis 1 -max_retries 999 -full_reporting "\
"-static_ports -avg_mbytes_per_sec=$avg_rate -max_mbytes_per_sec=$max_rate "\
"-num_ports=$threads -throttler_log_time_millis=200 "\
"-transfer_id=$$"
extendWdtOptions
WDTBIN="$WDT_BINARY $WDTBIN_OPTS"

BASEDIR=/dev/shm/wdtTest_$USER
#BASEDIR=/data/wdt/tmpWDT
mkdir -p $BASEDIR
DIR=`mktemp -d --tmpdir=$BASEDIR`
echo "Testing in $DIR"

#pkill -x wdt

mkdir $DIR/src
mkdir $DIR/extsrc

#cp -R wdt folly /usr/bin /usr/lib /usr/lib64 /usr/libexec /usr/share $DIR/src
#cp -R wdt folly /usr/bin /usr/lib /usr/lib64 /usr/libexec $DIR/src
#cp -R wdt folly /usr/bin /usr/lib $DIR/src
cp -R wdt folly /usr/bin $DIR/src
#cp -R wdt folly $DIR/src
# Removing symlinks which point to the same source tree
for link in `find -L $DIR/src -xtype l`
do
  real_path=`readlink -e $link`;
  if [[ $real_path =~ ^$DIR/src/* ]]
  then
    echo "Removing symlink $link"
    rm $link
  fi
done
#cp -R wdt $DIR/src

#for size in 1k 64K 512K 1M 16M 256M 512M 1G
#for size in 512K 1M 16M 256M 512M 1G
# In Mbytes
for size in .0009765625 .0625 0.5 1 16 256
do
    base=inp$size
    for i in {1..8}
    do
      $WDT_GEN_FILES -stats_source="$WDT_GEN_BIGRAMS" \
        -directory="$DIR/src" -filename="$base.$i" -gen_size_mb="$size"
    done
done
fallocate -l 3G $DIR/src/file_big
echo "done with setup"

# test symlink issues
(cd $DIR/src ; touch a; ln -s doesntexist badlink; dd if=/dev/zero of=c bs=1024 count=1; mkdir d; ln -s ../d d/e; ln -s ../c d/a)
(cd $DIR/extsrc; mkdir TestDir; mkdir TestDir/test; cd TestDir; echo "Text1" >> file1; cd test; echo "Text2" >> file1; ln -s $DIR/extsrc/TestDir; cp -R $DIR/extsrc/TestDir $DIR/src)

# Various smaller tests if the bigger one fails and logs are too hard to read:
#cp wdt/wdtlib.cpp wdt/wdtlib.h $DIR/src
#cp wdt/*.cpp $DIR/src
#cp /usr/bin/* $DIR/src
#cp wdt/wdtlib.cpp $DIR/src/a
#cp wdt/wdtlib.h  $DIR/src/b
#head -30 wdt/wdtlib.cpp >  $DIR/src/c
# Can't have both client and server send to stdout in parallel or log lines
# get mangled/are missing - so we redirect the server one
$WDTBIN -minloglevel=1 -directory $DIR/dst > $DIR/server.log 2>&1 &
# client now retries connects so no need wait for server to be up
pidofreceiver=$!
# Only 1 socket (single threaded send/receive)
#$WDTBIN -num_sockets=1 -directory $DIR/src -destination ::1
# Normal

#time trickle -d 1000 -u 1000 $WDTBIN -directory $DIR/src -destination $HOSTNAME |& tee $DIR/client.log
if [ $delay -gt 0 ] ; then
  (sleep 3; kill -STOP $pidofreceiver; sleep $delay; kill -CONT $pidofreceiver ) &
fi
echo "$WDTBIN -directory $DIR/src -destination $HOSTNAME |& tee $DIR/client.log"
time $WDTBIN -directory $DIR/src -destination $HOSTNAME |& tee $DIR/client.log

$WDTBIN -directory $DIR/dst_symlinks >> $DIR/server.log 2>&1 &
echo "$WDTBIN -follow_symlinks -directory $DIR/src -destination $HOSTNAME |& tee -a $DIR/client.log"
time $WDTBIN -follow_symlinks -directory $DIR/src -destination $HOSTNAME |& tee -a $DIR/client.log

# rsync test:
#time rsync --stats -v -W -r $DIR/src/ $DIR/dst/

# No need to wait for transfer to finish, client now exits when last byte is saved

if [ $DO_VERIFY -eq 1 ] ; then
    echo "Verifying for run without follow_symlinks"
    echo "Checking for difference `date`"

    NUM_FILES=`(cd $DIR/dst ; ( find . -type f | wc -l))`
    echo "Transfered `du -ks $DIR/dst` kbytes across $NUM_FILES files"

    (cd $DIR/src ; ( find . -type f -print0 | xargs -0 md5sum | sort ) \
        > ../src.md5s )
    (cd $DIR/dst ; ( find . -type f -print0 | xargs -0 md5sum | sort ) \
        > ../dst.md5s )

    echo "Should be no diff"
    (cd $DIR; diff -u src.md5s dst.md5s)
    STATUS=$?

    echo "Verifying for run with follow_symlinks"
    echo "Checking for difference `date`"

    NUM_FILES=`(cd $DIR/dst_symlinks; ( find . -type f | wc -l))`
    echo "Transfered `du -ks $DIR/dst_symlinks` kbytes across $NUM_FILES files"

    (cd $DIR/src ; ( find -L . -type f -print0 | xargs -0 md5sum | sort ) \
        > ../src_symlinks.md5s )
    (cd $DIR/dst_symlinks ; ( find . -type f -print0 | xargs -0 md5sum \
        | sort ) > ../dst_symlinks.md5s )

    echo "Should be no diff"
    (cd $DIR; diff -u src_symlinks.md5s dst_symlinks.md5s)
    SYMLINK_STATUS=$?
    if [ $STATUS -eq 0 ] ; then
      STATUS=$SYMLINK_STATUS
    fi
#(cd $DIR; ls -lR src/ dst/ )
else
    echo "Skipping independant verification"
    STATUS=0
fi

echo "Server logs:"
cat $DIR/server.log
if [ $keeplog -ne 0 ]; then
  cp $DIR/client.log client.log
fi
if [ $STATUS -eq 0 ] ; then
  echo "Good run, deleting logs in $DIR"
  find $DIR -type d | xargs chmod 755 # cp -r makes lib/locale not writeable somehow
  rm -rf $DIR
else
  echo "Bad run ($STATUS) - keeping full logs and partial transfer in $DIR"
fi

exit $STATUS


#! /bin/sh
# Invoke the regular script after finding an XFS dir to write to

XFS=`df -T 2> /dev/null | awk '$2=="xfs" {print $7}'|head -1`

if [ -z "$XFS" ] ; then
    echo "No XFS fs found. Skipping test."
    exit 0
fi

echo "Found XFS on $XFS"

if [ ! -w $XFS ] ; then
    XFS=$XFS/users/$USER
    if [ ! -w $XFS ] ; then
        echo "Can't write to either base xfs nor $XFS. Skipping test."
        exit 0
    fi
    DIR=$XFS/wdtTest
else
    DIR=$XFS/wdtTest_$USER
fi

set -x
exec `dirname $0`/wdt_e2e_simple_test.sh -d $DIR


#! /bin/bash

source `dirname "$0"`/common_functions.sh
setBinaries

transfer() {
  $WDT_BIN -skip_writes > $DIR/server${TEST_COUNT}.log 2>&1 &
  pidofreceiver=$!
  $WDT_BIN -destination=$HOSTNAME -directory=$SRC_DIR \
    2>&1 | tee $DIR/client${TEST_COUNT}.log
  SENDER_STATUS=$?
  wait $pidofreceiver
  RECEIVER_STATUS=$?
  if [ $SENDER_STATUS -ne $1 ]; then
    printServerLog
    echo "Expected sender status to be $1, but got $SENDER_STATUS"
    wdtExit 1
  fi
  if [ $RECEIVER_STATUS -ne $2 ]; then
    printServerLog
    echo "Expected receiver status to be $2, but got $RECEIVER_STATUS"
    wdtExit 1
  fi
  TEST_COUNT=$((TEST_COUNT + 1))
}

STARTING_PORT=22356
WDTBIN_OPTS="-ipv6 -start_port=$STARTING_PORT -read_timeout_millis=100 \
-write_timeout_millis=100 -connect_timeout_millis=100 -transfer_id=wdt \
-max_accept_retries=10"
extendWdtOptions
WDT_BIN="$WDT_BINARY $WDTBIN_OPTS"

BASEDIR=/tmp/wdtTest_$USER

mkdir -p $BASEDIR
DIR=`mktemp -d $BASEDIR/XXXXXX`
echo "Testing in $DIR"

TEST_COUNT=0

echo "Testing directory permission error"
SRC_DIR=$DIR/inaccessible_dir
mkdir $SRC_DIR
chmod 000 $SRC_DIR
transfer 7 7  # BYTE_SOURCE_READ_ERROR

echo "Testing file permission error"
SRC_DIR=$DIR/src1
mkdir $SRC_DIR
touch $SRC_DIR/inaccessible_file
chmod 000 $SRC_DIR/inaccessible_file
transfer 7 7  # BYTE_SOURCE_READ_ERROR

echo "Testing port block and file permission issue"
blockDportByRejecting $STARTING_PORT
transfer 7 7 # read error more "interesting" than transient network error
undoLastIpTableChange

echo "Test passed, removing $DIR"
rm -rf $DIR
wdtExit 0


#! /bin/bash

source `dirname "$0"`/common_functions.sh

setBinaries
setDirectory

# create a big file
dd if=/dev/zero of="$SRC_DIR/file" bs=536870912 count=1

WDTBIN_OPTS="-static_ports -num_ports=2 -transfer_id=wdt"
extendWdtOptions

TEST_COUNT=0
# start the server
$WDT_RECEIVER $WDTBIN_OPTS -skip_writes \
-connect_timeout_millis 100 \
-read_timeout_millis=200 > "$DIR/server${TEST_COUNT}.log" 2>&1 &
pidofreceiver=$!

# block 22356 for small duration so that file is transferred through 22357
blockDportByDropping 22356
# start client
$WDT_SENDER $WDTBIN_OPTS -destination localhost -directory "$DIR/src" \
-block_size_mbytes=-1 -avg_mbytes_per_sec=100 \
2>&1 | tee "$DIR/client${TEST_COUNT}.log" &
pidofsender=$!
sleep 0.1
undoLastIpTableChange
sleep 5
# block 22357 in the middle
blockDportByDropping 22357
waitForTransferEnd

echo "Test passed, deleting directory $DIR"
rm -rf "$DIR"
wdtExit 0


#! /bin/bash

source `dirname "$0"`/common_functions.sh
setBinaries
BASEDIR=/tmp/wdtTest_$USER
setDirectory

# Simple error test for 2 receiver

WDTBIN_OPTS="-num_ports=1 -enable_download_resumption"
WDTBIN_OPTS+=" -directory $DIR/dst"
WDTBIN="$WDT_BINARY $WDTBIN_OPTS"

# Use -fork so this blocks until url is generated/receiver started
$WDTBIN -fork -abort_after_seconds=3 > "$DIR/url1"
# Second transfer
$WDTBIN -abort_after_seconds=5 > "$DIR/url2"

STATUS=$?

echo "2nd transfer status $STATUS - should be TRANSFER_LOG_ACQUIRE_ERROR 25"

ls -l $DIR/url*

URL2=$(cat "$DIR/url2")

if [ ! -z "$URL2" ] ; then
  echo "There shouldn't be a url in 2nd run - there was $URL2"
  STATUS=-1
fi

rm -rf "$DIR"

if [ $STATUS -eq 25 ] ; then
  echo "Good run!"
  exit 0
else
  echo "Bad run! $STATUS"
  exit 1
fi


#! /bin/bash

# This is to regress/test the max ~22000Mbytes/sec transfer rate
# to get the highest (> 20G) rates you need:
# a) /dev/shm to be almost empty
# b) nothing else much running on your dev server
# c) run this a couple times

# modified from wdt_e2e_test.sh and fbonly/wdt_prof.sh

source `dirname "$0"`/common_functions.sh

echo "Run this sender performance test from fbcode/ directory"
echo "or give full path to wdt binary to use and staging directory"

#set -x
#set -o pipefail

if [ -z "$1" ]; then
  WDTBIN="_bin/wdt/wdt"
else
  WDTBIN="$1"
fi

if [ -z "$2" ]; then
    BASEDIR=/dev/shm/wdtTest_$USER
else
    BASEDIR=$2
fi

AWK=gawk

# TEST_COUNT is an environment variable. It is set up by the python benchmarking
# script.
if [ -z $TEST_COUNT ]; then
  TEST_COUNT=16
fi

# WDT_THROUGHPUT is an env var. Set it to overwrite the expected 16Gbyte/sec
if [ -z $WDT_THROUGHPUT ]; then
  WDT_THROUGHPUT=16000
fi

if [ -z "$HOSTNAME" ] ; then
    echo "HOSTNAME not set, will try with 'localhost'"
    HOSTNAME=localhost
else
    echo "Will self connect to HOSTNAME=$HOSTNAME"
fi

REMOTE=$HOSTNAME
SKIP_WRITES="true"

# TODO: switch to url

# Without throttling:
#WDTBIN_OPTS="-sleep_millis 1 -max_retries 3 -num_sockets 13"

#CPU normalization
NUM_CPU=`grep processor /proc/cpuinfo|wc -l`
# leave one cpu alone for the rest (or typically 2 because of /2)
NUM_THREADS=`echo $NUM_CPU/2|bc`
echo "Using $NUM_THREADS threads (on each sender, receiver) for $NUM_CPU CPUs"

# With, still gets almost same max (21G) with throttling set high enough
WDTBIN_OPTS="-sleep_millis 1 -max_retries 3 -num_ports $NUM_THREADS
-transfer_id=$$ -encryption_type=none
-exit_on_bad_flags=false -skip_fadvise
--avg_mbytes_per_sec=26000 --max_mbytes_per_sec=26001 --enable_checksum=false"
extendWdtOptions
CLIENT_PROFILE_FORMAT="%Uuser %Ssystem %Eelapsed %PCPU (%Xtext+%Ddata \
%Mmax)k\n%Iinputs+%Ooutputs (%Fmajor+%Rminor)pagefaults %Wswaps\nCLIENT_PROFILE %U \
%S %e"
SERVER_PROFILE_FORMAT="%Uuser %Ssystem %Eelapsed %PCPU (%Xtext+%Ddata \
%Mmax)k\n%Iinputs+%Ooutputs (%Fmajor+%Rminor)pagefaults %Wswaps\nSERVER_PROFILE %U \
%S %e"

WDTNAME=`basename $WDTBIN`
WDTCMD="$WDTBIN $WDTBIN_OPTS"

mkdir -p $BASEDIR
DIR=`mktemp -d --tmpdir=$BASEDIR`
echo "Testing $WDTBIN in $DIR"
if [ -z "$DIR" ]; then
  echo "Unable to create dir in $BASEDIR"
  exit 1
fi

pkill -x wdt
pkill -x wdt_fb
pkill -x $WDTNAME

mkdir $DIR/src

for size in 1k 64K 512K 1M 16M 256M 512M
do
    base=inp$size
    echo dd if=/dev/... of=$DIR/src/$base.1 bs=$size count=1
    dd if=/dev/zero of=$DIR/src/$base.1 bs=$size count=1
    for i in {2..32}
    do
        cp $DIR/src/$base.1 $DIR/src/$base.$i
    done
done
echo "Done with staging src test files"

#set -e

# TODO: add a test that will fail if the listening socket is closed in
# long running mode upon error (to make sure the code doesn't
# regress/handles that case ok) by adding: but that fails for other reasons
#             -max_accept_retries=1 -accept_timeout_millis=1 \

/usr/bin/time -f "$SERVER_PROFILE_FORMAT" $WDTCMD -directory $DIR/dst \
              -run_as_daemon=true -skip_writes=$SKIP_WRITES  \
                       > $DIR/server.log 2>&1 &

# wait for server to be up
#while [ `/bin/true | nc $REMOTE 22356; echo $?` -eq 1 ]
#do
#  echo "Server not up yet...`date`..."
#  sleep 0.5
#done
#echo "Server is up on $REMOTE 22356 - `date` - starting client run"

for ((i = 1; i <= TEST_COUNT; i++))
do
  echo "starting ${i}th run"
  TWO_PHASE_ARG=""
  # every other run will be two_phases
  [ $(($i % 2)) -eq 0 ] && TWO_PHASE_ARG="-two_phases"

  /usr/bin/time -f "$CLIENT_PROFILE_FORMAT" $WDTCMD -directory $DIR/src \
  -destination $REMOTE $TWO_PHASE_ARG |& tee $DIR/client$i.log
  THROUGHPUT=`$AWK 'match($0, /.*Total sender throughput = ([0-9.]+)/, res) \
  {print res[1]} END {}' $DIR/client$i.log`
  echo "THROUGHPUT $THROUGHPUT"
  TRANSFER_TIME=`$AWK 'match($0, /.*Total sender time = ([0-9.]+)/, res) \
  {print res[1]} END {}' $DIR/client$i.log`
  echo "TRANSFER_TIME $TRANSFER_TIME"
done

pkill -x $WDTNAME
# echo -n e | nc $REMOTE 22356 # end command is gone

echo "Server logs:"
cat $DIR/server.log

# Todo : count how many runs are above X... and which

MAXRATE=`$AWK 'BEGIN {max=0} match($0, /.*Total sender throughput = ([0-9.]+)/, res) {rate=res[1]; if (rate>max) max=rate} END {print int(max+.5)}' $DIR/client*.log`

ALLSPEEDS=`$AWK 'match($0, /.*Total sender throughput = ([0-9.]+)/, res) {printf("%s ", res[1])}' $DIR/client*.log`

echo "Deleting logs and staging in $DIR"
rm -rf $DIR

echo "Speed for all runs: $ALLSPEEDS"

# Normalize by CPU / number of threads
# (32 cores leaves 15 threads and can do ~20-22G, use 16 for margin of noise)
EXPECTED_SPEED=`echo $WDT_THROUGHPUT*$NUM_THREADS/16|bc`

echo "Best throughput for $WDTBIN"
if [ "$MAXRATE" -lt "$EXPECTED_SPEED" ]; then
    echo "Regression: $MAXRATE is too slow - before top"
    atop -l 1 1 | head -40
    echo "Regression: $MAXRATE is too slow for $NUM_CPU cpus ($EXPECTED_SPEED)"
    exit 1
else
    echo "Good rate $MAXRATE for $NUM_CPU cpus (threshold $EXPECTED_SPEED)"
    exit 0
fi

# all done


#! /bin/sh

# This script is mostly copied from wdt_e2e_test script. It has the same set up
# procedure. It is modified to support windtunnel integration.
# TODO: refactor this and e2e script to remove duplication of set up code

# TEST_COUNT is an environment variable. It is set up by the python benchmarking
# script.
if [ -z $TEST_COUNT ]; then
  TEST_COUNT=1
fi

WDTBIN_OPTS="-minloglevel=0 -num_ports=8 -enable_checksum=false"
if [ -z "$1" ]; then
  WDTBIN="_bin/wdt/wdt $WDTBIN_OPTS"
else
  WDTBIN="$1 $WDTBIN_OPTS"
fi
CLIENT_PROFILE_FORMAT="%Uuser %Ssystem %Eelapsed %PCPU (%Xtext+%Ddata \
%Mmax)k\n%Iinputs+%Ooutputs (%Fmajor+%Rminor)pagefaults %Wswaps\nCLIENT_PROFILE %U \
%S %e"
SERVER_PROFILE_FORMAT="%Uuser %Ssystem %Eelapsed %PCPU (%Xtext+%Ddata \
%Mmax)k\n%Iinputs+%Ooutputs (%Fmajor+%Rminor)pagefaults %Wswaps\nSERVER_PROFILE %U \
%S %e"

BASEDIR=/dev/shm/tmpWDT
mkdir -p $BASEDIR
DIR=`mktemp -d --tmpdir=$BASEDIR`
echo "Testing in $DIR"

pkill -x wdt

mkdir $DIR/src

cp -R wdt folly $DIR/src
for size in 1k 64K 512K 1M 16M 256M 512M
do
    base=inp$size
    echo dd if=/dev/... of=$DIR/src/$base.1 bs=$size count=1
    dd if=/dev/zero of=$DIR/src/$base.1 bs=$size count=1
    for i in {2..8}
    do
        cp $DIR/src/$base.1 $DIR/src/$base.$i
    done
done
echo "done with setup"

for ((i = 1; i <= TEST_COUNT; i++))
do
  TWO_PHASE_ARG=""
  # every other run will be two_phases
  [ $(($i % 2)) -eq 0 ] && TWO_PHASE_ARG="-two_phases"
  /usr/bin/time -f "$SERVER_PROFILE_FORMAT" $WDTBIN -directory $DIR/dst$i > \
  $DIR/server$i.log 2>&1 &

  # wait for server to be up
  #while [ `/bin/true | nc $HOSTNAME 22356; echo $?` -eq 1 ]
  #do
  #  echo "Server not up yet...`date`..."
  #  sleep 0.5
  #done
  #echo "Server is up on $HOSTNAME 22356 - `date` - starting client run"

  /usr/bin/time -f "$CLIENT_PROFILE_FORMAT" $WDTBIN -directory $DIR/src \
  -destination $HOSTNAME $TWO_PHASE_ARG |& tee $DIR/client$i.log
  echo -n e | nc -4 $HOSTNAME 22356
  rm -rf $DIR/dst$i
  THROUGHPUT=`awk 'match($0, /.*Total sender throughput = ([0-9.]+)/, res) \
  {print res[1]} END {}' $DIR/client$i.log`
  echo "THROUGHPUT $THROUGHPUT"
  TRANSFER_TIME=`awk 'match($0, /.*Total sender time =  ([0-9.]+)/, res) \
  {print res[1]} END {}' $DIR/client$i.log`
  echo "TRANSFER_TIME $TRANSFER_TIME"
done

echo "Server logs:"
for ((i = 1; i <= TEST_COUNT; i++))
do
  cat $DIR/server$i.log
done

find $DIR -type d | xargs chmod 755 # cp -r makes lib/locale not writeable somehow
echo "Deleting $DIR"
rm -rf $DIR


#! /bin/bash

source `dirname "$0"`/common_functions.sh

startNewTransfer() {
  echo "Starting receiver:"
  echo "$WDTBIN_SERVER $EXTRA_SERVER_ARGS -directory $DIR/dst${TEST_COUNT}"
  $WDTBIN_SERVER $EXTRA_SERVER_ARGS -directory $DIR/dst${TEST_COUNT} > \
  $DIR/server${TEST_COUNT}.log 2>&1 &
  pidofreceiver=$!
  echo "Starting sender:"
  echo "$WDTBIN_CLIENT $EXTRA_CLIENT_ARGS"
  $WDTBIN_CLIENT $EXTRA_CLIENT_ARGS |& tee -a $DIR/client${TEST_COUNT}.log &
  pidofsender=$!
  unset EXTRA_SERVER_ARGS
  unset EXTRA_CLIENT_ARGS
}

processTransferFinish() {
  waitForTransferEnd
  verifyTransferAndCleanup
  undoLastIpTableChange
  TEST_COUNT=$((TEST_COUNT + 1))
}

usage="
The possible options to this script are
-s sender protocol version
-r receiver protocol version
-p start port
-x extraoptions
"

#protocol versions, used to check version negotiation
#version 0 represents default version
SENDER_PROTOCOL_VERSION=0
RECEIVER_PROTOCOL_VERSION=0
STARTING_PORT=24000

while getopts ":s:p:r:h" opt; do
  case $opt in
    s) SENDER_PROTOCOL_VERSION="$OPTARG"
    ;;
    r) RECEIVER_PROTOCOL_VERSION="$OPTARG"
    ;;
    p) STARTING_PORT="$OPTARG"
    ;;
    h) echo "$usage"
       exit 0
    ;;
    \?) echo "$usage"
       exit 1
    ;;
  esac
done

setBinaries

echo "sender protocol version $SENDER_PROTOCOL_VERSION, receiver protocol \
version $RECEIVER_PROTOCOL_VERSION"

if [ $RECEIVER_PROTOCOL_VERSION -ne 0 ] && [ $RECEIVER_PROTOCOL_VERSION -lt 23 \
] && [ -z $NO_ENCRYPT ]; then
  echo "Can not support encryption for receiver with version \
$RECEIVER_PROTOCOL_VERSION"
  # exit with 0 to make the test pass
  exit 0
fi

threads=4
ERROR_COUNT=25
TEST_COUNT=0

BASEDIR=/dev/shm/wdtTest_$USER

mkdir -p $BASEDIR
DIR=`mktemp -d --tmpdir=$BASEDIR`
echo "Testing in $DIR"

WDTBIN_OPTS="-enable_perf_stat_collection -ipv6 -start_port=$STARTING_PORT \
-avg_mbytes_per_sec=15 -max_mbytes_per_sec=20 -run_as_daemon=false \
-full_reporting -read_timeout_millis=495 -write_timeout_millis=495 \
-progress_report_interval_millis=-1 -abort_check_interval_millis=100 \
-treat_fewer_port_as_error -exit_on_bad_flags=false -skip_fadvise \
-connect_timeout_millis 100 -transfer_id $$ -num_ports=$threads"
extendWdtOptions
WDTBIN_SERVER="$WDT_RECEIVER $WDTBIN_OPTS \
  -protocol_version=$RECEIVER_PROTOCOL_VERSION"
WDTBIN_CLIENT="$WDT_SENDER $WDTBIN_OPTS -destination $HOSTNAME \
  -protocol_version=$SENDER_PROTOCOL_VERSION -directory $DIR/src"
#pkill -x wdt

mkdir -p $DIR/src/dir1
cp -R folly $DIR/src/dir1
for ((i = 2; i <= 50; i++))
do
  mkdir $DIR/src/dir${i}
  cp -R $DIR/src/dir1 $DIR/src/dir${i}
done

# Testing with different start ports
echo "Testing with different start ports in sender and receiver"
EXTRA_CLIENT_ARGS="-start_port=$((STARTING_PORT + 1))"
startNewTransfer
processTransferFinish

# Testing with different less number of threads in sender
echo "Testing with less number of threads in client"
EXTRA_CLIENT_ARGS="-num_ports=$((threads - 1))"
startNewTransfer
processTransferFinish

echo "Testing with more number of threads in client"
EXTRA_CLIENT_ARGS="-num_ports=$((threads + 1))"
startNewTransfer
processTransferFinish

# Blocking sender port before transfer by
echo "Testing with port blocked before transfer(1)"
blockSportByDropping "$STARTING_PORT"
startNewTransfer
processTransferFinish

echo "Testing with port blocked before transfer(2)"
blockDportByDropping "$STARTING_PORT"
startNewTransfer
processTransferFinish

# Blocking a port in the middle of the transfer
echo "Testing by blocking a port in the middle of the transfer(1)"
startNewTransfer
sleep 5
echo "blocking $STARTING_PORT"
blockSportByDropping "$STARTING_PORT"
processTransferFinish

echo "Testing by blocking a port in the middle of the transfer(2)"
startNewTransfer
sleep 5
PORT_TO_BLOCK=$((STARTING_PORT + 1))
echo "blocking $PORT_TO_BLOCK"
blockDportByDropping "$PORT_TO_BLOCK"
processTransferFinish

echo "Testing by blocking a port in the middle of the transfer and more \
client threads"
EXTRA_CLIENT_ARGS="-num_ports=$((threads + 1))"
startNewTransfer
sleep 5
PORT_TO_BLOCK=$((STARTING_PORT + 1))
echo "blocking $PORT_TO_BLOCK"
blockDportByDropping "$PORT_TO_BLOCK"
processTransferFinish

echo "Testing by blocking a port in the middle of the transfer and less \
client threads"
EXTRA_CLIENT_ARGS="-num_ports=$((threads - 1))"
startNewTransfer
sleep 5
PORT_TO_BLOCK=$((STARTING_PORT + 1))
echo "blocking $PORT_TO_BLOCK"
blockDportByDropping "$PORT_TO_BLOCK"
processTransferFinish

# Simulating network glitches by rejecting packets
echo "Simulating network glitches by rejecting packets"
startNewTransfer
simulateNetworkGlitchesByRejecting
processTransferFinish

# Simulating network glitches by dropping packets
echo "Simulating network glitches by dropping packets"
startNewTransfer
simulateNetworkGlitchesByDropping
processTransferFinish

echo "Good run, deleting logs in $DIR"
rm -rf "$DIR"

wdtExit 0


#! /bin/bash

# Tests for option types can not be run in the same process because gflags
# can not be reset to default condition. This script runs all the tests one by
# one in separate processes

set -e
set -o pipefail

if [ "$#" -eq 0 ]; then
  binaries=(_bin/wdt/option_type_test_long_flags \
    _bin/wdt/short_flags/option_type_test_short_flags)
else
  binaries=$*
fi

for binary in "${binaries[@]}"
do
  ALL_TESTS=$($binary --gtest_list_tests|tail -n +2 |  sed 's/^ *//g')
  for test_name in $ALL_TESTS
  do
    echo "$binary --gtest_filter=*.$test_name"
    $binary --gtest_filter="*.$test_name"
    LAST_STATUS=$?
    if [ $LAST_STATUS -ne 0 ] ; then
      exit $LAST_STATUS
    fi
  done
done
exit 0


#! /bin/bash

if [ "$#" -eq 0 ]; then
  binary=_bin/wdt/wdt_url_test
else
  binary=$1
fi

set -o pipefail
set -e

searchStr='Generated a transfer id'
transferId1=$($binary -v=1 2>&1 | grep "$searchStr" | head -1 | \
  awk '{print $10}')
transferId2=$($binary -v=1 2>&1 | grep "$searchStr" | head -1 | \
  awk '{print $10}')

if [ "$transferId1" == "$transferId2" ]; then
  echo "Failed to get different transfer-id $transferId1"
  exit 1
fi
echo "Test successful $transferId1 $transferId2"
exit 0


#! /bin/sh

#set -e
# modified from wdt_e2e_test.sh and fbonly/wdt_prof.sh

SRCHOST=$1
DSTHOST=$2
if [ -z $DSTHOST ] ; then
    echo "Usage: $0 srchost desthost"
    exit 1
fi

#SKIP_WRITES="false"
SKIP_WRITES="true"

echo "Run from ~fbcode (wdt's parent dir). Skip writes is $SKIP_WRITES"

#WDTBIN_OPTS="-minloglevel 2 -sleep_millis 10 -max_retries 5 -num_sockets 15"
WDTBIN_OPTS="-sleep_millis 1 -max_retries 3 -num_ports 16"
#WDTBIN_OPTS="-sleep_millis 1 -max_retries 3 -num_sockets 16 -ipv4=true"

BASEDIR=/dev/shm/wdtTest_$USER

# Version with profiler:
#WDTNAME="wdt_prof"
#WDTDIR="_bin/wdt/fbonly"
# Non profiler version
WDTNAME="wdt"
WDTORIGDIR="_bin/wdt"
WDTDIR="$BASEDIR/_bin/wdt"
WDTBIN="$WDTDIR/$WDTNAME"
WDTCMD="$WDTBIN $WDTBIN_OPTS"

# For prod
REMOTEUSER="root"
RSHDST="ssh -l $REMOTEUSER $DSTHOST"
RSHSRC="ssh -l $REMOTEUSER $SRCHOST"
SCP="scp"
NC="nc -4" # ipv4 only

#BASEDIR=/data/wdt/tmpWDT
mkdir -p $BASEDIR
DIR=`mktemp -d --tmpdir=$BASEDIR`
echo "Testing in $DIR"

CMD="pkill -x $WDTNAME; mkdir -p $DIR/src ; mkdir -p $WDTDIR"
$RSHSRC $CMD
$RSHDST $CMD
$SCP $WDTORIGDIR/$WDTNAME $REMOTEUSER@$DSTHOST:$WDTBIN
$SCP $WDTORIGDIR/$WDTNAME $REMOTEUSER@$SRCHOST:$WDTBIN

# Start server/recipient first (so it's likely ready by the time we are done
# staging the src)
echo "Starting server on destination side $DSTHOST"
$RSHDST "date; $WDTCMD -directory $DIR/dst -skip_writes=$SKIP_WRITES > $DIR/server.log 2>&1 &"

#cp -R wdt folly /usr/bin /usr/lib /usr/lib64 /usr/libexec /usr/share $DIR/src
#cp -R wdt folly /usr/bin /usr/lib /usr/lib64 /usr/libexec $DIR/src
# TODO get a better test/load generation - remote execute the for below for ex
#$RSHSRC "cp -R /usr/bin /usr/share $DIR/src"

#$RSHSRC "cp -R /usr/bin /usr/lib /usr/lib64 $DIR/src"
echo "Staging on source side"
$RSHSRC "dd if=/dev/zero of=$DIR/src/big.1 bs=16M count=1; for i in {2..64} ; do ln $DIR/src/big.1 $DIR/src/big.\$i; done; ls -lh $DIR/src"

echo 'done with setup'

echo "Staging copy done on $SRCHOST, checking $DSTHOST"

# wait for server to be up
while [ `$RSHSRC "/bin/true | $NC $DSTHOST 22356; echo $?"` -eq 1 ]
do
 echo "Destination server not up yet...`date`..."
 sleep 0.5
done
echo "Server is up on $DSTHOST 22356 - `date`"

# Only 1 socket (single threaded send/receive)
#$WDTCMD -num_sockets=1 -directory $DIR/src -destination ::1
# Normal

#time trickle -d 1000 -u 1000 $WDTCMD -directory $DIR/src -destination $HOSTNAME |& tee $DIR/client.log
$RSHSRC "time $WDTCMD -directory $DIR/src -destination $DSTHOST |& tee $DIR/client.log"

echo "2nd run of client"
$RSHSRC "time $WDTCMD -directory $DIR/src -destination $DSTHOST |& tee $DIR/client2.log"

echo "3nd run of client with 2 phases"
$RSHSRC "time  $WDTCMD -directory $DIR/src -destination $DSTHOST -two_phases |& tee $DIR/client3.log"

echo "Skipping independant verification"
STATUS=0

echo "Making the server end gracefully"
$RSHSRC "echo -n e | $NC $DSTHOST 22356"

echo "Server logs:"
$RSHDST "cat $DIR/server.log"

echo "Deleting logs in $DIR"
$RSHSRC "find $DIR -type d | xargs chmod 755; rm -rf $DIR"
$RSHDST "pkill -x wdt; rm -rf $DIR"

echo "All done with testing from $SRCHOST to $DSTHOST"

exit $STATUS


#! /bin/bash

source `dirname "$0"`/common_functions.sh

setBinaries
setDirectory
generateRandomFiles "$SRC_DIR" .015625 # in Mbytes: 16k smallest file

WDTBIN_OPTS="-fork -abort_after_seconds 8"
extendWdtOptions

TEST_COUNT=0
# Test stdin pipe:

( $WDT_RECEIVER $WDTBIN_OPTS -directory $DIR/dst${TEST_COUNT} \
    2> "$DIR/server${TEST_COUNT}.log" ; cd $SRC_DIR; ls -1) \
 | $WDT_SENDER $WDTBIN_OPTS -directory $SRC_DIR -manifest - - \
    2>&1 | tee "$DIR/client${TEST_COUNT}.log"

checkLastCmdStatus
verifyTransferAndCleanup

TEST_COUNT=1

( $WDT_RECEIVER $WDTBIN_OPTS -directory $DIR/dst${TEST_COUNT} \
    2> "$DIR/server${TEST_COUNT}.log" ; cd $SRC_DIR ) \
 | $WDT_SENDER $WDTBIN_OPTS -directory $SRC_DIR -manifest - - \
    2>&1 | tee "$DIR/client${TEST_COUNT}.log"

checkLastCmdStatus

if [ $(ls -1 $DIR/dst${TEST_COUNT} | wc -l) -ne 0 ] ; then
  echo "This shouldnt have transferred any files (empty fileInfo)"
  wdtExit 1
fi

echo "Test passed, deleting directory $DIR"
rm -rf "$DIR"
wdtExit 0


# shellcheck disable=SC2148
# runs wdt with certain parameters for thread, average transfer rate, and peak rate
# peak rate is just a multiplier over average rate here
# This thread can be used by specifying a multiplier and the results of the
# logs for this experiment can be plotted.
BASE_DIR="/tmp/wdt_plots"
DATADIR=/dev/shm/tmpWDT
mkdir -p $BASE_DIR
DIR=`mktemp -d --tmpdir=$DATADIR`
echo "Testing in $DIR"
pkill -x wdt
mkdir $DIR/src
cp -R wdt folly /usr/share $DIR/src
# Removing symlinks which point to the same source tree
for link in `find -L $DIR/src -xtype l`
do
  real_path=`$REALPATH $link`;
  if [[ $real_path =~ ^$DIR/src/* ]]
  then
    rm $link
  fi
done
for size in 1k 64K 512K 1M 16M 256M 512M
do
    base=inp$size
    echo dd if=/dev/... of=$DIR/src/$base.1 bs=$size count=1
    dd if=/dev/zero of=$DIR/src/$base.1 bs=$size count=1
    for i in {2..8}
    do
        cp $DIR/src/$base.1 $DIR/src/$base.$i
    done
done
echo "done with setup"
printf "Starting the receiver"
WDTBIN="_bin/wdt/wdt -directory=$DIR/dst"
$WDTBIN -minloglevel=1 -directory $DIR/dst > $DIR/server.log 2>&1 &
pidofreceiver=$!
doit() {
  transfer_rate=$1
  peak_rate="$(echo "$transfer_rate*$2" | bc -l)"
  delay=$3
  printf "Running experiment with (Avg_Rate, Peak_Rate, Delay)=%s,%s,%s\n" "$transfer_rate" "$peak_rate" "$delay"
  echo $transfer_rate
  WDTBIN_OPTS="-minloglevel=0 -sleep_millis 1 -max_retries 999 -full_reporting -avg_mbytes_per_sec=$transfer_rate -max_mbytes_per_sec=$peak_rate -throttler_log_time_millis=200"
  if [ $delay -gt 0 ] ; then
    (sleep 3; kill -STOP $pidofreceiver; sleep $delay; kill -CONT $pidofreceiver ) &
  fi
  echo "$WDTBIN $WDTBIN_OPTS -directory $DIR/src -destination $HOSTNAME |& tee $DIR/client.log"
  time $WDTBIN $WDTBIN_OPTS -directory $DIR/src -destination $HOSTNAME |& tee $DIR/client.log
  grep  Throttler:Transfer_Rates "$DIR/client.log" | awk -F '::' '{print $2}' > wdt_run_plot
  gnuplot << EOF
    set xlabel "Time(sec)"
    set ylabel "Throughput (MB)"
    set term png
    set output "plot_{$transfer_rate}_{$peak_rate}.png"
    plot "wdt_run_plot" using 1:2 with lines title "Avg Throughput", "wdt_run_plot" using 1:3 with lines title "Instant throughput"
EOF
  mv "plot_{$transfer_rate}_{$peak_rate}.png" "$BASE_DIR/plot_{$transfer_rate}_{$peak_rate}.png"
  cp "$DIR/client.log" "$BASE_DIR/plot_{$transfer_rate}_{$peak_rate}.logs"
  rm "wdt_run_plot"
  printf "Created the plot at $BASE_DIR/plot_{$transfer_rate}_{$peak_rate}.png"
  rm -rf "$DIR/dst/*"
  sleep 2
}
if [[ ! -d "$BASE_DIR" ]] ; then
  mkdir "$BASE_DIR"
  printf "Creating a directory $BASE_DIR"
else
  rm -rf "$BASE_DIR/*"
  printf "Deleteing contents of $BASE_DIR"
fi
delay=5 #introduce delay of 5 seconds
max_runs=1 #number of runs for averaging
start=100
end=200
for (( num_runs=1; num_runs<=$max_runs; num_runs+=1 ))
do
  for (( t=$start; t<= $end; t+=100))
  do
    for mul in {1.1,1.2,1.3,1.5}
    do
      doit $t $mul $delay
    done
  done
done
printf "Killing the receiver"
kill -9 $pidofreceiver
rm -rf "$DIR/src"
rm -rf "$DIR/dst"


#!/bin/sh

# Temporary compiled binary
OUT_FILE="tempbin"

# Optional temporary compiled WebAssembly
OUT_WASM="temp.wasm"

# Source files to compile using Emscripten.
IN_FILES="examples/emscripten.c"

# Emscripten build using emcc.
emscripten_emcc_build() {
  # Compile the same example as above
  CC_FLAGS="-Wall -Wextra -Wshadow -Werror -Os -g0 -flto"
  emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
  # Did compilation work?
  if [ $? -ne 0 ]; then
    echo "Compiling ${IN_FILES}: FAILED"
    exit 1
  fi
  echo "Compiling ${IN_FILES}: PASSED"
  rm -f $OUT_WASM
}

# Emscripten build using docker.
emscripten_docker_build() {
  docker container run --rm \
    --volume $PWD:/code \
    --workdir /code \
    emscripten/emsdk:latest \
    emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
  # Did compilation work?
  if [ $? -ne 0 ]; then
      echo "Compiling ${IN_FILES} (using docker): FAILED"
    exit 1
  fi
  echo "Compiling ${IN_FILES} (using docker): PASSED"
  rm -f $OUT_WASM
}

# Try Emscripten build using emcc or docker.
try_emscripten_build() {
  which emcc > /dev/null
  if [ $? -eq 0 ]; then
    emscripten_emcc_build
    return $?
  fi

  which docker > /dev/null
  if [ $? -eq 0 ]; then
    emscripten_docker_build
    return $?
  fi

  echo "(Skipping Emscripten test)"
}

# Amalgamate the sources
./create_single_file_decoder.sh
# Did combining work?
if [ $? -ne 0 ]; then
  echo "Single file decoder creation script: FAILED"
  exit 1
fi
echo "Single file decoder creation script: PASSED"

# Compile the generated output
cc -Wall -Wextra -Wshadow -Werror -Os -g0 -o $OUT_FILE examples/simple.c
# Did compilation work?
if [ $? -ne 0 ]; then
  echo "Compiling simple.c: FAILED"
  exit 1
fi
echo "Compiling simple.c: PASSED"

# Run then delete the compiled output
./$OUT_FILE
retVal=$?
rm -f $OUT_FILE
# Did the test work?
if [ $retVal -ne 0 ]; then
  echo "Running simple.c: FAILED"
  exit 1
fi
echo "Running simple.c: PASSED"

# Try Emscripten build if emcc or docker command is available.
try_emscripten_build

exit 0


#!/bin/sh

# Where to find the sources (only used to copy zstd.h)
ZSTD_SRC_ROOT="../../lib"

# Temporary compiled binary
OUT_FILE="tempbin"

# Optional temporary compiled WebAssembly
OUT_WASM="temp.wasm"

# Source files to compile using Emscripten.
IN_FILES="zstd.c examples/roundtrip.c"

# Emscripten build using emcc.
emscripten_emcc_build() {
  # Compile the same example as above
  CC_FLAGS="-Wall -Wextra -Wshadow -Werror -Os -g0 -flto"
  emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
  # Did compilation work?
  if [ $? -ne 0 ]; then
    echo "Compiling ${IN_FILES}: FAILED"
    exit 1
  fi
  echo "Compiling ${IN_FILES}: PASSED"
  rm -f $OUT_WASM
}

# Emscripten build using docker.
emscripten_docker_build() {
  docker container run --rm \
    --volume $PWD:/code \
    --workdir /code \
    emscripten/emsdk:latest \
    emcc $CC_FLAGS -s WASM=1 -I. -o $OUT_WASM $IN_FILES
  # Did compilation work?
  if [ $? -ne 0 ]; then
      echo "Compiling ${IN_FILES} (using docker): FAILED"
    exit 1
  fi
  echo "Compiling ${IN_FILES} (using docker): PASSED"
  rm -f $OUT_WASM
}

# Try Emscripten build using emcc or docker.
try_emscripten_build() {
  which emcc > /dev/null
  if [ $? -eq 0 ]; then
    emscripten_emcc_build
    return $?
  fi

  which docker > /dev/null
  if [ $? -eq 0 ]; then
    emscripten_docker_build
    return $?
  fi

  echo "(Skipping Emscripten test)"
}

# Amalgamate the sources
./create_single_file_library.sh
# Did combining work?
if [ $? -ne 0 ]; then
  echo "Single file library creation script: FAILED"
  exit 1
fi
echo "Single file library creation script: PASSED"

# Copy the header to here (for the tests)
cp "$ZSTD_SRC_ROOT/zstd.h" examples/zstd.h

# Compile the generated output
cc -Wall -Wextra -Werror -Wshadow -pthread -I. -Os -g0 -o $OUT_FILE zstd.c examples/roundtrip.c
# Did compilation work?
if [ $? -ne 0 ]; then
  echo "Compiling roundtrip.c: FAILED"
  exit 1
fi
echo "Compiling roundtrip.c: PASSED"

# Run then delete the compiled output
./$OUT_FILE
retVal=$?
rm -f $OUT_FILE
# Did the test work?
if [ $retVal -ne 0 ]; then
  echo "Running roundtrip.c: FAILED"
  exit 1
fi
echo "Running roundtrip.c: PASSED"

# Try Emscripten build if emcc or docker command is available.
try_emscripten_build

exit 0