#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

SCRIPT="$(realpath "$0")"
SCRIPTPATH="$(dirname "$SCRIPT")"
RUNNER_FRAMEWORK_DIR="$SCRIPTPATH/FBTestRunnerApp/Frameworks"
XCODE_PATH=$(xcode-select -p)

if [ ! -d "$RUNNER_FRAMEWORK_DIR" ]; then
  mkdir "$RUNNER_FRAMEWORK_DIR"
fi

rm -rf "${RUNNER_FRAMEWORK_DIR:-}/*"
cp -rf "$XCODE_PATH/Platforms/iPhoneSimulator.platform/Developer/Library/Frameworks/XCTest.framework" "$RUNNER_FRAMEWORK_DIR"
cp -rf "$XCODE_PATH/Platforms/iPhoneSimulator.platform/Developer/Library/PrivateFrameworks/XCTAutomationSupport.framework" "$RUNNER_FRAMEWORK_DIR"


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

#
# Build script for `idx`. Run using `npm run build`.
#

set -e
set -u

ROOT_DIR=$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")

babel "$ROOT_DIR/src" --out-dir "$ROOT_DIR/lib" --ignore test.js --copy-files

# Strip `@providesModule` from lib/**/*.js.
find "$ROOT_DIR/lib" -type f -name '*.js' -exec \
  sed -i '' '/^ \* @providesModule /d' {} \;


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -e

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

# make sure we run from the root of the repo
pushd "$SCRIPT_DIR" > /dev/null

# We need to record the date that the documentation was last modified to put in our man
# pages. Unfortunately that information is only available reliably from `git`, which we don't have
# access to from other distributions of the infer source code. Such source distributions should
# distribute the "configure" script too. The idea is to bake this date inside "configure" so that
# it's available at build time. We do that by generating an m4 macro that hardcodes the date we
# compute in this script for "configure" to find.
MAN_LAST_MODIFIED_M4=m4/__GENERATED__ac_check_infer_man_last_modified.m4
printf 'generating %s' "$MAN_LAST_MODIFIED_M4... "
if test -d '.git' ; then
  # date at which the man pages were last modified, to record in the manpages themselves
  MAN_FILES=(
      infer/src/base/CommandLineOption.ml
      infer/src/base/Config.ml
  )
  MAN_DATE=$(git log -n 1 --pretty=format:%cd --date=short -- "${MAN_FILES[@]}")
  INFER_MAN_LAST_MODIFIED=${INFER_MAN_LAST_MODIFIED:-$MAN_DATE}
else
  echo 'no git repository detected; setting last modified date to today'
  # best effort: get today's date
  INFER_MAN_LAST_MODIFIED=${INFER_MAN_LAST_MODIFIED:-$(date +%Y-%m-%d)}
fi

printf "AC_DEFUN([AC_CHECK_INFER_MAN_LAST_MODIFIED],\n" > "$MAN_LAST_MODIFIED_M4"
printf "[INFER_MAN_LAST_MODIFIED=%s\n" "$INFER_MAN_LAST_MODIFIED" >> "$MAN_LAST_MODIFIED_M4"
printf " AC_SUBST([INFER_MAN_LAST_MODIFIED])\n" >> "$MAN_LAST_MODIFIED_M4"
printf "])\n" >> "$MAN_LAST_MODIFIED_M4"
printf 'done\n'

# older versions of `autoreconf` only support including macros via acinclude.m4
ACINCLUDE="acinclude.m4"
printf "generating $ACINCLUDE..."
cat m4/*.m4 > "$ACINCLUDE"
printf " done\n"

printf "generating ./configure script..."
autoreconf -fi
printf " done\n"

echo ""
echo "you may now run the following commands to build Infer:"
echo ""
echo "  ./configure"
echo "  make"
echo ""
echo 'run `./configure --help` for more options'


#!/usr/bin/env bash

# Convenience script to build Infer when using opam

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -e
set -o pipefail
set -u

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
INFER_ROOT="$SCRIPT_DIR"
DEPENDENCIES_DIR="$INFER_ROOT/facebook/dependencies"
PLATFORM="$(uname)"
SANDCASTLE=${SANDCASTLE:-}
NCPU="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)"
INFER_OPAM_DEFAULT_SWITCH="4.14.0+flambda"
INFER_OPAM_DEFAULT_SWITCH_OPTIONS="--package=ocaml-variants.4.14.0+options,ocaml-option-flambda"
INFER_OPAM_SWITCH=${INFER_OPAM_SWITCH:-$INFER_OPAM_DEFAULT_SWITCH}
INFER_OPAM_SWITCH_OPTIONS=${INFER_OPAM_SWITCH_OPTIONS:-$INFER_OPAM_DEFAULT_SWITCH_OPTIONS}
PLUGIN_DIR="$INFER_ROOT/facebook-clang-plugins"
PLUGIN_SETUP_SCRIPT=${PLUGIN_SETUP_SCRIPT:-setup.sh}
PLUGIN_SETUP="${PLUGIN_DIR}/clang/${PLUGIN_SETUP_SCRIPT}"

function usage() {
  echo "Usage: $0 [options] [targets] [-- options for ./configure]"
  echo
  echo " targets:"
  echo "   all      build everything (default)"
  echo "   clang    build C and Objective-C analyzer"
  echo "   erlang   build Erlang analyzer"
  echo "   hack     build Hack analyzer"
  echo "   java     build Java analyzer"
  echo "   python   build Python analyzer"
  echo
  echo " options:"
  echo "   -h,--help             show this message"
  echo "   --no-opam-lock        do not use the opam/infer.opam.locked file and let opam resolve dependencies"
  echo "   --only-setup-opam     initialize opam, install the opam dependencies of infer, and exit"
  echo "   --user-opam-switch    use the current opam switch to install infer (default: $INFER_OPAM_DEFAULT_SWITCH)"
  echo "   -y,--yes              automatically agree to everything"
  echo "   --                    stop parsing options and pass the remaining ones to ./configure"
  echo
  echo " examples:"
  echo "    $0                                 # build infer with default options"
  echo "    $0 -- --help                       # display the options from ./configure"
  echo "    $0 java                            # build only the Java analyzer"
  echo "    $0 -y -- --disable-clang-analyzers # build everything but clang support, agree to all questions"
}

# arguments
CONFIGURE_OPTS=()
BUILD_CLANG=${BUILD_CLANG:-no}
BUILD_ERLANG=${BUILD_ERLANG:-no}
BUILD_HACK=${BUILD_HACK:-no}
BUILD_JAVA=${BUILD_JAVA:-no}
BUILD_PYTHON=${BUILD_PYTHON:-no}
INTERACTIVE=${INTERACTIVE:-yes}
JOBS=${JOBS:-$NCPU}
ONLY_SETUP_OPAM=${ONLY_SETUP_OPAM:-no}
USE_OPAM_LOCK=${USE_OPAM_LOCK:-yes}
USER_OPAM_SWITCH=no

ORIG_ARGS="$*"

function build_all() {
  BUILD_CLANG=yes
  BUILD_ERLANG=yes
  BUILD_HACK=yes
  BUILD_JAVA=yes
  BUILD_PYTHON=yes
}

while [[ $# -gt 0 ]]; do
  opt_key="$1"
  case $opt_key in
    all)
      build_all
      shift
      continue
      ;;
    clang)
      BUILD_CLANG=yes
      shift
      continue
      ;;
    erlang)
      BUILD_ERLANG=yes
      shift
      continue
      ;;
    hack)
      BUILD_HACK=yes
      shift
      continue
      ;;
    java)
      BUILD_JAVA=yes
      shift
      continue
      ;;
    python)
      BUILD_PYTHON=yes
      shift
      continue
      ;;
    -h|--help)
      usage
      exit 0
     ;;
    --no-opam-lock)
      USE_OPAM_LOCK=no
      shift
      continue
     ;;
    --user-opam-switch)
      USER_OPAM_SWITCH=yes
      shift
      continue
     ;;
    --only-setup-opam)
      ONLY_SETUP_OPAM=yes
      shift
      continue
     ;;
    -y|--yes)
      INTERACTIVE=no
      shift
      continue
     ;;
    --)
      shift
      CONFIGURE_OPTS+=("$*")
      shift $#
      continue
     ;;
     *)
      usage
      exit 1
  esac
  shift
done

if [ "$BUILD_CLANG" == "no" ] && [ "$BUILD_ERLANG" == "no" ] && \
   [ "$BUILD_HACK" == "no" ] && [ "$BUILD_JAVA" == "no" ] && \
   [ "$BUILD_PYTHON" == "no" ]; then
  build_all
fi

# enable --yes option for some commands in non-interactive mode
YES=
if [ "$INTERACTIVE" == "no" ]; then
  YES=--yes
fi
# --yes by default for opam commands except if we are using the user's opam switch
if [ "$INTERACTIVE" == "no" ] || [ "$USER_OPAM_SWITCH" == "no" ]; then
    export OPAMYES=true
fi

setup_opam () {
    opam var root 1>/dev/null 2>/dev/null || opam init --reinit --bare --no-setup &&
    opam_retry opam_switch_create_if_needed "$INFER_OPAM_SWITCH" "$INFER_OPAM_SWITCH_OPTIONS" &&
    opam switch set "$INFER_OPAM_SWITCH"
}

install_opam_deps () {
    local locked=
    if [ "$USE_OPAM_LOCK" == yes ]; then
        locked=.locked
    fi
    opam install --deps-only "$INFER_ROOT"/opam/infer.opam$locked
}

echo "initializing opam... " >&2
. "$INFER_ROOT"/scripts/opam_utils.sh
if [ "$USER_OPAM_SWITCH" == "no" ]; then
    setup_opam
fi
eval $(SHELL=bash opam env)
echo >&2
echo "installing infer dependencies; this can take up to 30 minutes... " >&2
opam_retry install_opam_deps

if [ "$ONLY_SETUP_OPAM" == "yes" ]; then
  exit 0
fi

echo "preparing build... " >&2
./autogen.sh > /dev/null

# we want to add these before the ones passed explicitly by the user, if only because they
# correspond to options that appear earlier on the command line of this script so it's the more
# natural behaviour
CONFIGURE_PREPEND_OPTS=""
if [ "$BUILD_CLANG" == "no" ]; then
  CONFIGURE_PREPEND_OPTS+=" --disable-c-analyzers"
fi
if [ "$BUILD_ERLANG" == "no" ]; then
  CONFIGURE_PREPEND_OPTS+=" --disable-erlang-analyzers"
fi
if [ "$BUILD_HACK" == "no" ]; then
  CONFIGURE_PREPEND_OPTS+=" --disable-hack-analyzers"
fi
if [ "$BUILD_JAVA" == "no" ]; then
  CONFIGURE_PREPEND_OPTS+=" --disable-java-analyzers"
fi
if [ "$BUILD_PYTHON" == "no" ]; then
  CONFIGURE_PREPEND_OPTS+=" --disable-python-analyzers"
fi

set -x
# empty arrays trigger unbound variable errors, use horrible syntax to work around it
./configure $CONFIGURE_PREPEND_OPTS ${CONFIGURE_OPTS[@]+"${CONFIGURE_OPTS[@]}"}
set +x

# test if clang is enabled and if so are we about to build it? we cannot rely on $BUILD_CLANG
# because the user can also set --disable-clang-analyzers
if [ $(grep BUILD_C_ANALYZERS Makefile.autoconf | cut -d ' ' -f 3) = "yes" ]; then
  if ! "$PLUGIN_SETUP" --only-check-install; then
    echo ""
    echo "  Warning: you are not using a release of Infer. The C and"
    echo "  Objective-C analyses require a custom clang to be compiled"
    echo "  now. This step takes ~30-60 minutes, possibly more."
    echo ""
    echo "  To speed this along, you are encouraged to use a release of"
    echo "  Infer instead:"
    echo ""
    echo "  http://fbinfer.com/docs/getting-started"
    echo ""
    echo "  If you are only interested in analyzing Java programs, simply"
    echo "  run this script with only the \"java\" argument:"
    echo ""
    echo "  $0 java"
    echo ""

    confirm="n"
    printf "Are you sure you want to compile clang? (y/N) "
    if [ "$INTERACTIVE" == "no" ]; then
        confirm="y"
        echo "$confirm"
    else
        read confirm
    fi

    if [ "x$confirm" != "xy" ]; then
        exit 0
    fi

    # only run this script if we are definitely building clang
    facebook-clang-plugins/clang/src/prepare_clang_src.sh
  fi
fi

make -j "$JOBS" opt || (
  echo >&2
  echo '  compilation failure; you can try running' >&2
  echo >&2
  echo '    make clean' >&2
  echo "    '$0' $ORIG_ARGS" >&2
  echo >&2
  exit 1)

echo
echo "*** Success! Infer is now built in '$SCRIPT_DIR/infer/bin/'."
echo '*** Install infer on your system with `make install`.'
echo
echo '*** If you plan to hack on infer, check out CONTRIBUTING.md to setup your dev environment.'


#!/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -e

infer -- make -C 00 clean all
infer -- make -C 01 clean all
infer explore --no-source-preview --select 1
infer -- make -C 02 clean all
infer explore --no-source-preview --select 0
infer -- make -C 03 clean all
infer explore --no-source-preview --select 1
infer --eradicate-only -- make -C 04 clean all


#!/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# Simple installation script for llvm/clang.

set -e
set -o pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CLANG_RELATIVE_SRC="src/download/llvm-project/llvm"
CLANG_SRC="${CLANG_SRC:-$SCRIPT_DIR/$CLANG_RELATIVE_SRC}"
CLANG_PREFIX="$SCRIPT_DIR/install"
CLANG_INSTALLED_VERSION_FILE="$SCRIPT_DIR/installed.version"
PATCHELF=${PATCHELF:-patchelf}
PLATFORM=$(uname)
PLATFORM_ENV=${PLATFORM_ENV:-}
STRIP=${STRIP:-strip}
CMAKE=${CMAKE:-cmake}
ZLIB=${ZLIB:-$CLANG_SRC}

NCPUS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)"
JOBS="${JOBS:-$(($NCPUS>=8?$NCPUS/4:2))}"

SHASUM=${SHASUM:-shasum -a 256}

usage () {
    echo "Usage: $0 [-chr]"
    echo
    echo " options:"
    echo "    -c,--only-check-install    check if recompiling clang is needed"
    echo "    -h,--help                  show this message"
    echo "    -n,--ninja                 use ninja for building"
    echo "    -p,--clang-hash            print the installed clang hash"
    echo "    -r,--only-record-install   do not install clang but pretend we did"
    echo "    -s,--sequential-link       only use one process for linking (ninja only)"
}

clang_hash () {
    if [ "$CLANG_HASH_USE_GIT" = "yes" ]; then
        HASH=$(git -C "$CLANG_SRC" rev-parse HEAD)
        echo "$HASH"
    else
        pushd "$SCRIPT_DIR" > /dev/null
        HASH=$($SHASUM setup.sh src/prepare_clang_src.sh | $SHASUM)
        printf "%s" "$HASH" | cut -d ' ' -f 1
        popd > /dev/null
    fi
}

check_installed () {
    pushd "$SCRIPT_DIR" > /dev/null
    HASH=$(clang_hash)
    RESULT=1
    if [ -f "$CLANG_INSTALLED_VERSION_FILE" ]; then
        FILE_HASH=$(cat "$CLANG_INSTALLED_VERSION_FILE")
        if [ "$HASH" == "$FILE_HASH" ]; then
            RESULT=0
        fi
    fi
    popd > /dev/null
    return $RESULT
}

record_installed () {
    pushd "$SCRIPT_DIR" > /dev/null
    HASH=$(clang_hash)
    echo $HASH > "$CLANG_INSTALLED_VERSION_FILE"
    popd > /dev/null
}

ONLY_CHECK=
ONLY_RECORD=
PRINT_CLANG_HASH=
USE_NINJA=
SEQUENTIAL_LINK=

while [[ $# -gt 0 ]]; do
    opt_key="$1"
    case $opt_key in
        -p|--clang-hash)
            PRINT_CLANG_HASH=yes
            shift
            continue
            ;;
        -c|--only-check-install)
            ONLY_CHECK=yes
            shift
            continue
            ;;
        -r|--only-record-install)
            ONLY_RECORD=yes
            shift
            continue
            ;;
        -n|--ninja)
            USE_NINJA=yes
            shift
            continue
            ;;
        -s|--sequential-link)
            SEQUENTIAL_LINK=yes
            shift
            continue
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        *)
            usage
            exit 2
    esac
    shift
done

if [ "$PRINT_CLANG_HASH" = "yes" ]; then
    clang_hash
    exit 0
fi

if [ "$ONLY_RECORD" = "yes" ]; then
    record_installed
    exit 0
fi

if check_installed; then
    # already installed
    if [ "$ONLY_CHECK" = "yes" ]; then
        exit 0
    fi
    echo "Clang is already installed according to $CLANG_INSTALLED_VERSION_FILE"
    echo "Nothing to do, exiting."
    exit 0
else
    if [ "$ONLY_CHECK" = "yes" ]; then
        exit 1
    fi
fi

set -x

if [[ x"$MAKEFLAGS" != x ]]; then
    echo "WARNING: MAKEFLAGS var was set to: $MAKEFLAGS"
    echo "         which may (and will!) interfere with the setup"
    echo "         ... ignoring MAKEFLAGS."
    unset MAKEFLAGS
fi

if [[ x"$DESTDIR" != x ]]; then
    echo "WARNING: DESTDIR var was set to: $DESTDIR"
    echo "         but the setup script is not designed to handle"
    echo "         relocation of the installation."
    echo "         ... ignoring DESTDIR."
    unset DESTDIR
fi

if [[ "$PLATFORM" = "Linux" ]] && [[ -n "${PLATFORM_ENV}" ]] ; then
    CXXFLAGS="$CXXFLAGS -DHAVE_RPC_XDR_H=0 -D_GLIBCXX_INCLUDE_NEXT_C_HEADERS -Wl,-rpath-link,${PLATFORM_ENV}/lib"
fi

CMAKE_ARGS=(
  -DCMAKE_BUILD_TYPE=Release
  -DCMAKE_C_FLAGS="$CFLAGS $CMAKE_C_FLAGS"
  -DCMAKE_CXX_FLAGS="$CXXFLAGS $CPPFLAGS $CMAKE_CXX_FLAGS"
  -DCMAKE_INSTALL_PREFIX="$CLANG_PREFIX"
  -DLLVM_BUILD_EXTERNAL_COMPILER_RT=On
  -DLLVM_BUILD_TOOLS=Off
  -DLLVM_ENABLE_ASSERTIONS=Off
  -DLLVM_ENABLE_EH=On
  -DLLVM_ENABLE_RTTI=On
  -DLLVM_INCLUDE_DOCS=Off
  -DLLVM_INCLUDE_EXAMPLES=Off
  -DLLVM_INCLUDE_TESTS=Off
  -DLLVM_TARGETS_TO_BUILD="X86;AArch64;ARM;Mips"
)

if [ "$PLATFORM" = "Darwin" ]; then
    CMAKE_ARGS+=(
      -DLLVM_ENABLE_LIBCXX=On
      -DCMAKE_SHARED_LINKER_FLAGS="$LDFLAGS $CMAKE_SHARED_LINKER_FLAGS"
      -DLLVM_BUILD_LLVM_DYLIB=ON
    )
else
    CMAKE_ARGS+=(
      -DCMAKE_SHARED_LINKER_FLAGS="$LDFLAGS $CMAKE_SHARED_LINKER_FLAGS -lstdc++ -fPIC"
    )
fi

if [[ "$PLATFORM" = "Linux" ]] && [[ -n "${PLATFORM_ENV}" ]] ; then
    # Please note that this case only applies to infer/master platform builds
    # Prevent CMAKE from adding -isystem /usr/include for platform builds
    CMAKE_ARGS+=(
        -DLLVM_ENABLE_PROJECTS="clang;compiler-rt"
        -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi"
        -DZLIB_INCLUDE_DIR="$ZLIB/include"
    )
else
    CMAKE_ARGS+=(-DLLVM_ENABLE_PROJECTS="clang;compiler-rt;openmp"
                 -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi"
    )
fi

if [ "$USE_NINJA" = "yes" ]; then
    CMAKE_GENERATOR="Ninja"
    BUILD_BIN="ninja"
    # Do not set a 'j' build default for Ninja (let Ninja decide)
    BUILD_ARGS=""
else
    CMAKE_GENERATOR="Unix Makefiles"
    BUILD_BIN="make"
    BUILD_ARGS="-j $JOBS"
fi

if [ "$SEQUENTIAL_LINK" = "yes" ]; then
    if [[ x"$USE_NINJA" = x ]]; then
        echo "Linking with a single process is only supported with the Ninja generator."
        echo "Unable to proceed, exiting."
        exit 1
    fi
    # For Ninja, the compile jobs is the number of CPUs *not* $JOBS
    CMAKE_ARGS+=(
        -DCMAKE_JOB_POOLS:STRING="compile=$NCPUS;link=1"
        -DCMAKE_JOB_POOL_COMPILE:STRING="compile"
        -DCMAKE_JOB_POOL_LINK:STRING="link"
    )
fi

# start the installation
if [ ! -d "$CLANG_SRC" ]; then
    echo "Clang src (${CLANG_SRC}) missing, please run src/prepare_clang_src.sh"
    exit 1
fi

if [ -n "$CLANG_TMP_DIR" ]; then
    TMP=$CLANG_TMP_DIR
else
    TMP=`mktemp -d ${TMPDIR-/tmp}/clang-setup.XXXXXX`
fi
pushd "$TMP"

mkdir -p build
pushd build

# workaround install issue with ocaml llvm bindings and ocamldoc
mkdir -p docs/ocamldoc/html

$CMAKE -G "$CMAKE_GENERATOR" "$CLANG_SRC" "${CMAKE_ARGS[@]}" $CLANG_CMAKE_ARGS

$BUILD_BIN $BUILD_ARGS

echo "testing clang build"
./bin/clang --version

# "uninstall" previous clang
rm -fr "$CLANG_PREFIX"

$BUILD_BIN $BUILD_ARGS install

popd # build
popd # $TMP

# On Linux, copy __config_site to install directory. This way we don't need additional -I statements
CONFIG_SITE="$CLANG_PREFIX/include/x86_64-unknown-linux-gnu/c++/v1/__config_site"
if [[ "$PLATFORM" = "Linux" ]] && [[ -f "$CONFIG_SITE" ]]; then
    cp -f "$CONFIG_SITE" "$CLANG_PREFIX/include/c++/v1/__config_site"
fi

# delete libs not needed by Infer
if [ "$KEEP_LIBS" != "yes" ]; then
    rm -v "$CLANG_PREFIX"/lib/libclang*
    rm -v "$CLANG_PREFIX"/lib/libLLVM*
fi

# brutally strip everything, ignore errors
set +e
find "$CLANG_PREFIX"/{bin,lib} -type f -exec "$STRIP" -x \{\} \;

if [[ "$PLATFORM" = "Linux" ]] && [[ -n "${PLATFORM_ENV}" ]]; then
    # patch binaries to use platform_env rpath, ignore errors
    find "$CLANG_PREFIX"/{bin,lib} -type f -exec "$PATCHELF" --set-rpath "${PLATFORM_ENV}/lib" \{\} \;
fi
set -e

echo "testing installed clang"
"$CLANG_PREFIX"/bin/clang --version

echo "deleting temp dir '$CLANG_TMP_DIR'..."
if [ -n "$CLANG_TMP_DIR" ]; then
    rm -rf "$TMP/*"
else
    rm -rf "$TMP"
fi

record_installed


#!/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# Download llvm, clang and needed libraries

set -e
set -o pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SHASUM=${SHASUM:-shasum -a 256}
PATCH=${PATCH:-patch}

LLVM_VER="16.0.0"
LLVM_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VER}/llvm-project-${LLVM_VER}.src.tar.xz"
LLVM_SHA="9a56d906a2c81f16f06efc493a646d497c53c2f4f28f0cb1f3c8da7f74350254"
LLVM_FILE="llvm-project.src.tar.xz"
CLANG_PREBUILD_PATCHES=(
    "$SCRIPT_DIR/err_ret_local_block.patch"
    "$SCRIPT_DIR/mangle_suppress_errors.patch"
    "$SCRIPT_DIR/AArch64SVEACLETypes.patch"
)

mkdir -p "${SCRIPT_DIR}/download"
pushd "${SCRIPT_DIR}/download" >/dev/null

if ! echo "${LLVM_SHA}  ${LLVM_FILE}" | $SHASUM -c ; then
  echo "wrong checksum for $LLVM_FILE, downloading it again"
  rm -f "$LLVM_FILE"
fi

if [ ! -f "${LLVM_FILE}" ]; then
    curl -L "${LLVM_URL}" --output "${LLVM_FILE}"
fi
echo "${LLVM_SHA}  ${LLVM_FILE}" | $SHASUM -c

rm -rf "llvm-project-${LLVM_VER}.src" "llvm-project"

tar xf "${LLVM_FILE}"
mv "llvm-project-${LLVM_VER}.src" "llvm-project"

if [ "$SKIP_PATCH" != "yes" ]; then
    # apply prebuild patch
    for PATCH_FILE in "${CLANG_PREBUILD_PATCHES[@]}"; do
        "$PATCH" --force -p 1 < "$PATCH_FILE"
    done
fi

popd >/dev/null


#!/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# Script to validate Yojson outputs w.r.t. ATD specifications.
# This works by running a given 'converter' to parse and pretty-print the outputs, then observing the difference with gunzip + pretty-print.

if [ "$2" == "--std" ]; then
    CONVERTER=("$1" --std --pretty)
    YDUMP=(ydump -std)
    shift 2
else
    CONVERTER=("$1" --pretty)
    YDUMP=(ydump)
    shift
fi

while [ -n "$1" ]
do
    if [[ "$1" == *.gz ]]; then
        DUMP_CMD=(gunzip -c "$1")
    else
        DUMP_CMD=(cat "$1")
    fi

    if ! diff -q <("${DUMP_CMD[@]}" | "${YDUMP[@]}") <("${CONVERTER[@]}" "$1" /dev/stdout) >/dev/null 2>&1; then
        echo "The file '$1' does not respect the ATD format implemented by $CONVERTER."
        echo "Here is the command that shows the problem:"
        echo -n "  diff <("
        printf '%s ' "${DUMP_CMD[@]}"
        echo -n "| "
        printf '"%s" ' "${YDUMP[@]}"
        echo -n ") <("
        printf '"%s" ' "${CONVERTER[@]}" "$1"
        echo "/dev/stdout)"
        exit 2
    fi

    shift;
done


#!/bin/sh
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

./build/binioutest | bdump


#/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
"$@" | bdump


#/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
"$@"


# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

#!/bin/bash

TESTFILE="$1"
shift
TESTNAME=`basename "$TESTFILE"`

OUTFILE="$TESTFILE.out"
EXPFILE="$TESTFILE.exp"
DIFFFILE="$TESTFILE.diff"

if [[ $VERBOSE > 0 ]]
then
    DEST=/dev/stdout
else
    DEST=/dev/null
fi

if $* 2>&1 | tee "$OUTFILE" > $DEST
then
    if diff <(cat "$EXPFILE" 2> /dev/null || true) "$OUTFILE" > "$DIFFFILE"
    then
        echo "[+] $TESTNAME succeeded"
        rm "$DIFFFILE"
        exit 0
    else
        echo "[-] $TESTNAME failed (unexpected output)"
        printf '\033[1;31m\n'
        if [[ $LIMIT > 0 ]]
        then
            tail -n $LIMIT "$DIFFFILE"
        else
            cat "$DIFFFILE"
        fi
        printf '\033[0m\n'
        rm "$DIFFFILE"
        exit 1
    fi
else
    echo "[-] $TESTNAME failed (error $?)"
    exit 2
fi


#!/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -o pipefail
set -u

basedir=$(dirname "${BASH_SOURCE[0]}")
out=$("${basedir}"/erlang.escript "$@")
exit_code=$?
echo "$out"
if [ $exit_code = 0 ]; then
    sh -c "$out"
    exit $?
fi
exit $exit_code


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

mkdir -p jar

while read -r file ; do
    name=${file%.*}
    javac "$name.java"
    jar cvfe "$name.jar" "$name" *.class
    rm *.class
    mv "$name.jar" jar
done < <(ls | grep .java)


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# create a release of the Infer Java annotations to send to oss.sonatype.org

set -x
set -e

# This fixes 'gpg: signing failed: Inappropriate ioctl for device' error
# during signing
export GPG_TTY=$(tty)

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
ROOTDIR="$( cd $DIR/.. && pwd)"

# check if we're in the open-source repo
if ! git remote get-url origin | grep -q "\bgithub\.com\b"; then
    echo "Please run this script from the Git repo for open-source Infer!"
    exit 1
fi

echo "Starting release..."

( cd "$ROOTDIR/infer/annotations" && \
      mvn -e release:clean release:prepare && \
      mvn -e release:perform -DpushChanges=false )


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -e
set -o pipefail
set -u
set -x

if [ "$#" != "1" ]; then
  set +x
  echo "Usage: $0 version_tag" 1>&2
  echo 1>&2
  echo "Example: $0 v0.11.0" 1>&2
  exit 1
fi

VERSION=$1

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
ROOT_DIR="$SCRIPT_DIR"/..
NCPUS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)"
PLATFORM=$(uname)
if [ "$PLATFORM" == 'Darwin' ]; then
    RELEASE_NAME=infer-osx-"$VERSION"
else
    RELEASE_NAME=infer-linux64-"$VERSION"
fi
RELEASE_TARBALL="$RELEASE_NAME".tar.xz

# configurable stuff
DRYRUN=${DRYRUN:-no}
JOBS=${JOBS:-$NCPUS}

pushd "$ROOT_DIR"
rm -fr "$RELEASE_NAME"

./build-infer.sh --only-setup-opam
eval $(opam env)
touch .release
./autogen.sh
./configure \
    --prefix="/$RELEASE_NAME"

make -j "$JOBS" \
    install-with-libs \
    BUILD_MODE=opt \
    DESTDIR="$ROOT_DIR" \
    libdir_relative_to_bindir=../lib
popd

if [ "$DRYRUN" = "no" ]; then
    installed_version="$(./"$RELEASE_NAME"/bin/infer --version | head -1 | cut -d ' ' -f 3)"
    if [ "$installed_version" != "$VERSION" ]; then
        set +x
        printf "Infer reports the wrong version number: got '%s' but expected '%s'\n" \
               "$installed_version" "$VERSION" 1>&2
        exit 1
    fi

    # trick so that the String-Who-Must-Not-Be-Named doesn't appear verbatim in the script
    FBDASHONLY=$(printf "%s%s" 'FB-O' 'NLY')
    if grep -Ir "$FBDASHONLY" "$RELEASE_NAME"; then
        set +x
        echo "Found files marked $FBDASHONLY" 1>&2
        exit 1
    fi
fi

tar cJf "$RELEASE_TARBALL" "$RELEASE_NAME"
rm -fr "$RELEASE_NAME"

# special GitHub sauce for later steps to find the tarball
echo
echo "::set-output name=tarball-path::$RELEASE_TARBALL"


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# USAGE
#
# 1. Follow instructions in https://source.android.com/setup/build/downloading to download
#    the AOSP source and in https://source.android.com/setup/build/building to get proprietary
#    binaries.  You will need lots of space (at least 100Gb).  Let <android-root> be its root as
#    an *absolute path*.
#
# 2. Replace <android-root>/prebuilts/jdk/jdk9/<your OS>/bin/javac with the following script.
#
#    #!/bin/bash
#    infer -q --capture --continue --starvation-only --no-starvation \
#      --project-root <android-root> --results-dir <android-root>/infer-out -- \
#      /usr/local/bin/javac "$@"
#
#    Here, my local installation of java is in /usr/local/, change accordingly.  I used a
#    Java *8* installation without problems, YMMV.
#
# 3. From <android-root> do
#
#    $ . build/envsetup.sh
#    $ export TEMPORARY_DISABLE_PATH_RESTRICTIONS=true
#    $ cd libcore/ojluni
#    $ mm -j1 javac-check
#
#    ... and wait.  It took me ~22h.
#
# 4. From <android-root> run
#
#    $ infer analyze --starvation-only --dev-android-strict-mode
#
# 5. From <android-root> run this script, capturing stdout.
#
#    $ <infer-root>/scripts/make-strict-mode.sh > \
#        <infer-root>/infer/src/concurrency/StrictModeModels.ml
#
# 6. You may need to adapt the optional ~actuals_pred argument for methods in the above ML file.
#    The aim is to avoid false positives when there is an overloaded method with different
#    signatures and it so happens that one of the versions makes a violation when another does not.
#    Recompile Infer.

SOURCE_FILES=$(grep "error:" infer-out/report.txt | cut -f1 -d: | sort -u | grep -v test )
MATCHERS=""

cat <<EOF
(*
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 *)

open! IStd
open ConcurrencyModels

EOF

for SOURCE_FILE in $SOURCE_FILES ; do
  PACKAGE=$(grep -E "^package " $SOURCE_FILE | cut -f2 -d' ' | cut -f1 -d\;)

  if [[ $PACKAGE != android.* ]] && [[ $PACKAGE != androidx.* ]] && [[ $PACKAGE != java.* ]] ; then
    continue
  fi

  BASENAME=$(basename $SOURCE_FILE )
  CLASS=${BASENAME%.*}

  if ! grep -q -E "public.*class.* ${CLASS}" $SOURCE_FILE ; then
    continue
  fi

  HIDE=$(grep -B 2 -E "public.*class.* ${CLASS}" $SOURCE_FILE | grep '@hide')
  if [ ! -z "$HIDE" ] ; then
    continue
  fi

  FULLCLASSNAME="${PACKAGE}.${CLASS}"
  METHOD_REXP="^  Method \`.* $CLASS\."
  METHODS=$(grep -E "$METHOD_REXP" infer-out/report.txt | cut -f2 -d. | cut -f1 -d\` | sort -u)

  if [ -z "$METHODS" ] ; then
    continue
  fi

  HEADER=""
  MATCHER="is_${CLASS}_method"

  for METHOD in $METHODS; do
    METHODNAME=$(echo $METHOD | cut -f1 -d\( )

    if ! grep -q -E "public.*${METHODNAME}" $SOURCE_FILE ; then
      continue
    fi

    if [ -z "$HEADER" ] ; then
      echo "(* $SOURCE_FILE *)"
      echo "let ${MATCHER} ="
      echo "  is_call_of_class \"${FULLCLASSNAME}\""
      echo "    ["
      HEADER=true
    fi

    echo "      \"${METHODNAME}\"; (* $METHOD *)"
  done

  if [ ! -z "$HEADER" ] ; then
    echo "    ]"
    echo '  |> Staged.unstage'
    echo
    MATCHERS="$MATCHERS $MATCHER"
  fi
done

echo
echo "let is_strict_mode_violation ="
echo "  let matchers = ["
for M in $MATCHERS ; do
  echo "    $M ;"
done
echo "    ]"
echo "  in"
echo "  fun tenv pn actuals -> List.exists matchers ~f:(fun m -> m tenv pn actuals)"


#!/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# source this script to import its utility functions

opam_retry () {
  "$@" || {
    echo >&2;
    printf '*** `%s` failed\n' "$*" >&2;
    echo '*** Updating opam then retrying' >&2;
    opam update &&
    "$@" || {
      echo >&2;
      printf '*** ERROR: `%s` failed\n' "$*" >&2;
      exit 1
    }
  }
}

opam_failed () {
    local command=$1
    echo
    printf '*** ERROR: %s failed\n' "$command" >&2
    printf "*** ERROR: Try running \$(opam update) then running this script again\n" >&2
    exit 1
}

opam_require_version_2 () {
    local status=0
    local version=0
    { version=$(opam --version 2>/dev/null); status=$?; }
    if [ "$status" != 0 ]; then
        printf '*** ERROR: `opam --version` failed, please install opam version 2\n' >&2
        env >&2
        exit 1
    fi
    case $version in
        2*) ;;
        *)
            printf '*** ERROR: opam version "%s" is not supported, please install opam version 2\n' "$version" >&2
            printf '*** NOTE: opam is "%s"\n' "$(which opam)" >&2
            env >&2
            exit 1
    esac
}

# assumes opam is available and initialized
opam_switch_create_if_needed () {
    local switch=$1
    local options=$2
    local switch_exists=no
    printf "looking if switch %s exists in this list:\n" "$switch";
    opam switch list --short
    for installed_switch in $(opam switch list --short); do
        if [ "$installed_switch" == "$switch" ]; then
            switch_exists=yes
            break
        fi
    done
    printf "verdict: %s\n" $switch_exists;
    if [ "$switch_exists" = "no" ]; then
        if [ -e "$(opam var root)/$switch" ] ; then
            rm -rf "$(opam var root)/$switch" || true
        fi
        opam switch create "$switch" $options
    fi
}

opam_require_version_2

# removes packages that cannot be found by ocamlfind
opam_remove_broken_package () {
    local pkg="$1"
    if ! ocamlfind query "${pkg}"; then
        echo "ocamlfind cannot find ${pkg} package. Removing the package..."
        opam remove "${pkg}" || true
    fi
}


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# Usage: set_libso_path.sh [LIBSO_DIR] [TARGET]
#
# This changes MacOSX's executable [TARGET] to use shared libraries in
# [LIBSO_DIR] when rpath has been set to [LIBSO_DIR].

set -e
set -o pipefail
set -u

LIBSO_DIR=$1
TARGET=$2

TMP=$( mktemp )
trap "rm $TMP" EXIT

otool -L "$TARGET" | tail -n +2 > "$TMP"
while IFS='' read -r line || [[ -n "$line" ]]; do
  LIB_PATH=$( echo $line | awk '{print $1}' )
  LIB_FILE=$( basename "${LIB_PATH}" )
  if [ -f "${LIBSO_DIR}/${LIB_FILE}" ]; then
    install_name_tool -change "${LIB_PATH}" "@rpath/${LIB_FILE}" "$TARGET"
  fi
done < "$TMP"


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# wrap execution of sledge with time and memory limits
# and add status entries to report in case of an unexpected exit

# usage: wrap.sh <timeout(sec)> <memout(GB)> <command> <testdir/testname>

set -u

timeout=$1
memout=$(( $2*1024*1024 ))
command=${@: 3: $#-3}
test=${@: -1}

unknown_error () {
  echo "((name $test)(entry(Status(UnknownError \"$1\"))))" >> $test.sexp
}

assert () {
  echo "((name $test)(entry(Status(Assert \"$1\"))))" >> $test.sexp
}

abort () {
  if grep -q "Assertion failed" $test.err;
  then
    assert "$(\
      grep "Assertion failed" $test.err \
      | sed 's/</\</g;s/>/\>/g;s/Assertion failed: (\(.*\)), function.*/\1/' \
      | sed 's/"/\\\"/g' )"
  else
    echo "((name $test)(entry(Status Abort)))" >> $test.sexp
  fi
}

timeout () {
  echo "((name $test)(entry(Status Timeout)))" >> $test.sexp
}

memout () {
  echo "((name $test)(entry(Status Memout)))" >> $test.sexp
}

(
  ulimit -t $timeout -v $memout
  $command -report $test.sexp $test 1> $test.out 2> $test.err
) &> /dev/null
status=$?

case $status in
  ( 0 | 1 ) ;;
  ( 132 ) unknown_error "illegal instruction" ;;
  ( 136 ) unknown_error "floating-point exception" ;;
  ( 139 ) unknown_error "segmentation violation" ;;
  ( 127 ) memout ;;
  ( 134 ) abort ;;
  ( 137 | 152 ) timeout ;;
  ( * ) unknown_error "exit $status" ;;
esac

(test -f $test.sexp && grep -q "Status" $test.sexp) \
|| unknown_error "exit $status"

exit $status


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

sledge=$PWD/cli/sledge_cli.exe

export SLEDGE_CONFIG=$PWD/../../test/config

line=$(\
  $sledge help -recursive -expand \
    | grep -n "== subcommands ===" \
    | cut -d : -f1,1)

line=$(($line+1))

$sledge help -recursive | sed -e "s/sledge_cli.exe/sledge/g"

$sledge h -r -e \
  | tail -n +$line \
  | sed -e "s/  \(.*\)  .*/\1/g;/^ *$/d" \
  | while read cmd; do \
      printf "\n====== sledge $cmd ======\n\n"; \
      $sledge $cmd -help | sed -e "s/sledge_cli.exe/sledge/g"; \
    done


#!/bin/sh -ex

if test "$#" != 1; then
    echo "Usage: $0 <prefix>"
    exit 1
fi

PREFIX=$1

dune install "--prefix=$PREFIX" --release

for pkg in $(basename -s .opam *.opam); do
    case "$pkg" in
    llvm)
        cp META.llvm.in "$PREFIX/lib/llvm/META";;
    llvm_*)
        target=$(echo "$pkg" | cut -d_ -f2-)
        sed "s/@TARGET@/$target/g" META.llvm_TARGET.in > "$PREFIX/lib/llvm_$target/META";;
    *)
        echo "Something went wrong while processing $pkg. Please report."
        exit 1;;
    esac
done


#!/bin/sh -e

if test "$(dirname $0)" != '.'; then
    echo "The script must be executed from its current directory."
    exit 1
fi

if test "$#" -ne 1; then
    echo "Usage: $0 <llvm-config>"
    exit 1
fi

llvm_config=$1
default_mode=
support_static_mode=false
support_shared_mode=false

llvm_config() {
    "$llvm_config" $@
}

if llvm_config --link-static; then
    default_mode=static
    support_static_mode=true
fi

if llvm_config --link-shared; then
    default_mode=shared
    support_shared_mode=true
fi

if test -z "$default_mode"; then
    echo "Something is wrong with the llvm-config command provided."
    exit 1
fi

base_cflags=$(llvm_config --cflags)
ldflags="$(llvm_config --ldflags) -lstdc++"
llvm_targets=$(llvm_config --targets-built)

rm -rf src
cp -r llvm-project/llvm/bindings/ocaml src

create_dune_file() {
    findlibname=$1
    dirname=$2
    modname=$3
    cfile=$4
    depends=$5
    components=$6
    extra_deps=$7

    if test "$dirname" = "backends"; then
        basedir=src/$dirname/$components
        test ! -d "$basedir" && mkdir "$basedir"
        sed "s/@TARGET@/$components/g" "src/$dirname/llvm_backend.ml.in" > "$basedir/$modname.ml"
        sed "s/@TARGET@/$components/g" "src/$dirname/llvm_backend.mli.in" > "$basedir/$modname.mli"
        sed "s/@TARGET@/$components/g" "src/$dirname/backend_ocaml.c" > "$basedir/$cfile.c"
        cflags="$base_cflags \"-DTARGET=$components\""
    else
        basedir=src/$dirname
        cflags=$base_cflags
    fi

    test ! -d "$basedir/common" && mkdir "$basedir/common"
    cp "$basedir/$modname.mli" "$basedir/common"

    echo "
(library
 (name $modname)
 (public_name $findlibname)
 (wrapped false)
 (virtual_modules $modname)
 (libraries $depends)
 (default_implementation $findlibname.$default_mode))
" > "$basedir/common/dune"

    if $support_shared_mode; then
        test ! -d "$basedir/shared" && mkdir "$basedir/shared"
        cp "$basedir/$modname.ml" "$basedir/shared"
        cp "$basedir/$cfile.c" "$basedir/shared"

        echo "
(library
 (name ${modname}_shared)
 (public_name $findlibname.shared)
 (implements $findlibname)
 (foreign_stubs
  (language c)
  (names ${cfile})
  (extra_deps ${extra_deps})
  (flags ($cflags -I../../llvm)))
 (c_library_flags ($ldflags $(llvm_config --system-libs --link-shared --libs $components))))
" >> "$basedir/shared/dune"
    fi

    if $support_static_mode; then
        test ! -d "$basedir/static" && mkdir "$basedir/static"
        cp "$basedir/$modname.ml" "$basedir/static"
        cp "$basedir/$cfile.c" "$basedir/static"

        echo "
(library
 (name ${modname}_static)
 (public_name $findlibname.static)
 (implements $findlibname)
 (foreign_stubs
  (language c)
  (names ${cfile})
  (extra_deps ${extra_deps})
  (flags ($cflags -I../../llvm)))
 (c_library_flags ($ldflags $(llvm_config --system-libs --link-static --libs $components))))
" >> "$basedir/static/dune"
    fi

    rm "$basedir/$modname.ml"
    rm "$basedir/$cfile.c"
}

# ------------------ public name -------- directory ---------------- module name -------- C file name --------- OCaml dependencies -------------- LLVM components (for the linker) - extra-deps
create_dune_file     llvm                 llvm                       llvm                 llvm_ocaml            ""                                "core support"                     ../llvm_ocaml.h
create_dune_file     llvm.analysis        analysis                   llvm_analysis        analysis_ocaml        "llvm"                            "analysis"
create_dune_file     llvm.bitreader       bitreader                  llvm_bitreader       bitreader_ocaml       "llvm"                            "bitreader"
create_dune_file     llvm.bitwriter       bitwriter                  llvm_bitwriter       bitwriter_ocaml       "llvm unix"                       "bitwriter"
create_dune_file     llvm.executionengine executionengine            llvm_executionengine executionengine_ocaml "llvm llvm.target ctypes.foreign" "executionengine mcjit native"
create_dune_file     llvm.ipo             transforms/ipo             llvm_ipo             ipo_ocaml             "llvm"                            "ipo"
create_dune_file     llvm.irreader        irreader                   llvm_irreader        irreader_ocaml        "llvm"                            "irreader"
create_dune_file     llvm.scalar_opts     transforms/scalar_opts     llvm_scalar_opts     scalar_opts_ocaml     "llvm"                            "scalaropts"
create_dune_file     llvm.transform_utils transforms/utils           llvm_transform_utils transform_utils_ocaml "llvm"                            "transformutils"
create_dune_file     llvm.vectorize       transforms/vectorize       llvm_vectorize       vectorize_ocaml       "llvm"                            "vectorize"
create_dune_file     llvm.passmgr_builder transforms/passmgr_builder llvm_passmgr_builder passmgr_builder_ocaml "llvm"                            "ipo"
create_dune_file     llvm.target          target                     llvm_target          target_ocaml          "llvm"                            "target"                           ../../llvm/llvm_ocaml.h
create_dune_file     llvm.linker          linker                     llvm_linker          linker_ocaml          "llvm"                            "linker"
create_dune_file     llvm.all_backends    all_backends               llvm_all_backends    all_backends_ocaml    "llvm"                            "$llvm_targets"

for target in $llvm_targets; do
    touch "llvm_${target}.opam"
    create_dune_file "llvm_$target"       backends                   "llvm_$target"       "${target}_ocaml"     "llvm"                            "$target"
done


#!/bin/bash

# Copies the 'demangle' library, excluding 'DemangleConfig.h', to llvm. If no
# llvm directory is specified, then assume a monorepo layout.

set -e

FILES="ItaniumDemangle.h StringView.h Utility.h README.txt"
LLVM_DEMANGLE_DIR=$1

if [[ -z "$LLVM_DEMANGLE_DIR" ]]; then
    LLVM_DEMANGLE_DIR="../../../llvm/include/llvm/Demangle"
fi

if [[ ! -d "$LLVM_DEMANGLE_DIR" ]]; then
    echo "No such directory: $LLVM_DEMANGLE_DIR" >&2
    exit 1
fi

read -p "This will overwrite the copies of $FILES in $LLVM_DEMANGLE_DIR; are you sure? [y/N]" -n 1 -r ANSWER
echo

if [[ $ANSWER =~ ^[Yy]$ ]]; then
    for I in $FILES ; do
        cp $I $LLVM_DEMANGLE_DIR/$I
    done
fi


#!/usr/bin/env node

/**
 * Copyright (c) Facebook, Inc. and its affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

'use strict';

require('./jscodeshift.js');


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -eo pipefail
./gradlew -q packageLibs
mv build/distributions/lambda.zip build/lambda-lib.zip

./gradlew build

ARTIFACT_BUCKET=ktfmt-online-formatter-lambda
TEMPLATE=format.yaml
aws cloudformation package --template-file $TEMPLATE --s3-bucket $ARTIFACT_BUCKET --output-template-file out.yml
aws cloudformation deploy --template-file out.yml --stack-name ktfmt --capabilities CAPABILITY_NAMED_IAM --region=us-east-2


#!/bin/bash -xe

# Author(s) - anirbanr-fb
# Borrowed from
# https://kudu.apache.org/docs/installation.html#build_from_source

# We need to specify a specific version of Kerberos so we don't break certain
# non-Kudu components that enforce a specific Kerberos version.
KRB5_VERSION=1.18.2-12.el8.x86_64  # NOTE: only correct on CentOS 8!

# INSTALL ALL DEPENDENCIES
sudo dnf install -y autoconf automake cyrus-sasl-devel cyrus-sasl-gssapi \
  cyrus-sasl-plain flex gcc gcc-c++ gdb git java-1.8.0-openjdk \
  libtool make openssl-devel patch pkgconfig redhat-lsb-core rsync unzip \
  vim-common which cmake doxygen libcap-devel \
  krb5-server-${KRB5_VERSION} krb5-workstation-${KRB5_VERSION}

# DOWNLOAD AND BUILD THE THIRD-PARTY libraries
# IF THINGS BREAK IN MIDDLE
# wipe out: you can use the big hammer!
#  rm -f thirdparty/{src,installed,build}
#  rerun
./thirdparty/build-if-necessary.sh

mkdir -p build/release
cd build/release
cmake -DCMAKE_BUILD_TYPE=release ../..
make -j20


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Enables the Red Hat devtoolset on RHEL 6 based systems and executes the
# arguments. On non-RHEL 6 systems, the arguments are executed without changes
# to the environment.
# USAGE: ./enable_devtoolset.sh <command> <args>...

set -e

if [[ "$OSTYPE" =~ ^linux ]] && \
   [[ "$(lsb_release -irs)" =~ (CentOS|RedHatEnterpriseServer)[[:space:]]+6\.[[:digit:]]+ ]]; then
  # Invoke the inner script, which may do some additional customization within
  # the devtoolset.
  ROOT=$(cd $(dirname "$BASH_SOURCE") ; pwd)
  scl enable devtoolset-3 "$ROOT/enable_devtoolset_inner.sh $*"
else
  $@
fi


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Executes the provided arguments within the context of a Red Hat devtoolset.
#
# This script should not be used directly; it is called by enable_devtoolset.sh.

set -e

# If ccache was on the PATH and CC/CXX have not already been set, set them to
# devtoolset-3 specific ccache helper scripts (thus enabling ccache).
if [ $(which ccache 2> /dev/null) -a ! "$CC" -a ! "$CXX" ]; then
  ROOT=$(cd $(dirname "$BASH_SOURCE") ; pwd)
  export CC="$ROOT/ccache-devtoolset-3/cc"
  export CXX="$ROOT/ccache-devtoolset-3/c++"
fi

# Execute the arguments.
$*


#!/bin/bash -e
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Script which embeds a piece of raw data into a C++ source file. Functionally
# the same as xxd -i, but inserts custom namespace and variable names.

IN_FILE=$1
OUT_FILE=$2

echo "// header generated by build-support/generate_precompiled_xxd.sh" > $OUT_FILE
echo "namespace kudu {" >> $OUT_FILE
echo "namespace codegen {" >> $OUT_FILE

echo "extern const char precompiled_ll_data[] = {" >> $OUT_FILE
xxd -i - < $IN_FILE >> $OUT_FILE
# LLVM requires the binary to be null terminated.
echo ", 0x00" >> $OUT_FILE
echo "};" >> $OUT_FILE

LEN=$(wc -c $IN_FILE | awk '{print $1}')
echo "extern const unsigned int precompiled_ll_len = ${LEN};" >> $OUT_FILE

echo "} // namespace codegen" >> $OUT_FILE
echo "} // namespace kudu" >> $OUT_FILE


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Script which tries to determine the most recent git hash in the current
# branch which was checked in by gerrit. This commit hash is printed to
# stdout.
#
# It does so by looking for the 'Reviewed-on' tag added by gerrit. This is
# more foolproof than trying to guess at the "origin/" branch name, since the
# developer might be working on some local topic branch.
set -e

git log --grep='Reviewed-on: ' -n1 --pretty=format:%H


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

ROOT=$(cd $(dirname $BASH_SOURCE)/..; pwd)

TMP=$(mktemp -t kudu-lint.XXXXXXXXX)
trap "rm $TMP" EXIT

ONLY_CHANGED=false

for flag in "$@" ; do
  case $flag in
    --changed-only | -c)
      ONLY_CHANGED=true
      ;;
    *)
      echo unknown flag: $flag
      exit 1
      ;;
  esac
done

if $ONLY_CHANGED; then
  FILES=$(git diff --name-only $($ROOT/build-support/get-upstream-commit.sh)  \
    | egrep  '\.(cc|h)$' | grep -v "gutil\|trace_event\|x509_check_host")
  if [ -z "$FILES" ]; then
    echo No source files changed
    exit 0
  fi
else
  FILES=$(find $ROOT/src -name '*.cc' -or -name '*.h' | grep -v "\.pb\.\|\.service\.\|\.proxy\.\|\.krpc\.\|gutil\|trace_event\|kudu_export\.h\|x509_check_host")
fi

cd $ROOT

$ROOT/thirdparty/installed/common/bin/cpplint.py \
  --verbose=4 \
  --filter=-whitespace/comments,-readability/todo,-readability/inheritance,-build/header_guard,-build/include_order,-legal/copyright,-build/c++11,-readability/nolint \
  $FILES 2>&1 | grep -v 'Done processing' | tee $TMP

NUM_ERRORS=$(grep "Total errors found" $TMP | awk '{print $4}')

if [[ "$NUM_ERRORS" -ne 0 ]]; then
  exit 1
fi


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Reports a test run to the central test server, which records
# the results in a database. This is what drives our "flaky test dashboard".
# This script does blocking network IO, so if you are running it from the
# context of a build, you may want to run it in the background.
#
# Note that this may exit with a non-zero code if the network is flaky or the
# test result server is down.

set -e

ROOT=$(dirname $BASH_SOURCE)/..

# Verify and parse command line and options
if [ $# -ne 3 ]; then
  echo "usage: $0 <path/to/test> <path/to/log> <exit-code>"
  echo
  echo The \$TEST_RESULT_SERVER environment variable may be used
  echo to specify where to report the tests.
  exit 1
fi
TEST_EXECUTABLE=$1
LOGFILE=$2
STATUS=$3
TEST_RESULT_SERVER=${TEST_RESULT_SERVER:-localhost:8080}
REPORT_TIMEOUT=${REPORT_TIMEOUT:-10}

# On Jenkins, we'll have this variable set. Otherwise,
# report the build ID as non-jenkins.
BUILD_ID=${BUILD_TAG:-non-jenkins}

# Figure out the current git revision, and append a "-dirty" tag if it's
# not a pristine checkout
REVISION=$(cd $ROOT && git rev-parse HEAD)
if ! ( cd $ROOT && git diff --quiet .  && git diff --cached --quiet . ) ; then
  REVISION="${REVISION}-dirty"
fi

BUILD_ROOT=$(dirname $TEST_EXECUTABLE)/..

# Parse out our "build config" - a space-separated list of tags
# which include the cmake build type as well as the list of configured
# sanitizers

CMAKECACHE=$BUILD_ROOT/CMakeCache.txt
BUILD_CONFIG=$(grep '^CMAKE_BUILD_TYPE:' $CMAKECACHE | cut -f 2 -d=)
if grep -q "KUDU_USE_ASAN:UNINITIALIZED=1" $CMAKECACHE ; then
  BUILD_CONFIG="$BUILD_CONFIG asan"
fi
if grep -q "KUDU_USE_TSAN:UNINITIALIZED=1" $CMAKECACHE ; then
  BUILD_CONFIG="$BUILD_CONFIG tsan"
fi
if grep -q "KUDU_USE_UBSAN:UNINITIALIZED=1" $CMAKECACHE ; then
  BUILD_CONFIG="$BUILD_CONFIG ubsan"
fi

# We sometimes have flaky infrastructure where NTP is broken. In that case
# do not report it as a failed test.
if zgrep -q 'Clock considered unsynchronized' $LOGFILE ; then
  echo Not reporting test that failed due to NTP issues.
  exit 1
fi

# Only upload a log if the test failed.
# This saves some space on S3, network bandwidth, etc, and we don't
# have a lot of use for the logs of successful tests anyway.
if [ "$STATUS" -ne 0 ]; then
  LOG_PARAM="-F log=@$LOGFILE"
else
  LOG_PARAM=""
fi

curl -s \
    --max-time $REPORT_TIMEOUT \
    $LOG_PARAM \
    -F "build_id=$BUILD_ID" \
    -F "hostname=$(hostname)" \
    -F "test_name=$(basename $TEST_EXECUTABLE)" \
    -F "status=$STATUS" \
    -F "revision=$REVISION" \
    -F "build_config=$BUILD_CONFIG" \
    http://$TEST_RESULT_SERVER/add_result


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Script which wraps running a test and redirects its output to a
# test log directory.
#
# If KUDU_COMPRESS_TEST_OUTPUT is non-empty, then the logs will be
# gzip-compressed while they are written.
#
# If KUDU_FLAKY_TEST_ATTEMPTS is set, and either the test being run matches one
# of the lines in the file KUDU_FLAKY_TEST_LIST or KUDU_RETRY_ALL_FAILED_TESTS
# is non-zero, then the test will be retried on failure up to the specified
# number of times. This can be used in the gerrit workflow to prevent annoying
# false -1s caused by tests that are known to be flaky in master.
#
# If KUDU_REPORT_TEST_RESULTS is non-zero, then tests are reported to the
# central test server.
#
# If KUDU_MEASURE_TEST_CPU_CONSUMPTION is non-zero, then tests will be wrapped
# in a 'perf stat' command which measures the CPU consumption on a periodic
# basis, writing the output into '<test>.<shard>.perf.txt' logs in the test log
# directory. This is handy in order to determine an appropriate value for the
# 'PROCESSORS' property passed to 'ADD_KUDU_TEST' in CMakeLists.txt.

# Path to the test executable or script to be run.
# May be relative or absolute.
TEST_PATH=$1

# Absolute path to the root source directory. This script is expected to live within it.
SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/.. ; pwd)

# Absolute path to the root build directory. The test path is expected to be within it.
BUILD_ROOT=$(cd $(dirname "$TEST_PATH")/.. ; pwd)

TEST_LOGDIR=$BUILD_ROOT/test-logs
mkdir -p $TEST_LOGDIR

TEST_DEBUGDIR=$BUILD_ROOT/test-debug
mkdir -p $TEST_DEBUGDIR

TEST_DIRNAME=$(cd $(dirname $TEST_PATH); pwd)
TEST_FILENAME=$(basename $TEST_PATH)
ABS_TEST_PATH=$TEST_DIRNAME/$TEST_FILENAME
shift
# Remove path and extension (if any).

# The "short" test name doesn't include the shard number.
SHORT_TEST_NAME=$(echo $TEST_FILENAME | perl -pe 's/\..+?$//')

# The full test name does include the shard number if the test is sharded.
if [ ${GTEST_TOTAL_SHARDS:-0} -gt 1 ]; then
  TEST_NAME=${SHORT_TEST_NAME}.${GTEST_SHARD_INDEX:?}
else
  TEST_NAME=${SHORT_TEST_NAME}
fi

# Determine whether the user has chosen to retry all failed tests, or whether
# the test is a known flaky by comparing against the user-specified list.
TEST_EXECUTION_ATTEMPTS=1
if [ -n "$KUDU_RETRY_ALL_FAILED_TESTS" ]; then
  echo "Will retry all failed tests"
  TEST_IS_RETRYABLE=1
elif [ -n "$KUDU_FLAKY_TEST_LIST" ]; then
  if [ -f "$KUDU_FLAKY_TEST_LIST" ] && grep -q --count --line-regexp "$SHORT_TEST_NAME" "$KUDU_FLAKY_TEST_LIST"; then
    TEST_IS_RETRYABLE=1
  else
    echo "Flaky test list file $KUDU_FLAKY_TEST_LIST missing"
  fi
fi

if [ -n "$TEST_IS_RETRYABLE" ]; then
  TEST_EXECUTION_ATTEMPTS=${KUDU_FLAKY_TEST_ATTEMPTS:-1}
  echo $TEST_NAME is a retryable test. Will attempt running it
  echo up to $TEST_EXECUTION_ATTEMPTS times.
fi

# We run each test in its own subdir to avoid core file related races.
TEST_WORKDIR=$BUILD_ROOT/test-work/$TEST_NAME
mkdir -p $TEST_WORKDIR
pushd $TEST_WORKDIR >/dev/null || exit 1
rm -f *

set -o pipefail

LOGFILE=$TEST_LOGDIR/$TEST_NAME.txt
XMLFILE=$TEST_LOGDIR/$TEST_NAME.xml
export GTEST_OUTPUT="xml:$XMLFILE" # Enable JUnit-compatible XML output.

# Remove both the compressed and uncompressed output, so the developer
# doesn't accidentally get confused and read output from a prior test
# run.
rm -f $LOGFILE $LOGFILE.gz

if [ -n "$KUDU_COMPRESS_TEST_OUTPUT" ] && [ "$KUDU_COMPRESS_TEST_OUTPUT" -ne 0 ] ; then
  pipe_cmd=gzip
  LOGFILE=${LOGFILE}.gz
else
  pipe_cmd=cat
fi

# Set a 15-minute timeout for tests run via 'make test'.
# This keeps our jenkins builds from hanging in the case that there's
# a deadlock or anything.
#
# NOTE: this should be kept in sync with the default value of ARG_TIMEOUT
# in the definition of ADD_KUDU_TEST in the top-level CMakeLists.txt.
KUDU_TEST_TIMEOUT=${KUDU_TEST_TIMEOUT:-900}

# Allow for collecting core dumps.
KUDU_TEST_ULIMIT_CORE=${KUDU_TEST_ULIMIT_CORE:-0}
ulimit -c $KUDU_TEST_ULIMIT_CORE

# Run the actual test.
for ATTEMPT_NUMBER in $(seq 1 $TEST_EXECUTION_ATTEMPTS) ; do
  if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
    # If the test fails, the test output may or may not be left behind,
    # depending on whether the test cleaned up or exited immediately. Either
    # way we need to clean it up. We do this by comparing the data directory
    # contents before and after the test runs, and deleting anything new.
    #
    # The comm program requires that its two inputs be sorted.
    TEST_TMPDIR_BEFORE=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
  fi

  # gtest won't overwrite old junit test files, resulting in a build failure
  # even when retries are successful.
  rm -f $XMLFILE

  if [[ $OSTYPE =~ ^darwin ]]; then
    #
    # For builds on MacOS X 10.11, neither (g)addr2line nor atos translates
    # address into line number.  The atos utility is able to do that only
    # if load address (-l <addr>) or reference to currently running process
    # (-p <pid>) is given.  This is so even for binaries linked with
    # PIE disabled.
    #
    addr2line_filter=cat
  else
    addr2line_filter="$SOURCE_ROOT/build-support/stacktrace_addr2line.pl $ABS_TEST_PATH"
  fi
  echo "Running $TEST_NAME, redirecting output into $LOGFILE" \
       "(attempt ${ATTEMPT_NUMBER}/$TEST_EXECUTION_ATTEMPTS)"
  test_command=$ABS_TEST_PATH
  if [ -n "$KUDU_MEASURE_TEST_CPU_CONSUMPTION" ] && [ "$KUDU_MEASURE_TEST_CPU_CONSUMPTION" -ne 0 ] ; then
    perf_file=$TEST_LOGDIR/$TEST_NAME.perf.txt
    echo "Will wrap test in perf-stat, output to ${perf_file}..."
    test_command="perf stat -I1000 -e task-clock -o $perf_file $test_command"
  fi
  $test_command "$@" --test_timeout_after $KUDU_TEST_TIMEOUT 2>&1 \
    | $addr2line_filter \
    | $pipe_cmd > $LOGFILE
  STATUS=$?

  # An empty gzipped file is corrupt because even a logically empty gzipped file
  # should contain at least the gzip header. If we detect this, treat the test
  # log output as a plain text file.
  if [[ -f $LOGFILE && ! -s $LOGFILE && $LOGFILE =~ .gz$ ]]; then
    RENAMED_LOGFILE=${LOGFILE%%.gz}
    mv $LOGFILE $RENAMED_LOGFILE
    LOGFILE=$RENAMED_LOGFILE
  fi

  # TSAN doesn't always exit with a non-zero exit code due to a bug:
  # mutex errors don't get reported through the normal error reporting infrastructure.
  # So we make sure to detect this and exit 1.
  #
  # Additionally, certain types of failures won't show up in the standard JUnit
  # XML output from gtest. We assume that gtest knows better than us and our
  # regexes in most cases, but for certain errors we delete the resulting xml
  # file and let our own post-processing step regenerate it.
  export GREP=$(which egrep)
  if zgrep --silent "ThreadSanitizer|Leak check.*detected leaks" $LOGFILE ; then
    echo ThreadSanitizer or leak check failures in $LOGFILE
    STATUS=1
    rm -f $XMLFILE
  fi

  if [ $ATTEMPT_NUMBER -lt $TEST_EXECUTION_ATTEMPTS ]; then
    # Now delete any new test output.
    TEST_TMPDIR_AFTER=$(find $TEST_TMPDIR -maxdepth 1 -type d | sort)
    DIFF=$(comm -13 <(echo "$TEST_TMPDIR_BEFORE") \
                    <(echo "$TEST_TMPDIR_AFTER"))
    for DIR in $DIFF; do
      # Multiple tests may be running concurrently. To avoid deleting the
      # wrong directories, constrain to only directories beginning with the
      # test name.
      #
      # This may delete old test directories belonging to this test, but
      # that's not typically a concern when rerunning flaky tests.
      if [[ $DIR =~ ^$TEST_TMPDIR/$TEST_NAME ]]; then
        echo Deleting leftover flaky test directory "$DIR"
        rm -Rf "$DIR"
      fi
    done
  fi

  if [ -n "$KUDU_REPORT_TEST_RESULTS" ] && [ "$KUDU_REPORT_TEST_RESULTS" -ne 0 ]; then
    echo Reporting results
    $SOURCE_ROOT/build-support/report-test.sh "$ABS_TEST_PATH" "$LOGFILE" "$STATUS" &

    # On success, we'll do "best effort" reporting, and disown the subprocess.
    # On failure, we want to upload the failed test log. So, in that case,
    # wait for the report-test.sh job to finish, lest we accidentally run
    # a test retry and upload the wrong log.
    if [ "$STATUS" -eq "0" ]; then
      disown
    else
      wait
    fi
  fi

  if [ "$STATUS" -eq "0" ]; then
    break
  elif [ "$ATTEMPT_NUMBER" -lt "$TEST_EXECUTION_ATTEMPTS" ]; then
    echo Test failed attempt number $ATTEMPT_NUMBER
    echo Will retry...
  fi
done

# If we have a LeakSanitizer report, and XML reporting is configured, add a new test
# case result to the XML file for the leak report. Otherwise Jenkins won't show
# us which tests had LSAN errors.
if zgrep --silent "ERROR: LeakSanitizer: detected memory leaks" $LOGFILE ; then
    echo Test had memory leaks. Editing XML
    perl -p -i -e '
    if (m#</testsuite>#) {
      print "<testcase name=\"LeakSanitizer\" status=\"run\" classname=\"LSAN\">\n";
      print "  <failure message=\"LeakSanitizer failed\" type=\"\">\n";
      print "    See txt log file for details\n";
      print "  </failure>\n";
      print "</testcase>\n";
    }' $XMLFILE
fi

# Capture and compress core file and binary.
COREFILES=$(ls | grep ^core)
if [ -n "$COREFILES" ]; then
  echo Found core dump. Saving executable and core files.
  gzip < $ABS_TEST_PATH > "$TEST_DEBUGDIR/$TEST_NAME.gz" || exit $?
  for COREFILE in $COREFILES; do
    gzip < $COREFILE > "$TEST_DEBUGDIR/$TEST_NAME.$COREFILE.gz" || exit $?
  done
  # Pull in any .so files as well.
  for LIB in $(ldd $ABS_TEST_PATH | grep $BUILD_ROOT | awk '{print $3}'); do
    LIB_NAME=$(basename $LIB)
    gzip < $LIB > "$TEST_DEBUGDIR/$LIB_NAME.gz" || exit $?
  done
fi

popd >/dev/null
rm -Rf $TEST_WORKDIR

exit $STATUS


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

ROOT=$(cd $(dirname $BASH_SOURCE)/..; pwd)

LAST_COMMITTED_REV=$($ROOT/build-support/get-upstream-commit.sh)
if [ -z "$LAST_COMMITTED_REV" ]; then
  echo "ERROR: could not find the last committed upstream changelist"
  exit 1
fi

$ROOT/build-support/clang_tidy_gerrit.py \
  --no-gerrit \
  --rev-range \
  "${LAST_COMMITTED_REV}"


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This script is invoked from the Jenkins builds to build Kudu
# and run all the unit tests.
#
# Environment variables may be used to customize operation:
#   BUILD_TYPE: Default: DEBUG
#     Maybe be one of ASAN|TSAN|DEBUG|RELEASE|COVERAGE|LINT|IWYU
#
#   KUDU_ALLOW_SLOW_TESTS   Default: 1
#     Runs the "slow" version of the unit tests. Set to 0 to
#     run the tests more quickly.
#
#   TEST_TMPDIR   Default: /tmp/kudutest-$UID
#     Specifies the temporary directory where tests should write their
#     data. It is expected that following the completion of all tests, this
#     directory is empty (i.e. every test cleaned up after itself).
#
#   RUN_FLAKY_ONLY    Default: 0
#     Only runs tests which have failed recently, if this is 1.
#     Used by the kudu-flaky-tests jenkins build.
#
#   KUDU_FLAKY_TEST_ATTEMPTS  Default: 1
#     If more than 1, will fetch the list of known flaky tests
#     from the kudu-test jenkins job, and allow those tests to
#     be flaky in this build.
#
#   TEST_RESULT_SERVER  Default: none
#     The host:port pair of a server running test_result_server.py.
#     This must be configured for flaky test resistance or test result
#     reporting to work.
#
#   ENABLE_DIST_TEST  Default: 0
#     If set to 1, will submit C++ and Java tests to be run by the
#     distributed test runner instead of running them locally.
#     This requires that $DIST_TEST_HOME be set to a working dist_test
#     checkout (and that dist_test itself be appropriately configured to
#     point to a cluster)
#
#   BUILD_JAVA        Default: 1
#     Build and test java code if this is set to 1.
#
#   BUILD_MAVEN       Default: 0
#     When building java code, build with Maven if this is set to 1.
#
#   BUILD_GRADLE      Default: 1
#     When building java code, build with Gradle if this is set to 1.
#
#   BUILD_PYTHON       Default: 1
#     Build and test the Python wrapper of the client API.
#
#   BUILD_PYTHON3      Default: 1
#     Build and test the Python wrapper of the client API in Python. This
#     option is not mutually exclusive from BUILD_PYTHON. If both options
#     are set (default), then both will be run.
#
#   PIP_INSTALL_FLAGS  Default: ""
#     Extra flags which are passed to 'pip install' when setting up the build
#     environment for the Python wrapper.
#
#   MVN_FLAGS          Default: ""
#     Extra flags which are passed to 'mvn' when building and running Java
#     tests. This can be useful, for example, to choose a different maven
#     repository location.
#
#   GRADLE_FLAGS       Default: ""
#     Extra flags which are passed to 'gradle' when running Gradle commands.
#
#   ERROR_ON_TEST_FAILURE    Default: 1
#     Whether test failures will cause this script to return an error.

# If a commit messages contains a line that says 'DONT_BUILD', exit
# immediately.
DONT_BUILD=$(git show|egrep '^\s{4}DONT_BUILD$')
if [ "x$DONT_BUILD" != "x" ]; then
    echo "*** Build not requested. Exiting."
    exit 1
fi

set -e
# We pipe our build output to a log file with tee.
# This bash setting ensures that the script exits if the build fails.
set -o pipefail
# gather core dumps
ulimit -c unlimited

BUILD_TYPE=${BUILD_TYPE:-DEBUG}
BUILD_TYPE=$(echo "$BUILD_TYPE" | tr a-z A-Z) # capitalize
BUILD_TYPE_LOWER=$(echo "$BUILD_TYPE" | tr A-Z a-z)

# Set up defaults for environment variables.
DEFAULT_ALLOW_SLOW_TESTS=1

# TSAN builds are pretty slow, so don't do SLOW tests unless explicitly
# requested. Setting KUDU_USE_TSAN influences the thirdparty build.
if [ "$BUILD_TYPE" = "TSAN" ]; then
  DEFAULT_ALLOW_SLOW_TESTS=0
  export KUDU_USE_TSAN=1
fi

export KUDU_FLAKY_TEST_ATTEMPTS=${KUDU_FLAKY_TEST_ATTEMPTS:-1}
export KUDU_ALLOW_SLOW_TESTS=${KUDU_ALLOW_SLOW_TESTS:-$DEFAULT_ALLOW_SLOW_TESTS}
export KUDU_COMPRESS_TEST_OUTPUT=${KUDU_COMPRESS_TEST_OUTPUT:-1}
export TEST_TMPDIR=${TEST_TMPDIR:-/tmp/kudutest-$UID}
BUILD_JAVA=${BUILD_JAVA:-1}
BUILD_MAVEN=${BUILD_MAVEN:-0}
BUILD_GRADLE=${BUILD_GRADLE:-1}
BUILD_PYTHON=${BUILD_PYTHON:-1}
BUILD_PYTHON3=${BUILD_PYTHON3:-1}
ERROR_ON_TEST_FAILURE=${ERROR_ON_TEST_FAILURE:-1}

# Ensure that the test data directory is usable.
mkdir -p "$TEST_TMPDIR"
if [ ! -w "$TEST_TMPDIR" ]; then
  echo "Error: Test output directory ($TEST_TMPDIR) is not writable on $(hostname) by user $(whoami)"
  exit 1
fi

SOURCE_ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd)
BUILD_ROOT=$SOURCE_ROOT/build/$BUILD_TYPE_LOWER

# Remove testing artifacts from the previous run before we do anything
# else. Otherwise, if we fail during the "build" step, Jenkins will
# archive the test logs from the previous run, thinking they came from
# this run, and confuse us when we look at the failed build.
rm -rf $BUILD_ROOT
mkdir -p $BUILD_ROOT

# Same for the Java tests, which aren't inside BUILD_ROOT
rm -rf $SOURCE_ROOT/java/*/target
rm -rf $SOURCE_ROOT/java/*/build

list_flaky_tests() {
  local url="http://$TEST_RESULT_SERVER/list_failed_tests?num_days=3&build_pattern=%25kudu-test%25"
  >&2 echo Fetching flaky test list from "$url" ...
  curl -s --show-error "$url"
  return $?
}

TEST_LOGDIR="$BUILD_ROOT/test-logs"
TEST_DEBUGDIR="$BUILD_ROOT/test-debug"

cleanup() {
  echo Cleaning up all build artifacts...
  $SOURCE_ROOT/build-support/jenkins/post-build-clean.sh
}
# If we're running inside Jenkins (the BUILD_ID is set), then install
# an exit handler which will clean up all of our build results.
if [ -n "$BUILD_ID" ]; then
  trap cleanup EXIT
fi

export TOOLCHAIN_DIR=/opt/toolchain
if [ -d "$TOOLCHAIN_DIR" ]; then
  PATH=$TOOLCHAIN_DIR/apache-maven-3.0/bin:$PATH
fi

THIRDPARTY_TYPE=
if [ "$BUILD_TYPE" = "TSAN" ]; then
  THIRDPARTY_TYPE=tsan
fi
$SOURCE_ROOT/build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh $THIRDPARTY_TYPE

THIRDPARTY_BIN=$(pwd)/thirdparty/installed/common/bin
export PPROF_PATH=$THIRDPARTY_BIN/pprof

if which ccache >/dev/null ; then
  CLANG=$(pwd)/build-support/ccache-clang/clang
else
  CLANG=$(pwd)/thirdparty/clang-toolchain/bin/clang
fi

# Configure the build
#
# ASAN/TSAN can't build the Python bindings because the exported Kudu client
# library (which the bindings depend on) is missing ASAN/TSAN symbols.
cd $BUILD_ROOT
if [ "$BUILD_TYPE" = "ASAN" ]; then
  USE_CLANG=1
  CMAKE_BUILD=fastdebug
  EXTRA_BUILD_FLAGS="-DKUDU_USE_ASAN=1 -DKUDU_USE_UBSAN=1"
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
elif [ "$BUILD_TYPE" = "TSAN" ]; then
  USE_CLANG=1
  CMAKE_BUILD=fastdebug
  EXTRA_BUILD_FLAGS="-DKUDU_USE_TSAN=1"
  EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -LE no_tsan"
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
elif [ "$BUILD_TYPE" = "COVERAGE" ]; then
  USE_CLANG=1
  CMAKE_BUILD=debug
  EXTRA_BUILD_FLAGS="-DKUDU_GENERATE_COVERAGE=1"
  DO_COVERAGE=1

  # We currently dont capture coverage for Java or Python.
  BUILD_JAVA=0
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
elif [ "$BUILD_TYPE" = "LINT" ]; then
  CMAKE_BUILD=debug
elif [ "$BUILD_TYPE" = "IWYU" ]; then
  USE_CLANG=1
  CMAKE_BUILD=debug
else
  # Must be DEBUG or RELEASE
  CMAKE_BUILD=$BUILD_TYPE
fi

# Assemble the cmake command line, starting with environment variables.

# There's absolutely no reason to rebuild the thirdparty tree; we just ran
# build-if-necessary.sh above.
CMAKE="env NO_REBUILD_THIRDPARTY=1"

# If using clang, we have to change the compiler via environment variables.
if [ -n "$USE_CLANG" ]; then
  CMAKE="$CMAKE CC=$CLANG CXX=$CLANG++"
fi

# This will be a passthrough for systems without devtoolset.
CMAKE="$CMAKE $SOURCE_ROOT/build-support/enable_devtoolset.sh"

CMAKE="$CMAKE $THIRDPARTY_BIN/cmake"
CMAKE="$CMAKE -DCMAKE_BUILD_TYPE=$CMAKE_BUILD"

# On distributed tests, force dynamic linking even for release builds. Otherwise,
# the test binaries are too large and we spend way too much time uploading them
# to the test slaves.
if [ "$ENABLE_DIST_TEST" == "1" ]; then
  CMAKE="$CMAKE -DKUDU_LINK=dynamic"
fi
if [ -n "$EXTRA_BUILD_FLAGS" ]; then
  CMAKE="$CMAKE $EXTRA_BUILD_FLAGS"
fi
CMAKE="$CMAKE $SOURCE_ROOT"
$CMAKE

# Create empty test logs or else Jenkins fails to archive artifacts, which
# results in the build failing.
mkdir -p Testing/Temporary
mkdir -p $TEST_LOGDIR

# Short circuit for LINT builds.
if [ "$BUILD_TYPE" = "LINT" ]; then
  make lint | tee $TEST_LOGDIR/lint.log
  exit $?
fi

# Short circuit for IWYU builds: run the include-what-you-use tool on the files
# modified since the last committed changelist committed upstream.
if [ "$BUILD_TYPE" = "IWYU" ]; then
  make iwyu | tee $TEST_LOGDIR/iwyu.log
  exit $?
fi

# Only enable test core dumps for certain build types.
if [ "$BUILD_TYPE" != "ASAN" ]; then
  export KUDU_TEST_ULIMIT_CORE=unlimited
fi

# If we are supposed to be resistant to flaky tests, we need to fetch the
# list of tests to ignore
if [ "$KUDU_FLAKY_TEST_ATTEMPTS" -gt 1 ]; then
  echo Fetching flaky test list...
  export KUDU_FLAKY_TEST_LIST=$BUILD_ROOT/flaky-tests.txt
  mkdir -p $(dirname $KUDU_FLAKY_TEST_LIST)
  echo -n > $KUDU_FLAKY_TEST_LIST
    if [ -n "$TEST_RESULT_SERVER" ] && \
        list_flaky_tests > $KUDU_FLAKY_TEST_LIST ; then
    echo Will retry flaky tests up to $KUDU_FLAKY_TEST_ATTEMPTS times:
    cat $KUDU_FLAKY_TEST_LIST
    echo ----------
  else
    echo Unable to fetch flaky test list. Disabling flaky test resistance.
    export KUDU_FLAKY_TEST_ATTEMPTS=1
  fi
fi

# our tests leave lots of data lying around, clean up before we run
if [ -d "$TEST_TMPDIR" ]; then
  rm -Rf $TEST_TMPDIR/*
fi

# actually do the build
echo
echo Building C++ code.
echo ------------------------------------------------------------
NUM_PROCS=$(getconf _NPROCESSORS_ONLN)
make -j$NUM_PROCS 2>&1 | tee build.log

# If compilation succeeds, try to run all remaining steps despite any failures.
set +e

# Run tests
if [ "$RUN_FLAKY_ONLY" == "1" ] ; then
  if [ -z "$TEST_RESULT_SERVER" ]; then
    echo Must set TEST_RESULT_SERVER to use RUN_FLAKY_ONLY
    exit 1
  fi
  echo
  echo Running flaky tests only:
  echo ------------------------------------------------------------
  if ! ( set -o pipefail ;
         list_flaky_tests | tee $BUILD_ROOT/flaky-tests.txt) ; then
    echo Could not fetch flaky tests list.
    exit 1
  fi
  test_regex=$(perl -e '
    chomp(my @lines = <>);
    print join("|", map { "^" . quotemeta($_) } @lines);
   ' $BUILD_ROOT/flaky-tests.txt)
  if [ -z "$test_regex" ]; then
    echo No tests are flaky.
    exit 0
  fi
  EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -R $test_regex"

  # We don't support detecting java and python flaky tests at the moment.
  echo Disabling Java and python build since RUN_FLAKY_ONLY=1
  BUILD_PYTHON=0
  BUILD_PYTHON3=0
  BUILD_JAVA=0
fi

TESTS_FAILED=0
EXIT_STATUS=0
FAILURES=""

# If we're running distributed C++ tests, submit them asynchronously while
# we run the Java and Python tests.
if [ "$ENABLE_DIST_TEST" == "1" ]; then
  echo
  echo Submitting C++ distributed-test job.
  echo ------------------------------------------------------------
  # dist-test uses DIST_TEST_JOB_PATH to define where to output it's id file.
  export DIST_TEST_JOB_PATH=$BUILD_ROOT/c-dist-test-job-id
  rm -f $DIST_TEST_JOB_PATH
  if ! $SOURCE_ROOT/build-support/dist_test.py --no-wait run ; then
    EXIT_STATUS=1
    FAILURES="$FAILURES"$'Could not submit C++ distributed test job\n'
  fi
  # Still need to run a few non-dist-test-capable tests locally.
  EXTRA_TEST_FLAGS="$EXTRA_TEST_FLAGS -L no_dist_test"
fi

if ! $THIRDPARTY_BIN/ctest -j$NUM_PROCS $EXTRA_TEST_FLAGS ; then
  TESTS_FAILED=1
  FAILURES="$FAILURES"$'C++ tests failed\n'
fi

if [ "$DO_COVERAGE" == "1" ]; then
  echo
  echo Generating coverage report...
  echo ------------------------------------------------------------
  if ! $SOURCE_ROOT/thirdparty/installed/common/bin/gcovr \
      -r $SOURCE_ROOT \
      --gcov-filter='.*src#kudu.*' \
      --gcov-executable=$SOURCE_ROOT/build-support/llvm-gcov-wrapper \
      --xml \
      > $BUILD_ROOT/coverage.xml ; then
    EXIT_STATUS=1
    FAILURES="$FAILURES"$'Coverage report failed\n'
  fi
fi

if [ "$BUILD_JAVA" == "1" ]; then
  echo
  echo Building and testing java...
  echo ------------------------------------------------------------

  # Make sure we use JDK8
  export JAVA_HOME=$JAVA8_HOME
  export PATH=$JAVA_HOME/bin:$PATH
  pushd $SOURCE_ROOT/java
  set -x

  # Run the full Maven build.
  if [ "$BUILD_MAVEN" == "1" ]; then
    EXTRA_MVN_FLAGS="-B"
    EXTRA_MVN_FLAGS="$EXTRA_MVN_FLAGS -Dsurefire.rerunFailingTestsCount=3"
    EXTRA_MVN_FLAGS="$EXTRA_MVN_FLAGS -Dfailsafe.rerunFailingTestsCount=3"
    EXTRA_MVN_FLAGS="$EXTRA_MVN_FLAGS -Dmaven.javadoc.skip"
    EXTRA_MVN_FLAGS="$EXTRA_MVN_FLAGS $MVN_FLAGS"
    if ! mvn $EXTRA_MVN_FLAGS clean verify ; then
      TESTS_FAILED=1
      FAILURES="$FAILURES"$'Java Maven build/test failed\n'
    fi
  fi

  # Run the full Gradle build.
  if [ "$BUILD_GRADLE" == "1" ]; then
    export EXTRA_GRADLE_FLAGS="--console=plain"
    EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS --no-daemon"
    EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS --continue"
    EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS -DrerunFailingTestsCount=3"
    # KUDU-2524: temporarily disable scalafmt until we can work out its JDK
    # incompatibility issue.
    EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS -DskipFormat"
    EXTRA_GRADLE_FLAGS="$EXTRA_GRADLE_FLAGS $GRADLE_FLAGS"
    # If we're running distributed Java tests, submit them asynchronously.
    if [ "$ENABLE_DIST_TEST" == "1" ]; then
      echo
      echo Submitting Java distributed-test job.
      echo ------------------------------------------------------------
      # dist-test uses DIST_TEST_JOB_PATH to define where to output it's id file.
      export DIST_TEST_JOB_PATH=$BUILD_ROOT/java-dist-test-job-id
      rm -f $DIST_TEST_JOB_PATH
      if ! $SOURCE_ROOT/build-support/dist_test.py --no-wait java run-all ; then
        EXIT_STATUS=1
        FAILURES="$FAILURES"$'Could not submit Java distributed test job\n'
      fi
    else
      # TODO: Run `gradle check` in BUILD_TYPE DEBUG when static code analysis is fixed
      if ! ./gradlew $EXTRA_GRADLE_FLAGS clean test ; then
        TESTS_FAILED=1
        FAILURES="$FAILURES"$'Java Gradle build/test failed\n'
      fi
    fi
  fi

  # Run a script to verify the contents of the JARs to ensure the shading and
  # packaging is correct.
  $SOURCE_ROOT/build-support/verify_jars.pl .

  set +x
  popd
fi

if [ "$BUILD_PYTHON" == "1" ]; then
  echo
  echo Building and testing python.
  echo ------------------------------------------------------------

  # Failing to compile the Python client should result in a build failure.
  set -e
  export KUDU_HOME=$SOURCE_ROOT
  export KUDU_BUILD=$BUILD_ROOT
  pushd $SOURCE_ROOT/python

  # Create a sane test environment.
  rm -Rf $KUDU_BUILD/py_env
  virtualenv $KUDU_BUILD/py_env
  source $KUDU_BUILD/py_env/bin/activate

  # Old versions of pip (such as the one found in el6) default to pypi's http://
  # endpoint which no longer exists. The -i option lets us switch to the
  # https:// endpoint in such cases.
  #
  # Unfortunately, in these old versions of pip, -i doesn't appear to apply
  # recursively to transitive dependencies installed via a direct dependency's
  # "python setup.py" command. Therefore we have no choice but to upgrade to a
  # new version of pip to proceed.
  #
  # Beginning with pip 10, Python 2.6 is no longer supported. Attempting to
  # upgrade to pip 10 on Python 2.6 yields syntax errors. We don't need any new
  # pip features, so let's pin to the last pip version to support Python 2.6.
  pip install -i https://pypi.python.org/simple $PIP_INSTALL_FLAGS --upgrade 'pip < 10.0.0b1'

  # New versions of pip raise an exception when upgrading old versions of
  # setuptools (such as the one found in el6). The workaround is to upgrade
  # setuptools on its own, outside of requirements.txt, and with the pip version
  # check disabled.
  pip install --disable-pip-version-check $PIP_INSTALL_FLAGS --upgrade 'setuptools >= 0.8'

  # One of our dependencies is pandas, installed via requirements.txt below. It
  # depends on numpy, and if we don't install numpy directly, the pandas
  # installation will install the latest numpy which is incompatible with Python 2.6.
  #
  # To work around this, we need to install a 2.6-compatible version of numpy
  # before installing pandas. Listing such a numpy version in requirements.txt
  # doesn't work; it needs to be explicitly installed here.
  #
  # Installing numpy may involve some compiler work, so we must pass in the
  # current values of CC and CXX.
  #
  # See https://github.com/numpy/numpy/releases/tag/v1.12.0 for more details.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS 'numpy <1.12.0'

  # We've got a new pip and new setuptools. We can now install the rest of the
  # Python client's requirements.
  #
  # Installing the Cython dependency may involve some compiler work, so we must
  # pass in the current values of CC and CXX.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt

  # We need to install Pandas manually since its not a required package but is needed
  # to run all of the tests.
  # pandas 0.18 dropped support for python 2.6.
  #
  # See https://pandas.pydata.org/pandas-docs/version/0.23.0/whatsnew.html#v0-18-0-march-13-2016
  # for more details.
  pip install $PIP_INSTALL_FLAGS "pandas<0.18"

  # Delete old Cython extensions to force them to be rebuilt.
  rm -Rf build kudu_python.egg-info kudu/*.so

  # Build the Python bindings. This assumes we run this script from base dir.
  CC=$CLANG CXX=$CLANG++ python setup.py build_ext
  set +e

  # Run the Python tests.
  if ! python setup.py test \
      --addopts="kudu --junit-xml=$TEST_LOGDIR/python_client.xml" \
      2> $TEST_LOGDIR/python_client.log ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Python tests failed\n'
  fi

  deactivate
  popd
fi

if [ "$BUILD_PYTHON3" == "1" ]; then
  echo
  echo Building and testing python 3.
  echo ------------------------------------------------------------

  # Failing to compile the Python client should result in a build failure.
  set -e
  export KUDU_HOME=$SOURCE_ROOT
  export KUDU_BUILD=$BUILD_ROOT
  pushd $SOURCE_ROOT/python

  # Create a sane test environment.
  rm -Rf $KUDU_BUILD/py_env
  virtualenv -p python3 $KUDU_BUILD/py_env
  source $KUDU_BUILD/py_env/bin/activate

  # Old versions of pip (such as the one found in el6) default to pypi's http://
  # endpoint which no longer exists. The -i option lets us switch to the
  # https:// endpoint in such cases.
  #
  # Unfortunately, in these old versions of pip, -i doesn't appear to apply
  # recursively to transitive dependencies installed via a direct dependency's
  # "python setup.py" command. Therefore we have no choice but to upgrade to a
  # new version of pip to proceed.
  pip install -i https://pypi.python.org/simple $PIP_INSTALL_FLAGS --upgrade pip

  # New versions of pip raise an exception when upgrading old versions of
  # setuptools (such as the one found in el6). The workaround is to upgrade
  # setuptools on its own, outside of requirements.txt, and with the pip version
  # check disabled.
  pip install --disable-pip-version-check $PIP_INSTALL_FLAGS --upgrade 'setuptools >= 0.8'

  # We've got a new pip and new setuptools. We can now install the rest of the
  # Python client's requirements.
  #
  # Installing the Cython dependency may involve some compiler work, so we must
  # pass in the current values of CC and CXX.
  CC=$CLANG CXX=$CLANG++ pip install $PIP_INSTALL_FLAGS -r requirements.txt

  # We need to install Pandas manually since its not a required package but is needed
  # to run all of the tests.
  pip install $PIP_INSTALL_FLAGS pandas

  # Delete old Cython extensions to force them to be rebuilt.
  rm -Rf build kudu_python.egg-info kudu/*.so

  # Build the Python bindings. This assumes we run this script from base dir.
  CC=$CLANG CXX=$CLANG++ python setup.py build_ext
  set +e

  # Run the Python tests.
  if ! python setup.py test \
      --addopts="kudu --junit-xml=$TEST_LOGDIR/python3_client.xml" \
      2> $TEST_LOGDIR/python3_client.log ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Python 3 tests failed\n'
  fi

  deactivate
  popd
fi

# If we submitted the tasks earlier, go fetch the results now
if [ "$ENABLE_DIST_TEST" == "1" ]; then
  echo
  echo Fetching previously submitted C++ dist-test results...
  echo ------------------------------------------------------------
  C_DIST_TEST_ID=`cat $BUILD_ROOT/c-dist-test-job-id`
  if ! $DIST_TEST_HOME/bin/client watch $C_DIST_TEST_ID ; then
    TESTS_FAILED=1
    FAILURES="$FAILURES"$'Distributed C++ tests failed\n'
  fi
  DT_DIR=$TEST_LOGDIR/dist-test-out
  rm -Rf $DT_DIR
  $DIST_TEST_HOME/bin/client fetch --artifacts -d $DT_DIR $C_DIST_TEST_ID
  # Fetching the artifacts expands each log into its own directory.
  # Move them back into the main log directory
  rm -f $DT_DIR/*zip
  for arch_dir in $DT_DIR/* ; do
    mv $arch_dir/build/$BUILD_TYPE_LOWER/test-logs/* $TEST_LOGDIR
    rm -Rf $arch_dir
  done

  if [ "$BUILD_GRADLE" == "1" ]; then
    echo
    echo Fetching previously submitted Java dist-test results...
    echo ------------------------------------------------------------
    JAVA_DIST_TEST_ID=`cat $BUILD_ROOT/java-dist-test-job-id`
    if ! $DIST_TEST_HOME/bin/client watch $JAVA_DIST_TEST_ID ; then
      TESTS_FAILED=1
      FAILURES="$FAILURES"$'Distributed Java tests failed\n'
    fi
    DT_DIR=$TEST_LOGDIR/java-dist-test-out
    rm -Rf $DT_DIR
    $DIST_TEST_HOME/bin/client fetch --artifacts -d $DT_DIR $JAVA_DIST_TEST_ID
    # Fetching the artifacts expands each log into its own directory.
    # Move them back into the main log directory
    rm -f $DT_DIR/*zip
    for arch_dir in $DT_DIR/* ; do
      mv $arch_dir/build/java/test-logs/* $TEST_LOGDIR
      rm -Rf $arch_dir
    done
  fi
fi

if [ "$TESTS_FAILED" != "0" -o "$EXIT_STATUS" != "0" ]; then
  echo
  echo Tests failed, making sure we have XML files for all tests.
  echo ------------------------------------------------------------

  # Tests that crash do not generate JUnit report XML files.
  # We go through and generate a kind of poor-man's version of them in those cases.
  for GTEST_OUTFILE in $TEST_LOGDIR/*.txt.gz; do
    TEST_EXE=$(basename $GTEST_OUTFILE .txt.gz)
    GTEST_XMLFILE="$TEST_LOGDIR/$TEST_EXE.xml"
    if [ ! -f "$GTEST_XMLFILE" ]; then
      echo "JUnit report missing:" \
           "generating fake JUnit report file from $GTEST_OUTFILE and saving it to $GTEST_XMLFILE"
      zcat $GTEST_OUTFILE | $SOURCE_ROOT/build-support/parse_test_failure.py -x > $GTEST_XMLFILE
    fi
  done
else
  # If all tests passed, ensure that they cleaned up their test output.
  TEST_TMPDIR_CONTENTS=$(ls $TEST_TMPDIR)
  if [ -n "$TEST_TMPDIR_CONTENTS" ]; then
    echo "All tests passed, yet some left behind their test output."
    echo "TEST_TMPDIR: $TEST_TMPDIR"
    find $TEST_TMPDIR -ls
    EXIT_STATUS=1
  fi
fi

set -e

if [ -n "$FAILURES" ]; then
  echo
  echo
  echo ======================================================================
  echo Failure summary
  echo ======================================================================
  echo $FAILURES
  echo
  echo
fi

# If any of the tests failed and we are honoring test failures, set the exit
# status to 1. Note that it may have already been set to 1 above.
if [ "$ERROR_ON_TEST_FAILURE" == "1" -a "$TESTS_FAILED" == "1" ]; then
  EXIT_STATUS=1
fi

exit $EXIT_STATUS


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Script which, when run from Jenkins, will hunt and kill any other jenkins
# processes running on the same machine from a different build.
#
# This is done by looking for all of the processes owned by the jenkins user,
# checking their BUILD_ID environment variable, and killing any whose BUILD_ID
# does not match the current $BUILD_ID environment.  This assumes that there are
# no concurrent builds configured in Jenkins, for obvious reasons.
#
# Set $DRY_RUN before running this script to just see what would be killed.

set -e

JENKINS_USER=${JENKINS_USER:-jenkins}
CURRENT_BUILD=$BUILD_ID

if [ "$USER" != "$JENKINS_USER" ]; then
  echo Not running as user \'$JENKINS_USER\'
  exit 1
fi

if [ -z "$CURRENT_BUILD" ]; then
  echo Not running in the context of a Jenkins build
  exit 1
fi

JENKINS_PIDS=$(pgrep -u $JENKINS_USER)
for pid in $JENKINS_PIDS; do
  cmdline=$(ps h -p $pid -o cmd || echo '[pid exited]')
  build_env=$(cat /proc/$pid/environ 2>/dev/null | tr '\0' '\n' | egrep '^BUILD_ID=' || :)
  if [ -z "$build_env" ]; then
    # Some Jenkins processes, like the slave itself, don't have a BUILD_ID
    # set. We shouldn't kill those.
    echo "Process $pid ($cmdline) not associated with any build. Skipping..."
    continue
  fi
  build_id=$(echo $build_env | cut -d= -f2)
  if [ "$build_id" != "$CURRENT_BUILD" ]; then
    echo "Killing zombie process $pid (from build $build_id)"
    ps -fww -p $pid || :
    if [ -z "$DRY_RUN" ]; then
      kill -9 $pid || :
    fi
    echo ----------
  else
    echo "pid $pid ($cmdline) is from the current build. Not killing"
  fi
done


#!/bin/bash -x
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Script which runs on Jenkins slaves after the build/test to clean up
# disk space used by build artifacts. Our build tends to "make clean"
# before running anyway, so doing the post-build cleanup shouldn't
# hurt our build times. It does, however, save a fair amount of disk
# space in the Jenkins workspace disk. This can help prevent our EC2
# slaves from filling up and causing spurious failures.

ROOT=$(cd $(dirname "$BASH_SOURCE")/../..; pwd)
cd $ROOT

# Note that we use simple shell commands instead of "make clean"
# or "mvn clean". This is more foolproof even if something ends
# up partially compiling, etc.

# Clean up intermediate object files in the src tree
find build/latest/src -name \*.o -exec rm -f {} \;

# Clean up the actual build artifacts
rm -Rf build/latest/bin build/latest/lib

# Clean up any java build artifacts
find java -name \*.jar -delete -o -name \*.class -delete