#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# build-thirdparty.sh builds and installs thirdparty dependencies into prefix
# directories within the thirdparty directory. Three prefix directories are
# used, corresponding to build type:
#
#   * /thirdparty/installed/common - prefix directory for libraries and binary tools
#                                    common to all build types, e.g. CMake, C dependencies.
#   * /thirdparty/installed/uninstrumented - prefix directory for libraries built with
#                                            normal options (no sanitizer instrumentation).
#   * /thirdparty/installed/tsan - prefix directory for libraries built
#                                  with thread sanitizer instrumentation.
#
# Environment variables which can be set when calling build-thirdparty.sh:
#   * EXTRA_CFLAGS - additional flags passed to the C compiler.
#   * EXTRA_CXXFLAGS - additional flags passed to the C++ compiler.
#   * EXTRA_LDFLAGS - additional flags passed to the linker.
#   * EXTRA_LIBS - additional libraries to link.
#   * EXTRA_MAKEFLAGS - additional flags passed to make.
#   * PARALLEL - parallelism to use when compiling (defaults to number of cores).

set -ex

TP_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)

source $TP_DIR/vars.sh
source $TP_DIR/build-definitions.sh

# Before doing anything, run the pre-flight check for missing dependencies.
# This avoids the most common issues people have with building (if they don't
# read the docs)
$TP_DIR/preflight.py

################################################################################

if [ "$#" = "0" ]; then
  ARGS_TO_PRINT="common uninstrumented tsan"

  F_COMMON=1
  F_UNINSTRUMENTED=1
  F_TSAN=1
else
  ARGS_TO_PRINT="$*"
  REQUESTED_EXPLICIT_DEPENDENCIES=1

  # Parse the command line for specific dependencies or dependency groups.
  for arg in $*; do
    case $arg in
      # Dependency groups.
      "common")         F_COMMON=1 ;;
      "uninstrumented") F_UNINSTRUMENTED=1 ;;
      "tsan")           F_TSAN=1 ;;

      # Dependencies.
      "cmake")        F_CMAKE=1 ;;
      "gflags")       F_GFLAGS=1 ;;
      "glog")         F_GLOG=1 ;;
      "gmock")        F_GMOCK=1 ;;
      "gperftools")   F_GPERFTOOLS=1 ;;
      "libev")        F_LIBEV=1 ;;
      "lz4")          F_LZ4=1 ;;
      "bitshuffle")   F_BITSHUFFLE=1 ;;
      "protobuf")     F_PROTOBUF=1 ;;
      "rapidjson")    F_RAPIDJSON=1 ;;
      "snappy")       F_SNAPPY=1 ;;
      "zlib")         F_ZLIB=1 ;;
      "squeasel")     F_SQUEASEL=1 ;;
      "mustache")     F_MUSTACHE=1 ;;
      "gsg")          F_GSG=1 ;;
      "gcovr")        F_GCOVR=1 ;;
      "curl")         F_CURL=1 ;;
      "crcutil")      F_CRCUTIL=1 ;;
      "libunwind")    F_LIBUNWIND=1 ;;
      "llvm")         F_LLVM=1 ;;
      "trace-viewer") F_TRACE_VIEWER=1 ;;
      "nvml")         F_NVML=1 ;;
      "boost")        F_BOOST=1 ;;
      "breakpad")     F_BREAKPAD=1 ;;
      "sparsehash")   F_SPARSEHASH=1 ;;
      "sparsepp")     F_SPARSEPP=1 ;;
      "thrift")       F_THRIFT=1 ;;
      "bison")        F_BISON=1 ;;
      "hadoop")       F_HADOOP=1 ;;
      "hive")         F_HIVE=1 ;;
      "sentry")       F_SENTRY=1 ;;
      "coredumper")   F_COREDUMPER=1 ;;
      *)              echo "Unknown module: $arg"; exit 1 ;;
    esac
  done
fi

################################################################################

finish() {
  # Run the post-flight checks.
  $TP_DIR/postflight.py

  echo "---------------------"
  echo "Thirdparty dependencies '$ARGS_TO_PRINT' built and installed successfully"
  exit 0
}

for PREFIX_DIR in $PREFIX_COMMON $PREFIX_DEPS $PREFIX_DEPS_TSAN; do
  mkdir -p $PREFIX_DIR/include

  # PREFIX_COMMON is for header-only libraries.
  if [ "$PREFIX_DIR" != "$PREFIX_COMMON" ]; then
    mkdir -p $PREFIX_DIR/lib

    # On some systems, autotools installs libraries to lib64 rather than lib.  Fix
    # this by setting up lib64 as a symlink to lib.  We have to do this step first
    # to handle cases where one third-party library depends on another.
    ln -nsf "$PREFIX_DIR/lib" "$PREFIX_DIR/lib64"
  fi
done

# Incorporate the value of these standard compilation environment variables into
# the EXTRA_* environment variables.
EXTRA_CFLAGS="$CFLAGS $EXTRA_CFLAGS"
EXTRA_CXXFLAGS="$CXXFLAGS $EXTRA_CXXFLAGS"
EXTRA_LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
EXTRA_LIBS="$LIBS $EXTRA_LIBS"

# We use -O2 instead of -O3 for thirdparty since benchmarks indicate
# that the benefits of a smaller code size outweight the benefits of
# more inlining.
#
# We also enable -fno-omit-frame-pointer so that profiling tools which
# use frame-pointer based stack unwinding can function correctly.
EXTRA_CFLAGS="-fno-omit-frame-pointer $EXTRA_CFLAGS"
EXTRA_CXXFLAGS="-fno-omit-frame-pointer -O2 $EXTRA_CXXFLAGS"

if [[ "$OSTYPE" =~ ^linux ]]; then
  OS_LINUX=1
  DYLIB_SUFFIX="so"
  PARALLEL=${PARALLEL:-$(grep -c processor /proc/cpuinfo)}

  if [ -d "$OPENSSL_WORKAROUND_DIR" ]; then
    # If the el6 workaround openssl is present, we must build dependencies
    # against that version of openssl, not the system version, because at test
    # runtime we use the workaround openssl.
    OPENSSL_CFLAGS="-I$OPENSSL_WORKAROUND_DIR/usr/include"
    OPENSSL_LDFLAGS="-L$OPENSSL_WORKAROUND_DIR/usr/lib64 -Wl,-rpath,$OPENSSL_WORKAROUND_DIR/usr/lib64"
  fi
elif [[ "$OSTYPE" == "darwin"* ]]; then
  OS_OSX=1
  DYLIB_SUFFIX="dylib"
  PARALLEL=${PARALLEL:-$(sysctl -n hw.ncpu)}

  # Kudu builds with C++11, which on OS X requires using libc++ as the standard
  # library implementation. Some of the dependencies do not compile against
  # libc++ by default, so we specify it explicitly.
  EXTRA_CXXFLAGS="$EXTRA_CXXFLAGS -stdlib=libc++"
  EXTRA_LDFLAGS="$EXTRA_LDFLAGS -stdlib=libc++"
  EXTRA_LIBS="$EXTRA_LIBS -lc++ -lc++abi"

  # Build against the Macports or Homebrew OpenSSL versions, in order to match
  # the Kudu build.
  if ! OPENSSL_CFLAGS=$(pkg-config --cflags openssl); then
    # If OpenSSL is built via Homebrew, pkg-config does not report on cflags.
    homebrew_openssl_dir=/usr/local/opt/openssl
    if [ -d $homebrew_openssl_dir ]; then
      OPENSSL_CFLAGS="-I$homebrew_openssl_dir/include"
      OPENSSL_LDFLAGS="-L$homebrew_openssl_dir/lib"
    fi
  fi

  # TSAN doesn't work on macOS. If it was explicitly asked for, respond with an
  # error. Otherwise, just disable it silently.
  if [ -n "$F_TSAN" ]; then
    if [ -n "$REQUESTED_EXPLICIT_DEPENDENCIES" ]; then
      echo TSAN does not work on macOS
      exit 1
    else
      unset F_TSAN
    fi
  fi
else
  echo Unsupported platform $OSTYPE
  exit 1
fi

### Detect and enable 'ninja' instead of 'make' for faster builds.
if which ninja-build > /dev/null ; then
  NINJA=ninja-build
  EXTRA_CMAKE_FLAGS=-GNinja
elif which ninja > /dev/null ; then
  NINJA=ninja
  EXTRA_CMAKE_FLAGS=-GNinja
fi

### Build common tools and header-only libraries

PREFIX=$PREFIX_COMMON
MODE_SUFFIX=""

# Add tools to path
export PATH=$PREFIX/bin:$PATH

: '
if [ -n "$F_COMMON" -o -n "$F_CMAKE" ]; then
  build_cmake
fi

if [ -n "$F_COMMON" -o -n "$F_RAPIDJSON" ]; then
  build_rapidjson
fi

if [ -n "$F_COMMON" -o -n "$F_GSG" ]; then
  build_cpplint
fi

if [ -n "$F_COMMON" -o -n "$F_GCOVR" ]; then
  build_gcovr
fi

if [ -n "$F_COMMON" -o -n "$F_TRACE_VIEWER" ]; then
  build_trace_viewer
fi
'

if [ -n "$F_COMMON" -o -n "$F_SPARSEHASH" ]; then
  build_sparsehash
fi

if [ -n "$F_COMMON" -o -n "$F_SPARSEPP" ]; then
  build_sparsepp
fi

if [ -n "$F_COMMON" -o -n "$F_BISON" ]; then
  build_bison
fi

: '
# Install Hadoop, Hive, and Sentry by symlinking their source directories (which
# are pre-built) into $PREFIX/opt.
if [ -n "$F_COMMON" -o -n "$F_HADOOP" ]; then
  mkdir -p $PREFIX/opt
  ln -nsf $HADOOP_SOURCE $PREFIX/opt/hadoop
fi

if [ -n "$F_COMMON" -o -n "$F_HIVE" ]; then
  mkdir -p $PREFIX/opt
  ln -nsf $HIVE_SOURCE $PREFIX/opt/hive
fi

if [ -n "$F_COMMON" -o -n "$F_SENTRY" ]; then
  mkdir -p $PREFIX/opt
  ln -nsf $SENTRY_SOURCE $PREFIX/opt/sentry
fi
'

### Build C dependencies without instrumentation

PREFIX=$PREFIX_DEPS
MODE_SUFFIX=""

save_env

# Enable debug symbols so that stacktraces and linenumbers are available at runtime.
EXTRA_CFLAGS="-g $EXTRA_CFLAGS"
EXTRA_CXXFLAGS="-g $EXTRA_CXXFLAGS"

if [ -n "$OS_LINUX" ] && [ -n "$F_UNINSTRUMENTED" -o -n "$F_LIBUNWIND" ]; then
  build_libunwind
fi

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_ZLIB" ]; then
  build_zlib
fi

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_LZ4" ]; then
  build_lz4
fi

: '
if [ -n "$F_UNINSTRUMENTED" -o -n "$F_BITSHUFFLE" ]; then
  build_bitshuffle
fi
'

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_LIBEV" ]; then
  build_libev
fi

: '
if [ -n "$F_UNINSTRUMENTED" -o -n "$F_SQUEASEL" ]; then
  build_squeasel
fi
'

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_CURL" ]; then
  build_curl
fi

: '
if [ -n "$OS_LINUX" ] && [ -n "$F_UNINSTRUMENTED" -o -n "$F_NVML" ]; then
  build_nvml
fi
'

restore_env

### Build C++ dependencies without instrumentation
: '
# Clang is used by all builds so it is part of the 'common' library group even
# though its LLVM libraries are installed to $PREFIX_DEPS.
if [ -n "$F_COMMON" -o -n "$F_LLVM" ]; then
  build_llvm normal
fi
'

save_env

# Enable debug symbols so that stacktraces and linenumbers are available at
# runtime. LLVM is compiled without debug symbols as they take up more than
# 20GiB of disk space.
EXTRA_CFLAGS="-g $EXTRA_CFLAGS"
EXTRA_CXXFLAGS="-g $EXTRA_CXXFLAGS"

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_GFLAGS" ]; then
  build_gflags
fi

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_GLOG" ]; then
  build_glog
fi

: '
if [ -n "$F_UNINSTRUMENTED" -o -n "$F_GPERFTOOLS" ]; then
  build_gperftools
fi
'

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_GMOCK" ]; then
  build_gmock
fi

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_PROTOBUF" ]; then
  build_protobuf
fi

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_SNAPPY" ]; then
  build_snappy
fi

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_CRCUTIL" ]; then
  build_crcutil
fi

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_BOOST" ]; then
  build_boost normal
fi

: '
if [ -n "$F_UNINSTRUMENTED" -o -n "$F_MUSTACHE" ]; then
  build_mustache
fi

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_BREAKPAD" ]; then
  build_breakpad
fi

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_THRIFT" ]; then
  build_thrift
fi
'

if [ -n "$F_UNINSTRUMENTED" -o -n "$F_COREDUMPER" ]; then
  build_coredumper
fi

restore_env

# If we're on macOS best to exit here, otherwise single dependency builds will try to
# build the tsan version of the dependency (and fail).

if [[ "$OSTYPE" == "darwin"* ]]; then
  echo "Not building tsan dependencies on macOS."
  finish
fi

### Build dependencies with TSAN instrumentation

# Achieving good results with TSAN requires that:
# 1. The C++ standard library should be instrumented with TSAN.
# 2. Dependencies which internally use threads or synchronization be
#    instrumented with TSAN.
# 3. As a corollary to 1, the C++ standard library requires that all shared
#    objects linked into an executable be built against the same version of the
#    C++ standard library version.
#
# At the very least, we must build our own C++ standard library. We use libc++
# because it's easy to build with clang, which has better TSAN support than gcc.
#
# To satisfy all of the above requirements, we first build libc++ instrumented
# with TSAN, then build a second copy of every C++ dependency against that
# libc++. Later on in the build process, Kudu is also built against libc++.
#
# Special flags for TSAN builds:
#   * -fsanitize=thread -  enable the thread sanitizer during compilation.
#   * -L ... - add the instrumented libc++ to the library search paths.
#   * -isystem ... - Add libc++ headers to the system header search paths.
#   * -nostdinc++ - Do not automatically link the system C++ standard library.
#   * -Wl,-rpath,... - Add instrumented libc++ location to the rpath so that it
#                      can be found at runtime.

if which ccache >/dev/null ; then
  CLANG="$TP_DIR/../build-support/ccache-clang/clang"
  CLANGXX="$TP_DIR/../build-support/ccache-clang/clang++"
else
  CLANG="$TP_DIR/clang-toolchain/bin/clang"
  CLANGXX="$TP_DIR/clang-toolchain/bin/clang++"
fi
export CC=$CLANG
export CXX=$CLANGXX

PREFIX=$PREFIX_DEPS_TSAN
MODE_SUFFIX=".tsan"

save_env

# Build the C (non-C++) dependencies with TSAN instrumentation.
EXTRA_CFLAGS="-fsanitize=thread $EXTRA_CFLAGS"

# Enable debug symbols so that stacktraces and linenumbers are available at runtime.
EXTRA_CFLAGS="-g $EXTRA_CFLAGS"
EXTRA_CXXFLAGS="-g $EXTRA_CXXFLAGS"

if [ -n "$F_TSAN" -o -n "$F_ZLIB" ]; then
  build_zlib
fi

if [ -n "$OS_LINUX" ] && [ -n "$F_TSAN" -o -n "$F_LIBUNWIND" ]; then
  build_libunwind
fi

if [ -n "$F_TSAN" -o -n "$F_LZ4" ]; then
  build_lz4
fi

if [ -n "$F_TSAN" -o -n "$F_BITSHUFFLE" ]; then
  build_bitshuffle
fi

if [ -n "$F_TSAN" -o -n "$F_LIBEV" ]; then
  build_libev
fi

if [ -n "$F_TSAN" -o -n "$F_SQUEASEL" ]; then
  build_squeasel
fi

if [ -n "$F_TSAN" -o -n "$F_CURL" ]; then
  build_curl
fi

if [ -n "$OS_LINUX" ] && [ -n "$F_TSAN" -o -n "$F_NVML" ]; then
  build_nvml
fi

restore_env

### Build C++ dependencies with TSAN instrumentation

# Build libc++abi first as it is a dependency for libc++. Its build has no
# built-in support for sanitizers, so we build it regularly.
if [ -n "$F_TSAN" -o -n "$F_LLVM" ]; then
  build_libcxxabi
fi

save_env

# Build libc++ with TSAN enabled.
if [ -n "$F_TSAN" -o -n "$F_LLVM" ]; then
  build_libcxx tsan
fi

# Build the rest of the dependencies against the TSAN-instrumented libc++
# instead of the system's C++ standard library.
EXTRA_CXXFLAGS="-isystem $PREFIX/include/c++/v1 $EXTRA_CXXFLAGS"
EXTRA_LDFLAGS="-L$PREFIX/lib $EXTRA_LDFLAGS"
EXTRA_LDFLAGS="-Wl,-rpath,$PREFIX/lib $EXTRA_LDFLAGS"

# Build the rest of the dependencies with TSAN instrumentation.
EXTRA_CFLAGS="-fsanitize=thread $EXTRA_CFLAGS"
EXTRA_CXXFLAGS="-fsanitize=thread $EXTRA_CXXFLAGS"
EXTRA_CXXFLAGS="-DTHREAD_SANITIZER $EXTRA_CXXFLAGS"

if [ -n "$F_TSAN" -o -n "$F_LLVM" ]; then
  build_llvm tsan
fi

# LLVM is told to use libc++ explicitly and thus doesn't need these, but the
# rest of the dependencies need them.
#
# Note: -nostdinc++ is necessary to prevent C++ headers from using #include_next
# to chain the host's C++ headers. However, using it means we need to also use
# -Qunused-arguments because clang raises an unused argument warning when it
# detects -nostdinc++ on a link line, and there's no way to prevent that when
# passing -nostdinc++ to cmake via -DCMAKE_CXX_FLAGS [1].
#
# 1. https://gitlab.kitware.com/cmake/cmake/issues/12652
EXTRA_CXXFLAGS="-Qunused-arguments -nostdinc++ $EXTRA_CXXFLAGS"
EXTRA_LDFLAGS="-stdlib=libc++ $EXTRA_LDFLAGS"

# Enable debug symbols so that stacktraces and linenumbers are available at
# runtime. LLVM is compiled without debug symbols because the LLVM debug symbols
# take up more than 20GiB of disk space.
EXTRA_CFLAGS="-g $EXTRA_CFLAGS"
EXTRA_CXXFLAGS="-g $EXTRA_CXXFLAGS"

if [ -n "$F_TSAN" -o -n "$F_PROTOBUF" ]; then
  build_protobuf
fi

if [ -n "$F_TSAN" -o -n "$F_GFLAGS" ]; then
  build_gflags
fi

if [ -n "$F_TSAN" -o -n "$F_GLOG" ]; then
  build_glog
fi

if [ -n "$F_TSAN" -o -n "$F_GMOCK" ]; then
  build_gmock
fi

if [ -n "$F_TSAN" -o -n "$F_SNAPPY" ]; then
  build_snappy
fi

if [ -n "$F_TSAN" -o -n "$F_CRCUTIL" ]; then
  build_crcutil
fi

if [ -n "$F_TSAN" -o -n "$F_BOOST" ]; then
  build_boost tsan
fi

: '
if [ -n "$F_TSAN" -o -n "$F_MUSTACHE" ]; then
  build_mustache
fi

if [ -n "$F_TSAN" -o -n "$F_BREAKPAD" ]; then
  build_breakpad
fi

if [ -n "$F_TSAN" -o -n "$F_THRIFT" ]; then
  build_thrift
fi
'

if [ -n "$F_TSAN" -o -n "$F_COREDUMPER" ]; then
  build_coredumper
fi

restore_env

finish


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# autoreconf calls are necessary to fix hard-coded aclocal versions in the
# configure scripts that ship with the projects.

set -e

TP_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)

source $TP_DIR/vars.sh

if [[ "$OSTYPE" =~ ^linux ]]; then
  OS_LINUX=1
fi

delete_if_wrong_patchlevel() {
  local DIR=$1
  local PATCHLEVEL=$2
  if [ ! -f $DIR/patchlevel-$PATCHLEVEL ]; then
    echo It appears that $DIR is missing the latest local patches.
    echo Removing it so we re-download it.
    rm -Rf $DIR
  fi
}

unzip_to_source() {
  local FILENAME=$1
  local SOURCE=$2
  unzip -q "$FILENAME"
  # Parse out the unzipped top directory
  DIR_NAME=`unzip -qql "$FILENAME" | awk 'NR==1 {print $4}' | sed -e 's|^[/]*\([^/]*\).*|\1|'`
  # If the unzipped directory is the wrong name, move it.
  if [ "$SOURCE" != "$TP_SOURCE_DIR/$DIR_NAME" ]; then
    mv "$TP_SOURCE_DIR/$DIR_NAME" "$SOURCE"
  fi
}

fetch_and_expand() {
  local FILENAME=$1
  local SOURCE=$2
  local URL_PREFIX=$3

  if [ -z "$FILENAME" ]; then
    echo "Error: Must specify file to fetch"
    exit 1
  fi

  if [ -z "$URL_PREFIX" ]; then
    echo "Error: Must specify url prefix to fetch"
    exit 1
  fi

  TAR_CMD=tar
  if [[ "$OSTYPE" == "darwin"* ]] && which gtar &>/dev/null; then
    TAR_CMD=gtar
  fi

  FULL_URL="${URL_PREFIX}/${FILENAME}"

  SUCCESS=0
  # Loop in case we encounter an error.
  for attempt in 1 2 3; do
    if [ -r "$FILENAME" ]; then
      echo "Archive $FILENAME already exists. Not re-downloading archive."
    else
      echo "Fetching $FILENAME from $FULL_URL"
      if ! curl --retry 3 -L -O "$FULL_URL"; then
        echo "Error downloading $FILENAME"
        rm -f "$FILENAME"

        # Pause for a bit before looping in case the server throttled us.
        sleep 5
        continue
      fi
    fi

    echo "Unpacking $FILENAME to $SOURCE"
    if [[ "$FILENAME" =~ \.zip$ ]]; then
      if ! unzip_to_source "$FILENAME" "$SOURCE"; then
        echo "Error unzipping $FILENAME, removing file"
        rm "$FILENAME"
        continue
      fi
    elif [[ "$FILENAME" =~ \.(tar\.gz|tgz)$ ]]; then
      if ! $TAR_CMD xf "$FILENAME"; then
        echo "Error untarring $FILENAME, removing file"
        rm "$FILENAME"
        continue
      fi
    else
      echo "Error: unknown file format: $FILENAME"
      exit 1
    fi

    SUCCESS=1
    break
  done

  if [ $SUCCESS -ne 1 ]; then
    echo "Error: failed to fetch and unpack $FILENAME"
    exit 1
  fi

  # Allow for not removing previously-downloaded artifacts.
  # Useful on a low-bandwidth connection.
  if [ -z "$NO_REMOVE_THIRDPARTY_ARCHIVES" ]; then
    echo "Removing $FILENAME"
    rm $FILENAME
  fi
  echo
}

fetch_with_url_and_patch() {
  local FILENAME=$1
  local SOURCE=$2
  local PATCH_LEVEL=$3
  local URL_PREFIX=$4
  # Remaining args are expected to be a list of patch commands

  delete_if_wrong_patchlevel $SOURCE $PATCH_LEVEL
  if [ ! -d $SOURCE ]; then
    fetch_and_expand $FILENAME $SOURCE $URL_PREFIX
    pushd $SOURCE
    shift 4
    # Run the patch commands
    for f in "$@"; do
      eval "$f"
    done
    touch patchlevel-$PATCH_LEVEL
    popd
    echo
  fi
}

# Call fetch_with_url_and_patch with the default dependency URL source.
fetch_and_patch() {
  local FILENAME=$1
  local SOURCE=$2
  local PATCH_LEVEL=$3

  shift 3
  fetch_with_url_and_patch \
    $FILENAME \
    $SOURCE \
    $PATCH_LEVEL \
    $DEPENDENCY_URL \
    "$@"
}

mkdir -p $TP_SOURCE_DIR
cd $TP_SOURCE_DIR

#GPERFTOOLS_PATCHLEVEL=2
#fetch_and_patch \
# gperftools-${GPERFTOOLS_VERSION}.tar.gz \
# $GPERFTOOLS_SOURCE \
# $GPERFTOOLS_PATCHLEVEL \
# "patch -p1 < $TP_DIR/patches/gperftools-Replace-namespace-base-with-namespace-tcmalloc.patch" \
# "patch -p1 < $TP_DIR/patches/gperftools-unbreak-memz.patch" \
# "autoreconf -fvi"

RAPIDJSON_PATCHLEVEL=0
fetch_and_patch \
 rapidjson-${RAPIDJSON_VERSION}.zip \
 $RAPIDJSON_SOURCE \
 $RAPIDJSON_PATCHLEVEL

echo "---------------"
echo "Thirdparty dependencies downloaded successfully"


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# autoreconf calls are necessary to fix hard-coded aclocal versions in the
# configure scripts that ship with the projects.

set -e

TP_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)

source $TP_DIR/vars.sh

if [[ "$OSTYPE" =~ ^linux ]]; then
  OS_LINUX=1
fi

delete_if_wrong_patchlevel() {
  local DIR=$1
  local PATCHLEVEL=$2
  if [ ! -f $DIR/patchlevel-$PATCHLEVEL ]; then
    echo It appears that $DIR is missing the latest local patches.
    echo Removing it so we re-download it.
    rm -Rf $DIR
  fi
}

unzip_to_source() {
  local FILENAME=$1
  local SOURCE=$2
  unzip -q "$FILENAME"
  # Parse out the unzipped top directory
  DIR_NAME=`unzip -qql "$FILENAME" | awk 'NR==1 {print $4}' | sed -e 's|^[/]*\([^/]*\).*|\1|'`
  # If the unzipped directory is the wrong name, move it.
  if [ "$SOURCE" != "$TP_SOURCE_DIR/$DIR_NAME" ]; then
    mv "$TP_SOURCE_DIR/$DIR_NAME" "$SOURCE"
  fi
}

fetch_and_expand() {
  local FILENAME=$1
  local SOURCE=$2
  local URL_PREFIX=$3

  if [ -z "$FILENAME" ]; then
    echo "Error: Must specify file to fetch"
    exit 1
  fi

  if [ -z "$URL_PREFIX" ]; then
    echo "Error: Must specify url prefix to fetch"
    exit 1
  fi

  TAR_CMD=tar
  if [[ "$OSTYPE" == "darwin"* ]] && which gtar &>/dev/null; then
    TAR_CMD=gtar
  fi

  FULL_URL="${URL_PREFIX}/${FILENAME}"

  SUCCESS=0
  # Loop in case we encounter an error.
  for attempt in 1 2 3; do
    if [ -r "$FILENAME" ]; then
      echo "Archive $FILENAME already exists. Not re-downloading archive."
    else
      echo "Fetching $FILENAME from $FULL_URL"
      if ! curl --retry 3 -L -O "$FULL_URL"; then
        echo "Error downloading $FILENAME"
        rm -f "$FILENAME"

        # Pause for a bit before looping in case the server throttled us.
        sleep 5
        continue
      fi
    fi

    echo "Unpacking $FILENAME to $SOURCE"
    if [[ "$FILENAME" =~ \.zip$ ]]; then
      if ! unzip_to_source "$FILENAME" "$SOURCE"; then
        echo "Error unzipping $FILENAME, removing file"
        rm "$FILENAME"
        continue
      fi
    elif [[ "$FILENAME" =~ \.(tar\.gz|tgz)$ ]]; then
      if ! $TAR_CMD xf "$FILENAME"; then
        echo "Error untarring $FILENAME, removing file"
        rm "$FILENAME"
        continue
      fi
    else
      echo "Error: unknown file format: $FILENAME"
      exit 1
    fi

    SUCCESS=1
    break
  done

  if [ $SUCCESS -ne 1 ]; then
    echo "Error: failed to fetch and unpack $FILENAME"
    exit 1
  fi

  # Allow for not removing previously-downloaded artifacts.
  # Useful on a low-bandwidth connection.
  if [ -z "$NO_REMOVE_THIRDPARTY_ARCHIVES" ]; then
    echo "Removing $FILENAME"
    rm $FILENAME
  fi
  echo
}

fetch_with_url_and_patch() {
  local FILENAME=$1
  local SOURCE=$2
  local PATCH_LEVEL=$3
  local URL_PREFIX=$4
  # Remaining args are expected to be a list of patch commands

  delete_if_wrong_patchlevel $SOURCE $PATCH_LEVEL
  if [ ! -d $SOURCE ]; then
    fetch_and_expand $FILENAME $SOURCE $URL_PREFIX
    pushd $SOURCE
    shift 4
    # Run the patch commands
    for f in "$@"; do
      eval "$f"
    done
    touch patchlevel-$PATCH_LEVEL
    popd
    echo
  fi
}

# Call fetch_with_url_and_patch with the default dependency URL source.
fetch_and_patch() {
  local FILENAME=$1
  local SOURCE=$2
  local PATCH_LEVEL=$3

  shift 3
  fetch_with_url_and_patch \
    $FILENAME \
    $SOURCE \
    $PATCH_LEVEL \
    $DEPENDENCY_URL \
    "$@"
}

mkdir -p $TP_SOURCE_DIR
cd $TP_SOURCE_DIR

GLOG_PATCHLEVEL=3
fetch_and_patch \
 glog-${GLOG_VERSION}.tar.gz \
 $GLOG_SOURCE \
 $GLOG_PATCHLEVEL \
 "patch -p0 < $TP_DIR/patches/glog-issue-198-fix-unused-warnings.patch" \
 "patch -p0 < $TP_DIR/patches/glog-issue-54-dont-build-tests.patch" \
 "patch -p1 < $TP_DIR/patches/glog-fix-symbolization.patch" \
 "autoreconf -fvi"

GMOCK_PATCHLEVEL=0
fetch_and_patch \
 googletest-release-${GMOCK_VERSION}.tar.gz \
 $GMOCK_SOURCE \
 $GMOCK_PATCHLEVEL

GFLAGS_PATCHLEVEL=0
fetch_and_patch \
 gflags-${GFLAGS_VERSION}.tar.gz \
 $GFLAGS_SOURCE \
 $GFLAGS_PATCHLEVEL

: '
GPERFTOOLS_PATCHLEVEL=2
fetch_and_patch \
 gperftools-${GPERFTOOLS_VERSION}.tar.gz \
 $GPERFTOOLS_SOURCE \
 $GPERFTOOLS_PATCHLEVEL \
 "patch -p1 < $TP_DIR/patches/gperftools-Replace-namespace-base-with-namespace-tcmalloc.patch" \
 "patch -p1 < $TP_DIR/patches/gperftools-unbreak-memz.patch" \
 "autoreconf -fvi"
'

PROTOBUF_PATCHLEVEL=0
fetch_and_patch \
 protobuf-${PROTOBUF_VERSION}.tar.gz \
 $PROTOBUF_SOURCE \
 $PROTOBUF_PATCHLEVEL \
 "autoreconf -fvi"

# Returns 0 if cmake should be patched to work around this bug [1].
#
# Currently only SLES 12 SP0 is known to be vulnerable, and since the workaround
# hurts cmake performance, we apply it only if absolutely necessary.
#
# 1. https://gitlab.kitware.com/cmake/cmake/issues/15873.
needs_patched_cmake() {
  if [ ! -e /etc/SuSE-release ]; then
    # Not a SUSE distro.
    return 1
  fi
  if ! grep -q "SUSE Linux Enterprise Server 12" /etc/SuSE-release; then
    # Not SLES 12.
    return 1
  fi
  if ! grep -q "PATCHLEVEL = 0" /etc/SuSE-release; then
    # Not SLES 12 SP0.
    return 1
  fi
  return 0
}

: '
CMAKE_PATCHLEVEL=1
CMAKE_PATCHES=""
if needs_patched_cmake; then \
 CMAKE_PATCHES="patch -p1 < $TP_DIR/patches/cmake-issue-15873-dont-use-select.patch"
fi

fetch_and_patch \
 cmake-${CMAKE_VERSION}.tar.gz \
 $CMAKE_SOURCE \
 $CMAKE_PATCHLEVEL \
 "$CMAKE_PATCHES"
'

SNAPPY_PATCHLEVEL=0
fetch_and_patch \
 snappy-${SNAPPY_VERSION}.tar.gz \
 $SNAPPY_SOURCE \
 $SNAPPY_PATCHLEVEL \
 "autoreconf -fvi"

ZLIB_PATCHLEVEL=0
fetch_and_patch \
 zlib-${ZLIB_VERSION}.tar.gz \
 $ZLIB_SOURCE \
 $ZLIB_PATCHLEVEL

LIBEV_PATCHLEVEL=0
fetch_and_patch \
 libev-${LIBEV_VERSION}.tar.gz \
 $LIBEV_SOURCE \
 $LIBEV_PATCHLEVEL

: '
RAPIDJSON_PATCHLEVEL=0
fetch_and_patch \
 rapidjson-${RAPIDJSON_VERSION}.zip \
 $RAPIDJSON_SOURCE \
 $RAPIDJSON_PATCHLEVEL

SQUEASEL_PATCHLEVEL=0
fetch_and_patch \
 squeasel-${SQUEASEL_VERSION}.tar.gz \
 $SQUEASEL_SOURCE \
 $SQUEASEL_PATCHLEVEL

MUSTACHE_PATCHLEVEL=0
fetch_and_patch \
 mustache-${MUSTACHE_VERSION}.tar.gz \
 $MUSTACHE_SOURCE \
 $MUSTACHE_PATCHLEVEL

GSG_PATCHLEVEL=1
fetch_and_patch \
 google-styleguide-${GSG_VERSION}.tar.gz \
 $GSG_SOURCE \
 $GSG_PATCHLEVEL \
 "patch -p1 < $TP_DIR/patches/google-styleguide-cpplint.patch"

GCOVR_PATCHLEVEL=0
fetch_and_patch \
 gcovr-${GCOVR_VERSION}.tar.gz \
 $GCOVR_SOURCE \
 $GCOVR_PATCHLEVEL
'

CURL_PATCHLEVEL=0
fetch_and_patch \
 curl-${CURL_VERSION}.tar.gz \
 $CURL_SOURCE \
 $CURL_PATCHLEVEL \
 "autoreconf -fvi"

CRCUTIL_PATCHLEVEL=1
fetch_and_patch \
 crcutil-${CRCUTIL_VERSION}.tar.gz \
 $CRCUTIL_SOURCE \
 $CRCUTIL_PATCHLEVEL \
 "patch -p0 < $TP_DIR/patches/crcutil-fix-libtoolize-on-osx.patch"

LIBUNWIND_PATCHLEVEL=1
fetch_and_patch \
 libunwind-${LIBUNWIND_VERSION}.tar.gz \
 $LIBUNWIND_SOURCE \
 $LIBUNWIND_PATCHLEVEL \
 "patch -p1 < $TP_DIR/patches/libunwind-trace-cache-destructor.patch"

: '
PYTHON_PATCHLEVEL=0
fetch_and_patch \
 python-${PYTHON_VERSION}.tar.gz \
 $PYTHON_SOURCE \
 $PYTHON_PATCHLEVEL

LLVM_PATCHLEVEL=2
fetch_and_patch \
 llvm-${LLVM_VERSION}-iwyu-${IWYU_VERSION}.src.tar.gz \
 $LLVM_SOURCE \
 $LLVM_PATCHLEVEL \
  "patch -p1 < $TP_DIR/patches/llvm-fix-amazon-linux.patch" \
  "patch -p1 < $TP_DIR/patches/llvm-add-iwyu.patch" \
  "patch -p1 < $TP_DIR/patches/llvm-iwyu-nocurses.patch" \
  "patch -p1 < $TP_DIR/patches/llvm-iwyu-include-picker.patch" \
  "patch -d tools/clang/tools/include-what-you-use -p1 < $TP_DIR/patches/llvm-iwyu-llvm-6-compat.patch" \
  "patch -d projects/compiler-rt -p1 < $TP_DIR/patches/llvm-tsan-disable-trace-switching-after-multithreaded-for.patch"
'

LZ4_PATCHLEVEL=1
fetch_and_patch \
 lz4-lz4-$LZ4_VERSION.tar.gz \
 $LZ4_SOURCE \
 $LZ4_PATCHLEVEL \
 "patch -p1 < $TP_DIR/patches/lz4-0001-Fix-cmake-build-to-use-gnu-flags-on-clang.patch"

: '
BITSHUFFLE_PATCHLEVEL=0
fetch_and_patch \
 bitshuffle-${BITSHUFFLE_VERSION}.tar.gz \
 $BITSHUFFLE_SOURCE \
 $BITSHUFFLE_PATCHLEVEL

TRACE_VIEWER_PATCHLEVEL=0
fetch_and_patch \
 kudu-trace-viewer-${TRACE_VIEWER_VERSION}.tar.gz \
 $TRACE_VIEWER_SOURCE \
 $TRACE_VIEWER_PATCHLEVEL

NVML_PATCHLEVEL=0
fetch_and_patch \
 nvml-${NVML_VERSION}.tar.gz \
 $NVML_SOURCE \
 $NVML_PATCHLEVEL
'

BOOST_PATCHLEVEL=1
fetch_and_patch \
 boost_${BOOST_VERSION}.tar.gz \
 $BOOST_SOURCE \
 $BOOST_PATCHLEVEL \
 "patch -p0 < $TP_DIR/patches/boost-issue-12179-fix-compilation-errors.patch"

# Return 0 if the current system appears to be el6 (either CentOS or proper RHEL)
needs_openssl_workaround() {
  test -f /etc/redhat-release || return 1
  rel="$(cat /etc/redhat-release)"
  pat="(CentOS|Red Hat Enterprise).* release 6.*"
  [[ "$rel" =~ $pat ]]
  return $?
}
if needs_openssl_workaround && [ ! -d "$OPENSSL_WORKAROUND_DIR" ] ; then
  echo Building on el6: installing OpenSSL from CentOS 6.4.
  $TP_DIR/install-openssl-el6-workaround.sh
fi

BREAKPAD_PATCHLEVEL=1
fetch_and_patch \
 breakpad-${BREAKPAD_VERSION}.tar.gz \
 $BREAKPAD_SOURCE \
 $BREAKPAD_PATCHLEVEL \
 "patch -p1 < $TP_DIR/patches/breakpad-add-basic-support-for-dwz-dwarf-extension.patch"

SPARSEHASH_PATCHLEVEL=2
fetch_and_patch \
 sparsehash-c11-${SPARSEHASH_VERSION}.tar.gz \
 $SPARSEHASH_SOURCE \
 $SPARSEHASH_PATCHLEVEL \
 "patch -p1 < $TP_DIR/patches/sparsehash-0001-Add-compatibily-for-gcc-4.x-in-traits.patch"

SPARSEPP_PATCHLEVEL=0
fetch_and_patch \
 sparsepp-${SPARSEPP_VERSION}.tar.gz \
 $SPARSEPP_SOURCE \
 $SPARSEPP_PATCHLEVEL

THRIFT_PATCHLEVEL=0
fetch_and_patch \
 $THRIFT_NAME.tar.gz \
 $THRIFT_SOURCE \
 $THRIFT_PATCHLEVEL

BISON_PATCHLEVEL=0
fetch_and_patch \
 $BISON_NAME.tar.gz \
 $BISON_SOURCE \
 $BISON_PATCHLEVEL
 # This would normally call autoreconf, but it does not succeed with
 # autoreconf 2.69-11 (RHEL 7): "autoreconf: 'configure.ac' or 'configure.in' is required".

HIVE_PATCHLEVEL=0
fetch_and_patch \
 $HIVE_NAME-stripped.tar.gz \
 $HIVE_SOURCE \
 $HIVE_PATCHLEVEL

HADOOP_PATCHLEVEL=0
fetch_and_patch \
 $HADOOP_NAME-stripped.tar.gz \
 $HADOOP_SOURCE \
 $HADOOP_PATCHLEVEL

SENTRY_PATCHLEVEL=0
fetch_and_patch \
 $SENTRY_NAME.tar.gz \
 $SENTRY_SOURCE \
 $SENTRY_PATCHLEVEL

COREDUMPER_PATCHLEVEL=3
fetch_with_url_and_patch \
 $COREDUMPER_NAME.tar.gz \
 $COREDUMPER_SOURCE \
 $COREDUMPER_PATCHLEVEL \
 https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/google-coredumper \
 "patch -p0 < $TP_DIR/patches/coredumper_p0.patch" \
 "patch -p0 < $TP_DIR/patches/coredumper_p1.patch" \
 "patch -p0 < $TP_DIR/patches/coredumper_p2.patch"

echo "---------------"
echo "Thirdparty dependencies downloaded successfully"


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# This script serves to workaround a problematic OpenSSL ABI change
# made between RHEL 6.4 and 6.5. Namely:
#
#  RHEL 6.4's OpenSSL library is built with no symbol versioning. For example:
#  $ objdump -T libssl.so | grep SSL_CTX_new
#    0000000000037110 g    DF .text  0000000000000577  Base        SSL_CTX_new
#
#  RHEL 6.5's OpenSSL library has symbol versions. For example:
#  $ objdump -T /usr/lib64/libssl.so | grep SSL_CTX_new
#    0000003ae8243610 g    DF .text  0000000000000597  libssl.so.10 SSL_CTX_new
#
# Thus, if we build Kudu on RHEL 6.5 or later, the resulting binaries expect
# the versioned symbols in libssl and will not run on RHEL 6.4 or earlier:
#
#  $ objdump -T kudu-tserver | grep SSL_CTX_new
#    0000000000000000      DF *UND*     0000000000000000  libssl.so.10 SSL_CTX_new
#
# In contrast, if a binary is built not expecting versioned symbols, the runtime
# linker can still resolve those symbols by choosing the versioned ones. Thus,
# binaries built against RHEL 6.4 are forward-compatible to later versions, but
# not vice versa.
#
# Note that Kudu cannot simply be built on RHEL 6.4 because the devtoolset toolchain is
# not available. So, given that we want to produce binaries that run on RHEL 6.4,
# we need to perform a workaround such that our binaries built on 6.6 don't depend
# on the versioned symbols in OpenSSL. This script provides such a workaround.
#
# The workaround itself is quite simple: we download the OpenSSL RPMs from CentOS 6.4
# and unpack them into a directory in thirdparty/. If we then build against those
# the resulting binaries can run on either el6.4 or el6.6.

set -e

TP_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
source $TP_DIR/vars.sh

mkdir -p $OPENSSL_WORKAROUND_DIR
cd $OPENSSL_WORKAROUND_DIR

# Clean up any previous leftovers.
rm -Rf usr etc

# Download and unpack OpenSSL RPMs from CentOS 6.4.
#
# We have mirrored these in our S3 bucket, but the original sources are in
# http://vault.centos.org/6.4/os/x86_64/Packages/ .
for FILENAME in openssl-1.0.0-27.el6.x86_64.rpm openssl-devel-1.0.0-27.el6.x86_64.rpm ; do
  FULL_URL="${CLOUDFRONT_URL_PREFIX}/${FILENAME}"
  if [ -r "$FILENAME" ]; then
    echo $FILENAME already exists. Not re-downloading.
  else
    echo "Fetching $FILENAME from $FULL_URL"
    curl -L -O "${FULL_URL}"
  fi

  echo "Unpacking $FILENAME"
  rpm2cpio $FILENAME | cpio -idm
done


#!/bin/bash

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Packages Hadoop for inclusion in the Kudu thirdparty build.
#
# Strips jars and unecessary documentation. This makes the download smaller, and
# improves HMS startup time.
#
# Summary:
# 1. Download and unpack the Hadoop binary tarball (optional)
# 2. Strip out unnecessary jars
# 3. Repackage the tarball (optional)

set -ex

function usage() {
  cat <<EOF
  Usage:
    ${0} -h | --help
    ${0} [-d] [-r] <ARTIFACT>
    -h | --help - This message
    -d | --download - Optional
      Download and unpack the binary tarball.
    -r | --repackage - Optional
      Repackage the tarball.
    <ARTIFACT> - Required
      The artifact name.
EOF
}

OPTS=`getopt -o hdr -l "help,download,repackage" -n 'parse-options' -- "$@"`
OPTS_RESULT=$?
if [ ${OPTS_RESULT} != 0 ]; then
  echo "Failed parsing options." >&2
  exit ${OPTS_RESULT}
fi

while true; do
  case "$1" in
    -d | --download) DOWNLOAD=True; shift;;
    -r | --repackage) REPACKAGE=True; shift;;
    -h | --help) usage; exit;;
    --) shift; break;;
    *) break ;;
  esac
done

ARTIFACT="${1}"

if [ -z "${ARTIFACT}" ]; then
  usage
  exit
fi

if [ -n "${DOWNLOAD}" ]; then
  curl --retry 3 -L -O https://archive.apache.org/dist/hadoop/common/$ARTIFACT/$ARTIFACT.tar.gz
  tar xf $ARTIFACT.tar.gz
fi

DIRS="client"
DIRS="$DIRS common/jdiff"
DIRS="$DIRS common/sources"
DIRS="$DIRS hdfs/sources"
DIRS="$DIRS httpfs"
DIRS="$DIRS kms"
DIRS="$DIRS mapreduce/lib-examples"
DIRS="$DIRS mapreduce/sources"
DIRS="$DIRS tools"

for DIR in $DIRS; do
  rm -rf $ARTIFACT/share/hadoop/$DIR
done

for PROJECT in avro commons-math3 snappy; do
  rm -f $ARTIFACT/share/hadoop/common/lib/*$PROJECT*.jar
  rm -f $ARTIFACT/share/hadoop/hdfs/lib/*$PROJECT*.jar
done

rm -f $ARTIFACT/share/hadoop/yarn/*.jar
rm -rf $ARTIFACT/share/hadoop/yarn/lib
rm -rf $ARTIFACT/share/hadoop/yarn/sources
rm -rf $ARTIFACT/share/hadoop/yarn/timelineservice
rm -rf $ARTIFACT/share/doc

if [ -n "${REPACKAGE}" ]; then
  tar czf $ARTIFACT-stripped.tar.gz $ARTIFACT
fi


#!/bin/bash

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Packages Hive for inclusion in the Kudu thirdparty build.
#
# Strips jars unnecessary for running the Hive Metastore. This makes the download
# smaller, and improves HMS startup time.
#
# Summary:
# 1. Download and unpack the Hive binary tarball (optional)
# 2. Strip out unnecessary jars
# 3. Repackage the tarball (optional)

set -ex

function usage() {
  cat <<EOF
  Usage:
    ${0} -h | --help
    ${0} [-d] [-r] <ARTIFACT>
    -h | --help - This message
    -d | --download - Optional
      Download and unpack the binary tarball.
    -r | --repackage - Optional
      Repackage the tarball.
    -v | --version VERSION - Optional
      The Hive version.
    <ARTIFACT> - Required
      The artifact name.
EOF
}

OPTS=`getopt -o hdrv: -l "help,download,repackage,version:" -n 'parse-options' -- "$@"`
OPTS_RESULT=$?
if [ ${OPTS_RESULT} != 0 ]; then
  echo "Failed parsing options." >&2
  exit ${OPTS_RESULT}
fi

while true; do
  case "$1" in
    -d | --download) DOWNLOAD=True; shift;;
    -r | --repackage) REPACKAGE=True; shift;;
    -v | --version) VERSION=$2; shift; shift;;
    -h | --help) usage; exit;;
    --) shift; break;;
    *) break ;;
  esac
done

ARTIFACT="${1}"

if [ -z "${ARTIFACT}" ]; then
  usage
  exit
fi

if [ -n "${DOWNLOAD}" ]; then
  curl --retry 3 -L -O https://archive.apache.org/dist/hive/hive-$VERSION/$ARTIFACT.tar.gz
  tar xf $ARTIFACT.tar.gz
fi

PROJECTS="accumulo"
PROJECTS="$PROJECTS aether"
PROJECTS="$PROJECTS avatica"
PROJECTS="$PROJECTS avro"
PROJECTS="$PROJECTS calcite"
PROJECTS="$PROJECTS curator"
PROJECTS="$PROJECTS druid"
PROJECTS="$PROJECTS ecj"
PROJECTS="$PROJECTS fastutil"
PROJECTS="$PROJECTS groovy"
PROJECTS="$PROJECTS hbase"
PROJECTS="$PROJECTS htrace"
PROJECTS="$PROJECTS icu4j"
PROJECTS="$PROJECTS jcodings"
PROJECTS="$PROJECTS jersey"
PROJECTS="$PROJECTS jetty"
PROJECTS="$PROJECTS jsp"
PROJECTS="$PROJECTS maven"
PROJECTS="$PROJECTS parquet"
PROJECTS="$PROJECTS slider"
PROJECTS="$PROJECTS snappy"
PROJECTS="$PROJECTS zookeeper"

for PROJECT in $PROJECTS; do
  rm -f $ARTIFACT/lib/*$PROJECT*.jar
done

rm -rf $ARTIFACT/auxlib
rm -rf $ARTIFACT/hcatalog/bin
rm -rf $ARTIFACT/hcatalog/etc
rm -rf $ARTIFACT/hcatalog/libexec
rm -rf $ARTIFACT/hcatalog/sbin
rm -rf $ARTIFACT/hcatalog/share/doc
rm -rf $ARTIFACT/hcatalog/share/webhcat
rm -rf $ARTIFACT/jdbc
rm -rf $ARTIFACT/lib/php
rm -rf $ARTIFACT/lib/python
rm -rf $ARTIFACT/lib/py

if [ -n "${REPACKAGE}" ]; then
  # Remove the 'apache-' prefix and '-bin' suffix in order to normalize the
  # tarball with Hadoop and the rest of our thirdparty dependencies.
  mv $ARTIFACT hive-$VERSION
  tar czf hive-$VERSION-stripped.tar.gz hive-$VERSION
fi


#!/bin/bash

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Packages LLVM for inclusing in the Kudu thirdparty build.
#
# Our llvm tarball includes clang, extra clang tools, lld, and compiler-rt.
#
# See http://clang.llvm.org/get_started.html and http://lld.llvm.org/ for
# details on how they're laid out in the llvm tarball.
#
# Summary:
# 1.  Unpack the llvm tarball
# 2.  Unpack the clang tarball as tools/clang (rename from cfe-<version> to clang)
# 3.  Unpack the extra clang tools tarball as tools/clang/tools/extra
# 4.  Unpack the lld tarball as tools/lld
# 5.  Unpack the compiler-rt tarball as projects/compiler-rt
# 6.  Unpack the libc++ tarball as projects/libcxx
# 7.  Unpack the libc++abi tarball as projects/libcxxabi
# 9.  Unpack the IWYU tarball in tools/clang/tools/include-what-you-use
# 10. Create new tarball from the resulting source tree
#
# Usage:
#  $ env VERSION=6.0.0 IWYU_VERSION=0.9 thirdparty/package-llvm.sh

set -eux

for ARTIFACT in llvm cfe compiler-rt libcxx libcxxabi lld clang-tools-extra; do
  wget http://releases.llvm.org/$VERSION/$ARTIFACT-$VERSION.src.tar.xz
  tar xf $ARTIFACT-$VERSION.src.tar.xz
  rm $ARTIFACT-$VERSION.src.tar.xz
done

IWYU_TAR=include-what-you-use-${IWYU_VERSION}.src.tar.gz
wget https://include-what-you-use.org/downloads/$IWYU_TAR
tar xf $IWYU_TAR
rm $IWYU_TAR

mv cfe-$VERSION.src llvm-$VERSION.src/tools/clang
mv clang-tools-extra-$VERSION.src llvm-$VERSION.src/tools/clang/tools/extra
mv lld-$VERSION.src llvm-$VERSION.src/tools/lld
mv compiler-rt-$VERSION.src llvm-$VERSION.src/projects/compiler-rt
mv libcxx-$VERSION.src llvm-$VERSION.src/projects/libcxx
mv libcxxabi-$VERSION.src llvm-$VERSION.src/projects/libcxxabi
mv include-what-you-use llvm-$VERSION.src/tools/clang/tools/

tar czf llvm-$VERSION-iwyu-$IWYU_VERSION.src.tar.gz llvm-$VERSION.src


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

if [ -z "$TP_DIR" ]; then
   echo "TP_DIR variable not set, check your scripts"
   exit 1
fi

TP_SOURCE_DIR="$TP_DIR/src"
TP_BUILD_DIR="$TP_DIR/build"

# This URL corresponds to the CloudFront Distribution for the S3
# bucket cloudera-thirdparty-libs which is directly accessible at
# http://cloudera-thirdparty-libs.s3.amazonaws.com/
CLOUDFRONT_URL_PREFIX=http://d3dr9sfxru4sde.cloudfront.net

# Third party dependency downloading URL, default to the CloudFront
# Distribution URL.
DEPENDENCY_URL=${DEPENDENCY_URL:-$CLOUDFRONT_URL_PREFIX}

PREFIX_COMMON=$TP_DIR/installed/common
PREFIX_DEPS=$TP_DIR/installed/uninstrumented
PREFIX_DEPS_TSAN=$TP_DIR/installed/tsan

GFLAGS_VERSION=2.2.0
GFLAGS_NAME=gflags-$GFLAGS_VERSION
GFLAGS_SOURCE=$TP_SOURCE_DIR/$GFLAGS_NAME

GLOG_VERSION=0.3.5
GLOG_NAME=glog-$GLOG_VERSION
GLOG_SOURCE=$TP_SOURCE_DIR/$GLOG_NAME

GMOCK_VERSION=1.8.0
GMOCK_NAME=googletest-release-$GMOCK_VERSION
GMOCK_SOURCE=$TP_SOURCE_DIR/$GMOCK_NAME

GPERFTOOLS_VERSION=2.6.90
GPERFTOOLS_NAME=gperftools-$GPERFTOOLS_VERSION
GPERFTOOLS_SOURCE=$TP_SOURCE_DIR/$GPERFTOOLS_NAME

PROTOBUF_VERSION=3.4.1
PROTOBUF_NAME=protobuf-$PROTOBUF_VERSION
PROTOBUF_SOURCE=$TP_SOURCE_DIR/$PROTOBUF_NAME

# Note: CMake gets patched on SLES12SP0. When changing the CMake version, please check if
# cmake-issue-15873-dont-use-select.patch needs to be updated.
CMAKE_VERSION=3.9.0
CMAKE_NAME=cmake-$CMAKE_VERSION
CMAKE_SOURCE=$TP_SOURCE_DIR/$CMAKE_NAME

SNAPPY_VERSION=1.1.4
SNAPPY_NAME=snappy-$SNAPPY_VERSION
SNAPPY_SOURCE=$TP_SOURCE_DIR/$SNAPPY_NAME

LZ4_VERSION=r130
LZ4_NAME=lz4-lz4-$LZ4_VERSION
LZ4_SOURCE=$TP_SOURCE_DIR/$LZ4_NAME

# from https://github.com/kiyo-masui/bitshuffle
# Hash of git: 55f9b4caec73fa21d13947cacea1295926781440
BITSHUFFLE_VERSION=55f9b4c
BITSHUFFLE_NAME=bitshuffle-$BITSHUFFLE_VERSION
BITSHUFFLE_SOURCE=$TP_SOURCE_DIR/$BITSHUFFLE_NAME

ZLIB_VERSION=1.2.8
ZLIB_NAME=zlib-$ZLIB_VERSION
ZLIB_SOURCE=$TP_SOURCE_DIR/$ZLIB_NAME

LIBEV_VERSION=4.20
LIBEV_NAME=libev-$LIBEV_VERSION
LIBEV_SOURCE=$TP_SOURCE_DIR/$LIBEV_NAME

RAPIDJSON_VERSION=0.11
RAPIDJSON_NAME=rapidjson-$RAPIDJSON_VERSION
RAPIDJSON_SOURCE=$TP_SOURCE_DIR/$RAPIDJSON_NAME

# Hash of the squeasel git revision to use.
# (from http://github.com/cloudera/squeasel)
#
# To re-build this tarball use the following in the squeasel repo:
#  export NAME=squeasel-$(git rev-parse HEAD)
#  git archive HEAD --prefix=$NAME/ -o /tmp/$NAME.tar.gz
#  s3cmd put -P /tmp/$NAME.tar.gz s3://cloudera-thirdparty-libs/$NAME.tar.gz
SQUEASEL_VERSION=9335b81317a6451d5a37c5dc7ec088eecbf68c82
SQUEASEL_NAME=squeasel-$SQUEASEL_VERSION
SQUEASEL_SOURCE=$TP_SOURCE_DIR/$SQUEASEL_NAME

# Hash of the mustache git revision to use.
# (from https://github.com/henryr/cpp-mustache)
#
# To re-build this tarball use the following in the mustache repo:
#  export NAME=mustache-$(git rev-parse HEAD)
#  git archive HEAD --prefix=$NAME/ -o /tmp/$NAME.tar.gz
#  s3cmd put -P /tmp/$NAME.tar.gz s3://cloudera-thirdparty-libs/$NAME.tar.gz
MUSTACHE_VERSION=87a592e8aa04497764c533acd6e887618ca7b8a8
MUSTACHE_NAME=mustache-$MUSTACHE_VERSION
MUSTACHE_SOURCE=$TP_SOURCE_DIR/$MUSTACHE_NAME

# git revision of google style guide:
# https://github.com/google/styleguide
# git archive --prefix=google-styleguide-$(git rev-parse HEAD)/ -o /tmp/google-styleguide-$(git rev-parse HEAD).tgz HEAD
GSG_VERSION=7a179d1ac2e08a5cc1622bec900d1e0452776713
GSG_NAME=google-styleguide-$GSG_VERSION
GSG_SOURCE=$TP_SOURCE_DIR/$GSG_NAME

GCOVR_VERSION=3.0
GCOVR_NAME=gcovr-$GCOVR_VERSION
GCOVR_SOURCE=$TP_SOURCE_DIR/$GCOVR_NAME

CURL_VERSION=7.59.0
CURL_NAME=curl-$CURL_VERSION
CURL_SOURCE=$TP_SOURCE_DIR/$CURL_NAME

# Hash of the crcutil git revision to use.
# (from http://github.com/adembo/crcutil)
#
# To re-build this tarball use the following in the crcutil repo:
#  export NAME=crcutil-$(git rev-parse HEAD)
#  git archive HEAD --prefix=$NAME/ -o /tmp/$NAME.tar.gz
#  s3cmd put -P /tmp/$NAME.tar.gz s3://cloudera-thirdparty-libs/$NAME.tar.gz
CRCUTIL_VERSION=42148a6df6986a257ab21c80f8eca2e54544ac4d
CRCUTIL_NAME=crcutil-$CRCUTIL_VERSION
CRCUTIL_SOURCE=$TP_SOURCE_DIR/$CRCUTIL_NAME

LIBUNWIND_VERSION=1.5.0
LIBUNWIND_NAME=libunwind-$LIBUNWIND_VERSION
LIBUNWIND_SOURCE=$TP_SOURCE_DIR/$LIBUNWIND_NAME

# See package-llvm.sh for details on the LLVM tarball.
LLVM_VERSION=6.0.0
LLVM_NAME=llvm-$LLVM_VERSION.src
LLVM_SOURCE=$TP_SOURCE_DIR/$LLVM_NAME

# The include-what-you-use is built along with LLVM in its source tree.
IWYU_VERSION=0.9

# Python 2.7 is required to build LLVM 3.6+. It is only built and installed if
# the system Python version is not 2.7.
PYTHON_VERSION=2.7.13
PYTHON_NAME=python-$PYTHON_VERSION
PYTHON_SOURCE=$TP_SOURCE_DIR/$PYTHON_NAME

# Our trace-viewer repository is separate since it's quite large and
# shouldn't change frequently. We upload the built artifacts (HTML/JS)
# when we need to roll to a new revision.
#
# The source can be found in the 'kudu' branch of https://github.com/cloudera/catapult
# and built with "tracing/kudu-build.sh" included within the repository.
TRACE_VIEWER_VERSION=21d76f8350fea2da2aa25cb6fd512703497d0c11
TRACE_VIEWER_NAME=kudu-trace-viewer-$TRACE_VIEWER_VERSION
TRACE_VIEWER_SOURCE=$TP_SOURCE_DIR/$TRACE_VIEWER_NAME

NVML_VERSION=1.1
NVML_NAME=nvml-$NVML_VERSION
NVML_SOURCE=$TP_SOURCE_DIR/$NVML_NAME

BOOST_VERSION=1_61_0
BOOST_NAME=boost_$BOOST_VERSION
BOOST_SOURCE=$TP_SOURCE_DIR/$BOOST_NAME

OPENSSL_WORKAROUND_DIR="$TP_DIR/installed/openssl-el6-workaround"

# The breakpad source artifact is created using the script found in
# scripts/make-breakpad-src-archive.sh
BREAKPAD_VERSION=9eac2058b70615519b2c4d8c6bdbfca1bd079e39
BREAKPAD_NAME=breakpad-$BREAKPAD_VERSION
BREAKPAD_SOURCE=$TP_SOURCE_DIR/$BREAKPAD_NAME

# Hash of the sparsehash-c11 git revision to use.
# (from http://github.com/sparsehash/sparsehash-c11)
#
# To re-build this tarball use the following in the sparsehash-c11 repo:
#  export NAME=sparsehash-c11-$(git rev-parse HEAD)
#  git archive HEAD --prefix=$NAME/ -o /tmp/$NAME.tar.gz
#  s3cmd put -P /tmp/$NAME.tar.gz s3://cloudera-thirdparty-libs/$NAME.tar.gz
SPARSEHASH_VERSION=47a55825ca3b35eab1ca22b7ab82b9544e32a9af
SPARSEHASH_NAME=sparsehash-c11-$SPARSEHASH_VERSION
SPARSEHASH_SOURCE=$TP_SOURCE_DIR/$SPARSEHASH_NAME

# Hash of the sparsepp git revision to use.
# (from https://github.com/greg7mdp/sparsepp)
#
# To re-build this tarball use the following in the sparsepp repo:
#  export NAME=sparsepp-$(git rev-parse HEAD)
#  git archive HEAD --prefix=$NAME/ -o /tmp/$NAME.tar.gz
#  s3cmd put -P /tmp/$NAME.tar.gz s3://cloudera-thirdparty-libs/$NAME.tar.gz
SPARSEPP_VERSION=824860bb76893d163efbcff330734b9f62eecb17
SPARSEPP_NAME=sparsepp-$SPARSEPP_VERSION
SPARSEPP_SOURCE=$TP_SOURCE_DIR/$SPARSEPP_NAME

THRIFT_VERSION=0.11.0
THRIFT_NAME=thrift-$THRIFT_VERSION
THRIFT_SOURCE=$TP_SOURCE_DIR/$THRIFT_NAME

BISON_VERSION=3.0.5
BISON_NAME=bison-$BISON_VERSION
BISON_SOURCE=$TP_SOURCE_DIR/$BISON_NAME

# TODO(dan): bump to a release version once HIVE-17747 and HIVE-16886/HIVE-18526
# are published. The SHA below is the current head of branch-2.
# Note: The Hive release binary tarball is stripped of unnecessary jars before
# being uploaded. See thirdparty/package-hive.sh for details.
HIVE_VERSION=498021fa15186aee8b282d3c032fbd2cede6bec4
HIVE_NAME=hive-$HIVE_VERSION
HIVE_SOURCE=$TP_SOURCE_DIR/$HIVE_NAME

# Note: The Hadoop release tarball is stripped of unnecessary jars before being
# uploaded. See thirdparty/package-hadoop.sh for details.
HADOOP_VERSION=2.8.5
HADOOP_NAME=hadoop-$HADOOP_VERSION
HADOOP_SOURCE=$TP_SOURCE_DIR/$HADOOP_NAME

# TODO(dan): bump to a release version once SENTRY-2371 is published. The SHA
# below is the current head of the master branch.
SENTRY_VERSION=2c9a927a9e87cba0e4c0f34fc0b55887c6636927
SENTRY_NAME=apache-sentry-$SENTRY_VERSION-bin
SENTRY_SOURCE=$TP_SOURCE_DIR/$SENTRY_NAME

COREDUMPER_VERSION=1.2.1
COREDUMPER_NAME=coredumper-$COREDUMPER_VERSION
COREDUMPER_SOURCE=$TP_SOURCE_DIR/$COREDUMPER_NAME


#!/bin/bash -e
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
################################################################################
# Script to make a source archive from the Breakpad project, which does not run
# releases.
#
# We export only checked-in files, without the git metadata, and construct
# the expected directory structure to compile it. Then we tar it up.

BREAKPAD_REPO=https://chromium.googlesource.com/breakpad/breakpad
LSS_REPO=https://chromium.googlesource.com/linux-syscall-support
LSS_REVISION=3f6478ac95edf86cd3da300c2c0d34a438f5dbeb

TOPDIR=breakpad-repos

rm -rf "$TOPDIR"
mkdir $TOPDIR
pushd $TOPDIR

BREAKPAD_DIR=breakpad.git
git clone $BREAKPAD_REPO $BREAKPAD_DIR
pushd $BREAKPAD_DIR

REVISION=${1:-$(git rev-parse HEAD)}
NAME=breakpad-$REVISION

git checkout $REVISION
# Export the checked-in files, without .git metadata, into "../$NAME/".
git archive --format=tar --prefix=$NAME/ $REVISION | (cd .. && tar xf -)
popd

LSS_DIR=lss.git
git clone $LSS_REPO $LSS_DIR
pushd $LSS_DIR
# Export into the thirdparty directory under "../$NAME/".
git archive --format=tar --prefix=$NAME/src/third_party/lss/ $LSS_REVISION | (cd .. && tar xf -)
popd

FILENAME="$NAME.tar.gz"
tar czvf "$FILENAME" "$NAME/"
mv "$FILENAME" ..
popd
echo "Archive created at $(pwd)/$FILENAME"

exit 0


export NOVNC_REF='1.3.0'
export WEBSOCKIFY_REF='0.10.0'
export DEBIAN_FRONTEND=noninteractive

# Install FluxBox, VNC & noVNC
mkdir -p /opt/bin && chmod +x /dev/shm \
    && apt-get update && apt-get install -y unzip fluxbox x11vnc \
    && curl -L -o noVNC.zip "https://github.com/novnc/noVNC/archive/v${NOVNC_REF}.zip" \
    && unzip -x noVNC.zip \
    && rm -rf noVNC-${NOVNC_REF}/{docs,tests} \
    && mv noVNC-${NOVNC_REF} /opt/bin/noVNC \
    && cp /opt/bin/noVNC/vnc.html /opt/bin/noVNC/index.html \
    && rm noVNC.zip \
    && curl -L -o websockify.zip "https://github.com/novnc/websockify/archive/v${WEBSOCKIFY_REF}.zip" \
    && unzip -x websockify.zip \
    && rm websockify.zip \
    && rm -rf websockify-${WEBSOCKIFY_REF}/{docs,tests} \
    && mv websockify-${WEBSOCKIFY_REF} /opt/bin/noVNC/utils/websockify

# Patch noVNC

cat <<'EOF' > /opt/bin/noVNC/clip.patch
diff --git a/app/ui.js b/app/ui.js
index cb6a9fd..dbe42e0 100644
--- a/app/ui.js
+++ b/app/ui.js
@@ -951,6 +951,7 @@ const UI = {
     clipboardReceive(e) {
         Log.Debug(">> UI.clipboardReceive: " + e.detail.text.substr(0, 40) + "...");
         document.getElementById('noVNC_clipboard_text').value = e.detail.text;
+        navigator.clipboard.writeText(e.detail.text).catch(() => {});
         Log.Debug("<< UI.clipboardReceive");
     },

diff --git a/core/rfb.js b/core/rfb.js
index ea3bf58..fad57bc 100644
--- a/core/rfb.js
+++ b/core/rfb.js
@@ -176,6 +176,7 @@ export default class RFB extends EventTargetMixin {
             handleMouse: this._handleMouse.bind(this),
             handleWheel: this._handleWheel.bind(this),
             handleGesture: this._handleGesture.bind(this),
+            handleFocus: () => navigator.clipboard.readText().then(this.clipboardPasteFrom.bind(this)).catch(() => {})
         };

         // main setup
@@ -515,6 +516,7 @@ export default class RFB extends EventTargetMixin {
         this._canvas.addEventListener("gesturestart", this._eventHandlers.handleGesture);
         this._canvas.addEventListener("gesturemove", this._eventHandlers.handleGesture);
         this._canvas.addEventListener("gestureend", this._eventHandlers.handleGesture);
+        window.addEventListener('focus', this._eventHandlers.handleFocus);

         Log.Debug("<< RFB.connect");
     }
@@ -522,6 +524,7 @@ export default class RFB extends EventTargetMixin {
     _disconnect() {
         Log.Debug(">> RFB.disconnect");
         this._cursor.detach();
+        window.removeEventListener('focus', this._eventHandlers.handleFocus);
         this._canvas.removeEventListener("gesturestart", this._eventHandlers.handleGesture);
         this._canvas.removeEventListener("gesturemove", this._eventHandlers.handleGesture);
         this._canvas.removeEventListener("gestureend", this._eventHandlers.handleGesture);
EOF

cd /opt/bin/noVNC
git apply clip.patch

# Configure FluxBox menus
mkdir /root/.fluxbox
cat <<'EOF' > /root/.fluxbox/menu
  [begin] (fluxbox)
    [submenu] (Browsers) {}
      [exec] (Chromium) { /ms-playwright-agent/node_modules/.bin/playwright docker launch --endpoint http://127.0.0.1:5400 --browser chromium } <>
      [exec] (Firefox) { /ms-playwright-agent/node_modules/.bin/playwright docker launch --endpoint http://127.0.0.1:5400 --browser firefox  } <>
      [exec] (WebKit) { /ms-playwright-agent/node_modules/.bin/playwright docker launch --endpoint http://127.0.0.1:5400 --browser webkit  } <>
    [end]
    [include] (/etc/X11/fluxbox/fluxbox-menu)
  [end]
EOF

cat <<'EOF' > /root/.fluxbox/lastwallpaper
$center $full|/ms-playwright-agent/node_modules/playwright-core/lib/server/chromium/appIcon.png||:99
$center $full|/ms-playwright-agent/node_modules/playwright-core/lib/server/chromium/appIcon.png||:99.0
EOF


#!/bin/bash
set -e
SCREEN_WIDTH=1360
SCREEN_HEIGHT=1020
SCREEN_DEPTH=24
SCREEN_DPI=96
GEOMETRY="$SCREEN_WIDTH""x""$SCREEN_HEIGHT""x""$SCREEN_DEPTH"

nohup /usr/bin/xvfb-run --server-num=$DISPLAY_NUM \
     --listen-tcp \
     --server-args="-screen 0 "$GEOMETRY" -fbdir /var/tmp -dpi "$SCREEN_DPI" -listen tcp -noreset -ac +extension RANDR" \
     /usr/bin/fluxbox -display "$DISPLAY" >/dev/null 2>&1 &

for i in $(seq 1 500); do
  if xdpyinfo -display $DISPLAY >/dev/null 2>&1; then
    break
  fi
  echo "Waiting for Xvfb..."
  sleep 0.2
done

nohup x11vnc -noprimary -nosetprimary -forever -shared -rfbport 5900 -rfbportv6 5900 -display "$DISPLAY" >/dev/null 2>&1 &
nohup /opt/bin/noVNC/utils/novnc_proxy --listen 7900 --vnc localhost:5900 >/dev/null 2>&1 &

cd /ms-playwright-agent

NOVNC_UUID=$(cat /proc/sys/kernel/random/uuid)
echo "novnc is listening on http://127.0.0.1:7900?path=$NOVNC_UUID&resize=scale&autoconnect=1"

PW_UUID=$(cat /proc/sys/kernel/random/uuid)

# Make sure to re-start playwright server if something goes wrong.
# The approach taken from: https://stackoverflow.com/a/697064/314883

until npx playwright run-server --port=5400 --path=/$PW_UUID --proxy-mode=tether; do
  echo "Server crashed with exit code $?. Respawning.." >&2
  sleep 1
done


#!/bin/bash
set -e
set -x

if [[ $(arch) == "aarch64" ]]; then
  echo "ERROR: not supported on Linux Arm64"
  exit 1
fi

if [[ ! -f "/etc/os-release" ]]; then
  echo "ERROR: cannot install on unknown linux distribution (/etc/os-release is missing)"
  exit 1
fi

ID=$(bash -c 'source /etc/os-release && echo $ID')
if [[ "${ID}" != "ubuntu" ]]; then
  echo "ERROR: cannot install on $ID distribution - only Ubuntu is supported"
  exit 1
fi

# 1. make sure to remove old beta if any.
if dpkg --get-selections | grep -q "^google-chrome-beta[[:space:]]*install$" >/dev/null; then
  apt-get remove -y google-chrome-beta
fi

# 2. Update apt lists (needed to install curl and chrome dependencies)
apt-get update

# 3. Install curl to download chrome
if ! command -v curl >/dev/null; then
  apt-get install -y curl
fi

# 4. download chrome beta from dl.google.com and install it.
cd /tmp
curl -O https://dl.google.com/linux/direct/google-chrome-beta_current_amd64.deb
apt-get install -y ./google-chrome-beta_current_amd64.deb
rm -rf ./google-chrome-beta_current_amd64.deb
cd -
google-chrome-beta --version


#!/bin/bash
set -e
set -x

rm -rf "/Applications/Google Chrome Beta.app"
cd /tmp
curl -o ./googlechromebeta.dmg -k https://dl.google.com/chrome/mac/universal/beta/googlechromebeta.dmg
hdiutil attach -nobrowse -quiet -noautofsck -noautoopen -mountpoint /Volumes/googlechromebeta.dmg ./googlechromebeta.dmg
cp -rf "/Volumes/googlechromebeta.dmg/Google Chrome Beta.app" /Applications
hdiutil detach /Volumes/googlechromebeta.dmg
rm -rf /tmp/googlechromebeta.dmg

/Applications/Google\ Chrome\ Beta.app/Contents/MacOS/Google\ Chrome\ Beta --version


#!/bin/bash
set -e
set -x

if [[ $(arch) == "aarch64" ]]; then
  echo "ERROR: not supported on Linux Arm64"
  exit 1
fi

if [[ ! -f "/etc/os-release" ]]; then
  echo "ERROR: cannot install on unknown linux distribution (/etc/os-release is missing)"
  exit 1
fi

ID=$(bash -c 'source /etc/os-release && echo $ID')
if [[ "${ID}" != "ubuntu" ]]; then
  echo "ERROR: cannot install on $ID distribution - only Ubuntu is supported"
  exit 1
fi

# 1. make sure to remove old stable if any.
if dpkg --get-selections | grep -q "^google-chrome[[:space:]]*install$" >/dev/null; then
  apt-get remove -y google-chrome
fi

# 2. Update apt lists (needed to install curl and chrome dependencies)
apt-get update

# 3. Install curl to download chrome
if ! command -v curl >/dev/null; then
  apt-get install -y curl
fi

# 4. download chrome stable from dl.google.com and install it.
cd /tmp
curl -O https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
apt-get install -y ./google-chrome-stable_current_amd64.deb
rm -rf ./google-chrome-stable_current_amd64.deb
cd -
google-chrome --version


#!/bin/bash
set -e
set -x

rm -rf "/Applications/Google Chrome.app"
cd /tmp
curl -o ./googlechrome.dmg -k https://dl.google.com/chrome/mac/universal/stable/GGRO/googlechrome.dmg
hdiutil attach -nobrowse -quiet -noautofsck -noautoopen -mountpoint /Volumes/googlechrome.dmg ./googlechrome.dmg
cp -rf "/Volumes/googlechrome.dmg/Google Chrome.app" /Applications
hdiutil detach /Volumes/googlechrome.dmg
rm -rf /tmp/googlechrome.dmg
/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --version


#!/bin/bash

set -e
set -x

if [[ $(arch) == "aarch64" ]]; then
  echo "ERROR: not supported on Linux Arm64"
  exit 1
fi

if [[ ! -f "/etc/os-release" ]]; then
  echo "ERROR: cannot install on unknown linux distribution (/etc/os-release is missing)"
  exit 1
fi

ID=$(bash -c 'source /etc/os-release && echo $ID')
if [[ "${ID}" != "ubuntu" ]]; then
  echo "ERROR: cannot install on $ID distribution - only Ubuntu is supported"
  exit 1
fi

# 1. make sure to remove old beta if any.
if dpkg --get-selections | grep -q "^microsoft-edge-beta[[:space:]]*install$" >/dev/null; then
  apt-get remove -y microsoft-edge-beta
fi

# 2. Install curl to download Microsoft gpg key
if ! command -v curl >/dev/null; then
  apt-get update
  apt-get install -y curl
fi

# 3. Add the GPG key, the apt repo, update the apt cache, and install the package
curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg
install -o root -g root -m 644 /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/
sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge-dev.list'
rm /tmp/microsoft.gpg
apt-get update && apt-get install -y microsoft-edge-beta

microsoft-edge-beta --version


#!/bin/bash
set -e
set -x

cd /tmp
curl -o ./msedge_beta.pkg -k "$1"
# Note: there's no way to uninstall previously installed MSEdge.
# However, running PKG again seems to update installation.
sudo installer -pkg /tmp/msedge_beta.pkg -target /
rm -rf /tmp/msedge_beta.pkg
/Applications/Microsoft\ Edge\ Beta.app/Contents/MacOS/Microsoft\ Edge\ Beta --version


#!/bin/bash

set -e
set -x

if [[ $(arch) == "aarch64" ]]; then
  echo "ERROR: not supported on Linux Arm64"
  exit 1
fi

if [[ ! -f "/etc/os-release" ]]; then
  echo "ERROR: cannot install on unknown linux distribution (/etc/os-release is missing)"
  exit 1
fi

ID=$(bash -c 'source /etc/os-release && echo $ID')
if [[ "${ID}" != "ubuntu" ]]; then
  echo "ERROR: cannot install on $ID distribution - only Ubuntu is supported"
  exit 1
fi

# 1. make sure to remove old dev if any.
if dpkg --get-selections | grep -q "^microsoft-edge-dev[[:space:]]*install$" >/dev/null; then
  apt-get remove -y microsoft-edge-dev
fi

# 2. Install curl to download Microsoft gpg key
if ! command -v curl >/dev/null; then
  apt-get update
  apt-get install -y curl
fi

# 3. Add the GPG key, the apt repo, update the apt cache, and install the package
curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg
install -o root -g root -m 644 /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/
sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge-dev.list'
rm /tmp/microsoft.gpg
apt-get update && apt-get install -y microsoft-edge-dev

microsoft-edge-dev --version


#!/bin/bash
set -e
set -x

cd /tmp
curl -o ./msedge_dev.pkg -k "$1"
# Note: there's no way to uninstall previously installed MSEdge.
# However, running PKG again seems to update installation.
sudo installer -pkg /tmp/msedge_dev.pkg -target /
rm -rf /tmp/msedge_dev.pkg
/Applications/Microsoft\ Edge\ Dev.app/Contents/MacOS/Microsoft\ Edge\ Dev --version


#!/bin/bash

set -e
set -x

if [[ $(arch) == "aarch64" ]]; then
  echo "ERROR: not supported on Linux Arm64"
  exit 1
fi

if [[ ! -f "/etc/os-release" ]]; then
  echo "ERROR: cannot install on unknown linux distribution (/etc/os-release is missing)"
  exit 1
fi

ID=$(bash -c 'source /etc/os-release && echo $ID')
if [[ "${ID}" != "ubuntu" ]]; then
  echo "ERROR: cannot install on $ID distribution - only Ubuntu is supported"
  exit 1
fi

# 1. make sure to remove old stable if any.
if dpkg --get-selections | grep -q "^microsoft-edge-stable[[:space:]]*install$" >/dev/null; then
  apt-get remove -y microsoft-edge-stable
fi

# 2. Install curl to download Microsoft gpg key
if ! command -v curl >/dev/null; then
  apt-get update
  apt-get install -y curl
fi

# 3. Add the GPG key, the apt repo, update the apt cache, and install the package
curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg
install -o root -g root -m 644 /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/
sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/edge stable main" > /etc/apt/sources.list.d/microsoft-edge-stable.list'
rm /tmp/microsoft.gpg
apt-get update && apt-get install -y microsoft-edge-stable

microsoft-edge-stable --version


#!/bin/bash
set -e
set -x

cd /tmp
curl -o ./msedge_stable.pkg -k "$1"
# Note: there's no way to uninstall previously installed MSEdge.
# However, running PKG again seems to update installation.
sudo installer -pkg /tmp/msedge_stable.pkg -target /
rm -rf /tmp/msedge_stable.pkg
/Applications/Microsoft\ Edge.app/Contents/MacOS/Microsoft\ Edge --version


for package in  'lexical' \
                'lexical-react' \
                'lexical-yjs' \
                'lexical-list' \
                'lexical-table' \
                'lexical-file' \
                'lexical-clipboard' \
                'lexical-hashtag' \
                'lexical-history' \
                'lexical-selection' \
                'lexical-offset' \
                'lexical-code' \
                'lexical-plain-text' \
                'lexical-rich-text' \
                'lexical-utils' \
                'lexical-dragon' \
                'lexical-overflow' \
                'lexical-link' \
                'lexical-text' \
                'lexical-markdown' \
                'lexical-mark'
do
  gen-flow-files "./packages/${package}/src" --out-dir "./packages/${package}/dist"
done


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e
set -o pipefail

# litho-intellij-plugin
DIR=$(dirname "$0")
cd "$DIR"
rm -rf build
echo "./gradlew :litho-intellij-plugin:buildPlugin"
../gradlew :litho-intellij-plugin:buildPlugin &> /dev/null

cd build/distributions
mkdir tmp
unzip -uoq litho-intellij-plugin.zip -d tmp
cd tmp/litho-intellij-plugin/lib
mkdir tmp

for filename in ./*.jar; do
    [ -e "$filename" ] || continue
    unzip -uoq "$filename" -d tmp
done

rm ./*.jar

_JAR="litho-intellij-plugin.jar"
jar -cf "$_JAR" -C tmp .

echo "$(pwd)/$_JAR"

while getopts "p:" opt; do
  case $opt in
    p)
      cp "$_JAR" "$OPTARG"
    ;;
    \?)
      exit 1
    ;;
  esac
done


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# litho-intellij-plugin
DIR=$(dirname "$0")
cd "$DIR" || exit 1
echo "./gradlew :litho-intellij-plugin:test"
../gradlew :litho-intellij-plugin:test &> /dev/null


#!/usr/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

[ -z "$ANDROID_HOME" ] && ANDROID_HOME="/opt/android_sdk"

if env | grep -q ^ANDROID_HOME=
then
  echo "ANDROID_HOME env variable is already exported"
else
  echo "ANDROID_HOME env variable was not exported. setting it now"
  export ANDROID_HOME
fi

# New environment variables required
export ANDROID_SDK_ROOT=$ANDROID_HOME

function download() {
  if hash curl 2>/dev/null; then
    curl -s -L -o "$2" "$1"
  elif hash wget 2>/dev/null; then
    wget -O "$2" "$1"
  else
    echo >&2 "No supported download tool installed. Please get either wget or curl."
    exit
  fi
}

function installsdk() {
  getAndroidSDK

  PROXY_ARGS=""
  if [[ ! -z "$HTTPS_PROXY" ]]; then
    PROXY_HOST="$(echo "$HTTPS_PROXY" | cut -d : -f 1,1)"
    PROXY_PORT="$(echo "$HTTPS_PROXY" | cut -d : -f 2,2)"
    PROXY_ARGS="--proxy=http --proxy_host=$PROXY_HOST --proxy_port=$PROXY_PORT"
  fi

  echo y | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" $PROXY_ARGS "$@"
}

function getAndroidSDK {
  TMP=/tmp/sdk$$.zip

  echo "downloading sdk command line tools"
  download 'https://dl.google.com/android/repository/commandlinetools-linux-7302050_latest.zip' $TMP

  echo "clean $ANDROID_HOME"
  rm -r $ANDROID_HOME/*

  echo "unzipping to $ANDROID_HOME"
  unzip -qod $ANDROID_HOME/cmdline-tools/ $TMP

  echo "rename dir"
  mv $ANDROID_HOME/cmdline-tools/cmdline-tools $ANDROID_HOME/cmdline-tools/latest

  echo "deleting $TMP"
  rm $TMP
}

function installAndroidSDK {
  export PATH="$ANDROID_HOME/platform-tools:$ANDROID_HOME/tools:$ANDROID_HOME/tools/bin:$PATH"

  mkdir -p "$ANDROID_HOME/licenses/"
  echo > "$ANDROID_HOME/licenses/android-sdk-license"
  echo -n 24333f8a63b6825ea9c5514f83c2829b004d1fee > "$ANDROID_HOME/licenses/android-sdk-license"

  installsdk 'platforms;android-30' 'cmake;3.6.4111459' 'build-tools;30.0.2'
}


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script is used to codemod all default state update calls to the sync variant
# via a list of all specs. This is useful to keep current behavior, and then adopt
# async behavior with new callsites.
#
# NOTE: This script is not perfect, and will 1) convert calls that match state
# update calls in the same file 2) will miss state updates called from outside the
# particular ComponentrSpec.

all_specs=$1

if [[ -z $all_specs ]]; then
  # to generate this, you can run something like
  # egrep -lir --include=*Spec.{java,kt} --exclude=./buck* "@OnUpdateState\b" [directory_to_search] > all_specs_with_update_state.txt
  echo "Usage: $0 <all_specs_with_update_state.txt>"
  exit 1
fi

while read -r file; do
  echo "$file:"

  if ! matches=$(grep -A 1 "@OnUpdateState\b" "$file" | grep void) || [[ -z $matches ]]; then
    echo "! No matches for $file."
    continue
  fi

  while read -r m; do
    if [[ $m =~ static\ (<[a-zA-Z0-9\ ]+>\ )?void\ ([a-zA-Z0-9]+)\( ]]; then
      method=${BASH_REMATCH[2]}
      echo "  ${BASH_REMATCH[2]}"
    else
      echo "! Failed match for $m"
      continue
    fi

    # Replace callsites with sync variant
    sed -i "" -e 's/\(\.\s*\)'"$method"'(/\1'"$method"'Sync(/g' "$file"

  done <<< "$matches"
done < "$all_specs"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -ex
shopt -s globstar

# Deploys build artifacts to your local m2 repository.
VERSION="0.1.0-SNAPSHOT"
TMPDIR=/tmp/m2tmp

./gradlew assemble

mkdir -p $TMPDIR

for file in **/build/libs/*.jar; do
  ARTIFACT=$(basename "$file" | sed 's/\.jar$//')

  mvn install:install-file \
    -Dfile=$file \
    -DgroupId=com.facebook.litho \
    -DartifactId=$ARTIFACT \
    -Dversion=$VERSION \
    -Dpackaging=jar \
    -DcreateChecksum=true \
    -DlocalRepositoryPath=$TMPDIR
done

for file in **/build/outputs/**/*-release.aar; do
  ARTIFACT=$(basename "$file" | sed 's/-release\.aar$//')

  mvn install:install-file \
    -Dfile=$file \
    -DgroupId=com.facebook.litho \
    -DartifactId=$ARTIFACT \
    -Dversion=$VERSION \
    -Dpackaging=aar \
    -DcreateChecksum=true \
    -DlocalRepositoryPath=$TMPDIR
done

for file in $TMPDIR/**/maven-metadata-local.xml*; do
  RENAMED=$(echo $file | sed 's/-local\.xml/.xml/')
  mv $file $RENAMED
done

echo "Local artifact deployment complete."
echo "You can now upload you stuff, e.g. with "
echo "rsync --delete -avP /tmp/m2tmp/ $USER@remotehost:www/html/m2repo/"


#!/bin/sh
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

D8=/opt/android/sdk_D37111790/build-tools/29.0.2/d8
$D8 --min-api 24 --output classes_for_analysis.zip "$@"


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# Removes all `*.actual` suffixes from files in the directory above this.

set -e
set -x

THIS_DIRECTORY=$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)

for file in $(find "${THIS_DIRECTORY}/../" -name "*.actual"); do
  mv "$file" "${file%.actual}"
done


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -e

autoreconf --install


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# This script exists to generate the SSL certs used in unit tests.
# It shouldn't really ever be run again until the year 2041
# (when these certs expire)., unless you need to change
# or update attributes of the certificate (Common Name, Organization,
# whatever).

set -e

DAYS=10000
CA_KEY=ca_key.pem
CA_CERT=ca_cert.pem
CA_CERT_SRL=ca_cert.srl

# Generate Certificate Authority Key
openssl genrsa -out "${CA_KEY}" 2048

# Generate Self-Signed Certificate Authority Cert
openssl req -x509 -new -nodes \
    -key "${CA_KEY}" \
    -days "${DAYS}" \
    -out "${CA_CERT}" \
    -subj '/C=US/O=Asox/CN=Asox Certification Authority'

# CA serial number
echo 00000009 > ${CA_CERT_SRL}

function generateCert() {
    local KEYPREFIX="$1"
    local CN=$2
    local TEST_KEY="${KEYPREFIX}_key.pem"
    local TEST_CERT="${KEYPREFIX}_cert.pem"
    local TEST_CERT_CSR="${KEYPREFIX}_cert.csr"

    cat > conf.tmp <<EOF
prompt = no
[req]
distinguished_name = req_distinguished_name
req_extensions = v3_req

[req_distinguished_name]
countryName = US
stateOrProvinceName = Ohio
localityName = Hilliard
commonName = $CN

[v3_req]
# Extensions to add to a certificate request
# basicConstraints = CA:FALSE
# keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = IP:$3, IP:$4
EOF

    # Generate the test key
    openssl genrsa -out "${TEST_KEY}" 2048

    # Generate the test key certificate request
    openssl req -new -nodes \
	-config conf.tmp \
	-key "${TEST_KEY}" \
	-days "${DAYS}" \
	-out "${TEST_CERT_CSR}" \
	-subj '/C=US/O=Asox/CN=Asox Town Limited'

    # Sign the test key
    openssl x509 -req \
	-extensions v3_req \
	-extfile conf.tmp \
	-days "${DAYS}" \
	-in "${TEST_CERT_CSR}" \
	-CA "${CA_CERT}" \
	-CAkey "${CA_KEY}" \
	-out "${TEST_CERT}"

    # Clean up the signing request
    rm "${TEST_CERT_CSR}" conf.tmp
}


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# we assume the generate_key-inl.sh is in the same dir as this script
ROOT=$(dirname "$0")
. "$ROOT/generate_keys-inl.sh"

generateCert test Asox 127.0.0.1 ::1
generateCert broken Asox 0.0.0.0 ::0

# Clean up serial number
rm "${CA_CERT_SRL}"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

sudo apt-get install -y libdouble-conversion1 libgflags2 \
    libboost-program-options1.54.0 libboost-filesystem1.54.0 \
    libboost-system1.54.0 libboost-regex1.54.0 libboost-thread1.54.0 \
    libboost-context1.54.0 libgoogle-glog0 libevent-2.0-5 libjemalloc1

sudo apt-get purge -y gcc g++ ragel autoconf \
    git libtool python-dev libssl-dev libevent-dev \
    binutils-dev make libdouble-conversion-dev libgflags-dev \
    libgoogle-glog-dev libjemalloc-dev

sudo apt-get purge -y 'libboost.*-dev'
sudo apt-get autoremove --purge -y
sudo apt-get autoclean -y
sudo apt-get clean -y

if [[ "x$1" != "x" ]]; then
    PKG_DIR=$1/pkgs
    INSTALL_DIR=$1/install
    strip "$INSTALL_DIR"/bin/mcrouter
    rm -rf "$PKG_DIR"
    rm -rf "$INSTALL_DIR"/lib/*.a
    rm -rf "$INSTALL_DIR"/include
fi


# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

function die { printf "%s: %s\n" "$0" "$@"; exit 1; }

[ -n "$1" ] || die "PKG_DIR missing"
[ -n "$2" ] || die "INSTALL_DIR missing"

PKG_DIR="$1"
INSTALL_DIR="$2"
INSTALL_AUX_DIR="$3"
shift $#
MAKE_ARGS="$@ -j$(nproc)"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

mkdir -p "$PKG_DIR" "$INSTALL_DIR"

if [ -n "$3" ]; then
  mkdir -p "$INSTALL_AUX_DIR"
fi

cd "$PKG_DIR" || die "cd fail"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

MAKE_FILE="$1"
TARGET="$2"
PKG_DIR="${3%/}"/pkgs
INSTALL_DIR="${3%/}"/install
INSTALL_AUX_DIR="${3%/}"/install/aux

[ -n "$MAKE_FILE" ] || ( echo "Make file missing"; exit 1 )
[ -n "$TARGET" ] || ( echo "Target missing"; exit 1 )

mkdir -p "$PKG_DIR" "$INSTALL_DIR" "$INSTALL_AUX_DIR"

cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )

REPO_BASE_DIR="$(cd ../../ && pwd)" || die "Couldn't determine repo top dir"
export REPO_BASE_DIR

export LDFLAGS="-ljemalloc $LDFLAGS"

make "$TARGET" -j3 -f "$MAKE_FILE" PKG_DIR="$PKG_DIR" INSTALL_DIR="$INSTALL_DIR" INSTALL_AUX_DIR="$INSTALL_AUX_DIR"

printf "%s\n" "make $TARGET for $MAKE_FILE done"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

ORDER="$1"
PKG_DIR="${2%/}"/pkgs
INSTALL_DIR="${2%/}"/install
INSTALL_AUX_DIR="${2%/}"/install/aux
shift 2

mkdir -p "$PKG_DIR" "$INSTALL_DIR" "$INSTALL_AUX_DIR"

cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )

REPO_BASE_DIR="$(cd ../../ && pwd)" || die "Couldn't determine repo top dir"
export REPO_BASE_DIR

export LDFLAGS="-ljemalloc $LDFLAGS"

for script in $(ls "order_$ORDER/" | egrep '^[0-9]+_.*[^~]$' | sort -n); do
  "./order_$ORDER/$script" "$PKG_DIR" "$INSTALL_DIR" "$INSTALL_AUX_DIR" "$@"
done

printf "%s\n" "Mcrouter installed in $INSTALL_DIR/bin/mcrouter"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

[ -n "$1" ] || ( echo "Install dir missing"; exit 1 )

sudo yum install -y epel-release
sudo yum install -y centos-release-scl

# Install devtoolset version 4 for GCC 5.x
sudo yum install -y devtoolset-4-gcc devtoolset-4-gcc-c++

export CC=/opt/rh/devtoolset-4/root/usr/bin/gcc
export CXX=/opt/rh/devtoolset-4/root/usr/bin/c++

sudo yum install -y \
    autoconf \
    binutils-devel \
    bison \
    boost-devel \
    cmake \
    double-conversion-devel \
    flex \
    git \
    gflags-devel \
    glog-devel \
    gtest-devel \
    jemalloc-devel \
    libtool \
    libevent-devel \
    make \
    openssl-devel \
    python-devel \
    ragel

# Install automake-1.15 from Fedora
yum info automake-1.15-4.fc23 || sudo yum install -y "http://archives.fedoraproject.org/pub/archive/fedora/linux/releases/23/Everything/x86_64/os/Packages/a/automake-1.15-4.fc23.noarch.rpm"

cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )

./get_and_build_everything.sh centos-7.2 "$@"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

[ -n "$1" ] || ( echo "Install dir missing"; exit 1 )

sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo add-apt-repository -y ppa:george-edison55/cmake-3.x
sudo add-apt-repository -y ppa:chris-lea/libsodium
sudo add-apt-repository -y ppa:sickpig/boost

sudo apt-get update

sudo apt-get install -y \
    autoconf \
    binutils-dev \
    bison \
    cmake \
    flex \
    g++ \
    g++-5 \
    gcc \
    git \
    libboost1.58-dev \
    libboost-thread1.58-dev \
    libboost-filesystem1.58-dev \
    libboost-context1.58-dev \
    libboost-regex1.58-dev \
    libboost-program-options1.58-dev \
    libbz2-dev \
    libdouble-conversion-dev \
    libevent-dev \
    libgflags-dev \
    libgoogle-glog-dev \
    libjemalloc-dev \
    liblz4-dev \
    liblzma-dev \
    liblzma5 \
    libsnappy-dev \
    libsodium-dev \
    libssl-dev \
    libtool \
    make \
    pkg-config \
    python-dev \
    ragel \
    software-properties-common

# sudo apt-get upgrade -yq cmake

cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )

export CC="gcc-5"
export CXX="g++-5"

./get_and_build_everything.sh ubuntu-14.04 "$@"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

[ -n "$1" ] || ( echo "Install dir missing"; exit 1 )

sudo apt-get update

sudo apt-get install -y \
    autoconf \
    binutils-dev \
    bison \
    cmake \
    flex \
    g++ \
    gcc \
    git \
    libboost1.55-all-dev \
    libbz2-dev \
    libdouble-conversion-dev \
    libevent-dev \
    libgflags-dev \
    libgoogle-glog-dev \
    libjemalloc-dev \
    liblz4-dev \
    liblzma-dev \
    liblzma5 \
    libsnappy-dev \
    libsodium-dev \
    libssl-dev \
    libtool \
    make \
    pkg-config \
    python-dev \
    ragel

cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )

./get_and_build_everything.sh ubuntu-15.04 "$@"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

[ -n "$1" ] || ( echo "Install dir missing"; exit 1 )

sudo apt-get update

# Note libzstd-dev is not available on stock Ubuntu 14.04 or 15.04.
sudo apt-get install -y \
    autoconf \
    binutils-dev \
    bison \
    cmake \
    flex \
    g++ \
    gcc \
    git \
    libboost1.58-all-dev \
    libbz2-dev \
    libdouble-conversion-dev \
    libevent-dev \
    libgflags-dev \
    libgoogle-glog-dev \
    libjemalloc-dev \
    liblz4-dev \
    liblzma-dev \
    liblzma5 \
    libsnappy-dev \
    libsodium-dev \
    libssl-dev \
    libtool \
    libunwind8-dev \
    make \
    pkg-config \
    python-dev \
    ragel

cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )

./get_and_build_everything.sh ubuntu-16.04 "$@"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

BASE_DIR="$1"
TARGET="${2:-all}"

[ -n "$BASE_DIR" ] || ( echo "Base dir missing"; exit 1 )

sudo apt-get update

# Note libzstd-dev is not available on stock Ubuntu 14.04 or 15.04.
sudo apt-get install -y \
    autoconf \
    binutils-dev \
    bison \
    cmake \
    flex \
    g++ \
    gcc \
    git \
    libboost1.65-all-dev \
    libbz2-dev \
    libdouble-conversion-dev \
    libevent-dev \
    libgflags-dev \
    libgoogle-glog-dev \
    libjemalloc-dev \
    liblz4-dev \
    liblzma-dev \
    liblzma5 \
    libsnappy-dev \
    libsodium-dev \
    libssl-dev \
    libtool \
    libunwind8-dev \
    make \
    pkg-config \
    python-dev \
    ragel

cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )

./get_and_build_by_make.sh "Makefile_ubuntu-18.04" "$TARGET" "$BASE_DIR"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -ex

BASE_DIR="$1"
TARGET="${2:-all}"

[ -n "$BASE_DIR" ] || ( echo "Base dir missing"; exit 1 )

sudo apt-get update

# Note libzstd-dev is not available on stock Ubuntu 14.04 or 15.04.
sudo apt-get install -y \
    autoconf \
    binutils-dev \
    bison \
    cmake \
    flex \
    g++ \
    gcc \
    git \
    libboost1.71-all-dev \
    libbz2-dev \
    libdouble-conversion-dev \
    libevent-dev \
    libgflags-dev \
    libgoogle-glog-dev \
    libjemalloc-dev \
    liblz4-dev \
    liblzma-dev \
    liblzma5 \
    libsnappy-dev \
    libsodium-dev \
    libssl-dev \
    libtool \
    libunwind8-dev \
    make \
    pkg-config \
    python-dev \
    ragel \
    sudo

cd "$(dirname "$0")" || ( echo "cd fail"; exit 1 )

./get_and_build_by_make.sh "Makefile_ubuntu-20.04" "$TARGET" "$BASE_DIR"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

if [ $# -lt 2 ]; then
  echo -e \
    "Usage: ./$(basename "$0") package_dir install_dir [install_aux_dir]\n" \
    "  where\n" \
    "    package_dir - location where all packages should be download.\n" \
    "    install_dir - location where all packages should be installed.\n" \
    "                  Note: it should already contain a built version of mcrouter in it.\n" \
    "                        i.e. there should be a binary install_dir/bin/mcrouter\n\n" \
    "    install_aux_dir - aux location where all packages should be installed  \n\n" \
    "You can also alter script behavior by setting next environment variables:\n" \
    "  MEMCACHED_PORT - port that would be used by memcached (default: 15000)\n" \
    "  MCROUTER_PORT - port that would be used by mcrouter (default: 15001)\n" \
    "  RUN_DURATION - duration of each individual run in seconds (time during which we \n" \
    "                 collect performance measurements).\n" \
    "                 Note: it doesn't include startup, warmup and cleanup time.\n" \
    "                 (default: 30)\n" \
    "  TWEMPERF_BIN - path to twemperf binary.\n" \
    "                 By default we would download and build a version from github.\n" \
    "  MEMCACHED_BIN - path to memcached binary.\n" \
    "                  By default we would lookup up for installed memcached, if none \n" \
    "                  found, we would download and build it on our own."
  exit 0
fi

# This port will be used for running our own memcached instance.
MEMCACHED_PORT=${MEMCACHED_PORT:-15000}

# This port will be used for running mcrouter instance.
MCROUTER_PORT=${MCROUTER_PORT:-15001}

# Duration of individual workload run.
RUN_DURATION=${RUN_DURATION:-30}

sudo apt-get install -y wget gcc g++ libevent-dev make build-essential

source common.sh

# Disable verbose logging.
set +x

if [ ! -d "$INSTALL_DIR/bin" ]; then
  mkdir "$INSTALL_DIR/bin"
fi

# Ensure that we have twemperf installed.
if [ -z "$TWEMPERF_BIN" ]; then
  echo -n "TWEMPERF_BIN is not set..."
  TWEMPERF_BIN=$INSTALL_DIR/bin/mcperf
  if [ ! -e "$TWEMPERF_BIN" ]; then
    if [ ! -d twemperf ]; then
      echo -n "Downloading source code..."
      git clone https://github.com/twitter/twemperf || die "Failed to clone twemperf"
    else
      echo -n "Found cloned repo..."
    fi

    echo -n "Building it..."

    cd twemperf && autoreconf -i && ./configure && make -j 4 && cd .. || die "Failed to build twemperf"
    ln -s "$PKG_DIR/twemperf/src/mcperf" "$TWEMPERF_BIN" || die "Couldn't install twemperf into $TWEMPERF_BIN"

    echo "Twemperf was installed to $TWEMPERF_BIN"
  else
    echo "Found existing installation of twemperf in $TWEMPERF_BIN"
  fi
else
  echo "TWEMPERF_BIN is set to $TWEMPERF_BIN"
fi

# Ensure that we have memcached installed.
echo -n "Looking for memcached installation..."
if [ -z "$MEMCACHED_BIN" ]; then
  echo -n "MEMCACHED_BIN is not defined..."
  if MEMCACHED_BIN=$(which memcached); then
    echo "Found memcached: $MEMCACHED_BIN"
  else
    MEMCACHED_BIN=$INSTALL_DIR/bin/memcached
    if [ ! -e "$MEMCACHED_BIN" ]; then
      if [ ! -d memcached-1.4.24 ]; then
        echo -n "Downloading source code..."
        wget http://www.memcached.org/files/memcached-1.4.24.tar.gz &> /dev/null || die "Failed to download memcached source code!"
        tar -xvf memcached-1.4.24.tar.gz || die "Failed to extract memcached source code"
      fi
      echo -n "Building it..."

      cd memcached-1.4.24 && ./configure && make all && cd .. || die "Failed to build memcached!"
      ln -s "$PKG_DIR/memcached-1.4.24/memcached" "$MEMCACHED_BIN" || die "Couldn't install memcached into $MEMCACHED_BIN"

      echo "Memcached was installed to $MEMCACHED_BIN"
    else
      echo "Found existing installation of memcached in $MEMCACHED_BIN"
    fi
  fi
else
  echo "MEMCACHED_BIN is set to $MEMCACHED_BIN"
fi

# Check that we have mcrouter.
MCROUTER_BIN=$INSTALL_DIR/bin/mcrouter
if [ ! -e "$MCROUTER_BIN" ]; then
  die "Cannot find mcrouter in install dir, please build and install it before running loadtest!"
fi

# Workload runner, accepts next arguments, runs it for 30 seconds
#  $1 - number of connections.
#  $2 - qps.
#  $3 - method (e.g. get, set, gets, cas, etc.)
function run_workload() {
  # We're goin to start spawning child processes, set up a trap to properly clean them up.
  function cleanup() {
    kill "$MEMCACHED_PID" "$MCROUTER_PID" "$TWEMPERF_PID" &> /dev/null

    # Allow everything to shutdown.
    sleep 2
  }

  trap cleanup EXIT

  # Start memcached.
  "$MEMCACHED_BIN" -t 1 -p "$MEMCACHED_PORT" &> /dev/null &
  MEMCACHED_PID=$!

  # Allow memcached some time to startup.
  sleep 1

  # Check that memcached is running.
  echo version | nc 0 "$MEMCACHED_PORT" | grep "VERSION" &> /dev/null || die "Failed to start memcached!"

  "$MCROUTER_BIN" --config-str="{\"pools\":{\"A\":{\"servers\":[\"localhost:$MEMCACHED_PORT\"],\"server_timeout\":1000}},\"route\":\"Pool|A\"}" -p "$MCROUTER_PORT" &> /dev/null &
  MCROUTER_PID=$!

  # Allow mcrouter some time to startup.
  sleep 1

  # Check that mcrouter is running.
  echo version | nc 0 "$MCROUTER_PORT" | grep "mcrouter" &> /dev/null || die "Failed to start mcrouter!"

  # Calculate total number of calls to do.
  let "NUM_CALLS = $2 * ($RUN_DURATION + 6) / $1"

  "$TWEMPERF_BIN" --server=localhost --port="$MCROUTER_PORT" --num-conns="$1" --call-rate="$2" --num-calls="$NUM_CALLS" --method="$3" &> /dev/null &
  TWEMPERF_PID=$!

  # Wait for a couple of seconds.
  sleep 3

  # Collect CPU usage of mcrouter.
  PIDSTAT_OUT=$(pidstat -h -p "$MCROUTER_PID" "$RUN_DURATION" 1 | grep -v Linux | grep -v '#' | grep -v -e '^$' | awk '{print $7}')

  echo "$PIDSTAT_OUT"
}

echo "**************************************************************************"
echo "**************************************************************************"

echo -n "Running workload (num connections = 100, qps = 20000, method = set)... "
echo " Mcrouter CPU usage: $(run_workload 100 20000 set)%"

echo -n "Running workload (num connections = 100, qps = 40000, method = set)... "
echo " Mcrouter CPU usage: $(run_workload 100 40000 set)%"

echo -n "Running workload (num connections = 100, qps = 80000, method = set)... "
echo " Mcrouter CPU usage: $(run_workload 100 80000 set)%"

echo -n "Running workload (num connections = 100, qps = 20000, method = get)... "
echo " Mcrouter CPU usage: $(run_workload 100 20000 get)%"

echo -n "Running workload (num connections = 100, qps = 40000, method = get)... "
echo " Mcrouter CPU usage: $(run_workload 100 40000 get)%"

echo -n "Running workload (num connections = 100, qps = 80000, method = get)... "
echo " Mcrouter CPU usage: $(run_workload 100 80000 get)%"


#!/usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

## Set version
cd "$SCRIPT_DIR"/../
echo -n $(git tag|tail -1) > "$SCRIPT_DIR"/../VERSION

## Build
source /opt/rh/gcc-toolset-9/enable
sed -i '/m4_append.*mcrouter_version.*/d' "$SCRIPT_DIR"/../configure.ac
"$SCRIPT_DIR"/get_and_build_everything.sh centos-7.2 "$MCROUTER_DIR"

## Cleanup
rm -rf "$MCROUTER_DIR/pkgs"
rm -rf "$INSTALL_DIR/lib/*.a"
rm -rf "$INSTALL_DIR/include"

dnf remove -y   git \
                cmake \
                gcc-toolset* \
                autoconf \
                which \
                make \
                automake \
                wget \
                file \
                binutils-devel \
                glibc-devel \
                bzip2-devel \
                xz-devel \
                openssl-devel \
                python3-devel \
                gtest-devel \
                python3-pkgconfig

rm -rf "$MCROUTER_DIR/repo"