#!/bin/bash
########################################################################
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This script runs after Kudu binaries are built. It does:
# 1. For each binary, run $binary --helpxml and save the output to an XML file
# 2. For each generated XML file, run it through xsltproc two times:
#    a. Once to capture "supported" (stable) flags
#    b. Once to capture "unsupported" (evolving or experimental) flags
# 3. For each generated Asciidoc file, include it in either configuration_reference.adoc
#    or configuration_reference_unsupported.adoc
#
# Usage: make_docs.sh <output_dir>
########################################################################
set -e

usage() {
  echo usage: "$0 --build_root <path to build root> [--site <path to gh-pages checkout>]"
}

while [[ $# > 0 ]] ; do
    arg=$1
    case $arg in
      --help)
        usage
        exit 1
        ;;
      --build_root)
        BUILD_ROOT=$2
        shift
        shift
        ;;
      --site|-s)
        SITE=$2
        if [ -z "$SITE" ]; then
          usage
          exit 1
        fi
        shift
        shift
        if [ ! -d "$SITE"/.git ] || [ ! -d "$SITE/_layouts/" ]; then
          echo "path $SITE doesn't appear to be the root of a git checkout "
          echo "of the Kudu site. Expected to find .git/ and _layouts/ directories"
          exit 1
        fi
        SITE=$(cd $SITE && pwd)
        OUTPUT_DIR=$SITE/docs
        ;;
      --no-jekyll)
        NO_JEKYLL=1
        shift
        ;;
      *)
        echo unknown argument: $arg
        exit 1
    esac
done

if [ -z "$BUILD_ROOT" ]; then
  usage
  exit 1
fi

if [ -z "$SITE" ]; then
  OUTPUT_DIR=$BUILD_ROOT/docs
fi

GEN_DOC_DIR=$BUILD_ROOT/gen-docs
SOURCE_ROOT=$(cd $(dirname $0)/../../..; pwd)

if ! which ruby > /dev/null; then
  echo "ruby must be installed in order to build the docs."
  echo 1
fi

DOCS_SCRIPTS="$SOURCE_ROOT/docs/support/scripts"

# We must set GEM_PATH because bundler depends on it to find its own libraries.
export GEM_PATH="$BUILD_ROOT/gems"
echo GEM_PATH=$GEM_PATH

export PATH="$GEM_PATH/bin:$PATH"
echo PATH="$GEM_PATH/bin:$PATH"

BUNDLE="$GEM_PATH/bin/bundle"

echo "Locally installing ruby gems needed to build docs."
if [ ! -x "$BUNDLE" ]; then
  set -x
  gem install --no-ri --no-rdoc -q --install-dir "$GEM_PATH" bundler
  set +x
fi

set -x
cd "$BUILD_ROOT"
cp $DOCS_SCRIPTS/Gemfile .
cp $DOCS_SCRIPTS/Gemfile.lock .
$BUNDLE install --no-color --path "$GEM_PATH"
set +x

# We need the xsltproc package.
for requirement in "xsltproc"; do
  if ! which $requirement > /dev/null; then
    echo "$requirement is required, but cannot be found. Make sure it is in your path."
    exit 1
  fi
done

mkdir -p "$OUTPUT_DIR" "$GEN_DOC_DIR"

# Create config flag references for each of the binaries below
binaries=("kudu-master" \
          "kudu-tserver")

for binary in ${binaries[@]}; do
  echo "Running $binary --helpxml"

  (
    # Reset environment to avoid affecting the default flag values.
    for var in $(env | awk -F= '{print $1}' | egrep -i 'KUDU|GLOG'); do
      echo "unset $var"
      eval "unset $var"
    done

    # Create the XML file.
    # This command exits with a nonzero value.
    $BUILD_ROOT/bin/$binary --helpxml > ${GEN_DOC_DIR}/$(basename $binary).xml || true
  )

  # Create the supported config reference
  xsltproc \
    --stringparam binary $binary \
    --stringparam support-level supported \
    -o $GEN_DOC_DIR/${binary}_configuration_reference.adoc \
      $SOURCE_ROOT/docs/support/xsl/gflags_to_asciidoc.xsl \
    ${GEN_DOC_DIR}/$binary.xml
  INCLUSIONS_SUPPORTED+="include::${binary}_configuration_reference.adoc[leveloffset=+1]\n"

  # Create the unsupported config reference
  xsltproc \
    --stringparam binary $binary \
    --stringparam support-level unsupported \
    -o $GEN_DOC_DIR/${binary}_configuration_reference_unsupported.adoc \
      $SOURCE_ROOT/docs/support/xsl/gflags_to_asciidoc.xsl \
    ${GEN_DOC_DIR}/$binary.xml
  INCLUSIONS_UNSUPPORTED+="include::${binary}_configuration_reference_unsupported.adoc[leveloffset=+1]\n"
done

# Add the includes to the configuration reference files, replacing the template lines
cp $SOURCE_ROOT/docs/configuration_reference* $GEN_DOC_DIR/
sed -i "s#@@CONFIGURATION_REFERENCE@@#${INCLUSIONS_SUPPORTED}#" ${GEN_DOC_DIR}/configuration_reference.adoc
sed -i "s#@@CONFIGURATION_REFERENCE@@#${INCLUSIONS_UNSUPPORTED}#" ${GEN_DOC_DIR}/configuration_reference_unsupported.adoc

# Create tool references
echo "Running kudu --helpxml"
(
  # Reset environment to avoid affecting the default flag values.
  for var in $(env | awk -F= '{print $1}' | egrep -i 'KUDU|GLOG'); do
    echo "unset $var"
    eval "unset $var"
  done

  # Create the XML file.
  # This command exits with a nonzero value.
  $BUILD_ROOT/bin/kudu --helpxml > ${GEN_DOC_DIR}/kudu.xml || true
)

# Create the supported config reference
xsltproc \
-o $GEN_DOC_DIR/command_line_tools.adoc \
  $SOURCE_ROOT/docs/support/xsl/tool_to_asciidoc.xsl \
${GEN_DOC_DIR}/kudu.xml

# Add the includes to the cli tools reference files, replacing the template lines
cp $SOURCE_ROOT/docs/command_line_tools_reference.adoc $GEN_DOC_DIR/
sed -i "s#@@TOOLS_REFERENCE@@#include::command_line_tools.adoc[leveloffset=+1]\n#" ${GEN_DOC_DIR}/command_line_tools_reference.adoc

# If we're generating the web site, pass the template which causes us
# to generate Jekyll templates instead of full HTML.
if [ -n "$SITE" ]; then
    TEMPLATE_FLAG="-T $SOURCE_ROOT/docs/support/jekyll-templates"
else
    TEMPLATE_FLAG=""
fi

bundle exec asciidoctor -d book $TEMPLATE_FLAG \
    $SOURCE_ROOT/docs/*.adoc ${GEN_DOC_DIR}/*.adoc -D "$OUTPUT_DIR"

mkdir -p "$OUTPUT_DIR/images"
cp $SOURCE_ROOT/docs/images/* "$OUTPUT_DIR/images/"

echo
echo ----------------------------
echo "Docs built in $OUTPUT_DIR"

# If we're building the site, try to run Jekyll for them to make
# it a bit easier to quickly preview the results.
if [ -n "$SITE" ] && [ -z "$NO_JEKYLL" ]; then
  # We need to generate a config file which fakes the "github.url" property
  # so that relative links within the site work.
  BASE_URL="file://$SITE/_site/"
  TMP_CONFIG=$(mktemp -t config.XXXXXX)
  # Rename the config file. The template adds the random section
  # to the end of the file to be compatible with Mac OS. However,
  # Jekyll expects the file to end in yml.
  mv "$TMP_CONFIG" "$TMP_CONFIG.yml"
  TMP_CONFIG="$TMP_CONFIG.yml"
  trap "rm $TMP_CONFIG" EXIT
  printf "github:\n  url: %s" "$BASE_URL" > $TMP_CONFIG

  # Now rebuild the site itself.
  echo Attempting to re-build via Jekyll...
  bundle exec jekyll build --source "$SITE" --config "$TMP_CONFIG"

  # Output the URL so it's easy to click on from the terminal.
  echo ----------------------
  echo Rebuild successful. View your site at
  echo $BASE_URL/index.html
  echo ----------------------
fi


#!/bin/bash
########################################################################
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This script generates site documentation and Javadocs.
########################################################################
set -e

BUILD_TYPE=release
SOURCE_ROOT=$(cd $(dirname $0)/../../..; pwd)
BUILD_ROOT="$SOURCE_ROOT/build/$BUILD_TYPE"
SITE_OUTPUT_DIR="$BUILD_ROOT/site"

OPT_DOXYGEN=1 # By default, build doxygen docs.
OPT_JAVADOC=1 # By default, build javadocs.
OPT_FORCE='' # By default, don't overwrite the destination directory.

usage() {
  echo "Usage: $0 [--no-doxygen] [--no-javadoc] [--force]"
  echo "Specify --no-doxygen to skip generation of the C++ client API docs"
  echo "Specify --no-javadoc to skip generation of the Java API docs"
  echo "Specify --force to overwrite the destination directory, if it exists"
  exit 1
}

if [ $# -gt 0 ]; then
  for arg in $*; do
    case $arg in
      "--no-doxygen")  OPT_DOXYGEN='' ;;
      "--no-javadoc")  OPT_JAVADOC='' ;;
      "--force")       OPT_FORCE=1 ;;
      "--help")        usage ;;
      "-h")            usage ;;
      *)               echo "$0: Unknown command-line option: $arg"; usage ;;
    esac
  done
fi

if [ -n "$OPT_DOXYGEN" ] && ! which doxygen >/dev/null 2>&1; then
  echo "Error: doxygen not found."
  usage
fi

set -x

cd "$SOURCE_ROOT"

GIT_REMOTE=$(git config --get remote.origin.url)
if [ -z "$GIT_REMOTE" ]; then
  echo "Error: The 'origin' git remote is expected but is not defined"
  usage
fi

# Build Kudu thirdparty
$SOURCE_ROOT/build-support/enable_devtoolset.sh $SOURCE_ROOT/thirdparty/build-if-necessary.sh
echo "Successfully built third-party dependencies."

# Build the binaries so we can auto-generate the command-line references
mkdir -p "$BUILD_ROOT"
cd "$BUILD_ROOT"
rm -rf CMakeCache CMakeFiles/
$SOURCE_ROOT/build-support/enable_devtoolset.sh \
    $SOURCE_ROOT/thirdparty/installed/common/bin/cmake \
    -DNO_TESTS=1 \
    -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
    $SOURCE_ROOT
MAKE_TARGETS="kudu kudu-tserver kudu-master"
if [ -n "$OPT_DOXYGEN" ]; then
  MAKE_TARGETS="$MAKE_TARGETS doxygen"
fi
make -j$(getconf _NPROCESSORS_ONLN) $MAKE_TARGETS

# Check out the gh-pages repo into $SITE_OUTPUT_DIR
if [ -d "$SITE_OUTPUT_DIR" -a -n "$OPT_FORCE" ]; then
  rm -rf "$SITE_OUTPUT_DIR"
fi
git clone -q "$GIT_REMOTE" --reference "$SOURCE_ROOT" -b gh-pages --depth 1 "$SITE_OUTPUT_DIR"

# Build the docs using the styles from the Jekyll site
rm -Rf "$SITE_OUTPUT_DIR/docs"
$SOURCE_ROOT/docs/support/scripts/make_docs.sh --build_root $BUILD_ROOT --site "$SITE_OUTPUT_DIR"
if [ -f "$SITE_OUTPUT_DIR/docs/index.html" ]; then
  echo "Successfully built docs."
else
  echo "Docs failed to build."
  exit 1
fi

if [ -n "$OPT_JAVADOC" ]; then
  JAVADOC_SUBDIR="apidocs"
  cd "$SOURCE_ROOT/java"
  mvn clean install -DskipTests
  if mvn clean javadoc:aggregate | tee /dev/stdout | fgrep -q "Javadoc Warnings"; then
    echo "There are Javadoc warnings. Please fix them."
    exit 1
  fi

  if [ -f "$SOURCE_ROOT/java/target/site/$JAVADOC_SUBDIR/index.html" ]; then
    echo "Successfully built Javadocs."
  else
    echo "Javadocs failed to build."
    exit 1
  fi

  rm -Rf "$SITE_OUTPUT_DIR/$JAVADOC_SUBDIR"
  cp -a "$SOURCE_ROOT/java/target/site/$JAVADOC_SUBDIR" "$SITE_OUTPUT_DIR/"
fi

if [ -n "$OPT_DOXYGEN" ]; then
  CPP_CLIENT_API_SUBDIR="cpp-client-api"
  rm -Rf "$SITE_OUTPUT_DIR/$CPP_CLIENT_API_SUBDIR"
  cp -a "$BUILD_ROOT/docs/doxygen/client_api/html" "$SITE_OUTPUT_DIR/$CPP_CLIENT_API_SUBDIR"
fi

SITE_SUBDIRS="docs"
if [ -n "$OPT_JAVADOC" ]; then
  SITE_SUBDIRS="$SITE_SUBDIRS $JAVADOC_SUBDIR"
fi
if [ -n "$OPT_DOXYGEN" ]; then
  SITE_SUBDIRS="$SITE_SUBDIRS $CPP_CLIENT_API_SUBDIR"
fi

cd "$SITE_OUTPUT_DIR"
SITE_ARCHIVE="$SITE_OUTPUT_DIR/website_archive.zip"
zip -rq "$SITE_ARCHIVE" $SITE_SUBDIRS

set +x
echo "Generated web site at $SITE_OUTPUT_DIR"
echo "Docs zip generated at $SITE_ARCHIVE"
echo "To view live site locally, run: (cd $SITE_OUTPUT_DIR && ./site_tool jekyll serve)"


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

prepend_lines() {
  tail --lines=+2 $1 | perl -p -e "s,^,$2,g;"
}

printf "dist\tsys\tworkload\ttime\ttput\n"
for dist in zipfian uniform ; do
  for sys in hbase kudu ; do
    prepend_lines $dist-$sys/load-100M.log.tsv "${dist}\t${sys}\tload\t"
    prepend_lines $dist-$sys/run-workloada.log.tsv "${dist}\t${sys}\ta\t"
    prepend_lines $dist-$sys/run-workloadb.log.tsv "${dist}\t${sys}\tb\t"
    prepend_lines $dist-$sys/run-workloadc.log.tsv "${dist}\t${sys}\tc\t"
    prepend_lines $dist-$sys/run-workloadd.log.tsv "latest\t${sys}\td\t"
  done
done


#/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

EXPORTER="com.yahoo.ycsb.measurements.exporter.JSONArrayMeasurementsExporter"
COMMON_FLAGS="-p recordcount=100000000 -p columnfamily=family -cp /etc/hbase/conf -p exporter=$EXPORTER -p table=ycsb_100m"
OUT_DIR=uniform-hbase

mkdir -p $OUT_DIR
if true ; then
	./bin/ycsb load hbase10 $COMMON_FLAGS  -p exportfile=$OUT_DIR/load.json -p clientbuffering=true \
	  -P workloads/workloada -p recordcount=100000000 -threads 16 -s 2>&1 | tee $OUT_DIR/load-100M.log
fi
for x in a b c d ; do
  dist_param=
  if [ "$x" != "d" ]; then
    dist_param="-p requestdistribution=uniform"
  fi
  ./bin/ycsb run hbase10 -P workloads/workload$x -p recordcount=100000000 -p operationcount=10000000 \
      $COMMON_FLAGS -p exportfile=$OUT_DIR/$x.json \
      $dist_param \
      -threads 64 -s 2>&1 | tee $OUT_DIR/run-workload$x.log
done


#/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

EXPORTER="com.yahoo.ycsb.measurements.exporter.JSONArrayMeasurementsExporter"
COMMON_FLAGS="-p recordcount=100000000 -p exporter=$EXPORTER -p table_name=ycsb_100m -p masterQuorum=a1216"
OUT_DIR=uniform-kudu

mkdir -p $OUT_DIR
if true ; then
	./bin/ycsb load kudu $COMMON_FLAGS  -p exportfile=$OUT_DIR/load.json -p sync_ops=false \
	  -p pre_split_num_tablets=100 \
	  -P workloads/workloada -p recordcount=100000000 -threads 16 -s 2>&1 | tee $OUT_DIR/load-100M.log
fi
for x in a b c d ; do
  dist_param=
  if [ "$x" != "d" ]; then
    dist_param="-p requestdistribution=uniform"
  fi
  ./bin/ycsb run kudu -P workloads/workload$x -p recordcount=100000000 -p operationcount=10000000 -p sync_ops=true \
      $COMMON_FLAGS -p exportfile=$OUT_DIR/$x.json \
      $dist_param \
      -threads 64 -s 2>&1 | tee $OUT_DIR/run-workload$x.log
done


#/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

EXPORTER="com.yahoo.ycsb.measurements.exporter.JSONArrayMeasurementsExporter"
COMMON_FLAGS="-p recordcount=100000000 -p columnfamily=family -cp /etc/hbase/conf -p exporter=$EXPORTER -p table=ycsb_100m"
OUT_DIR=zipfian-hbase

mkdir -p $OUT_DIR
if true ; then
	./bin/ycsb load hbase10 $COMMON_FLAGS  -p exportfile=$OUT_DIR/load.json -p clientbuffering=true \
	  -P workloads/workloada -p recordcount=100000000 -threads 16 -s 2>&1 | tee $OUT_DIR/load-100M.log
fi
for x in a b c d ; do
  dist_param=
  if [ "$x" != "d" ]; then
    dist_param="-p requestdistribution=zipfian"
  fi
  ./bin/ycsb run hbase10 -P workloads/workload$x -p recordcount=100000000 -p operationcount=10000000 \
      $COMMON_FLAGS -p exportfile=$OUT_DIR/$x.json \
      $dist_param \
      -threads 64 -s 2>&1 | tee $OUT_DIR/run-workload$x.log
done


#/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

EXPORTER="com.yahoo.ycsb.measurements.exporter.JSONArrayMeasurementsExporter"
COMMON_FLAGS="-p recordcount=100000000 -p exporter=$EXPORTER -p table_name=ycsb_100m -p masterQuorum=a1216"
OUT_DIR=zipfian-kudu

mkdir -p $OUT_DIR
if true ; then
	./bin/ycsb load kudu $COMMON_FLAGS  -p exportfile=$OUT_DIR/load.json -p sync_ops=false \
	  -p pre_split_num_tablets=100 \
	  -P workloads/workloada -p recordcount=100000000 -threads 16 -s 2>&1 | tee $OUT_DIR/load-100M.log
fi
for x in a b c d ; do
  dist_param=
  if [ "$x" != "d" ]; then
    dist_param="-p requestdistribution=zipfian"
  fi
  ./bin/ycsb run kudu -P workloads/workload$x -p recordcount=100000000 -p operationcount=10000000 -p sync_ops=true \
      $COMMON_FLAGS -p exportfile=$OUT_DIR/$x.json \
      $dist_param \
      -threads 64 -s 2>&1 | tee $OUT_DIR/run-workload$x.log
done


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# This script tests for KUDU-1508 in the local kernel's implementation of ext4.
# To use it, you must compile hole_punch_range.c and make it available in the
# current working directory.
#
# This script must either be run as root, or as a user with sudo.

# Configurable options.
BLOCK_DEVICE_FILE=block
MOUNT_DIR=mnt
TEST_FILE=test
BLOCK_SIZES="1024 2048 4096"

# Run a command given in the first argument.
#
# If the second argument is set and the command fails, the exit code is
# returned and the command output is squelched.
#
# If the second argument is unset and the command fails, the command output is
# emitted and the script exits.
function run() {
  local CMD="$1"
  local FAIL_OK="$2"
  local OUT RETVAL

  OUT=$($CMD 2>&1)
  RETVAL=$?
  if [ $RETVAL -ne 0 ]; then
    if [ -n "$FAIL_OK" ]; then
      return $RETVAL
    fi

    echo "Command failed: $CMD"
    echo "Result: $OUT"
    exit $RETVAL
  fi
}

# Run an iteration of the test in a local test-only filesystem.
#
# Returns whether the filesystem was corrupted.
function run_test() {
  local BLOCK_SIZE=$1
  local BLOCK_NUM=$2

  local FILESYSTEM_SIZE=$(($BLOCK_SIZE * $BLOCK_NUM * 4))
  local TEST_PATH=$MOUNT_DIR/$TEST_FILE
  local FSCK_RESULT
  local INTERIOR_NODE_LAST_BLOCK

  # Clean up in preparation.
  run "sudo umount $MOUNT_DIR" fail_ok
  run "mkdir -p $MOUNT_DIR"
  run "rm -f $BLOCK_DEVICE_FILE"

  # Create the test filesystem and file.
  #
  # Note that 'fallocate --length <bytes>' doesn't appear to be recognized on
  # el6.6, despite what the help says.
  #
  # The 'sync' at the end speeds up subsequent hole punching.
  run "fallocate -l $FILESYSTEM_SIZE $BLOCK_DEVICE_FILE"
  run "mkfs -F -t ext4 -b $BLOCK_SIZE $BLOCK_DEVICE_FILE"
  run "sudo mount -o loop $BLOCK_DEVICE_FILE $MOUNT_DIR"
  run "sudo chown $EUID $MOUNT_DIR"
  run "dd if=/dev/zero of=$TEST_PATH conv=notrunc bs=$BLOCK_SIZE count=$BLOCK_NUM"
  run "sync"

  # Maximize the number of extents in the file by punching holes in every other
  # block.
  #
  # The 'sync' at the end makes the extent tree visible to debugfs.
  run "./hole_punch_range $TEST_PATH 0 $BLOCK_SIZE 2"
  run "sync"

  # Determine the block number of the last block in the first level 1 interior
  # node in the file's extent tree.
  while read LINE; do
    # Find the node. Its debugfs line will look something like this:
    #
    #  1/ 2   1/340      1 -    680 163969             680
    #
    # The last number in the line is the number of blocks pointed to by the
    # node, and since we're looking at the first level 1 interior node, it's
    # also the last block number pointed to by that node.
    if echo $LINE | grep -q "[[:space:]]*1/[[:space:]]*[[:digit:]]*[[:space:]]*1/"; then
      INTERIOR_NODE_LAST_BLOCK=$(echo $LINE | awk '{ print $NF }')
      break
    fi
  done < <(debugfs -R "dump_extents -n $TEST_FILE" $BLOCK_DEVICE_FILE 2> /dev/null)

  # If we can't find the last block number, the file does not have an extent
  # tree with at least two interior levels, and the subsequent corruption test
  # can be skipped.
  if [ -n "$INTERIOR_NODE_LAST_BLOCK" ]; then
    # Try to corrupt the filesystem by punching out all of the remaining blocks
    # belonging to this interior node.
    #
    # If the filesystem is vulnerable to KUDU-1508, it'll fail to update the
    # interior node's parent, corrupting the filesystem until the next fsck.
    run "./hole_punch_range $TEST_PATH 1 $INTERIOR_NODE_LAST_BLOCK 2"
  fi

  run "sudo umount $MOUNT_DIR"

  # Test the filesystem for corruption and return the result.
  run "fsck.ext4 $BLOCK_DEVICE_FILE -f -n" fail_ok
  FSCK_RESULT=$?

  # Clean up after the test.
  run "rmdir $MOUNT_DIR"
  run "rm -f $BLOCK_DEVICE_FILE"

  return $FSCK_RESULT
}

# Run the test with known bad parameters to see if this kernel is worth testing.
run_test 4096 $((16 * 1024))
if [[ $? -eq 0 ]]; then
  echo "This kernel is not vulnerable to KUDU-1508, skipping test"
  exit 0
fi

echo "This kernel is vulnerable to KUDU-1508, finding per-block size upper bounds"

# Now figure out, for each block size, what the max number of blocks is.
for BLOCK_SIZE in $BLOCK_SIZES; do
  MIN=0
  MAX=$((1024 * 16))
  CUR=
  echo "Block size $BLOCK_SIZE: searching for block number upper bound (MIN=$MIN,MAX=$MAX)"
  while [[ $MIN -lt $MAX ]] ; do
    CUR=$((($MAX + $MIN) / 2))
    run_test $BLOCK_SIZE $CUR
    if [[ $? -eq 0 ]]; then
      echo "Block size $BLOCK_SIZE: $CUR good (MIN=$MIN,MAX=$MAX)"
      MIN=$(($CUR + 1))
    else
      echo "Block size $BLOCK_SIZE: $CUR bad (MIN=$MIN,MAX=$MAX)"
      MAX=$(($CUR - 1))
    fi
  done

  echo "Block size $BLOCK_SIZE: upper bound found at $CUR"
done


#!/bin/bash -xe
########################################################################
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Run and compare benchmarks.
#
# Allows for running comparisons either locally or as part of a
# Jenkins job which integrates with a historical stats DB.
# Run this script with -help for usage information.
########################################################################

# Fail the job if any part fails, even when piping through 'tee', etc.
set -o pipefail

################################################################
# Constants
################################################################

MODE_JENKINS="jenkins"
MODE_LOCAL="local"

LOCAL_STATS_BASE="local-stats"

NUM_MT_TABLET_TESTS=5
MT_TABLET_TEST=mt-tablet-test
RPC_BENCH_TEST=RpcBenchBenchmark
CBTREE_TEST=cbtree-test
BLOOM_TEST=BloomfileBenchmark
MT_BLOOM_TEST=MultithreadedBloomfileBenchmark
WIRE_PROTOCOL_TEST=WireProtocolBenchmark
COMPACT_MERGE_BENCH=CompactBenchMerge
WITH_OVERLAP=Overlap
NO_OVERLAP=NoOverlap

MEMROWSET_BENCH=MemRowSetBenchmark
TS_INSERT_LATENCY=TabletServerInsertLatency
TS_8THREAD_BENCH=TabletServer8Threads
INSERT=Insert
SCAN_NONE_COMMITTED=ScanNoneCommitted
SCAN_ALL_COMMITTED=ScanAllCommitted

FS_SCANINSERT_MRS=FullStackScanInsertMRSOnly
FS_SCANINSERT_DISK=FullStackScanInsertWithDisk

DENSE_NODE_ITEST=DenseNodeItest

LOG_DIR_NAME=build/latest/bench-logs
OUT_DIR_NAME=build/latest/bench-out
HTML_FILE="benchmarks.html"

# Most tests will run this many times.
NUM_SAMPLES=${NUM_SAMPLES:-10}

################################################################
# Global variables
################################################################

BENCHMARK_MODE=$MODE_JENKINS # we default to "jenkins mode"
BASE_DIR=$(pwd)
LOGDIR=""
OUTDIR=""

################################################################
# Functions
################################################################

usage_and_die() {
  set +x
  echo "Usage: $0 [-local [git-hash-1 [git-hash-2 ...]]]"
  echo "       When -local is specified, perf of 1 or more git hashes are plotted."
  echo "       Otherwise, the script is run in 'Jenkins' mode and expects the"
  echo "       usual Jenkins environment variables to be defined, such as"
  echo "       BUILD_NUMBER and JOB_NAME."
  exit 1
}

ensure_cpu_scaling() {
  $BASE_DIR/src/kudu/scripts/ensure_cpu_scaling.sh "$@"
}

record_result() {
  local BUILD_IDENTIFIER=$1
  local TEST_NAME=$2
  local ITER=$3
  local VALUE=$4
  if [ $BENCHMARK_MODE = $MODE_JENKINS ]; then
    python write-jobs-stats-to-mysql.py $JOB_NAME $BUILD_IDENTIFIER $TEST_NAME $ITER $VALUE
  else
    local STATS_FILE="$OUTDIR/$LOCAL_STATS_BASE-$TEST_NAME.tsv"
    # Note: literal tabs in below string.
    echo "${TEST_NAME}	${VALUE}	${BUILD_IDENTIFIER}" >> "$STATS_FILE"
  fi
}

load_stats() {
  local TEST_NAME="$1"
  if [ "$BENCHMARK_MODE" = "$MODE_JENKINS" ]; then
    # Get last 4 weeks of stats
    python get-job-stats-from-mysql.py $TEST_NAME 28
  else
    # Convert MySQL wildcards to shell wildcards.
    local TEST_NAME=$(echo $TEST_NAME | perl -pe 's/%/*/g')
    local STATS_FILES=$(ls $OUTDIR/$LOCAL_STATS_BASE-$TEST_NAME.tsv)
    # Note: literal tabs in below string.
    echo "workload	runtime	build_number"
    for f in $STATS_FILES; do
      cat $f
    done
  fi
}

write_img_plot() {
  local INPUT_FILE=$1
  local TEST_NAME=$2
  # Rscript fails when there's only a header, so just skip
  if [ `wc -l $INPUT_FILE | cut -d ' ' -f1` -gt 1 ]; then
    Rscript jobs_runtime.R $INPUT_FILE $TEST_NAME
  fi
}

write_mttablet_img_plots() {
  local INPUT_FILE=$1
  local TEST_NAME=$2
  xvfb-run Rscript mt-tablet-test-graph.R $INPUT_FILE $TEST_NAME
}

build_kudu() {
  # PATH=<toolchain_stuff>:$PATH
  export TOOLCHAIN=/mnt/toolchain/toolchain.sh
  if [ -f "$TOOLCHAIN" ]; then
    source $TOOLCHAIN
  fi

  # Build thirdparty
  $BASE_DIR/build-support/enable_devtoolset.sh thirdparty/build-if-necessary.sh

  # PATH=<thirdparty_stuff>:<toolchain_stuff>:$PATH
  THIRDPARTY_BIN=$BASE_DIR/thirdparty/installed/common/bin
  export PPROF_PATH=$THIRDPARTY_BIN/pprof

  BUILD_TYPE=release

  # Build Kudu
  mkdir -p build/$BUILD_TYPE
  pushd build/$BUILD_TYPE
  rm -rf CMakeCache.txt CMakeFiles/

  $BASE_DIR/build-support/enable_devtoolset.sh $THIRDPARTY_BIN/cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ../..

  # clean up before we run
  rm -Rf /tmp/kudutpch1-$UID
  mkdir -p /tmp/kudutpch1-$UID

  NUM_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l)
  make -j${NUM_PROCS} 2>&1 | tee build.log
  popd

}

run_benchmarks() {
  # Create output directories if needed.
  mkdir -p "$LOGDIR"
  mkdir -p "$OUTDIR"

  # run all of the variations of mt-tablet-test
  ./build/latest/bin/mt-tablet-test \
    --gtest_filter=\*DoTestAllAtOnce\* \
    --num_counter_threads=0 \
    --tablet_test_flush_threshold_mb=32 \
    --num_slowreader_threads=0 \
    --flusher_backoff=1.0 \
    --flusher_initial_frequency_ms=1000 \
    --inserts_per_thread=1000000 \
    &> $LOGDIR/${MT_TABLET_TEST}.log

  # run rpc-bench test 5 times. 10 seconds per run
  for i in $(seq 1 $NUM_SAMPLES); do
    KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/rpc-bench \
      --gtest_filter=*BenchmarkCalls &> $LOGDIR/$RPC_BENCH_TEST$i.log
  done

  # run cbtree-test 5 times. 20 seconds per run
  for i in $(seq 1 $NUM_SAMPLES); do
    KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/cbtree-test \
      --gtest_filter=TestCBTree.TestScanPerformance &> $LOGDIR/${CBTREE_TEST}$i.log
  done

  # run bloomfile-test 5 times. ~3.3 seconds per run
  for i in $(seq 1 $NUM_SAMPLES); do
    ./build/latest/bin/bloomfile-test --benchmark_queries=10000000 --bloom_size_bytes=32768 \
      --n_keys=100000 --gtest_filter=*Benchmark &> $LOGDIR/$BLOOM_TEST$i.log
  done

  # run mt-bloomfile-test 5 times. 20-30 seconds per run.
  # We use 150M keys so the total bloom space is >150MB, and then
  # set the cache capacity to be only 100M to ensure that the cache
  # churns a lot.
  for i in $(seq 1 $NUM_SAMPLES); do
    ./build/latest/bin/mt-bloomfile-test --benchmark_queries=2000000 --bloom_size_bytes=4096 \
      --n_keys=150000000 --block_cache_capacity_mb=100 &> $LOGDIR/$MT_BLOOM_TEST$i.log
  done

  # run wire_protocol-test 5 times. 6 seconds per run
  for i in $(seq 1 $NUM_SAMPLES); do
    KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/wire_protocol-test --gtest_filter=*Benchmark \
      &> $LOGDIR/$WIRE_PROTOCOL_TEST$i.log
  done

  # run compaction-test 5 times, 6 seconds each
  for i in $(seq 1 $NUM_SAMPLES); do
    KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/compaction-test \
      --gtest_filter=TestCompaction.BenchmarkMerge* &> $LOGDIR/${COMPACT_MERGE_BENCH}$i.log
  done

  # run memrowset benchmark 5 times, ~10 seconds per run
  for i in $(seq 1 $NUM_SAMPLES) ; do
    ./build/latest/bin/memrowset-test --roundtrip_num_rows=10000000 \
        --gtest_filter=\*InsertCount\* &> $LOGDIR/${MEMROWSET_BENCH}$i.log
  done

  # Run single-threaded TS insert latency benchmark, 5-6 seconds per run
  for i in $(seq 1 $NUM_SAMPLES) ; do
    KUDU_ALLOW_SLOW_TESTS=true ./build/latest/bin/tablet_server-test \
      --gtest_filter=*MicroBench* \
      --single_threaded_insert_latency_bench_warmup_rows=1000 \
      --single_threaded_insert_latency_bench_insert_rows=10000 &> $LOGDIR/${TS_INSERT_LATENCY}$i.log
  done

  # Run multi-threaded TS insert benchmark
  for i in $(seq 1 $NUM_SAMPLES) ; do
    KUDU_ALLOW_SLOW_TESTS=1 build/latest/bin/tablet_server-stress-test \
      --num_inserts_per_thread=30000 -runtime_secs=0 &> $LOGDIR/${TS_8THREAD_BENCH}$i.log
  done

  # Run full stack scan/insert test using MRS only, ~26s each
  for i in $(seq 1 $NUM_SAMPLES) ; do
    ./build/latest/bin/full_stack-insert-scan-test \
      --gtest_filter=FullStackInsertScanTest.MRSOnlyStressTest \
      --concurrent_inserts=50 \
      --inserts_per_client=200000 \
      --rows_per_batch=10000 \
      &> $LOGDIR/${FS_SCANINSERT_MRS}$i.log
  done

  # Run full stack scan/insert test with disk, ~50s each
  for i in $(seq 1 $NUM_SAMPLES) ; do
    ./build/latest/bin/full_stack-insert-scan-test \
      --gtest_filter=FullStackInsertScanTest.WithDiskStressTest \
      --concurrent_inserts=50 \
      --inserts_per_client=200000 \
      --rows_per_batch=10000 \
      &> $LOGDIR/${FS_SCANINSERT_DISK}$i.log
  done

  # Run dense storage node test, ~150s each.
  #
  # Needs to run as root for measure_startup_drop_caches.
  for i in $(seq 1 $NUM_SAMPLES) ; do
    sudo ./build/latest/bin/dense_node-itest \
      --num_seconds=60 \
      --num_tablets=1000 \
      --measure_startup_sync \
      --measure_startup_drop_caches \
      --measure_startup_wait_for_bootstrap \
      &> $LOGDIR/${DENSE_NODE_ITEST}$i.log
  done
}

parse_and_record_all_results() {
  local BUILD_IDENTIFIER="$1"

  if [ -z "$BUILD_IDENTIFIER" ]; then
    echo "ERROR: BUILD_IDENTIFIER not defined"
    exit 1
  fi

  pushd src
  pushd kudu
  pushd scripts

  # parse the number of ms out of "[       OK ] MultiThreadedTabletTest/5.DoTestAllAtOnce (14966 ms)"
  local MT_TABLET_TEST_TIMINGS="${MT_TABLET_TEST}-timings"
  grep OK $LOGDIR/${MT_TABLET_TEST}.log | cut -d "(" -f2 | cut -d ")" -f1 | cut -d " " -f1 \
    > $LOGDIR/${MT_TABLET_TEST_TIMINGS}.txt

  # The tests go from 0 to NUM_MT_TABLET_TEST, but files start at line one so we add +1 to the line number.
  # Then using the timing we found, we multiply it by 1000 to gets seconds in float, then send it to MySQL
  for i in $(seq 0 $NUM_MT_TABLET_TESTS); do
    linenumber=$[ $i + 1 ]
    timing=`sed -n "${linenumber}p" $LOGDIR/${MT_TABLET_TEST_TIMINGS}.txt`
    record_result $BUILD_IDENTIFIER MultiThreadedTabletTest_$i 1 `echo $timing / 1000 | bc -l`
  done

  # parse out the real time from: "Time spent Insert 10000000 keys: real 16.438s user 16.164s  sys 0.229s"
  for i in $(seq 1 $NUM_SAMPLES); do
    real=`grep "Time spent Insert" $LOGDIR/${CBTREE_TEST}$i.log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ConcurrentBTreeScanInsert $i $real
  done

  for i in $(seq 1 $NUM_SAMPLES); do
    real=`grep "not frozen" $LOGDIR/${CBTREE_TEST}$i.log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ConcurrentBTreeScanNotFrozen $i $real
  done

  for i in $(seq 1 $NUM_SAMPLES); do
    real=`grep "(frozen" $LOGDIR/${CBTREE_TEST}$i.log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ConcurrentBTreeScanFrozen $i $real
  done

  # parse out the real time from "Time spent with overlap: real 0.557s user 0.546s sys 0.010s"
  for i in $(seq 1 $NUM_SAMPLES); do
    real=`grep "with overlap" $LOGDIR/${COMPACT_MERGE_BENCH}$i.log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ${COMPACT_MERGE_BENCH}${WITH_OVERLAP} $i $real
  done

  for i in $(seq 1 $NUM_SAMPLES); do
    real=`grep "without overlap" $LOGDIR/${COMPACT_MERGE_BENCH}$i.log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ${COMPACT_MERGE_BENCH}${NO_OVERLAP} $i $real
  done

  # parse out time from MRS benchmarks
  for i in $(seq 1 $NUM_SAMPLES); do
    log=$LOGDIR/${MEMROWSET_BENCH}$i.log
    real=`grep "Time spent Inserting" $log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ${MEMROWSET_BENCH}${INSERT} $i $real
    real=`grep "Time spent Scanning rows where none" $log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ${MEMROWSET_BENCH}${SCAN_NONE_COMMITTED} $i $real
    real=`grep "Time spent Scanning rows where all" $log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ${MEMROWSET_BENCH}${SCAN_ALL_COMMITTED} $i $real
  done

  # Parse out the real time from: "Time spent Running 10000000 queries: real 3.281s  user 3.273s sys 0.000s"
  for i in $(seq 1 $NUM_SAMPLES); do
    real=`grep "Time spent Running" $LOGDIR/$BLOOM_TEST$i.log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER $BLOOM_TEST $i $real
  done

  # Parse out the real time from: "Time spent Running 2000000 queries: real 28.193s user 26.903s sys 1.032s"
  # Many threads output their value, we keep the last;
  for i in $(seq 1 $NUM_SAMPLES); do
    real=`grep "Time spent Running" $LOGDIR/$MT_BLOOM_TEST$i.log | tail -n1 | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER $MT_BLOOM_TEST $i $real
  done

  # Parse out the real time from: "Time spent Converting to PB: real 5.962s  user 5.918s sys 0.025s"
  for i in $(seq 1 $NUM_SAMPLES); do
    real=`grep "Time spent Converting" $LOGDIR/$WIRE_PROTOCOL_TEST$i.log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER $WIRE_PROTOCOL_TEST $i $real
  done

  # parse the rate out of: "I1009 15:00:30.023576 27043 rpc-bench.cc:108] Reqs/sec:         84404.4"
  for i in $(seq 1 $NUM_SAMPLES); do
    rate=`grep Reqs $LOGDIR/$RPC_BENCH_TEST$i.log | cut -d ":" -f 5 | tr -d ' '`
    record_result $BUILD_IDENTIFIER $RPC_BENCH_TEST $i $rate
  done

  # parse latency numbers from single-threaded tserver benchmark
  for i in $(seq 1 $NUM_SAMPLES); do
    for metric in min mean percentile_95 percentile_99 percentile_99_9 ; do
      val=$(grep "\"$metric\": " $LOGDIR/${TS_INSERT_LATENCY}$i.log | awk '{print $2}' | sed -e 's/,//')
      record_result $BUILD_IDENTIFIER ${TS_INSERT_LATENCY}_$metric $i $val
    done
  done

  # parse latency and throughput numbers from multi-threaded tserver benchmark
  for i in $(seq 1 $NUM_SAMPLES); do
    local log=$LOGDIR/${TS_8THREAD_BENCH}$i.log
    for metric in min mean percentile_95 percentile_99 percentile_99_9 ; do
      val=$(grep "\"$metric\": " $log | awk '{print $2}' | sed -e 's/,//')
      record_result $BUILD_IDENTIFIER ${TS_8THREAD_BENCH}_${metric}_latency $i $val
    done
    rate=$(grep -o 'Throughput.*' $log | awk '{print $2}')
    record_result $BUILD_IDENTIFIER ${TS_8THREAD_BENCH}_throughput_wall $i $rate
    rate=$(grep -o 'CPU efficiency.*' $log | awk '{print $3}')
    record_result $BUILD_IDENTIFIER ${TS_8THREAD_BENCH}_throughput_cpu $i $rate
  done

  # parse scan timings for scans and inserts with MRS only
  for i in $(seq 1 $NUM_SAMPLES); do
    local log=$LOGDIR/${FS_SCANINSERT_MRS}$i.log
    insert=`grep "Time spent concurrent inserts" $log | ./parse_real_out.sh`
    scan_full=`grep "Time spent full schema scan" $log | ./parse_real_out.sh`
    scan_str=`grep "Time spent String projection" $log | ./parse_real_out.sh`
    scan_int32=`grep "Time spent Int32 projection" $log | ./parse_real_out.sh`
    scan_int64=`grep "Time spent Int64 projection" $log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_MRS}_insert $i $insert
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_MRS}_scan_full $i $scan_full
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_MRS}_scan_str $i $scan_str
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_MRS}_scan_int32 $i $scan_int32
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_MRS}_scan_int64 $i $scan_int64
  done

  # parse scan timings for scans and inserts with disk
  for i in $(seq 1 $NUM_SAMPLES); do
    local log=$LOGDIR/${FS_SCANINSERT_DISK}$i.log
    insert=`grep "Time spent concurrent inserts" $log | ./parse_real_out.sh`
    scan_full=`grep "Time spent full schema scan" $log | ./parse_real_out.sh`
    scan_str=`grep "Time spent String projection" $log | ./parse_real_out.sh`
    scan_int32=`grep "Time spent Int32 projection" $log | ./parse_real_out.sh`
    scan_int64=`grep "Time spent Int64 projection" $log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_DISK}_insert $i $insert
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_DISK}_scan_full $i $scan_full
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_DISK}_scan_str $i $scan_str
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_DISK}_scan_int32 $i $scan_int32
    record_result $BUILD_IDENTIFIER ${FS_SCANINSERT_DISK}_scan_int64 $i $scan_int64
  done

  # Parse timings and thread count
  for i in $(seq 1 $NUM_SAMPLES); do
    local log=$LOGDIR/${DENSE_NODE_ITEST}$i.log
    num_threads=`grep threads_running $log | cut -d ":" -f 5 | tr -d ' '`
    num_blocks=`grep log_block_manager_blocks_under_management $log | cut -d ":" -f 5 | tr -d ' '`
    num_bytes=`grep log_block_manager_bytes_under_management $log | cut -d ":" -f 5 | tr -d ' '`
    num_containers=`grep log_block_manager_containers $log | cut -d ":" -f 5 | tr -d ' '`
    num_full_containers=`grep log_block_manager_full_containers $log | cut -d ":" -f 5 | tr -d ' '`
    time_restarting_tserver=`grep "Time spent restarting tserver" $log | ./parse_real_out.sh`
    time_bootstrapping_tablets=`grep "Time spent bootstrapping tablets" $log | ./parse_real_out.sh`
    record_result $BUILD_IDENTIFIER ${DENSE_NODE_ITEST}_num_threads $i $num_threads
    record_result $BUILD_IDENTIFIER ${DENSE_NODE_ITEST}_num_blocks $i $num_blocks
    record_result $BUILD_IDENTIFIER ${DENSE_NODE_ITEST}_num_bytes $i $num_bytes
    record_result $BUILD_IDENTIFIER ${DENSE_NODE_ITEST}_num_containers $i $num_containers
    record_result $BUILD_IDENTIFIER ${DENSE_NODE_ITEST}_num_full_containers $i $num_full_containers
    record_result $BUILD_IDENTIFIER ${DENSE_NODE_ITEST}_time_restarting_tserver $i $time_restarting_tserver
    record_result $BUILD_IDENTIFIER ${DENSE_NODE_ITEST}_time_bootstrapping_tablets $i $time_bootstrapping_tablets
  done

  popd
  popd
  popd
}

generate_ycsb_plots() {
  local WORKLOAD=$1
  local PHASE=$2
  METRIC_NAME=ycsb-$PHASE-$WORKLOAD

  # first plot the overall stats for that phase
  OVERALL_FILENAME=$METRIC_NAME-OVERALL
  load_stats $OVERALL_FILENAME-runtime_ms > $OUTDIR/$OVERALL_FILENAME-runtime_ms.tsv
  write_img_plot $OUTDIR/$OVERALL_FILENAME-runtime_ms.tsv $OVERALL_FILENAME-runtime_ms
  load_stats $OVERALL_FILENAME-throughput_ops_sec > $OUTDIR/$OVERALL_FILENAME-throughput_ops_sec.tsv
  write_img_plot $OUTDIR/$OVERALL_FILENAME-throughput_ops_sec.tsv $OVERALL_FILENAME-throughput_ops_sec

  # now plot the individual operations
  OPS="INSERT UPDATE READ"

  for op in $OPS; do
    OP_FILENAME=$METRIC_NAME-$op
    load_stats $OP_FILENAME-average_latency_us > $OUTDIR/$OP_FILENAME-average_latency_us.tsv
    write_img_plot $OUTDIR/$OP_FILENAME-average_latency_us.tsv $OP_FILENAME-average_latency_us

    load_stats $OP_FILENAME-95th_latency_ms > $OUTDIR/$OP_FILENAME-95th_latency_ms.tsv
    write_img_plot $OUTDIR/$OP_FILENAME-95th_latency_ms.tsv $OP_FILENAME-95th_latency_ms

    load_stats $OP_FILENAME-99th_latency_ms > $OUTDIR/$OP_FILENAME-99th_latency_ms.tsv
    write_img_plot $OUTDIR/$OP_FILENAME-99th_latency_ms.tsv $OP_FILENAME-99th_latency_ms
  done
}

load_and_generate_plot() {
  local TEST_NAME=$1
  local PLOT_NAME=$2
  load_stats "$TEST_NAME" > $OUTDIR/$PLOT_NAME.tsv
  write_img_plot $OUTDIR/$PLOT_NAME.tsv $PLOT_NAME
}

load_stats_and_generate_plots() {
  pushd src
  pushd kudu
  pushd scripts

  load_and_generate_plot "%MultiThreadedTabletTest%" mt-tablet-test-runtime

  load_and_generate_plot ConcurrentBTreeScanInsert cb-tree-insert
  load_and_generate_plot ConcurrentBTreeScanNotFrozen cb-ctree-not-frozen
  load_and_generate_plot ConcurrentBTreeScanFrozen cb-ctree-frozen

  load_and_generate_plot "${COMPACT_MERGE_BENCH}%" compact-merge-bench

  load_and_generate_plot "${MEMROWSET_BENCH}${INSERT}" memrowset-bench-insert
  load_and_generate_plot "${MEMROWSET_BENCH}Scan%" memrowset-bench-scan

  load_and_generate_plot $BLOOM_TEST bloom-test
  load_and_generate_plot $MT_BLOOM_TEST mt-bloom-test

  load_and_generate_plot $WIRE_PROTOCOL_TEST wire-protocol-test

  load_and_generate_plot $RPC_BENCH_TEST rpc-bench-test

  load_and_generate_plot "${TS_INSERT_LATENCY}%" ts-insert-latency

  load_and_generate_plot "${TS_8THREAD_BENCH}%_latency" ts-8thread-insert-latency
  load_and_generate_plot "${TS_8THREAD_BENCH}%_throughput_%" ts-8thread-insert-throughput

  load_and_generate_plot "${FS_SCANINSERT_MRS}%_insert" fs-mrsonly-insert
  load_and_generate_plot "${FS_SCANINSERT_MRS}%_scan%" fs-mrsonly-scan
  load_and_generate_plot "${FS_SCANINSERT_DISK}%_insert" fs-withdisk-insert
  load_and_generate_plot "${FS_SCANINSERT_DISK}%_scan%" fs-withdisk-scan

  load_and_generate_plot "${DENSE_NODE_ITEST}_time%" dense-node-bench-times
  load_and_generate_plot "${DENSE_NODE_ITEST}_num%containers%" dense-node-bench-containers
  load_and_generate_plot "${DENSE_NODE_ITEST}_num_blocks%" dense-node-bench-blocks
  load_and_generate_plot "${DENSE_NODE_ITEST}_num_threads%" dense-node-bench-threads
  load_and_generate_plot "${DENSE_NODE_ITEST}_num_bytes%" dense-node-bench-bytes

  # Generate all the pngs for all the mt-tablet tests
  for i in $(seq 0 $NUM_MT_TABLET_TESTS); do
    cat $LOGDIR/${MT_TABLET_TEST}.log | ./graph-metrics.py MultiThreadedTabletTest/$i > $OUTDIR/test$i.tsv
    # Don't bail on failure (why not?)
    write_mttablet_img_plots $OUTDIR/test$i.tsv test$i || true
  done

  if [ "${BENCHMARK_MODE}" = "${MODE_JENKINS}" ]; then
    ################################################################
    # Plot the separately-recorded TPCH and YCSB graphs as well
    # (only for Jenkins)
    ################################################################

    # TPC-H 1 runs separately, let's just get those graphs
    load_and_generate_plot query_1_1gb tpch1-query
    load_and_generate_plot insert_1gb tpch1-insert

    # YCSB which runs the 5nodes_workload on a cluster
    # First we process the loading phase
    generate_ycsb_plots 5nodes_workload load

    # Then the running phase
    generate_ycsb_plots 5nodes_workload run
  fi

  # Move all the pngs to OUT_DIR.
  mv *.png $OUTDIR/

  # Generate an HTML file aggregating the PNGs.
  # Mostly for local usage, but somewhat useful to check the Jenkins runs too.
  pushd $OUTDIR/
  PNGS=$(ls *.png)
  echo -n > "$OUTDIR/$HTML_FILE"
  echo "<title>Kudu Benchmarks</title>" >> "$OUTDIR/$HTML_FILE"
  echo "<h1 align=center>Kudu Benchmarks</h1>" >> "$OUTDIR/$HTML_FILE"
  for png in $PNGS; do
    echo "<img src=$png><br>" >> "$OUTDIR/$HTML_FILE"
  done
  popd

  popd
  popd
  popd
}

build_run_record() {
  local BUILD_IDENTIFIER=$1
  build_kudu
  run_benchmarks
  parse_and_record_all_results "$BUILD_IDENTIFIER"
}

git_checkout() {
  local GIT_HASH=$1
  git checkout $GIT_HASH
}

run() {

  # Parse command-line options.
  if [ -n "$1" ]; then
    [ "$1" = "-local" ] || usage_and_die
    shift

    BENCHMARK_MODE=$MODE_LOCAL

    # If no hashes are provided, run against the current HEAD.
    if [ -z "$1" ]; then
      build_run_record "working_tree"
    else
      # Convert the passed-in git refs into their hashes.
      # This allows you to use "HEAD~3 HEAD" as arguments
      # and end up with those being evaluated with regard to
      # the _current_ branch, instead of evaluating the second
      # "HEAD" after checking out the first.
      local ref
      local hashes
      for ref in "$@" ; do
        hashes="$hashes $(git rev-parse "$ref")"
      done
      set $hashes
      while [ -n "$1" ]; do
        local GIT_HASH="$1"
        shift
        git_checkout "$GIT_HASH"
        build_run_record "$GIT_HASH"
      done
    fi

  else
    [ -n "$BUILD_NUMBER" ] || usage_and_die
    build_run_record "$BUILD_NUMBER"
  fi

  # The last step is the same for both modes.
  load_stats_and_generate_plots
}

################################################################
# main
################################################################

# Figure out where we are, store in global variables.
LOGDIR="$BASE_DIR/$LOG_DIR_NAME"
OUTDIR="$BASE_DIR/$OUT_DIR_NAME"

# Ensure we are in KUDU_HOME
if [ ! -f "$BASE_DIR/LICENSE.txt" ]; then
  set +x
  echo "Error: must run from top of Kudu source tree"
  usage_and_die
fi

# Set up environment.
ulimit -m $[3000*1000]
ulimit -c unlimited   # gather core dumps

# Set CPU governor, and restore it on exit.
old_governor=$(ensure_cpu_scaling performance)
restore_governor() {
  ensure_cpu_scaling $old_governor >/dev/null
}
trap restore_governor EXIT

# Kick off the benchmark script.
run $*

exit 0


#!/bin/bash -e
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Ensure that the CPU governor is set to a particular governor, outputting
# the prior governor on stdout.
#
# Without this, some of our tests end up having higher variance due to
# changing CPU speed during the test.
#
# Assumes that all CPUs are set to the same governor.
target_governor=$1
old_governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
for cpu_dir in /sys/devices/system/cpu/cpu[0-9]*/ ; do
    governor_file=$cpu_dir/cpufreq/scaling_governor
    governor=$(cat $governor_file)
    if [ "$governor" != "$target_governor" ]; then
        >&2 echo "CPU $cpu_dir not in '$target_governor' mode. Attempting to change"
        echo $target_governor | sudo tee $governor_file > /dev/null
        if [ $? -ne 0 ]; then
            >&2 echo Could not set $target_governor governor!
            >&2 echo Perhaps you need passwordless sudo for this user
            exit 1
        fi
    fi
done
echo $old_governor


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This script is invoked from a Jenkins job to build Kudu C++ components,
# Kudu Java client, kudu-jepsen, and run kudu-jepsen consistency tests
# against the freshly built Kudu binaries.
#
# The following environment variables may be used to customize the operation:
#   JAVA8_HOME          Default: /usr/lib/jvm/java-8-openjdk-amd64
#     Path to the JDK8 installation root.  Expecting to find 'java'
#     in the 'bin' sub-directory.  Java 8 or higher is required for the
#     maven-clojure-plugin.
#
#   M2_HOME             Default: /usr/share/maven
#     Path to the maven (3.3.6 and newer is required).
#
#   MVN_FLAGS           Default: ""
#     Extra flags which are passed to 'mvn' when building and running Java
#     tests. This can be useful, for example, to choose a different maven
#     repository location.
#
#   KUDU_MASTER_NODES   Default: ""
#     Set of master nodes for the Kudu cluster to run the Jepsen consistency
#     test. Those should be the same architecture and OS type as the
#     Jenkins machine where this build script is run.
#     NOTE: currently, the Jepsen test can run only at Debian nodes.
#
#   KUDU_TSERVER_NODES  Default: ""
#     Set of tablet server nodes for the Kudu cluster to run the Jepsen
#     consistency test. Those should be the same architecture and OS type
#     as the Jenkins machine where this build script is run.
#     NOTE: currently, the Jepsen test can run only at Debian nodes.
#
#   SSH_KEY             Default: ""
#     The absolute path to the file containing the private SSH key
#     for the root user at the master and tablet server nodes
#     (DB nodes in Jepsen terminology).
#     NOTE: the key should be in plain (i.e. non-encrypted) format.
#

#############################################################################
# Constants
#############################################################################

# Require Kudu binaries to be statically linked with the Kudu-specific libs:
# the kudu-jepsen does not distribute Kudu libraries.
KUDU_LINK=static

SRC_ROOT=$(readlink -f $(dirname $0)/../../..)
THIRDPARTY_BIN=$SRC_ROOT/thirdparty/installed/common/bin
NUM_PROCS=$(getconf _NPROCESSORS_ONLN)

#############################################################################
# Customizable parameters
#############################################################################

BUILD_TYPE=${BUILD_TYPE:-debug}
JAVA8_HOME=${JAVA8_HOME:-/usr/lib/jvm/java-8-openjdk-amd64}
M2_HOME=${M2_HOME:-/usr/share/maven}
MVN_FLAGS=${MVN_FLAGS:-}
KUDU_MASTER_NODES=${KUDU_MASTER_NODES:-}
KUDU_TSERVER_NODES=${KUDU_TSERVER_NODES:-}
SSH_KEY=${SSH_KEY:-}
ITER_NUM=${ITER_NUM:-1}

#############################################################################
# Main
#############################################################################

set -e
set -o pipefail
ulimit -c unlimited

cd $SRC_ROOT

echo
echo "Removing logs from prior runs"
echo "--------------------------------------------------------------------"
rm -rf java/kudu-jepsen/store

echo
echo "Building third-party components"
echo "--------------------------------------------------------------------"
$SRC_ROOT/build-support/enable_devtoolset.sh \
  $SRC_ROOT/thirdparty/build-if-necessary.sh

mkdir -p build/$BUILD_TYPE
pushd build
ln -sf $SRC_ROOT/build/$BUILD_TYPE latest
pushd $BUILD_TYPE

echo
echo "Building Kudu C++ components (no tests)"
echo "--------------------------------------------------------------------"
$SRC_ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_BIN/cmake \
  -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
  -DKUDU_LINK=$KUDU_LINK \
  -DNO_TESTS=1 \
  ../..
make -j$NUM_PROCS 2>&1 | tee build.log

export JAVA_HOME=$JAVA8_HOME
export PATH=$JAVA_HOME/bin:$M2_HOME/bin:$PATH
set -x

pushd $SRC_ROOT/java
echo
echo "Building Kudu Java packages and installing into local mvn repository"
echo "--------------------------------------------------------------------"
mvn $MVN_FLAGS -Pjepsen -DskipTests clean install

echo
echo "Building and running kudu-jepsen consistency tests"
echo "--------------------------------------------------------------------"
pushd kudu-jepsen
mvn $MVN_FLAGS clojure:run \
  -DmasterNodes="$KUDU_MASTER_NODES" \
  -DtserverNodes="$KUDU_TSERVER_NODES" \
  -DsshKeyPath="$SSH_KEY" \
  -DiterNum="$ITER_NUM"
popd
popd

set +x


#! /usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# This script takes a line in stdin and parses the real time out of it.
# Input example:
#  "Times for Insert 10000000 keys: real 16.438s	user 16.164s	sys 0.229s"
# Output:
#  16.438

awk -F 'real ' '{ print $2 }' | awk -F s '{ print $1 }'


#!/bin/bash -xe
########################################################################
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Run tpch benchmark and write results to the DB.
#
# Expects to find the following Jenkins environment variables set:
# - JOB_NAME
# - BUILD_NUMBER
# If these are not set, the script will still run but will not record
# the results in the MySQL database. Instead, it will output results
# into .tsv files in the kudu source root directory. This is useful for
# running this benchmark locally for testing / dev purposes.
#
# Optional environment variables to override (defaults set for Jenkins):
# - LINEITEM_TBL_PATH: Path to lineitem.tbl from the TPC-H suite.
# - KUDU_DATA_DIR: Directory to use for data storage.
# - TPCH_NUM_QUERY_ITERS: Number of TPC-H query iterations to run.
#########################################################################

##########################################################
# Constants
##########################################################
ROOT=$(readlink -f $(dirname $0)/../../..)

##########################################################
# Overridable params
##########################################################
LINEITEM_TBL_PATH=${LINEITEM_TBL_PATH:-/home/jdcryans/lineitem.tbl}
KUDU_DATA_DIR=${KUDU_DATA_DIR:-/data/2/tmp/kudutpch1-jenkins}
TPCH_NUM_QUERY_ITERS=${TPCH_NUM_QUERY_ITERS:-5}

##########################################################
# Functions
##########################################################
record_result() {
  local RECORD_STATS_SCRIPT=$ROOT/src/kudu/scripts/write-jobs-stats-to-mysql.py
  local TEST_NAME=$1
  local ITER=$2
  local VALUE=$3
  if [ -n "$JOB_NAME" ]; then
    # Jenkins.
    python $RECORD_STATS_SCRIPT $JOB_NAME $BUILD_NUMBER $TEST_NAME $ITER $VALUE
  else
    # Running locally.
    local STATS_FILE="$OUTDIR/tpch-$TEST_NAME.tsv"
    echo -e "${TEST_NAME}\t${ITER}\t${VALUE}" >> "$STATS_FILE"
  fi
}

ensure_cpu_scaling() {
  $ROOT/src/kudu/scripts/ensure_cpu_scaling.sh "$@"
}

##########################################################
# Main
##########################################################
if [ $TPCH_NUM_QUERY_ITERS -lt 2 ]; then
  echo "Error: TPCH_NUM_QUERY_ITERS must be 2 or greater"
  exit 1
fi

cd $ROOT

# Set up environment
set -o pipefail
ulimit -m $[3000*1000]
ulimit -c unlimited # gather core dumps

# Set CPU governor, and restore it on exit.
old_governor=$(ensure_cpu_scaling performance)
restore_governor() {
  ensure_cpu_scaling $old_governor >/dev/null
}
trap restore_governor EXIT

# PATH=<toolchain_stuff>:$PATH
export TOOLCHAIN=/mnt/toolchain/toolchain.sh
if [ -f "$TOOLCHAIN" ]; then
  source $TOOLCHAIN
fi

# Build thirdparty
$ROOT/build-support/enable_devtoolset.sh $ROOT/thirdparty/build-if-necessary.sh

# PATH=<thirdparty_stuff>:<toolchain_stuff>:$PATH
THIRDPARTY_BIN=$(pwd)/thirdparty/installed/common/bin
export PPROF_PATH=$THIRDPARTY_BIN/pprof

BUILD_TYPE=release

# Build Kudu
mkdir -p build/$BUILD_TYPE
pushd build/$BUILD_TYPE
rm -rf CMakeCache CMakeFiles/

$ROOT/build-support/enable_devtoolset.sh $THIRDPARTY_BIN/cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ../..

NUM_PROCS=$(cat /proc/cpuinfo | grep processor | wc -l)
make -j${NUM_PROCS} tpch1 2>&1 | tee build.log
popd

# Warming up the OS buffer.
cat $LINEITEM_TBL_PATH > /dev/null
cat $LINEITEM_TBL_PATH > /dev/null

OUTDIR=$ROOT/build/$BUILD_TYPE/tpch
rm -Rf $KUDU_DATA_DIR   # Clean up data dir.
mkdir -p $OUTDIR        # Create log file output dir.

$ROOT/build/$BUILD_TYPE/bin/tpch1 -logtostderr=1 \
                                  -tpch_path_to_data=$LINEITEM_TBL_PATH \
                                  -mini_cluster_base_dir=$KUDU_DATA_DIR \
                                  -tpch_num_query_iterations=$TPCH_NUM_QUERY_ITERS \
                                  $* \
                                  >$OUTDIR/benchmark.log 2>&1

cat $OUTDIR/benchmark.log
INSERT_TIME=$(grep "Time spent loading" $OUTDIR/benchmark.log | \
    perl -pe 's/.*Time spent loading: real ([0-9\.]+)s.*/\1/')
record_result insert_1gb 1 $INSERT_TIME

# We do not record the first iteration (#0) because we want to record the
# in-cache performance.
for iter in $(seq 1 $(expr $TPCH_NUM_QUERY_ITERS - 1)); do
  QUERY_TIME=$(grep "iteration # $iter" $OUTDIR/benchmark.log | \
      perl -pe "s/.*iteration # $iter: real ([0-9\.]+)s.*/\1/")
  record_result query_1_1gb $iter $QUERY_TIME
done


#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# A script that prints the first argument and ignores all the others.
echo "$1"


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# build-definitions.sh provides functions to build thirdparty dependencies.
# These functions do not take positional arguments, but individual builds may
# be influenced by setting environment variables:
#
# * PREFIX - the install destination directory.
# * MODE_SUFFIX - an optional suffix to add to each build directory's name.
# * EXTRA_CFLAGS - additional flags to pass to the C compiler.
# * EXTRA_CXXFLAGS - additional flags to pass to the C++ compiler.
# * EXTRA_LDFLAGS - additional flags to pass to the linker.
# * EXTRA_LIBS - additional libraries to link.
# * NINJA - if set, uses this instead of 'make' for cmake-configured libraries.
#           note that 'EXTRA_CMAKE_FLAGS" should be set to include -GNinja as
#           well to generate the proper build files.
# * EXTRA_CMAKE_FLAGS - extra flags to pass to cmake
#
# build-definitions.sh is meant to be sourced from build-thirdparty.sh, and
# relies on environment variables defined there and in vars.sh.

# Save the current build environment.
save_env() {
  _PREFIX=${PREFIX}
  _MODE_SUFFIX=${MODE_SUFFIX}
  _EXTRA_CFLAGS=${EXTRA_CFLAGS}
  _EXTRA_CXXFLAGS=${EXTRA_CXXFLAGS}
  _EXTRA_LDFLAGS=${EXTRA_LDFLAGS}
  _EXTRA_LIBS=${EXTRA_LIBS}
}

# Restore the most recently saved build environment.
restore_env() {
  PREFIX=${_PREFIX}
  MODE_SUFFIX=${_MODE_SUFFIX}
  EXTRA_CFLAGS=${_EXTRA_CFLAGS}
  EXTRA_CXXFLAGS=${_EXTRA_CXXFLAGS}
  EXTRA_LDFLAGS=${_EXTRA_LDFLAGS}
  EXTRA_LIBS=${_EXTRA_LIBS}
}

# Some versions of libtool are vulnerable to a bug[1] wherein clang's -stdlib
# parameter isn't passed through to the link command line. This causes the
# libtool to link the shared object against libstdc++ instead of libc++.
#
# The bug was only fixed in version 2.4.2 of libtool and el6 carries version
# 2.2.6, so we work around it here.
#
# 1. https://debbugs.gnu.org/db/10/10579.html
fixup_libtool() {
  if [[ ! "$EXTRA_LDFLAGS" =~ "-stdlib=libc++" ]]; then
    echo "libtool does not need to be fixed up: not using libc++"
    return
  fi
  if [ ! -f libtool ]; then
    echo "libtool not found"
    exit 1
  fi
  if ! grep -q -e 'postdeps=.*-lstdc++' libtool; then
    echo "libtool does not need to be fixed up: already configured for libc++"
    return
  fi

  # Modify a line like:
  #
  #   postdeps="-lfoo -lstdc++ -lbar -lstdc++ -lbaz"
  #
  # To become:
  #
  #   postdeps="-lfoo -lc++ -lbar -lc++ -lbaz"
  sed -i.before_fixup -e '/postdeps=/s/-lstdc++/-lc++/g' libtool
  echo "libtool has been fixed up"
}

build_cmake() {
  CMAKE_BDIR=$TP_BUILD_DIR/$CMAKE_NAME$MODE_SUFFIX
  mkdir -p $CMAKE_BDIR
  pushd $CMAKE_BDIR
  $CMAKE_SOURCE/bootstrap \
    --prefix=$PREFIX \
    --parallel=$PARALLEL
  # Unfortunately, cmake's bootstrap always uses Makefiles
  # and can't be configured to build with ninja.
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_libcxxabi() {
  LIBCXXABI_BDIR=$TP_BUILD_DIR/llvm-$LLVM_VERSION.libcxxabi$MODE_SUFFIX
  mkdir -p $LIBCXXABI_BDIR
  pushd $LIBCXXABI_BDIR
  rm -Rf CMakeCache.txt CMakeFiles/
  cmake \
    -DCMAKE_BUILD_TYPE=Release \
    -DCMAKE_INSTALL_PREFIX=$PREFIX \
    -DCMAKE_CXX_FLAGS="$EXTRA_CXXFLAGS $EXTRA_LDFLAGS" \
    -DLLVM_PATH=$LLVM_SOURCE \
    $EXTRA_CMAKE_FLAGS \
    $LLVM_SOURCE/projects/libcxxabi
  ${NINJA:-make} -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_libcxx() {
  local BUILD_TYPE=$1
  case $BUILD_TYPE in
    "tsan")
      SANITIZER_TYPE=Thread
      ;;
    *)
      echo "Unknown build type: $BUILD_TYPE"
      exit 1
      ;;
  esac

  LIBCXX_BDIR=$TP_BUILD_DIR/llvm-$LLVM_VERSION.libcxx$MODE_SUFFIX
  mkdir -p $LIBCXX_BDIR
  pushd $LIBCXX_BDIR
  rm -Rf CMakeCache.txt CMakeFiles/
  cmake \
    -DCMAKE_BUILD_TYPE=Release \
    -DCMAKE_INSTALL_PREFIX=$PREFIX \
    -DCMAKE_CXX_FLAGS="$EXTRA_CXXFLAGS" \
    -DCMAKE_EXE_LINKER_FLAGS="$EXTRA_LDFLAGS" \
    -DCMAKE_MODULE_LINKER_FLAGS="$EXTRA_LDFLAGS" \
    -DCMAKE_SHARED_LINKER_FLAGS="$EXTRA_LDFLAGS" \
    -DLLVM_PATH=$LLVM_SOURCE \
    -DLIBCXX_CXX_ABI=libcxxabi \
    -DLIBCXX_CXX_ABI_INCLUDE_PATHS=$LLVM_SOURCE/projects/libcxxabi/include \
    -DLIBCXX_CXX_ABI_LIBRARY_PATH=$PREFIX/lib \
    -DLLVM_USE_SANITIZER=$SANITIZER_TYPE \
    $EXTRA_CMAKE_FLAGS \
    $LLVM_SOURCE/projects/libcxx
  ${NINJA:-make} -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_or_find_python() {
  if [ -n "$PYTHON_EXECUTABLE" ]; then
    return
  fi

  # Build Python only if necessary.
  if [[ $(python2.7 -V 2>&1) =~ "Python 2.7." ]]; then
    PYTHON_EXECUTABLE=$(which python2.7)
  elif [[ $(python -V 2>&1) =~ "Python 2.7." ]]; then
    PYTHON_EXECUTABLE=$(which python)
  else
    PYTHON_BDIR=$TP_BUILD_DIR/$PYTHON_NAME$MODE_SUFFIX
    mkdir -p $PYTHON_BDIR
    pushd $PYTHON_BDIR
    $PYTHON_SOURCE/configure
    make -j$PARALLEL $EXTRA_MAKEFLAGS
    PYTHON_EXECUTABLE="$PYTHON_BDIR/python"
    popd
  fi
}

build_llvm() {
  local TOOLS_ARGS=
  local BUILD_TYPE=$1

  build_or_find_python

  # Always disabled; these subprojects are built standalone.
  TOOLS_ARGS="$TOOLS_ARGS -DLLVM_TOOL_LIBCXX_BUILD=OFF"
  TOOLS_ARGS="$TOOLS_ARGS -DLLVM_TOOL_LIBCXXABI_BUILD=OFF"

  # Disable some builds we don't care about.
  for arg in \
      CLANG_ENABLE_ARCMT \
      CLANG_TOOL_ARCMT_TEST_BUILD \
      CLANG_TOOL_C_ARCMT_TEST_BUILD \
      CLANG_TOOL_C_INDEX_TEST_BUILD \
      CLANG_TOOL_CLANG_CHECK_BUILD \
      CLANG_TOOL_CLANG_DIFF_BUILD \
      CLANG_TOOL_CLANG_FORMAT_VS_BUILD \
      CLANG_TOOL_CLANG_FUNC_MAPPING_BUILD \
      CLANG_TOOL_CLANG_FUZZER_BUILD \
      CLANG_TOOL_CLANG_IMPORT_TEST_BUILD \
      CLANG_TOOL_CLANG_OFFLOAD_BUNDLER_BUILD \
      CLANG_TOOL_CLANG_REFACTOR_BUILD \
      CLANG_TOOL_CLANG_RENAME_BUILD \
      CLANG_TOOL_DIAGTOOL_BUILD \
      COMPILER_RT_BUILD_LIBFUZZER \
      LLVM_INCLUDE_GO_TESTS \
      LLVM_POLLY_BUILD \
      LLVM_TOOL_BUGPOINT_BUILD \
      LLVM_TOOL_BUGPOINT_PASSES_BUILD \
      LLVM_TOOL_DSYMUTIL_BUILD \
      LLVM_TOOL_LLI_BUILD \
      LLVM_TOOL_LLVM_AS_FUZZER_BUILD \
      LLVM_TOOL_LLVM_BCANALYZER_BUILD \
      LLVM_TOOL_LLVM_CAT_BUILD \
      LLVM_TOOL_LLVM_CFI_VERIFY_BUILD \
      LLVM_TOOL_LLVM_C_TEST_BUILD \
      LLVM_TOOL_LLVM_CVTRES_BUILD \
      LLVM_TOOL_LLVM_CXXDUMP_BUILD \
      LLVM_TOOL_LLVM_CXXFILT_BUILD \
      LLVM_TOOL_LLVM_DEMANGLE_FUZZER_BUILD \
      LLVM_TOOL_LLVM_DIFF_BUILD \
      LLVM_TOOL_LLVM_DIS_BUILD \
      LLVM_TOOL_LLVM_DWARFDUMP_BUILD \
      LLVM_TOOL_LLVM_DWP_BUILD \
      LLVM_TOOL_LLVM_EXTRACT_BUILD \
      LLVM_TOOL_LLVM_GO_BUILD \
      LLVM_TOOL_LLVM_ISEL_FUZZER_BUILD \
      LLVM_TOOL_LLVM_JITLISTENER_BUILD \
      LLVM_TOOL_LLVM_MC_ASSEMBLE_FUZZER_BUILD \
      LLVM_TOOL_LLVM_MC_BUILD \
      LLVM_TOOL_LLVM_MC_DISASSEMBLE_FUZZER_BUILD \
      LLVM_TOOL_LLVM_MCMARKUP_BUILD \
      LLVM_TOOL_LLVM_MODEXTRACT_BUILD \
      LLVM_TOOL_LLVM_MT_BUILD \
      LLVM_TOOL_LLVM_NM_BUILD \
      LLVM_TOOL_LLVM_OBJCOPY_BUILD \
      LLVM_TOOL_LLVM_OBJDUMP_BUILD \
      LLVM_TOOL_LLVM_OPT_FUZZER_BUILD \
      LLVM_TOOL_LLVM_OPT_REPORT_BUILD \
      LLVM_TOOL_LLVM_PDBUTIL_BUILD \
      LLVM_TOOL_LLVM_PROFDATA_BUILD \
      LLVM_TOOL_LLVM_RC_BUILD \
      LLVM_TOOL_LLVM_READOBJ_BUILD \
      LLVM_TOOL_LLVM_RTDYLD_BUILD \
      LLVM_TOOL_LLVM_SHLIB_BUILD \
      LLVM_TOOL_LLVM_SIZE_BUILD \
      LLVM_TOOL_LLVM_SPECIAL_CASE_LIST_FUZZER_BUILD \
      LLVM_TOOL_LLVM_SPLIT_BUILD \
      LLVM_TOOL_LLVM_STRESS_BUILD \
      LLVM_TOOL_LLVM_STRINGS_BUILD \
      LLVM_TOOL_MSBUILD_BUILD \
      LLVM_TOOL_OBJ2YAML_BUILD \
      LLVM_TOOL_OPT_BUILD \
      LLVM_TOOL_OPT_VIEWER_BUILD \
      LLVM_TOOL_VERIFY_USELISTORDER_BUILD \
      LLVM_TOOL_XCODE_TOOLCHAIN_BUILD \
      LLVM_TOOL_YAML2OBJ_BUILD \
      ; do
    TOOLS_ARGS="$TOOLS_ARGS -D${arg}=OFF"
  done

  CLANG_CXXFLAGS="$EXTRA_CXXFLAGS"
  CLANG_LDFLAGS="$EXTRA_LDFLAGS"
  case $BUILD_TYPE in
    "normal")
      # Default build: core LLVM libraries, clang, compiler-rt, and all tools.

      # Set the gcc prefix based on the gcc that's on the path. Otherwise,
      # clang will auto-detect the newest devtoolset that is installed even
      # if it's not the current one on the path. This results in link errors
      # like:
      #   /opt/rh/devtoolset-3/root/usr/bin/ld:
      #     /opt/rh/devtoolset-6/root/usr/lib/gcc/x86_64-redhat-linux/6.2.1/crtbeginS.o:
      #     unrecognized relocation (0x2a) in section `.text'
      # ...as it tries to use the 'ld' from devtoolset-3 (on the path) to link libraries
      # coming from devtoolset-6.
      GCC_INSTALL_PREFIX=$(gcc -v 2>&1 | grep -o -- '--prefix=[^ ]*' | cut -f2 -d=)
      if [ -n "$GCC_INSTALL_PREFIX" ]; then
        TOOLS_ARGS="$TOOLS_ARGS -DGCC_INSTALL_PREFIX=$GCC_INSTALL_PREFIX"
      fi

      # Depend on zlib from the thirdparty tree. It's an optional dependency for
      # LLVM, but a required [1] one for IWYU. When TSAN is enabled these flags
      # are already set by build-thirdparty.sh in order to support the
      # thirdparty libc++, so it's not necessary to set them again.
      #
      # 1. https://github.com/include-what-you-use/include-what-you-use/issues/539
      CLANG_CXXFLAGS="$CLANG_CXXFLAGS -I$PREFIX/include"
      CLANG_LDFLAGS="$CLANG_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib"
      ;;
    "tsan")
      # Build just the core LLVM libraries, dependent on libc++.
      TOOLS_ARGS="$TOOLS_ARGS -DLLVM_ENABLE_LIBCXX=ON"
      TOOLS_ARGS="$TOOLS_ARGS -DLLVM_INCLUDE_TOOLS=OFF"
      TOOLS_ARGS="$TOOLS_ARGS -DLLVM_TOOL_COMPILER_RT_BUILD=OFF"

      # Configure for TSAN.
      TOOLS_ARGS="$TOOLS_ARGS -DLLVM_USE_SANITIZER=Thread"
      # Use the 'tblgen' from the non-TSAN build when building the TSAN
      # build, since it runs much faster.
      TOOLS_ARGS="$TOOLS_ARGS -DLLVM_TABLEGEN=$PREFIX_DEPS/bin/llvm-tblgen"
      ;;
    *)
      echo "Unknown LLVM build type: $BUILD_TYPE"
      exit 1
      ;;
  esac

  LLVM_BDIR=$TP_BUILD_DIR/llvm-$LLVM_VERSION$MODE_SUFFIX
  mkdir -p $LLVM_BDIR
  pushd $LLVM_BDIR

  # Rebuild the CMake cache every time.
  rm -Rf CMakeCache.txt CMakeFiles/

  # The LLVM build can fail if a different version is already installed
  # in the install prefix. It will try to link against that version instead
  # of the one being built.
  rm -Rf $PREFIX/include/{llvm*,clang*} \
         $PREFIX/lib/lib{LLVM,LTO,clang}* \
         $PREFIX/lib/clang/ \
         $PREFIX/lib/cmake/{llvm,clang}

  cmake \
    -DCMAKE_BUILD_TYPE=Release \
    -DCMAKE_INSTALL_PREFIX=$PREFIX \
    -DLLVM_INCLUDE_DOCS=OFF \
    -DLLVM_INCLUDE_EXAMPLES=OFF \
    -DLLVM_INCLUDE_TESTS=OFF \
    -DLLVM_INCLUDE_UTILS=OFF \
    -DLLVM_TARGETS_TO_BUILD=X86 \
    -DLLVM_ENABLE_RTTI=ON \
    -DCMAKE_CXX_FLAGS="$CLANG_CXXFLAGS" \
    -DCMAKE_EXE_LINKER_FLAGS="$CLANG_LDFLAGS" \
    -DCMAKE_MODULE_LINKER_FLAGS="$CLANG_LDFLAGS" \
    -DCMAKE_SHARED_LINKER_FLAGS="$CLANG_LDFLAGS" \
    -DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \
    $TOOLS_ARGS \
    $EXTRA_CMAKE_FLAGS \
    $LLVM_SOURCE

  ${NINJA:-make} -j$PARALLEL $EXTRA_MAKEFLAGS install

  if [[ "$BUILD_TYPE" == "normal" ]]; then
    # Create a link from Clang to thirdparty/clang-toolchain. This path is used
    # for all Clang invocations. The link can't point to the Clang installed in
    # the prefix directory, since this confuses CMake into believing the
    # thirdparty prefix directory is the system-wide prefix, and it omits the
    # thirdparty prefix directory from the rpath of built binaries.
    ln -sfn $LLVM_BDIR $TP_DIR/clang-toolchain
  fi
  popd
}

build_gflags() {
  GFLAGS_BDIR=$TP_BUILD_DIR/$GFLAGS_NAME$MODE_SUFFIX
  mkdir -p $GFLAGS_BDIR
  pushd $GFLAGS_BDIR
  rm -rf CMakeCache.txt CMakeFiles/
  cmake \
    -DCMAKE_BUILD_TYPE=Release \
    -DCMAKE_POSITION_INDEPENDENT_CODE=On \
    -DGFLAGS_NAMESPACE=gflags \
    -DCMAKE_INSTALL_PREFIX=$PREFIX \
    -DCMAKE_CXX_FLAGS="$EXTRA_CXXFLAGS" \
    -DCMAKE_EXE_LINKER_FLAGS="$EXTRA_LDFLAGS $EXTRA_LIBS" \
    -DCMAKE_MODULE_LINKER_FLAGS="$EXTRA_LDFLAGS $EXTRA_LIBS" \
    -DCMAKE_SHARED_LINKER_FLAGS="$EXTRA_LDFLAGS $EXTRA_LIBS" \
    -DBUILD_SHARED_LIBS=On \
    -DBUILD_STATIC_LIBS=On \
    -DREGISTER_INSTALL_PREFIX=Off \
    $EXTRA_CMAKE_FLAGS \
    $GFLAGS_SOURCE
  ${NINJA:-make} -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_libunwind() {
  LIBUNWIND_BDIR=$TP_BUILD_DIR/$LIBUNWIND_NAME$MODE_SUFFIX
  mkdir -p $LIBUNWIND_BDIR
  pushd $LIBUNWIND_BDIR
  # Disable minidebuginfo, which depends on liblzma, until/unless we decide to
  # add liblzma to thirdparty.
  CFLAGS="$EXTRA_CFLAGS" \
    $LIBUNWIND_SOURCE/configure \
    --disable-minidebuginfo \
    --with-pic \
    --prefix=$PREFIX
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_glog() {
  GLOG_BDIR=$TP_BUILD_DIR/$GLOG_NAME$MODE_SUFFIX
  mkdir -p $GLOG_BDIR
  pushd $GLOG_BDIR

  # glog depends on libunwind and gflags.
  #
  # Specifying -Wl,-rpath has different default behavior on GNU binutils ld vs.
  # the GNU gold linker. ld sets RPATH (due to defaulting to --disable-new-dtags)
  # and gold sets RUNPATH (due to defaulting to --enable-new-dtags). At the time
  # of this writing, contrary to the way RPATH is treated, when RUNPATH is
  # specified on a binary, glibc doesn't respect it for transitive (non-direct)
  # library dependencies (see https://sourceware.org/bugzilla/show_bug.cgi?id=13945).
  # So we must set RUNPATH for all deps-of-deps on the dep libraries themselves.
  #
  # This comment applies both here and the locations elsewhere in this script
  # where we add something to -Wl,-rpath.
  CXXFLAGS="$EXTRA_CXXFLAGS -I$PREFIX/include" \
    LDFLAGS="$EXTRA_LDFLAGS -L$PREFIX/lib -Wl,-rpath,$PREFIX/lib" \
    LIBS="$EXTRA_LIBS" \
    $GLOG_SOURCE/configure \
    --with-pic \
    --prefix=$PREFIX \
    --with-gflags=$PREFIX
  fixup_libtool
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_gperftools() {
  GPERFTOOLS_BDIR=$TP_BUILD_DIR/$GPERFTOOLS_NAME$MODE_SUFFIX
  mkdir -p $GPERFTOOLS_BDIR
  pushd $GPERFTOOLS_BDIR
  CFLAGS="$EXTRA_CFLAGS" \
    CXXFLAGS="$EXTRA_CXXFLAGS" \
    LDFLAGS="$EXTRA_LDFLAGS" \
    LIBS="$EXTRA_LIBS" \
    $GPERFTOOLS_SOURCE/configure \
    --enable-frame-pointers \
    --with-pic \
    --prefix=$PREFIX
  fixup_libtool
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_gmock() {
  GMOCK_SHARED_BDIR=$TP_BUILD_DIR/$GMOCK_NAME.shared$MODE_SUFFIX
  GMOCK_STATIC_BDIR=$TP_BUILD_DIR/$GMOCK_NAME.static$MODE_SUFFIX
  for SHARED in ON OFF; do
    if [ $SHARED = "ON" ]; then
      GMOCK_BDIR=$GMOCK_SHARED_BDIR
    else
      GMOCK_BDIR=$GMOCK_STATIC_BDIR
    fi
    mkdir -p $GMOCK_BDIR
    pushd $GMOCK_BDIR
    rm -rf CMakeCache.txt CMakeFiles/
    cmake \
      -DCMAKE_BUILD_TYPE=Debug \
      -DCMAKE_POSITION_INDEPENDENT_CODE=On \
      -DCMAKE_CXX_FLAGS="$EXTRA_CXXFLAGS" \
      -DCMAKE_EXE_LINKER_FLAGS="$EXTRA_LDFLAGS $EXTRA_LIBS" \
      -DCMAKE_MODULE_LINKER_FLAGS="$EXTRA_LDFLAGS $EXTRA_LIBS" \
      -DCMAKE_SHARED_LINKER_FLAGS="$EXTRA_LDFLAGS $EXTRA_LIBS" \
      -DBUILD_SHARED_LIBS=$SHARED \
      $EXTRA_CMAKE_FLAGS \
      $GMOCK_SOURCE/googlemock
    ${NINJA:-make} -j$PARALLEL $EXTRA_MAKEFLAGS
    popd
  done

  # Install gmock/gtest libraries and headers manually instead of using make
  # install. Make install results in libraries with a malformed lib name on
  # macOS.
  echo Installing gmock...
  cp -a $GMOCK_SHARED_BDIR/libgmock.$DYLIB_SUFFIX $PREFIX/lib/
  cp -a $GMOCK_STATIC_BDIR/libgmock.a $PREFIX/lib/
  rsync -av $GMOCK_SOURCE/googlemock/include/ $PREFIX/include/
  rsync -av $GMOCK_SOURCE/googletest/include/ $PREFIX/include/
}

build_protobuf() {
  PROTOBUF_BDIR=$TP_BUILD_DIR/$PROTOBUF_NAME$MODE_SUFFIX
  mkdir -p $PROTOBUF_BDIR
  pushd $PROTOBUF_BDIR
  CFLAGS="$EXTRA_CFLAGS" \
    CXXFLAGS="$EXTRA_CXXFLAGS" \
    LDFLAGS="$EXTRA_LDFLAGS" \
    LIBS="$EXTRA_LIBS" \
    $PROTOBUF_SOURCE/configure \
    --with-pic \
    --enable-shared \
    --enable-static \
    --prefix=$PREFIX
  fixup_libtool
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_snappy() {
  SNAPPY_BDIR=$TP_BUILD_DIR/$SNAPPY_NAME$MODE_SUFFIX
  mkdir -p $SNAPPY_BDIR
  pushd $SNAPPY_BDIR
  CFLAGS="$EXTRA_CFLAGS" \
    CXXFLAGS="$EXTRA_CXXFLAGS" \
    LDFLAGS="$EXTRA_LDFLAGS" \
    LIBS="$EXTRA_LIBS" \
    $SNAPPY_SOURCE/configure \
    --with-pic \
    --prefix=$PREFIX
  fixup_libtool
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_zlib() {
  ZLIB_BDIR=$TP_BUILD_DIR/$ZLIB_NAME$MODE_SUFFIX
  mkdir -p $ZLIB_BDIR
  pushd $ZLIB_BDIR

  # It doesn't appear possible to isolate source and build directories, so just
  # prepopulate the latter using the former.
  rsync -av --delete $ZLIB_SOURCE/ .

  CFLAGS="$EXTRA_CFLAGS -fPIC" \
    ./configure \
    --prefix=$PREFIX
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_lz4() {
  LZ4_BDIR=$TP_BUILD_DIR/$LZ4_NAME$MODE_SUFFIX
  mkdir -p $LZ4_BDIR
  pushd $LZ4_BDIR
  rm -Rf CMakeCache.txt CMakeFiles/
  CFLAGS="$EXTRA_CFLAGS" \
    cmake \
    -DCMAKE_BUILD_TYPE=release \
    -DBUILD_TOOLS=0 \
    -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \
    $EXTRA_CMAKE_FLAGS \
    $LZ4_SOURCE/cmake_unofficial
  ${NINJA:-make} -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_bitshuffle() {
  BITSHUFFLE_BDIR=$TP_BUILD_DIR/$BITSHUFFLE_NAME$MODE_SUFFIX
  mkdir -p $BITSHUFFLE_BDIR
  pushd $BITSHUFFLE_BDIR
  # bitshuffle depends on lz4, therefore set the flag I$PREFIX/include

  # This library has significant optimizations when built with -mavx2. However,
  # we still need to support non-AVX2-capable hardware. So, we build it twice,
  # once with the flag and once without, and use some linker tricks to
  # suffix the AVX2 symbols with '_avx2'. OSX doesn't have objcopy, so we only
  # do this trick on Linux.
  if [ -n "$OS_LINUX" ]; then
    arches="default avx2"
  else
    arches="default"
  fi
  to_link=""
  for arch in $arches ; do
    arch_flag=""
    if [ "$arch" == "avx2" ]; then
      arch_flag="-mavx2"
    fi
    tmp_obj=bitshuffle_${arch}_tmp.o
    dst_obj=bitshuffle_${arch}.o
    ${CC:-gcc} $EXTRA_CFLAGS $arch_flag -std=c99 -I$PREFIX/include -O3 -DNDEBUG -fPIC -c \
      "$BITSHUFFLE_SOURCE/src/bitshuffle_core.c" \
      "$BITSHUFFLE_SOURCE/src/bitshuffle.c" \
      "$BITSHUFFLE_SOURCE/src/iochain.c"
    # Merge the object files together to produce a combined .o file.
    ld -r -o $tmp_obj bitshuffle_core.o bitshuffle.o iochain.o
    # For the AVX2 symbols, suffix them.
    if [ "$arch" == "avx2" ]; then
      # Create a mapping file with '<old_sym> <suffixed_sym>' on each line.
      nm --defined-only --extern-only $tmp_obj | while read addr type sym ; do
        echo ${sym} ${sym}_${arch}
      done > renames.txt
      objcopy --redefine-syms=renames.txt $tmp_obj $dst_obj
    else
      mv $tmp_obj $dst_obj
    fi
    to_link="$to_link $dst_obj"
  done

  rm -f bitshuffle.a
  ar rs bitshuffle.a $to_link
  cp bitshuffle.a $PREFIX/lib/
  cp $BITSHUFFLE_SOURCE/src/bitshuffle.h $PREFIX/include/bitshuffle.h
  cp $BITSHUFFLE_SOURCE/src/bitshuffle_core.h $PREFIX/include/bitshuffle_core.h
  popd
}

build_libev() {
  LIBEV_BDIR=$TP_BUILD_DIR/$LIBEV_NAME$MODE_SUFFIX
  mkdir -p $LIBEV_BDIR
  pushd $LIBEV_BDIR
  CFLAGS="$EXTRA_CFLAGS" \
    CXXFLAGS="$EXTRA_CXXFLAGS" \
    $LIBEV_SOURCE/configure \
    --with-pic \
    --prefix=$PREFIX
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_rapidjson() {
  # just installing it into our prefix
  pushd $RAPIDJSON_SOURCE
  rsync -av --delete $RAPIDJSON_SOURCE/include/rapidjson/ $PREFIX/include/rapidjson/
  popd
}

build_squeasel() {
  # Mongoose's Makefile builds a standalone web server, whereas we just want
  # a static lib
  SQUEASEL_BDIR=$TP_BUILD_DIR/$SQUEASEL_NAME$MODE_SUFFIX
  mkdir -p $SQUEASEL_BDIR
  pushd $SQUEASEL_BDIR
  ${CC:-gcc} $EXTRA_CFLAGS $OPENSSL_CFLAGS $OPENSSL_LDFLAGS -std=c99 -O3 -DNDEBUG -fPIC -c "$SQUEASEL_SOURCE/squeasel.c"
  ar rs libsqueasel.a squeasel.o
  cp libsqueasel.a $PREFIX/lib/
  cp $SQUEASEL_SOURCE/squeasel.h $PREFIX/include/
  popd
}

build_mustache() {
  MUSTACHE_BDIR=$TP_BUILD_DIR/$MUSTACHE_NAME$MODE_SUFFIX
  mkdir -p $MUSTACHE_BDIR
  pushd $MUSTACHE_BDIR
  # We add $PREFIX/include for boost and $PREFIX_COMMON/include for rapidjson.
  ${CXX:-g++} $EXTRA_CXXFLAGS -I$PREFIX/include -I$PREFIX_COMMON/include -O3 -DNDEBUG -fPIC -c "$MUSTACHE_SOURCE/mustache.cc"
  ar rs libmustache.a mustache.o
  cp libmustache.a $PREFIX/lib/
  cp $MUSTACHE_SOURCE/mustache.h $PREFIX/include/
  popd
}

build_curl() {
  # Configure for a very minimal install - basically only HTTP(S), since we only
  # use this for testing our own HTTP/HTTPS endpoints at this point in time.
  CURL_BDIR=$TP_BUILD_DIR/$CURL_NAME$MODE_SUFFIX
  mkdir -p $CURL_BDIR
  pushd $CURL_BDIR

  # Note: curl shows a message asking for CPPFLAGS to be used for include
  # directories, not CFLAGS.
  CFLAGS="$EXTRA_CFLAGS" \
    CPPFLAGS="$EXTRA_CPPFLAGS $OPENSSL_CFLAGS" \
    LDFLAGS="$EXTRA_LDFLAGS $OPENSSL_LDFLAGS" \
    LIBS="$EXTRA_LIBS" \
    $CURL_SOURCE/configure \
    --prefix=$PREFIX \
    --disable-dict \
    --disable-file \
    --disable-ftp \
    --disable-gopher \
    --disable-imap \
    --disable-ipv6 \
    --disable-ldap \
    --disable-ldaps \
    --disable-manual \
    --disable-pop3 \
    --disable-rtsp \
    --disable-smtp \
    --disable-telnet \
    --disable-tftp \
    --without-librtmp \
    --without-libssh2
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_crcutil() {
  CRCUTIL_BDIR=$TP_BUILD_DIR/$CRCUTIL_NAME$MODE_SUFFIX
  mkdir -p $CRCUTIL_BDIR
  pushd $CRCUTIL_BDIR

  # The autogen.sh script makes it difficult to isolate source and build
  # directories, so just prepopulate the latter using the former.
  rsync -av --delete $CRCUTIL_SOURCE/ .
  ./autogen.sh

  CFLAGS="$EXTRA_CFLAGS" \
    CXXFLAGS="$EXTRA_CXXFLAGS" \
    LDFLAGS="$EXTRA_LDFLAGS" \
    LIBS="$EXTRA_LIBS" \
    ./configure \
    --prefix=$PREFIX
  fixup_libtool
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_breakpad() {
  BREAKPAD_BDIR=$TP_BUILD_DIR/$BREAKPAD_NAME$MODE_SUFFIX
  mkdir -p $BREAKPAD_BDIR
  pushd $BREAKPAD_BDIR

  CFLAGS="$EXTRA_CFLAGS" \
    CXXFLAGS="$EXTRA_CXXFLAGS" \
    LDFLAGS="$EXTRA_LDFLAGS" \
    LIBS="$EXTRA_LIBS" \
    $BREAKPAD_SOURCE/configure --prefix=$PREFIX
  make -j$PARALLEL $EXTRA_MAKEFLAGS install

  # Here we run a script to munge breakpad #include statements in-place. Sadly,
  # breakpad does not prefix its header file paths. Until we can solve that
  # issue upstream we use this "hack" to add breakpad/ as a prefix for all the
  # breakpad-related includes in the breakpad header files ourselves. See also
  # https://bugs.chromium.org/p/google-breakpad/issues/detail?id=721
  local BREAKPAD_INCLUDE_DIR=$PREFIX/include/breakpad
  pushd "$BREAKPAD_INCLUDE_DIR"
  find . -type f | xargs grep -l "#include" | \
    xargs perl -p -i -e '@pre = qw(client common google_breakpad processor third_party); for $p (@pre) { s{#include "$p/}{#include "breakpad/$p/}; }'
  popd

  popd
}

build_cpplint() {
  # Copy cpplint tool into bin directory
  cp $GSG_SOURCE/cpplint/cpplint.py $PREFIX/bin/cpplint.py
}

build_gcovr() {
  # Copy gcovr tool into bin directory
  cp -a $GCOVR_SOURCE/scripts/gcovr $PREFIX/bin/gcovr
}

build_trace_viewer() {
  echo Installing trace-viewer into the www directory
  cp -a $TRACE_VIEWER_SOURCE/tracing.* $TP_DIR/../www/
}

build_nvml() {
  NVML_BDIR=$TP_BUILD_DIR/$NVML_NAME$MODE_SUFFIX
  mkdir -p $NVML_BDIR
  pushd $NVML_BDIR

  # It doesn't appear possible to isolate source and build directories, so just
  # prepopulate the latter using the former.
  rsync -av --delete $NVML_SOURCE/ .
  cd src/

  # The embedded jemalloc build doesn't pick up the EXTRA_CFLAGS environment
  # variable, so we have to stick our flags into this config file.
  if ! grep -q -e "$EXTRA_CFLAGS" jemalloc/jemalloc.cfg ; then
    perl -p -i -e "s,(EXTRA_CFLAGS=\"),\$1$EXTRA_CFLAGS ," jemalloc/jemalloc.cfg
  fi
  for LIB in libvmem libpmem libpmemobj; do
    # Disable -Werror; it prevents jemalloc from building via clang.
    #
    # Add PREFIX/lib to the rpath; libpmemobj depends on libpmem at runtime.
    EXTRA_CFLAGS="$EXTRA_CFLAGS -Wno-error" \
      EXTRA_LDFLAGS="$EXTRA_LDFLAGS -Wl,-rpath,$PREFIX/lib" \
      make -j$PARALLEL $EXTRA_MAKEFLAGS DEBUG=0 $LIB

    # NVML doesn't allow configuring PREFIX -- it always installs into
    # DESTDIR/usr/lib. Additionally, the 'install' target builds all of
    # the NVML libraries, even though we only need the three libraries above.
    # So, we manually install the built artifacts.
    cp -a $NVML_BDIR/src/include/$LIB.h $PREFIX/include
    cp -a $NVML_BDIR/src/nondebug/$LIB.{so*,a} $PREFIX/lib
  done
  popd
}

build_boost() {
  local BUILD_TYPE=$1
  pushd $BOOST_SOURCE
  local TOOLSET="" # By default, use default toolset.
  # Boost only looks for this file here and in the user's $HOME directory.
  # To avoid adding this file to the source tree we would have to patch the auto-generated
  # project-config.jam that is generated by ./bootstrap which is a worse option.
  # This file is cleaned up before and after the build as to reduce the risk of polluting
  # the source tree.
  local USER_JAMFILE=./tools/build/src/user-config.jam

  if [ -e "$USER_JAMFILE" ]; then
    rm $USER_JAMFILE
  fi

  BOOST_BDIR=$TP_BUILD_DIR/boost-$BOOST_VERSION$MODE_SUFFIX

  # Compile with PIC and set nanosecond resolution for impala compatibility.
  BOOST_CFLAGS="$EXTRA_CFLAGS -fPIC -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG"
  BOOST_CXXFLAGS="$EXTRA_CXXFLAGS -fPIC -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG"
  BOOST_LDFLAGS="$EXTRA_LDFLAGS"

  case $BUILD_TYPE in
    "normal")
      ;;
    "tsan")
      BOOST_LDFLAGS="-stdlib=libc++ $BOOST_LDFLAGS"
      ;;
    *)
      echo "Unknown Boost build type: $BUILD_TYPE"
      exit 1
      ;;
  esac

  # If CC and CXX are set, set the compiler in user-config.jam.
  if [ -n "$CC" -a -n "$CXX" ]; then
    # Determine the name of the compiler referenced in $CC. This assumes the compiler
    # prints its name as the first word of the first line, which appears to work for gcc
    # and clang, even when they're called through ccache.
    local COMPILER=$($CC --version | awk 'NR==1 {print $1;}')

    # If the compiler binary used was 'cc' and not 'gcc', it will also report
    # itself as 'cc'. Coerce it to gcc.
    if [ "$COMPILER" = "cc" ]; then
      COMPILER=gcc
    fi

    TOOLSET="toolset=${COMPILER}"
    echo "Using $TOOLSET"
    echo "using ${COMPILER} : : $CXX ;" > $USER_JAMFILE
    echo "User jamfile location: $USER_JAMFILE"
    echo "User jamfile contents:"
    cat $USER_JAMFILE
  fi

  # Build the date_time boost lib.
  ./bootstrap.sh --prefix=$PREFIX threading=multi --with-libraries=date_time
  ./b2 clean $TOOLSET --build-dir="$BOOST_BDIR"
  ./b2 install variant=release link=static,shared --build-dir="$BOOST_BDIR" $TOOLSET -q -d0 \
    --debug-configuration \
    cflags="$BOOST_CFLAGS" \
    cxxflags="$BOOST_CXXFLAGS" \
    linkflags="$BOOST_LDFLAGS"

  if [ -e "$USER_JAMFILE" ]; then
    rm $USER_JAMFILE
  fi
  popd
}

build_sparsehash() {
  # This library is header-only, so we just copy the headers
  pushd $SPARSEHASH_SOURCE
  rsync -av --delete sparsehash/ $PREFIX/include/sparsehash/
  popd
}

build_sparsepp() {
  # This library is header-only, so we just copy the headers
  pushd $SPARSEPP_SOURCE
  rsync -av --delete sparsepp/ $PREFIX/include/sparsepp/
  popd
}

build_thrift() {
  THRIFT_BDIR=$TP_BUILD_DIR/$THRIFT_NAME$MODE_SUFFIX
  mkdir -p $THRIFT_BDIR
  pushd $THRIFT_BDIR
  rm -Rf CMakeCache.txt CMakeFiles/

  # Thrift depends on bison.
  #
  # Configure for a very minimal install - only the C++ client libraries are needed.
  # Thrift requires C++11 when compiled on Linux against libc++ (see cxxfunctional.h).
  CFLAGS="$EXTRA_CFLAGS" \
    CXXFLAGS="$EXTRA_CXXFLAGS -std=c++11" \
    LDFLAGS="$EXTRA_LDFLAGS" \
    LIBS="$EXTRA_LIBS" \
    cmake \
    -DBOOST_ROOT=$PREFIX \
    -DBUILD_C_GLIB=OFF \
    -DBUILD_COMPILER=ON \
    -DBUILD_CPP=ON \
    -DBUILD_EXAMPLES=OFF \
    -DBUILD_HASKELL=OFF \
    -DBUILD_JAVA=OFF \
    -DBUILD_PYTHON=OFF \
    -DBUILD_TESTING=OFF \
    -DBUILD_TUTORIALS=OFF \
    -DCMAKE_BUILD_TYPE=Release \
    -DCMAKE_INSTALL_PREFIX=$PREFIX \
    -DWITH_BOOSTTHREADS=OFF \
    -DWITH_LIBEVENT=OFF \
    -DWITH_OPENSSL=OFF \
    -DWITH_PLUGIN=OFF \
    $EXTRA_CMAKE_FLAGS \
    $THRIFT_SOURCE

  ${NINJA:-make} -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd

  # Install fb303.thrift into the share directory.
  mkdir -p $PREFIX/share/fb303/if
  cp $THRIFT_SOURCE/contrib/fb303/if/fb303.thrift $PREFIX/share/fb303/if
}

build_bison() {
  BISON_BDIR=$TP_BUILD_DIR/$BISON_NAME$MODE_SUFFIX
  mkdir -p $BISON_BDIR
  pushd $BISON_BDIR
  CFLAGS="$EXTRA_CFLAGS" \
    CXXFLAGS="$EXTRA_CXXFLAGS" \
    LDFLAGS="$EXTRA_LDFLAGS" \
    LIBS="$EXTRA_LIBS" \
    $BISON_SOURCE/configure \
    --prefix=$PREFIX
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}

build_coredumper() {
  COREDUMPER_BDIR=$TP_BUILD_DIR/$COREDUMPER_NAME$MODE_SUFFIX
  mkdir -p $COREDUMPER_BDIR
  pushd $COREDUMPER_BDIR
  CFLAGS="$EXTRA_CFLAGS" \
    CXXFLAGS="$EXTRA_CXXFLAGS" \
    LDFLAGS="$EXTRA_LDFLAGS" \
    LIBS="$EXTRA_LIBS" \
    $COREDUMPER_SOURCE/configure \
    --with-pic \
    --prefix=$PREFIX
  fixup_libtool
  make -j$PARALLEL $EXTRA_MAKEFLAGS install
  popd
}


#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Script which downloads and builds the thirdparty dependencies
# only if necessary.
#
# In a git repo, this uses git checksum information on the thirdparty
# tree. Otherwise, it uses a 'stamp file' approach.

set -e
set -o pipefail

DEPENDENCY_GROUPS=
case $1 in
  "")
    DEPENDENCY_GROUPS="common uninstrumented"
    ;;
  "tsan")
    DEPENDENCY_GROUPS="common tsan"
    ;;
  "all")
    DEPENDENCY_GROUPS="common uninstrumented tsan"
    ;;
  *)
    echo "Unknown build configuration: $1"
    exit 1
    ;;
esac

TP_DIR=$(dirname $BASH_SOURCE)
cd $TP_DIR

NEEDS_BUILD=
NEEDS_REHASH=

IS_IN_GIT=$(test -e ../.git && echo true || :)

if [ -n "$IS_IN_GIT" ]; then
  # Determine whether this subtree in the git repo has changed since thirdparty
  # was last built.
  CUR_THIRDPARTY_HASH=$(cd .. && git ls-tree -d HEAD thirdparty | awk '{print $3}')

  for GROUP in $DEPENDENCY_GROUPS; do
    LAST_BUILD_HASH=$(cat .build-hash.$GROUP || :)
    if [ "$CUR_THIRDPARTY_HASH" != "$LAST_BUILD_HASH" ]; then
      echo "Rebuilding thirdparty dependency group '$GROUP': the repository has changed since it was last built."
      echo "Old git hash: $LAST_BUILD_HASH"
      echo "New build hash: $CUR_THIRDPARTY_HASH"
      NEEDS_BUILD="$NEEDS_BUILD $GROUP"
      NEEDS_REHASH="$NEEDS_REHASH $GROUP"
    fi
  done

  if [ -z "$NEEDS_BUILD" ]; then
    # All the hashes matched. Determine whether the developer has any local changes.
    if ! ( git diff --quiet .  && git diff --cached --quiet . ) ; then
      echo "Rebuilding thirdparty dependency groups '$DEPENDENCY_GROUPS': there are local changes in the repository."
      NEEDS_BUILD="$DEPENDENCY_GROUPS"
    fi
  fi

else
  # If we aren't inside running inside a git repository (e.g. we are
  # part of a source distribution tarball) then we can't use git to find
  # out whether the build is clean. Instead, look at the ctimes of special
  # stamp files, and see if any files inside this directory have been
  # modified since then.
  for GROUP in $DEPENDENCY_GROUPS; do
    STAMP_FILE=.build-stamp.$GROUP
    if [ -f $STAMP_FILE ]; then
      CHANGED_FILE_COUNT=$(find . -cnewer $STAMP_FILE | wc -l)
      echo "$CHANGED_FILE_COUNT file(s) been modified since thirdparty dependency group '$GROUP' was last built."
      if [ $CHANGED_FILE_COUNT -gt 0 ]; then
        echo "Rebuilding."
        NEEDS_BUILD="$NEEDS_BUILD $GROUP"
      fi
    else
      echo "It appears that thirdparty dependency group '$GROUP' was never built. Building."
      NEEDS_BUILD="$NEEDS_BUILD $GROUP"
    fi
  done
fi

if [ -z "$NEEDS_BUILD" ]; then
  echo "Not rebuilding thirdparty. No changes since last build."
  exit 0
fi

# Remove the old hashes/stamps before building so that if the build is aborted
# and the repository is taken backwards in time to the point where the old
# hashes/stamps matched, the repository would still be rebuilt.
for GROUP in $NEEDS_BUILD; do
  rm -f .build-hash.$GROUP .build-stamp.$GROUP
done

# Download and build the necessary dependency groups.
./download-thirdparty.sh
./build-thirdparty.sh $NEEDS_BUILD

# The build succeeded. Update the appropriate hashes/stamps.
if [ -n "$IS_IN_GIT" ]; then
  for GROUP in $NEEDS_REHASH; do
    echo $CUR_THIRDPARTY_HASH > .build-hash.$GROUP
  done
else
  for GROUP in $NEEDS_BUILD; do
    touch .build-stamp.$GROUP
  done
fi