#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
# REQUIRE: db_bench binary exists in the current directory

# Exit Codes
EXIT_INVALID_ARGS=1
EXIT_NOT_COMPACTION_TEST=2
EXIT_UNKNOWN_JOB=3

# Size Constants
K=1024
M=$((1024 * K))
G=$((1024 * M))
T=$((1024 * G))

function display_usage() {
  echo "usage: benchmark.sh [--help] <test>"
  echo ""
  echo "These are the available benchmark tests:"
  echo -e "\tbulkload"
  echo -e "\tfillseq_disable_wal\t\tSequentially fill the database with no WAL"
  echo -e "\tfillseq_enable_wal\t\tSequentially fill the database with WAL"
  echo -e "\toverwrite"
  echo -e "\tupdaterandom"
  echo -e "\treadrandom"
  echo -e "\tmergerandom"
  echo -e "\tfilluniquerandom"
  echo -e "\tmultireadrandom"
  echo -e "\tfwdrange"
  echo -e "\trevrange"
  echo -e "\treadwhilewriting"
  echo -e "\treadwhilemerging"
  echo -e "\tfwdrangewhilewriting"
  echo -e "\trevrangewhilewriting"
  echo -e "\tfwdrangewhilemerging"
  echo -e "\trevrangewhilemerging"
  echo -e "\trandomtransaction"
  echo -e "\tuniversal_compaction"
  echo -e "\tdebug"
  echo ""
  echo "Generic enviroment Variables:"
  echo -e "\tJOB_ID\t\t\t\tAn identifier for the benchmark job, will appear in the results"
  echo -e "\tDB_DIR\t\t\t\tPath to write the database data directory"
  echo -e "\tWAL_DIR\t\t\t\tPath to write the database WAL directory"
  echo -e "\tOUTPUT_DIR\t\t\tPath to write the benchmark results to (default: /tmp)"
  echo -e "\tNUM_KEYS\t\t\tThe number of keys to use in the benchmark"
  echo -e "\tKEY_SIZE\t\t\tThe size of the keys to use in the benchmark (default: 20 bytes)"
  echo -e "\tVALUE_SIZE\t\t\tThe size of the values to use in the benchmark (default: 400 bytes)"
  echo -e "\tBLOCK_SIZE\t\t\tThe size of the database blocks in the benchmark (default: 8 KB)"
  echo -e "\tDB_BENCH_NO_SYNC\t\tDisable fsync on the WAL"
  echo -e "\tNUMACTL\t\t\t\tWhen defined use numactl --interleave=all"
  echo -e "\tNUM_THREADS\t\t\tThe number of threads to use (default: 64)"
  echo -e "\tMB_WRITE_PER_SEC\t\t\tRate limit for background writer"
  echo -e "\tNUM_NEXTS_PER_SEEK\t\t(default: 10)"
  echo -e "\tCACHE_SIZE\t\t\tSize of the block cache (default: 16GB)"
  echo -e "\tCACHE_NUMSHARDBITS\t\t\tNumber of shards for the block cache is 2 ** cache_numshardbits (default: 6)"
  echo -e "\tCOMPRESSION_MAX_DICT_BYTES"
  echo -e "\tCOMPRESSION_TYPE\t\tDefault compression(default: zstd)"
  echo -e "\tBOTTOMMOST_COMPRESSION\t\t(default: none)"
  echo -e "\tMIN_LEVEL_TO_COMPRESS\t\tValue for min_level_to_compress for Leveled"
  echo -e "\tCOMPRESSION_SIZE_PERCENT\tValue for compression_size_percent for Universal"
  echo -e "\tDURATION\t\t\tNumber of seconds for which the test runs"
  echo -e "\tWRITES\t\t\t\tNumber of writes for which the test runs"
  echo -e "\tWRITE_BUFFER_SIZE_MB\t\tThe size of the write buffer in MB (default: 128)"
  echo -e "\tTARGET_FILE_SIZE_BASE_MB\tThe value for target_file_size_base in MB (default: 128)"
  echo -e "\tMAX_BYTES_FOR_LEVEL_BASE_MB\tThe value for max_bytes_for_level_base in MB (default: 128)"
  echo -e "\tMAX_BACKGROUND_JOBS\t\tThe value for max_background_jobs (default: 16)"
  echo -e "\tCACHE_INDEX_AND_FILTER_BLOCKS\tThe value for cache_index_and_filter_blocks (default: 0)"
  echo -e "\tUSE_O_DIRECT\t\t\tUse O_DIRECT for user reads and compaction"
  echo -e "\tBYTES_PER_SYNC\t\t\tValue for bytes_per_sync, set to zero when USE_O_DIRECT is true"
  echo -e "\tSTATS_INTERVAL_SECONDS\t\tValue for stats_interval_seconds"
  echo -e "\tREPORT_INTERVAL_SECONDS\t\tValue for report_interval_seconds"
  echo -e "\tSUBCOMPACTIONS\t\t\tValue for subcompactions"
  echo -e "\tCOMPACTION_STYLE\t\tOne of leveled, universal, blob. Default is leveled."
  echo -e "\nEnvironment variables (mostly) for leveled compaction:"
  echo -e "\tLEVEL0_FILE_NUM_COMPACTION_TRIGGER\t\tValue for level0_file_num_compaction_trigger"
  echo -e "\tLEVEL0_SLOWDOWN_WRITES_TRIGGER\t\t\tValue for level0_slowdown_writes_trigger"
  echo -e "\tLEVEL0_STOP_WRITES_TRIGGER\t\t\tValue for level0_stop_writes_trigger"
  echo -e "\tPER_LEVEL_FANOUT\t\t\t\tValue for max_bytes_for_level_multiplier"
  echo -e "\tSOFT_PENDING_COMPACTION_BYTES_LIMIT_IN_GB\tThe value for soft_pending_compaction_bytes_limit in GB"
  echo -e "\tHARD_PENDING_COMPACTION_BYTES_LIMIT_IN_GB\tThe value for hard_pending_compaction_bytes_limit in GB"
  echo -e "\nEnvironment variables for universal compaction:"
  echo -e "\tUNIVERSAL_MIN_MERGE_WIDTH\tValue of min_merge_width option for universal"
  echo -e "\tUNIVERSAL_MAX_MERGE_WIDTH\tValue of min_merge_width option for universal"
  echo -e "\tUNIVERSAL_SIZE_RATIO\t\tValue of size_ratio option for universal"
  echo -e "\tUNIVERSAL_MAX_SIZE_AMP\t\tmax_size_amplification_percent for universal"
  echo -e "\tUNIVERSAL_ALLOW_TRIVIAL_MOVE\tSet allow_trivial_move to true for universal, default is false"
  echo -e "\nOptions for integrated BlobDB"
  echo -e "\tMIN_BLOB_SIZE\tValue for min_blob_size"
  echo -e "\tBLOB_FILE_SIZE\tValue for blob_file_size"
  echo -e "\tBLOB_COMPRESSION_TYPE\tValue for blob_compression_type"
  echo -e "\tBLOB_GC_AGE_CUTOFF\tValue for blob_garbage_collection_age_cutoff"
  echo -e "\tBLOB_GC_FORCE_THRESHOLD\tValue for blob_garbage_collection_force_threshold"
  echo -e "\tBLOB_FILE_STARTING_LEVEL\t\tBlob file starting level (default: 0)"
  echo -e "\tUSE_BLOB_CACHE\t\t\tEnable blob cache (default: 1)"
  echo -e "\tUSE_SHARED_BLOCK_AND_BLOB_CACHE\t\t\tUse the same backing cache for block cache and blob cache (default: 1)"
  echo -e "\tBLOB_CACHE_SIZE\t\t\tSize of the blob cache (default: 16GB)"
  echo -e "\tBLOB_CACHE_NUMSHARDBITS\t\t\tNumber of shards for the blob cache is 2 ** blob_cache_numshardbits (default: 6)"
  echo -e "\tPREPOPULATE_BLOB_CACHE\t\t\tPre-populate hot/warm blobs in blob cache (default: 0)"
}

if [ $# -lt 1 ]; then
  display_usage
  exit $EXIT_INVALID_ARGS
fi
bench_cmd=$1
shift
bench_args=$*

if [[ "$bench_cmd" == "--help" ]]; then
  display_usage
  exit
fi

job_id=${JOB_ID}

# Make it easier to run only the compaction test. Getting valid data requires
# a number of iterations and having an ability to run the test separately from
# rest of the benchmarks helps.
if [ "$COMPACTION_TEST" == "1" -a "$bench_cmd" != "universal_compaction" ]; then
  echo "Skipping $1 because it's not a compaction test."
  exit $EXIT_NOT_COMPACTION_TEST
fi

if [ -z $DB_DIR ]; then
  echo "DB_DIR is not defined"
  exit $EXIT_INVALID_ARGS
fi

if [ -z $WAL_DIR ]; then
  echo "WAL_DIR is not defined"
  exit $EXIT_INVALID_ARGS
fi

output_dir=${OUTPUT_DIR:-/tmp}
if [ ! -d $output_dir ]; then
  mkdir -p $output_dir
fi

report="$output_dir/report.tsv"
schedule="$output_dir/schedule.txt"

# all multithreaded tests run with sync=1 unless
# $DB_BENCH_NO_SYNC is defined
syncval="1"
if [ ! -z $DB_BENCH_NO_SYNC ]; then
  echo "Turning sync off for all multithreaded tests"
  syncval="0";
fi

compaction_style=${COMPACTION_STYLE:-leveled}
if [ $compaction_style = "leveled" ]; then
  echo Use leveled compaction
elif [ $compaction_style = "universal" ]; then
  echo Use universal compaction
elif [ $compaction_style = "blob" ]; then
  echo Use blob compaction
else
  echo COMPACTION_STYLE is :: $COMPACTION_STYLE :: and must be one of leveled, universal, blob
  exit $EXIT_INVALID_ARGS
fi

num_threads=${NUM_THREADS:-64}
mb_written_per_sec=${MB_WRITE_PER_SEC:-0}
# Only for tests that do range scans
num_nexts_per_seek=${NUM_NEXTS_PER_SEEK:-10}
cache_size=${CACHE_SIZE:-$(( 16 * $G ))}
cache_numshardbits=${CACHE_NUMSHARDBITS:-6}
compression_max_dict_bytes=${COMPRESSION_MAX_DICT_BYTES:-0}
compression_type=${COMPRESSION_TYPE:-zstd}
min_level_to_compress=${MIN_LEVEL_TO_COMPRESS:-"-1"}
compression_size_percent=${COMPRESSION_SIZE_PERCENT:-"-1"}

duration=${DURATION:-0}
writes=${WRITES:-0}

num_keys=${NUM_KEYS:-8000000000}
key_size=${KEY_SIZE:-20}
value_size=${VALUE_SIZE:-400}
block_size=${BLOCK_SIZE:-8192}
write_buffer_mb=${WRITE_BUFFER_SIZE_MB:-128}
target_file_mb=${TARGET_FILE_SIZE_BASE_MB:-128}
l1_mb=${MAX_BYTES_FOR_LEVEL_BASE_MB:-1024}
max_background_jobs=${MAX_BACKGROUND_JOBS:-16}
stats_interval_seconds=${STATS_INTERVAL_SECONDS:-60}
report_interval_seconds=${REPORT_INTERVAL_SECONDS:-1}
subcompactions=${SUBCOMPACTIONS:-1}
per_level_fanout=${PER_LEVEL_FANOUT:-8}

cache_index_and_filter=${CACHE_INDEX_AND_FILTER_BLOCKS:-0}
if [[ $cache_index_and_filter -eq 0 ]]; then
  cache_meta_flags=""
elif [[ $cache_index_and_filter -eq 1 ]]; then
  cache_meta_flags="\
  --cache_index_and_filter_blocks=$cache_index_and_filter \
  --cache_high_pri_pool_ratio=0.5 --cache_low_pri_pool_ratio=0"
else
  echo CACHE_INDEX_AND_FILTER_BLOCKS was $CACHE_INDEX_AND_FILTER_BLOCKS but must be 0 or 1
  exit $EXIT_INVALID_ARGS
fi

soft_pending_arg=""
if [ ! -z $SOFT_PENDING_COMPACTION_BYTES_LIMIT_IN_GB ]; then
  soft_pending_bytes=$( echo $SOFT_PENDING_COMPACTION_BYTES_LIMIT_IN_GB | \
    awk '{ printf "%.0f", $1 * GB }' GB=$G )
  soft_pending_arg="--soft_pending_compaction_bytes_limit=$soft_pending_bytes"
fi

hard_pending_arg=""
if [ ! -z $HARD_PENDING_COMPACTION_BYTES_LIMIT_IN_GB ]; then
  hard_pending_bytes=$( echo $HARD_PENDING_COMPACTION_BYTES_LIMIT_IN_GB | \
    awk '{ printf "%.0f", $1 * GB }' GB=$G )
  hard_pending_arg="--hard_pending_compaction_bytes_limit=$hard_pending_bytes"
fi

o_direct_flags=""
if [ ! -z $USE_O_DIRECT ]; then
  # Some of these flags are only supported in new versions and --undefok makes that work
  o_direct_flags="--use_direct_reads --use_direct_io_for_flush_and_compaction --prepopulate_block_cache=1"
  bytes_per_sync=0
else
  bytes_per_sync=${BYTES_PER_SYNC:-$(( 1 * M ))}
fi

univ_min_merge_width=${UNIVERSAL_MIN_MERGE_WIDTH:-2}
univ_max_merge_width=${UNIVERSAL_MAX_MERGE_WIDTH:-20}
univ_size_ratio=${UNIVERSAL_SIZE_RATIO:-1}
univ_max_size_amp=${UNIVERSAL_MAX_SIZE_AMP:-200}

if [ ! -z $UNIVERSAL_ALLOW_TRIVIAL_MOVE ]; then
  univ_allow_trivial_move=1
else
  univ_allow_trivial_move=0
fi

min_blob_size=${MIN_BLOB_SIZE:-0}
blob_file_size=${BLOB_FILE_SIZE:-$(( 256 * $M ))}
blob_compression_type=${BLOB_COMPRESSION_TYPE:-${compression_type}}
blob_gc_age_cutoff=${BLOB_GC_AGE_CUTOFF:-"0.25"}
blob_gc_force_threshold=${BLOB_GC_FORCE_THRESHOLD:-1}
blob_file_starting_level=${BLOB_FILE_STARTING_LEVEL:-0}
use_blob_cache=${USE_BLOB_CACHE:-1}
use_shared_block_and_blob_cache=${USE_SHARED_BLOCK_AND_BLOB_CACHE:-1}
blob_cache_size=${BLOB_CACHE_SIZE:-$(( 16 * $G ))}
blob_cache_numshardbits=${BLOB_CACHE_NUMSHARDBITS:-6}
prepopulate_blob_cache=${PREPOPULATE_BLOB_CACHE:-0}

# This script still works back to RocksDB 6.0
undef_params="\
use_blob_cache,\
use_shared_block_and_blob_cache,\
blob_cache_size,blob_cache_numshardbits,\
prepopulate_blob_cache,\
multiread_batched,\
cache_low_pri_pool_ratio,\
prepopulate_block_cache"

const_params_base="
  --undefok=$undef_params \
  --db=$DB_DIR \
  --wal_dir=$WAL_DIR \
  \
  --num=$num_keys \
  --key_size=$key_size \
  --value_size=$value_size \
  --block_size=$block_size \
  --cache_size=$cache_size \
  --cache_numshardbits=$cache_numshardbits \
  --compression_max_dict_bytes=$compression_max_dict_bytes \
  --compression_ratio=0.5 \
  --compression_type=$compression_type \
  --bytes_per_sync=$bytes_per_sync \
  $cache_meta_flags \
  $o_direct_flags \
  --benchmark_write_rate_limit=$(( 1024 * 1024 * $mb_written_per_sec )) \
  \
  --write_buffer_size=$(( $write_buffer_mb * M)) \
  --target_file_size_base=$(( $target_file_mb * M)) \
  --max_bytes_for_level_base=$(( $l1_mb * M)) \
  \
  --verify_checksum=1 \
  --delete_obsolete_files_period_micros=$((60 * M)) \
  --max_bytes_for_level_multiplier=$per_level_fanout \
  \
  --statistics=0 \
  --stats_per_interval=1 \
  --stats_interval_seconds=$stats_interval_seconds \
  --report_interval_seconds=$report_interval_seconds \
  --histogram=1 \
  \
  --memtablerep=skip_list \
  --bloom_bits=10 \
  --open_files=-1 \
  --subcompactions=$subcompactions \
  \
  $bench_args"

level_const_params="
  $const_params_base \
  --compaction_style=0 \
  --num_levels=8 \
  --min_level_to_compress=$min_level_to_compress \
  --level_compaction_dynamic_level_bytes=true \
  --pin_l0_filter_and_index_blocks_in_cache=1 \
  $soft_pending_arg \
  $hard_pending_arg \
"

# These inherit level_const_params because the non-blob LSM tree uses leveled compaction.
blob_const_params="
  $level_const_params \
  --enable_blob_files=true \
  --min_blob_size=$min_blob_size \
  --blob_file_size=$blob_file_size \
  --blob_compression_type=$blob_compression_type \
  --enable_blob_garbage_collection=true \
  --blob_garbage_collection_age_cutoff=$blob_gc_age_cutoff \
  --blob_garbage_collection_force_threshold=$blob_gc_force_threshold \
  --blob_file_starting_level=$blob_file_starting_level \
  --use_blob_cache=$use_blob_cache \
  --use_shared_block_and_blob_cache=$use_shared_block_and_blob_cache \
  --blob_cache_size=$blob_cache_size \
  --blob_cache_numshardbits=$blob_cache_numshardbits \
  --prepopulate_blob_cache=$prepopulate_blob_cache \
"

# TODO:
#   pin_l0_filter_and..., is this OK?
univ_const_params="
  $const_params_base \
  --compaction_style=1 \
  --num_levels=40 \
  --universal_compression_size_percent=$compression_size_percent \
  --pin_l0_filter_and_index_blocks_in_cache=1 \
  --universal_min_merge_width=$univ_min_merge_width \
  --universal_max_merge_width=$univ_max_merge_width \
  --universal_size_ratio=$univ_size_ratio \
  --universal_max_size_amplification_percent=$univ_max_size_amp \
  --universal_allow_trivial_move=$univ_allow_trivial_move \
"

if [ $compaction_style == "leveled" ]; then
  const_params="$level_const_params"
  l0_file_num_compaction_trigger=${LEVEL0_FILE_NUM_COMPACTION_TRIGGER:-4}
  l0_slowdown_writes_trigger=${LEVEL0_SLOWDOWN_WRITES_TRIGGER:-20}
  l0_stop_writes_trigger=${LEVEL0_STOP_WRITES_TRIGGER:-30}
elif [ $compaction_style == "universal" ]; then
  const_params="$univ_const_params"
  l0_file_num_compaction_trigger=${LEVEL0_FILE_NUM_COMPACTION_TRIGGER:-8}
  l0_slowdown_writes_trigger=${LEVEL0_SLOWDOWN_WRITES_TRIGGER:-20}
  l0_stop_writes_trigger=${LEVEL0_STOP_WRITES_TRIGGER:-30}
else
  # compaction_style == "blob"
  const_params="$blob_const_params"
  l0_file_num_compaction_trigger=${LEVEL0_FILE_NUM_COMPACTION_TRIGGER:-4}
  l0_slowdown_writes_trigger=${LEVEL0_SLOWDOWN_WRITES_TRIGGER:-20}
  l0_stop_writes_trigger=${LEVEL0_STOP_WRITES_TRIGGER:-30}
fi

l0_config="
  --level0_file_num_compaction_trigger=$l0_file_num_compaction_trigger \
  --level0_slowdown_writes_trigger=$l0_slowdown_writes_trigger \
  --level0_stop_writes_trigger=$l0_stop_writes_trigger"

# You probably don't want to set both --writes and --duration
if [ $duration -gt 0 ]; then
  const_params="$const_params --duration=$duration"
fi
if [ $writes -gt 0 ]; then
  const_params="$const_params --writes=$writes"
fi

params_w="$l0_config \
          --max_background_jobs=$max_background_jobs \
          --max_write_buffer_number=8 \
          $const_params"

params_bulkload="--max_background_jobs=$max_background_jobs \
                 --max_write_buffer_number=8 \
                 --allow_concurrent_memtable_write=false \
                 --level0_file_num_compaction_trigger=$((10 * M)) \
                 --level0_slowdown_writes_trigger=$((10 * M)) \
                 --level0_stop_writes_trigger=$((10 * M)) \
                 $const_params "

params_fillseq="--allow_concurrent_memtable_write=false \
                $params_w "

#
# Tune values for level and universal compaction.
# For universal compaction, these level0_* options mean total sorted of runs in
# LSM. In level-based compaction, it means number of L0 files.
#
params_level_compact="$const_params \
                --max_background_flushes=4 \
                --max_write_buffer_number=4 \
                --level0_file_num_compaction_trigger=4 \
                --level0_slowdown_writes_trigger=16 \
                --level0_stop_writes_trigger=20"

params_univ_compact="$const_params \
                --max_background_flushes=4 \
                --max_write_buffer_number=4 \
                --level0_file_num_compaction_trigger=8 \
                --level0_slowdown_writes_trigger=16 \
                --level0_stop_writes_trigger=20"

tsv_header="ops_sec\tmb_sec\tlsm_sz\tblob_sz\tc_wgb\tw_amp\tc_mbps\tc_wsecs\tc_csecs\tb_rgb\tb_wgb\tusec_op\tp50\tp99\tp99.9\tp99.99\tpmax\tuptime\tstall%\tNstall\tu_cpu\ts_cpu\trss\ttest\tdate\tversion\tjob_id\tgithash"

function get_cmd() {
  output=$1

  numa=""
  if [ ! -z $NUMACTL ]; then
    numa="numactl --interleave=all "
  fi

  # Try to use timeout when duration is set because some tests (revrange*) hang
  # for some versions (v6.10, v6.11).
  timeout_cmd=""
  if [ $duration -gt 0 ]; then
    if hash timeout ; then
      timeout_cmd="timeout $(( $duration + 600 ))"
    fi
  fi

  echo "/usr/bin/time -f '%e %U %S' -o $output $numa $timeout_cmd"
}

function month_to_num() {
    local date_str=$1
    date_str="${date_str/Jan/01}"
    date_str="${date_str/Feb/02}"
    date_str="${date_str/Mar/03}"
    date_str="${date_str/Apr/04}"
    date_str="${date_str/May/05}"
    date_str="${date_str/Jun/06}"
    date_str="${date_str/Jul/07}"
    date_str="${date_str/Aug/08}"
    date_str="${date_str/Sep/09}"
    date_str="${date_str/Oct/10}"
    date_str="${date_str/Nov/11}"
    date_str="${date_str/Dec/12}"
    echo $date_str
}

function start_stats {
  output=$1
  iostat -y -mx 1  >& $output.io &
  vmstat 1 >& $output.vm &
  # tail -1 because "ps | grep db_bench" returns 2 entries and we want the second
  while :; do ps aux | grep db_bench | grep -v grep | tail -1; sleep 10; done >& $output.ps &
  # This sets a global value
  pspid=$!

  while :; do
    b_gb=$( ls -l $DB_DIR 2> /dev/null | grep blob | awk '{ c += 1; b += $5 } END { printf "%.1f", b / (1024*1024*1024) }' )
    s_gb=$( ls -l $DB_DIR 2> /dev/null | grep sst | awk '{ c += 1; b += $5 } END { printf "%.1f", b / (1024*1024*1024) }' )
    l_gb=$( ls -l $WAL_DIR 2> /dev/null | grep log | awk '{ c += 1; b += $5 } END { printf "%.1f", b / (1024*1024*1024) }' )
    a_gb=$( ls -l $DB_DIR 2> /dev/null | awk '{ c += 1; b += $5 } END { printf "%.1f", b / (1024*1024*1024) }' )
    ts=$( date +%H%M%S )
    echo -e "${a_gb}\t${s_gb}\t${l_gb}\t${b_gb}\t${ts}"
    sleep 10
  done >& $output.sizes &
  # This sets a global value
  szpid=$!
}

function stop_stats {
  output=$1
  kill $pspid
  kill $szpid
  killall iostat
  killall vmstat
  sleep 1
  gzip $output.io
  gzip $output.vm

  am=$( sort -nk 1,1 $output.sizes | tail -1 | awk '{ print $1 }' )
  sm=$( sort -nk 2,2 $output.sizes | tail -1 | awk '{ print $2 }' )
  lm=$( sort -nk 3,3 $output.sizes | tail -1 | awk '{ print $3 }' )
  bm=$( sort -nk 4,4 $output.sizes | tail -1 | awk '{ print $4 }' )
  echo -e "max sizes (GB): $am all, $sm sst, $lm log, $bm blob" >> $output.sizes
}

function units_as_gb {
  size=$1
  units=$2

  case $units in
    MB)
      echo "$size" | awk '{ printf "%.1f", $1 / 1024.0 }'
      ;;
    GB)
      echo "$size"
      ;;
    TB)
      echo "$size" | awk '{ printf "%.1f", $1 * 1024.0 }'
      ;;
    *)
      echo "NA"
      ;;
  esac
}

function summarize_result {
  test_out=$1
  test_name=$2
  bench_name=$3

  # In recent versions these can be found directly via db_bench --version, --build_info but
  # grepping from the log lets this work on older versions.
  version="$( grep "RocksDB version:" "$DB_DIR"/LOG | head -1 | awk '{ printf "%s", $5 }' )"
  git_hash="$( grep "Git sha" "$DB_DIR"/LOG | head -1 | awk '{ printf "%s", substr($5, 1, 10) }' )"

  # Note that this function assumes that the benchmark executes long enough so
  # that "Compaction Stats" is written to stdout at least once. If it won't
  # happen then empty output from grep when searching for "Sum" will cause
  # syntax errors.
  date=$( grep ^Date: $test_out | awk '{ print $6 "-" $3 "-" $4 "T" $5 }' )
  my_date=$( month_to_num $date )
  uptime=$( grep ^Uptime\(secs $test_out | tail -1 | awk '{ printf "%.0f", $2 }' )
  stall_pct=$( grep "^Cumulative stall" $test_out| tail -1  | awk '{  print $5 }' )
  nstall=$( grep ^Stalls\(count\):  $test_out | tail -1 | awk '{ print $2 + $6 + $10 + $14 + $18 + $20 }' )

  if ! grep ^"$bench_name" "$test_out" > /dev/null 2>&1 ; then
    echo -e "failed\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t$test_name\t$my_date\t$version\t$job_id\t$git_hash"
    return
  fi

  # Output formats
  # V1: readrandom   :      10.218 micros/op 3131616 ops/sec; 1254.3 MB/s (176144999 of 176144999 found)
  # The MB/s is mssing for multireadrandom
  # V1a: multireadrandom :      10.164 micros/op 3148272 ops/sec; (177099990 of 177099990 found)
  # V1: overwrite    :       7.939 micros/op 125963 ops/sec;   50.5 MB/s
  # V2: overwrite    :       7.854 micros/op 127320 ops/sec 1800.001 seconds 229176999 operations;   51.0 MB/s

  format_version=$( grep ^"$bench_name" "$test_out" \
    | awk '{ if (NF >= 10 && $8 == "seconds") { print "V2" } else { print "V1" } }' )
  if [ $format_version == "V1" ]; then
    ops_sec=$( grep ^"$bench_name" "$test_out" | awk '{ print $5 }' )
    usecs_op=$( grep ^"$bench_name" "$test_out" | awk '{ printf "%.1f", $3 }' )
    if [ "$bench_name" == "multireadrandom" ]; then
      mb_sec="NA"
    else
      mb_sec=$( grep ^"$bench_name" "$test_out" | awk '{ print $7 }' )
    fi
  else
    ops_sec=$( grep ^"$bench_name" "$test_out" | awk '{ print $5 }' )
    usecs_op=$( grep ^"$bench_name" "$test_out" | awk '{ printf "%.1f", $3 }' )
    mb_sec=$( grep ^"$bench_name" "$test_out" | awk '{ print $11 }' )
  fi

  # For RocksDB version 4.x there are fewer fields but this still parses correctly
  # Cumulative writes: 242M writes, 242M keys, 18M commit groups, 12.9 writes per commit group, ingest: 95.96 GB, 54.69 MB/s
  cum_writes_gb_orig=$( grep "^Cumulative writes" "$test_out" | tail -1 | awk '{ for (x=1; x<=NF; x++) { if ($x == "ingest:") { printf "%.1f", $(x+1) } } }' )
  cum_writes_units=$( grep "^Cumulative writes" "$test_out" | tail -1 | awk '{ for (x=1; x<=NF; x++) { if ($x == "ingest:") { print $(x+2) } } }' | sed 's/,//g' )
  cum_writes_gb=$( units_as_gb "$cum_writes_gb_orig" "$cum_writes_units" )

  # Cumulative compaction: 1159.74 GB write, 661.03 MB/s write, 1108.89 GB read, 632.04 MB/s read, 6284.3 seconds
  cmb_ps=$( grep "^Cumulative compaction" "$test_out" | tail -1 | awk '{ printf "%.1f", $6 }' )
  sum_wgb_orig=$( grep "^Cumulative compaction" "$test_out" | tail -1 | awk '{ printf "%.1f", $3 }' )
  sum_wgb_units=$( grep "^Cumulative compaction" "$test_out" | tail -1 | awk '{ print $4 }' )
  sum_wgb=$( units_as_gb "$sum_wgb_orig" "$sum_wgb_units" )

  # Flush(GB): cumulative 97.193, interval 1.247
  flush_wgb=$( grep "^Flush(GB)" "$test_out" | tail -1 | awk '{ print $3 }' | tr ',' ' ' | awk '{ print $1 }' )

  if [[ "$sum_wgb" == "NA" || \
        "$cum_writes_gb" == "NA" || \
        "$cum_writes_gb_orig" == "0.0" || \
        -z "$cum_writes_gb_orig" || \
        -z "$flush_wgb" ]]; then
    wamp="NA"
  else
    wamp=$( echo "( $sum_wgb + $flush_wgb ) / $cum_writes_gb" | bc -l | awk '{ printf "%.1f", $1 }' )
  fi

  c_wsecs=$( grep "^ Sum" $test_out | tail -1 | awk '{ printf "%.0f", $15 }' )
  c_csecs=$( grep "^ Sum" $test_out | tail -1 | awk '{ printf "%.0f", $16 }' )

  lsm_size=$( grep "^ Sum" "$test_out" | tail -1 | awk '{ printf "%.0f%s", $3, $4 }' )
  blob_size=$( grep "^Blob file count:" "$test_out" | tail -1 | awk '{ printf "%.0f%s", $7, $8 }' )
  # Remove the trailing comma from blob_size: 3.0GB, -> 3.0GB
  blob_size="${blob_size/,/}"

  b_rgb=$( grep "^ Sum" $test_out | tail -1 | awk '{ printf "%.0f", $21 }' )
  b_wgb=$( grep "^ Sum" $test_out | tail -1 | awk '{ printf "%.0f", $22 }' )

  p50=$( grep "^Percentiles:" $test_out | tail -1 | awk '{ printf "%.1f", $3 }' )
  p99=$( grep "^Percentiles:" $test_out | tail -1 | awk '{ printf "%.0f", $7 }' )
  p999=$( grep "^Percentiles:" $test_out | tail -1 | awk '{ printf "%.0f", $9 }' )
  p9999=$( grep "^Percentiles:" $test_out | tail -1 | awk '{ printf "%.0f", $11 }' )
  pmax=$( grep "^Min: " $test_out | grep Median: | grep Max: | awk '{ printf "%.0f", $6 }' )

  # Use the last line because there might be extra lines when the db_bench process exits with an error
  time_out="$test_out".time
  u_cpu=$( tail -1 "$time_out" | awk '{ printf "%.1f", $2 / 1000.0 }' )
  s_cpu=$( tail -1 "$time_out" | awk '{ printf "%.1f", $3 / 1000.0  }' )

  rss="NA"
  if [ -f $test_out.stats.ps ]; then
    rss=$( awk '{ printf "%.1f\n", $6 / (1024 * 1024) }' "$test_out".stats.ps | sort -n | tail -1 )
  fi

  # if the report TSV (Tab Separate Values) file does not yet exist, create it and write the header row to it
  if [ ! -f "$report" ]; then
    echo -e "# ops_sec - operations per second" >> "$report"
    echo -e "# mb_sec - ops_sec * size-of-operation-in-MB" >> "$report"
    echo -e "# lsm_sz - size of LSM tree" >> "$report"
    echo -e "# blob_sz - size of BlobDB logs" >> "$report"
    echo -e "# c_wgb - GB written by compaction" >> "$report"
    echo -e "# w_amp - Write-amplification as (bytes written by compaction / bytes written by memtable flush)" >> "$report"
    echo -e "# c_mbps - Average write rate for compaction" >> "$report"
    echo -e "# c_wsecs - Wall clock seconds doing compaction" >> "$report"
    echo -e "# c_csecs - CPU seconds doing compaction" >> "$report"
    echo -e "# b_rgb - Blob compaction read GB" >> "$report"
    echo -e "# b_wgb - Blob compaction write GB" >> "$report"
    echo -e "# usec_op - Microseconds per operation" >> "$report"
    echo -e "# p50, p99, p99.9, p99.99 - 50th, 99th, 99.9th, 99.99th percentile response time in usecs" >> "$report"
    echo -e "# pmax - max response time in usecs" >> "$report"
    echo -e "# uptime - RocksDB uptime in seconds" >> "$report"
    echo -e "# stall% - Percentage of time writes are stalled" >> "$report"
    echo -e "# Nstall - Number of stalls" >> "$report"
    echo -e "# u_cpu - #seconds/1000 of user CPU" >> "$report"
    echo -e "# s_cpu - #seconds/1000 of system CPU" >> "$report"
    echo -e "# rss - max RSS in GB for db_bench process" >> "$report"
    echo -e "# test - Name of test" >> "$report"
    echo -e "# date - Date/time of test" >> "$report"
    echo -e "# version - RocksDB version" >> "$report"
    echo -e "# job_id - User-provided job ID" >> "$report"
    echo -e "# githash - git hash at which db_bench was compiled" >> "$report"
    echo -e $tsv_header >> "$report"
  fi

  echo -e "$ops_sec\t$mb_sec\t$lsm_size\t$blob_size\t$sum_wgb\t$wamp\t$cmb_ps\t$c_wsecs\t$c_csecs\t$b_rgb\t$b_wgb\t$usecs_op\t$p50\t$p99\t$p999\t$p9999\t$pmax\t$uptime\t$stall_pct\t$nstall\t$u_cpu\t$s_cpu\t$rss\t$test_name\t$my_date\t$version\t$job_id\t$git_hash" \
    >> "$report"
}

function run_bulkload {
  # This runs with a vector memtable and the WAL disabled to load faster. It is still crash safe and the
  # client can discover where to restart a load after a crash. I think this is a good way to load.
  echo "Bulk loading $num_keys random keys"
  log_file_name=$output_dir/benchmark_bulkload_fillrandom.log
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=fillrandom,stats \
       --use_existing_db=0 \
       --disable_auto_compactions=1 \
       --sync=0 \
       $params_bulkload \
       --threads=1 \
       --memtablerep=vector \
       --allow_concurrent_memtable_write=false \
       --disable_wal=1 \
       --seed=$( date +%s ) \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  eval $cmd
  summarize_result $log_file_name bulkload fillrandom

  echo "Compacting..."
  log_file_name=$output_dir/benchmark_bulkload_compact.log
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=compact,stats \
       --use_existing_db=1 \
       --disable_auto_compactions=1 \
       --sync=0 \
       $params_w \
       --threads=1 \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  eval $cmd
}

#
# Parameter description:
#
# $1 - 1 if I/O statistics should be collected.
# $2 - compaction type to use (level=0, universal=1).
# $3 - number of subcompactions.
# $4 - number of maximum background compactions.
#
function run_manual_compaction_worker {
  # This runs with a vector memtable and the WAL disabled to load faster.
  # It is still crash safe and the client can discover where to restart a
  # load after a crash. I think this is a good way to load.
  echo "Bulk loading $num_keys random keys for manual compaction."

  log_file_name=$output_dir/benchmark_man_compact_fillrandom_$3.log

  if [ "$2" == "1" ]; then
    extra_params=$params_univ_compact
  else
    extra_params=$params_level_compact
  fi

  # Make sure that fillrandom uses the same compaction options as compact.
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=fillrandom,stats \
       --use_existing_db=0 \
       --disable_auto_compactions=0 \
       --sync=0 \
       $extra_params \
       --threads=$num_threads \
       --compaction_measure_io_stats=$1 \
       --compaction_style=$2 \
       --subcompactions=$3 \
       --memtablerep=vector \
       --allow_concurrent_memtable_write=false \
       --disable_wal=1 \
       --max_background_compactions=$4 \
       --seed=$( date +%s ) \
       2>&1 | tee -a $log_file_name"

  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  eval $cmd

  summarize_result $log_file_namefillrandom_output_file man_compact_fillrandom_$3 fillrandom

  echo "Compacting with $3 subcompactions specified ..."

  log_file_name=$output_dir/benchmark_man_compact_$3.log

  # This is the part we're really interested in. Given that compact benchmark
  # doesn't output regular statistics then we'll just use the time command to
  # measure how long this step takes.
  cmd="{ \
       time ./db_bench --benchmarks=compact,stats \
       --use_existing_db=1 \
       --disable_auto_compactions=0 \
       --sync=0 \
       $extra_params \
       --threads=$num_threads \
       --compaction_measure_io_stats=$1 \
       --compaction_style=$2 \
       --subcompactions=$3 \
       --max_background_compactions=$4 \
       ;}
       2>&1 | tee -a $log_file_name"

  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  eval $cmd

  # Can't use summarize_result here. One way to analyze the results is to run
  # "grep real" on the resulting log files.
}

function run_univ_compaction {
  # Always ask for I/O statistics to be measured.
  io_stats=1

  # Values: kCompactionStyleLevel = 0x0, kCompactionStyleUniversal = 0x1.
  compaction_style=1

  # Define a set of benchmarks.
  subcompactions=(1 2 4 8 16)
  max_background_compactions=(16 16 8 4 2)

  i=0
  total=${#subcompactions[@]}

  # Execute a set of benchmarks to cover variety of scenarios.
  while [ "$i" -lt "$total" ]
  do
    run_manual_compaction_worker $io_stats $compaction_style ${subcompactions[$i]} \
      ${max_background_compactions[$i]}
    ((i++))
  done
}

function run_fillseq {
  # This runs with a vector memtable. WAL can be either disabled or enabled
  # depending on the input parameter (1 for disabled, 0 for enabled). The main
  # benefit behind disabling WAL is to make loading faster. It is still crash
  # safe and the client can discover where to restart a load after a crash. I
  # think this is a good way to load.

  # Make sure that we'll have unique names for all the files so that data won't
  # be overwritten.
  if [ $1 == 1 ]; then
    log_file_name="${output_dir}/benchmark_fillseq.wal_disabled.v${value_size}.log"
    test_name=fillseq.wal_disabled.v${value_size}
  else
    log_file_name="${output_dir}/benchmark_fillseq.wal_enabled.v${value_size}.log"
    test_name=fillseq.wal_enabled.v${value_size}
  fi

  # For Leveled compaction hardwire this to 0 so that data that is trivial-moved
  # to larger levels (3, 4, etc) will be compressed.
  if [ $compaction_style == "leveled" ]; then
    comp_arg="--min_level_to_compress=0"
  elif [ $compaction_style == "universal" ]; then
    if [ ! -z $UNIVERSAL_ALLOW_TRIVIAL_MOVE ]; then
      # See GetCompressionFlush where compression_size_percent < 1 means use the default
      # compression which is needed because trivial moves are enabled
      comp_arg="--universal_compression_size_percent=-1"
    else
      # See GetCompressionFlush where compression_size_percent > 0 means no compression.
      # Don't set anything here because compression_size_percent is set in univ_const_params
      comp_arg=""
    fi
  else
    # compaction_style == "blob"
    comp_arg="--min_level_to_compress=0"
  fi

  echo "Loading $num_keys keys sequentially"
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=fillseq,stats \
       $params_fillseq \
       $comp_arg \
       --use_existing_db=0 \
       --sync=0 \
       --threads=1 \
       --memtablerep=vector \
       --allow_concurrent_memtable_write=false \
       --disable_wal=$1 \
       --seed=$( date +%s ) \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  start_stats $log_file_name.stats
  eval $cmd
  stop_stats $log_file_name.stats

  # The constant "fillseq" which we pass to db_bench is the benchmark name.
  summarize_result $log_file_name $test_name fillseq
}

function run_lsm {
  # This flushes the memtable and L0 to get the LSM tree into a deterministic
  # state for read-only tests that will follow.
  echo "Flush memtable, wait, compact L0, wait"
  job=$1

  if [ $job = flush_mt_l0 ]; then
    benchmarks=levelstats,flush,waitforcompaction,compact0,waitforcompaction,memstats,levelstats
  elif [ $job = waitforcompaction ]; then
    benchmarks=levelstats,waitforcompaction,memstats,levelstats
  else
    echo Job unknown: $job
    exit $EXIT_NOT_COMPACTION_TEST
  fi

  log_file_name=$output_dir/benchmark_${job}.log
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=$benchmarks,stats \
       --use_existing_db=1 \
       --sync=0 \
       $params_w \
       --threads=1 \
       --seed=$( date +%s ) \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  start_stats $log_file_name.stats
  # waitforcompaction can hang with universal (compaction_style=1)
  # see bug https://github.com/facebook/rocksdb/issues/9275
  eval $cmd
  stop_stats $log_file_name.stats
  # Don't summarize, the log doesn't have the output needed for it
}

function run_change {
  output_name=$1
  grep_name=$2
  benchmarks=$3
  echo "Do $num_keys random $output_name"
  log_file_name="$output_dir/benchmark_${output_name}.t${num_threads}.s${syncval}.log"
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=$benchmarks,stats \
       --use_existing_db=1 \
       --sync=$syncval \
       $params_w \
       --threads=$num_threads \
       --merge_operator=\"put\" \
       --seed=$( date +%s ) \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  start_stats $log_file_name.stats
  eval $cmd
  stop_stats $log_file_name.stats
  summarize_result $log_file_name ${output_name}.t${num_threads}.s${syncval} $grep_name
}

function run_filluniquerandom {
  echo "Loading $num_keys unique keys randomly"
  log_file_name=$output_dir/benchmark_filluniquerandom.log
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=filluniquerandom,stats \
       --use_existing_db=0 \
       --sync=0 \
       $params_w \
       --threads=1 \
       --seed=$( date +%s ) \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  start_stats $log_file_name.stats
  eval $cmd
  stop_stats $log_file_name.stats
  summarize_result $log_file_name filluniquerandom filluniquerandom
}

function run_readrandom {
  echo "Reading $num_keys random keys"
  log_file_name="${output_dir}/benchmark_readrandom.t${num_threads}.log"
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=readrandom,stats \
       --use_existing_db=1 \
       $params_w \
       --threads=$num_threads \
       --seed=$( date +%s ) \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  start_stats $log_file_name.stats
  eval $cmd
  stop_stats $log_file_name.stats
  summarize_result $log_file_name readrandom.t${num_threads} readrandom
}

function run_multireadrandom {
  echo "Multi-Reading $num_keys random keys"
  log_file_name="${output_dir}/benchmark_multireadrandom.t${num_threads}.log"
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=multireadrandom,stats \
       --use_existing_db=1 \
       --threads=$num_threads \
       --batch_size=10 \
       $params_w \
       --seed=$( date +%s ) \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  start_stats $log_file_name.stats
  eval $cmd
  stop_stats $log_file_name.stats
  summarize_result $log_file_name multireadrandom.t${num_threads} multireadrandom
}

function run_readwhile {
  operation=$1
  echo "Reading $num_keys random keys while $operation"
  log_file_name="${output_dir}/benchmark_readwhile${operation}.t${num_threads}.log"
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench --benchmarks=readwhile${operation},stats \
       --use_existing_db=1 \
       --sync=$syncval \
       $params_w \
       --threads=$num_threads \
       --merge_operator=\"put\" \
       --seed=$( date +%s ) \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  start_stats $log_file_name.stats
  eval $cmd
  stop_stats $log_file_name.stats
  summarize_result $log_file_name readwhile${operation}.t${num_threads} readwhile${operation}
}

function run_rangewhile {
  operation=$1
  full_name=$2
  reverse_arg=$3
  log_file_name="${output_dir}/benchmark_${full_name}.t${num_threads}.log"
  time_cmd=$( get_cmd $log_file_name.time )
  echo "Range scan $num_keys random keys while ${operation} for reverse_iter=${reverse_arg}"
  cmd="$time_cmd ./db_bench --benchmarks=seekrandomwhile${operation},stats \
       --use_existing_db=1 \
       --sync=$syncval \
       $params_w \
       --threads=$num_threads \
       --merge_operator=\"put\" \
       --seek_nexts=$num_nexts_per_seek \
       --reverse_iterator=$reverse_arg \
       --seed=$( date +%s ) \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee -a $log_file_name"
  echo $cmd | tee $log_file_name
  start_stats $log_file_name.stats
  eval $cmd
  stop_stats $log_file_name.stats
  summarize_result $log_file_name ${full_name}.t${num_threads} seekrandomwhile${operation}
}

function run_range {
  full_name=$1
  reverse_arg=$2
  log_file_name="${output_dir}/benchmark_${full_name}.t${num_threads}.log"
  time_cmd=$( get_cmd $log_file_name.time )
  echo "Range scan $num_keys random keys for reverse_iter=${reverse_arg}"
  cmd="$time_cmd ./db_bench --benchmarks=seekrandom,stats \
       --use_existing_db=1 \
       $params_w \
       --threads=$num_threads \
       --seek_nexts=$num_nexts_per_seek \
       --reverse_iterator=$reverse_arg \
       --seed=$( date +%s ) \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee -a $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  start_stats $log_file_name.stats
  eval $cmd
  stop_stats $log_file_name.stats
  summarize_result $log_file_name ${full_name}.t${num_threads} seekrandom
}

function run_randomtransaction {
  echo "..."
  log_file_name=$output_dir/benchmark_randomtransaction.log
  time_cmd=$( get_cmd $log_file_name.time )
  cmd="$time_cmd ./db_bench $params_w --benchmarks=randomtransaction,stats \
       --num=$num_keys \
       --transaction_db \
       --threads=5 \
       --transaction_sets=5 \
       --report_file=${log_file_name}.r.csv \
       2>&1 | tee $log_file_name"
  if [[ "$job_id" != "" ]]; then
    echo "Job ID: ${job_id}" > $log_file_name
    echo $cmd | tee -a $log_file_name
  else
    echo $cmd | tee $log_file_name
  fi
  start_stats $log_file_name.stats
  eval $cmd
  stop_stats $log_file_name.stats
}

function now() {
  echo `date +"%s"`
}

echo "===== Benchmark ====="

# Run!!!
IFS=',' read -a jobs <<< $bench_cmd
# shellcheck disable=SC2068
for job in ${jobs[@]}; do

  if [ $job != debug ]; then
    echo "Starting $job (ID: $job_id) at `date`" | tee -a $schedule
  fi

  start=$(now)
  if [ $job = bulkload ]; then
    run_bulkload
  elif [ $job = flush_mt_l0 ]; then
    run_lsm flush_mt_l0
  elif [ $job = waitforcompaction ]; then
    run_lsm waitforcompaction
  elif [ $job = fillseq_disable_wal ]; then
    run_fillseq 1
  elif [ $job = fillseq_enable_wal ]; then
    run_fillseq 0
  elif [ $job = overwrite ]; then
    run_change overwrite overwrite overwrite
  elif [ $job = overwritesome ]; then
    # This uses a different name for overwrite results so it can be run twice in one benchmark run.
    run_change overwritesome overwrite overwrite
  elif [ $job = overwriteandwait ]; then
    run_change overwriteandwait overwrite overwrite,waitforcompaction
  elif [ $job = updaterandom ]; then
    run_change updaterandom updaterandom updaterandom
  elif [ $job = mergerandom ]; then
    run_change mergerandom mergerandom mergerandom
  elif [ $job = filluniquerandom ]; then
    run_filluniquerandom
  elif [ $job = readrandom ]; then
    run_readrandom
  elif [ $job = multireadrandom ]; then
    run_multireadrandom
  elif [ $job = fwdrange ]; then
    run_range $job false
  elif [ $job = revrange ]; then
    run_range $job true
  elif [ $job = readwhilewriting ]; then
    run_readwhile writing
  elif [ $job = readwhilemerging ]; then
    run_readwhile merging
  elif [ $job = fwdrangewhilewriting ]; then
    run_rangewhile writing $job false
  elif [ $job = revrangewhilewriting ]; then
    run_rangewhile writing $job true
  elif [ $job = fwdrangewhilemerging ]; then
    run_rangewhile merging $job false
  elif [ $job = revrangewhilemerging ]; then
    run_rangewhile merging $job true
  elif [ $job = randomtransaction ]; then
    run_randomtransaction
  elif [ $job = universal_compaction ]; then
    run_univ_compaction
  elif [ $job = debug ]; then
    num_keys=1000; # debug
    echo "Setting num_keys to $num_keys"
  else
    echo "unknown job $job"
    exit $EXIT_UNKNOWN_JOB
  fi
  end=$(now)

  if [ $job != debug ]; then
    echo "Completed $job (ID: $job_id) in $((end-start)) seconds" | tee -a $schedule
  fi

  echo -e $tsv_header
  tail -1 $report

done


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
# REQUIRE: db_bench binary exists in the current directory

dbdir=$1
odir=$2

# Size Constants
K=1024
M=$((1024 * K))

# Dynamic loader configuration
ld_library_path=${LD_LIBRARY_PATH:-""}

# Benchmark configuration
duration_rw=${DURATION_RW:-65}
duration_ro=${DURATION_RO:-65}
num_keys=${NUM_KEYS:-1000000}
num_threads=${NUM_THREADS:-16}
key_size=${KEY_SIZE:-20}
value_size=${VALUE_SIZE:-400}
mb_write_per_sec=${MB_WRITE_PER_SEC:-2}
ci_tests_only=${CI_TESTS_ONLY:-"false"}

# RocksDB configuration
compression_type=${COMPRESSION_TYPE:-lz4}
subcompactions=${SUBCOMPACTIONS:-1}
write_buffer_size_mb=${WRITE_BUFFER_SIZE_MB:-32}
target_file_size_base_mb=${TARGET_FILE_SIZE_BASE_MB:-32}
max_bytes_for_level_base_mb=${MAX_BYTES_FOR_LEVEL_BASE_MB:-128}
max_background_jobs=${MAX_BACKGROUND_JOBS:-8}
stats_interval_seconds=${STATS_INTERVAL_SECONDS:-20}
cache_index_and_filter_blocks=${CACHE_INDEX_AND_FILTER_BLOCKS:-0}
# USE_O_DIRECT doesn't need a default
bytes_per_sync=${BYTES_PER_SYNC:-$(( 1 * M ))}
# CACHE_SIZE_MB doesn't need a default
min_level_to_compress=${MIN_LEVEL_TO_COMPRESS:-"-1"}

compaction_style=${COMPACTION_STYLE:-leveled}
if [ "$compaction_style" = "leveled" ]; then
  echo Use leveled compaction
elif [ "$compaction_style" = "universal" ]; then
  echo Use universal compaction
elif [ "$compaction_style" = "blob" ]; then
  echo Use blob compaction
else
  echo COMPACTION_STYLE is :: "$COMPACTION_STYLE" :: and must be one of leveled, universal, blob
  exit 1
fi

# Leveled compaction configuration
level0_file_num_compaction_trigger=${LEVEL0_FILE_NUM_COMPACTION_TRIGGER:-4}
level0_slowdown_writes_trigger=${LEVEL0_SLOWDOWN_WRITES_TRIGGER:-20}
level0_stop_writes_trigger=${LEVEL0_STOP_WRITES_TRIGGER:-30}
per_level_fanout=${PER_LEVEL_FANOUT:-8}

# Universal compaction configuration
universal_min_merge_width=${UNIVERSAL_MIN_MERGE_WIDTH:-2}
universal_max_merge_width=${UNIVERSAL_MAX_MERGE_WIDTH:-20}
universal_size_ratio=${UNIVERSAL_SIZE_RATIO:-1}
universal_max_size_amp=${UNIVERSAL_MAX_SIZE_AMP:-200}
universal_compression_size_percent=${UNIVERSAL_COMPRESSION_SIZE_PERCENT:-"-1"}

# Integrated BlobDB configuration

min_blob_size=${MIN_BLOB_SIZE:-0}
blob_file_size=${BLOB_FILE_SIZE:-$(( 256 * M ))}
blob_compression_type=${BLOB_COMPRESSION_TYPE:-${compression_type}}
blob_gc_age_cutoff=${BLOB_GC_AGE_CUTOFF:-"0.25"}
blob_gc_force_threshold=${BLOB_GC_FORCE_THRESHOLD:-1}

# Arguments for dynamic loading
base_args=( LD_LIBRARY_PATH="$ld_library_path" )

# Arguments used for all tests
base_args+=( NUM_KEYS="$num_keys" )
base_args+=( NUM_THREADS="$num_threads" )
base_args+=( KEY_SIZE="$key_size" )
base_args+=( VALUE_SIZE="$value_size" )

base_args+=( SUBCOMPACTIONS="$subcompactions" )
base_args+=( COMPRESSION_TYPE="$compression_type" )
base_args+=( WRITE_BUFFER_SIZE_MB="$write_buffer_size_mb" )
base_args+=( TARGET_FILE_SIZE_BASE_MB="$target_file_size_base_mb" )
base_args+=( MAX_BYTES_FOR_LEVEL_BASE_MB="$max_bytes_for_level_base_mb" )
base_args+=( MAX_BACKGROUND_JOBS="$max_background_jobs" )
base_args+=( STATS_INTERVAL_SECONDS="$stats_interval_seconds" )
base_args+=( CACHE_INDEX_AND_FILTER_BLOCKS="$cache_index_and_filter_blocks" )
base_args+=( COMPACTION_STYLE="$compaction_style" )
base_args+=( BYTES_PER_SYNC="$bytes_per_sync" )

if [ -n "$USE_O_DIRECT" ]; then
  base_args+=( USE_O_DIRECT=1 )
fi

if [ -n "$NUMA" ]; then
  base_args+=( NUMACTL=1 )
fi

if [ -n "$CACHE_SIZE_MB" ]; then
  cacheb=$(( CACHE_SIZE_MB * M ))
  base_args+=( CACHE_SIZE="$cacheb" )
fi

if [ "$compaction_style" == "leveled" ]; then
  base_args+=( LEVEL0_FILE_NUM_COMPACTION_TRIGGER="$level0_file_num_compaction_trigger" )
  base_args+=( LEVEL0_SLOWDOWN_WRITES_TRIGGER="$level0_slowdown_writes_trigger" )
  base_args+=( LEVEL0_STOP_WRITES_TRIGGER="$level0_stop_writes_trigger" )
  base_args+=( PER_LEVEL_FANOUT="$per_level_fanout" )
elif [ "$compaction_style" == "universal" ]; then
  base_args+=( LEVEL0_FILE_NUM_COMPACTION_TRIGGER="$level0_file_num_compaction_trigger" )
  base_args+=( LEVEL0_SLOWDOWN_WRITES_TRIGGER="$level0_slowdown_writes_trigger" )
  base_args+=( LEVEL0_STOP_WRITES_TRIGGER="$level0_stop_writes_trigger" )
  base_args+=( UNIVERSAL_MIN_MERGE_WIDTH="$universal_min_merge_width" )
  base_args+=( UNIVERSAL_MAX_MERGE_WIDTH="$universal_max_merge_width" )
  base_args+=( UNIVERSAL_SIZE_RATIO="$universal_size_ratio" )
  base_args+=( UNIVERSAL_MAX_SIZE_AMP="$universal_max_size_amp" )
  if [ -n "$UNIVERSAL_ALLOW_TRIVIAL_MOVE" ]; then
    base_args+=( UNIVERSAL_ALLOW_TRIVIAL_MOVE=1 )
  fi
else
  # Inherit settings for leveled because index uses leveled LSM
  base_args+=( LEVEL0_FILE_NUM_COMPACTION_TRIGGER="$level0_file_num_compaction_trigger" )
  base_args+=( LEVEL0_SLOWDOWN_WRITES_TRIGGER="$level0_slowdown_writes_trigger" )
  base_args+=( LEVEL0_STOP_WRITES_TRIGGER="$level0_stop_writes_trigger" )
  base_args+=( PER_LEVEL_FANOUT="$per_level_fanout" )
  # Then add BlobDB specific settings
  base_args+=( MIN_BLOB_SIZE="$min_blob_size" )
  base_args+=( BLOB_FILE_SIZE="$blob_file_size" )
  base_args+=( BLOB_COMPRESSION_TYPE="$blob_compression_type" )
  base_args+=( BLOB_GC_AGE_CUTOFF="$blob_gc_age_cutoff" )
  base_args+=( BLOB_GC_FORCE_THRESHOLD="$blob_gc_force_threshold" )
fi

function usage {
  echo "usage: benchmark_compare.sh db_dir output_dir version+"
  echo -e "\tdb_dir\t\tcreate RocksDB database in this directory"
  echo -e "\toutput_dir\twrite output from performance tests in this directory"
  echo -e "\tversion+\tspace separated sequence of RocksDB versions to test."
  echo -e "\nThis expects that db_bench.\$version exists in \$PWD for each version in the sequence."
  echo -e "An example value for version+ is 6.23.0 6.24.0"
  echo ""
  echo -e "Environment variables for options"
  echo -e "\tNUM_KEYS\t\t\tnumber of keys to load"
  echo -e "\tKEY_SIZE\t\t\tsize of key"
  echo -e "\tVALUE_SIZE\t\t\tsize of value"
  echo -e "\tCACHE_SIZE_MB\t\t\tsize of block cache in MB"
  echo -e "\tDURATION_RW\t\t\tnumber of seconds for which each test runs, except for read-only tests"
  echo -e "\tDURATION_RO\t\t\tnumber of seconds for which each read-only test runs"
  echo -e "\tMB_WRITE_PER_SEC\t\trate limit for writer that runs concurrent with queries for some tests"
  echo -e "\tNUM_THREADS\t\t\tnumber of user threads"
  echo -e "\tCOMPRESSION_TYPE\t\tcompression type (zstd, lz4, none, etc)"
  echo -e "\tMIN_LEVEL_TO_COMPRESS\t\tmin_level_to_compress for leveled"
  echo -e "\tWRITE_BUFFER_SIZE_MB\t\tsize of write buffer in MB"
  echo -e "\tTARGET_FILE_SIZE_BASE_MB\tvalue for target_file_size_base in MB"
  echo -e "\tMAX_BYTES_FOR_LEVEL_BASE_MB\tvalue for max_bytes_for_level_base in MB"
  echo -e "\tMAX_BACKGROUND_JOBS\t\tvalue for max_background_jobs"
  echo -e "\tCACHE_INDEX_AND_FILTER_BLOCKS\tvalue for cache_index_and_filter_blocks"
  echo -e "\tUSE_O_DIRECT\t\t\tUse O_DIRECT for user reads and compaction"
  echo -e "\tBYTES_PER_SYNC\t\t\tValue for bytes_per_sync"
  echo -e "\tSTATS_INTERVAL_SECONDS\t\tvalue for stats_interval_seconds"
  echo -e "\tSUBCOMPACTIONS\t\t\tvalue for subcompactions"
  echo -e "\tCOMPACTION_STYLE\t\tCompaction style to use, one of: leveled, universal, blob"
  echo -e "\tCI_TESTS_ONLY\t\tRun a subset of tests tailored to a CI regression job, one of: true, false (default)"
  echo ""
  echo -e "\tOptions specific to leveled compaction:"
  echo -e "\t\tLEVEL0_FILE_NUM_COMPACTION_TRIGGER\tvalue for level0_file_num_compaction_trigger"
  echo -e "\t\tLEVEL0_SLOWDOWN_WRITES_TRIGGER\t\tvalue for level0_slowdown_writes_trigger"
  echo -e "\t\tLEVEL0_STOP_WRITES_TRIGGER\t\tvalue for level0_stop_writes_trigger"
  echo -e "\t\tPER_LEVEL_FANOUT\t\t\tvalue for max_bytes_for_level_multiplier"
  echo ""
  echo -e "\tOptions specific to universal compaction:"
  echo -e "\t\tSee LEVEL0_*_TRIGGER above"
  echo -e "\t\tUNIVERSAL_MIN_MERGE_WIDTH\t\tvalue of min_merge_width option for universal"
  echo -e "\t\tUNIVERSAL_MAX_MERGE_WIDTH\t\tvalue of min_merge_width option for universal"
  echo -e "\t\tUNIVERSAL_SIZE_RATIO\t\t\tvalue of size_ratio option for universal"
  echo -e "\t\tUNIVERSAL_MAX_SIZE_AMP\t\t\tmax_size_amplification_percent for universal"
  echo -e "\t\tUNIVERSAL_ALLOW_TRIVIAL_MOVE\t\tSet allow_trivial_move to true for universal, default is false"
  echo -e "\t\tUNIVERSAL_COMPRESSION_SIZE_PERCENT\tpercentage of LSM tree that should be compressed"
  echo ""
  echo -e "\tOptions for integrated BlobDB:"
  echo -e "\t\tMIN_BLOB_SIZE\t\t\t\tvalue for min_blob_size"
  echo -e "\t\tBLOB_FILE_SIZE\t\t\t\tvalue for blob_file_size"
  echo -e "\t\tBLOB_COMPRESSION_TYPE\t\t\tvalue for blob_compression_type"
  echo -e "\t\tBLOB_GC_AGE_CUTOFF\t\t\tvalue for blog_garbage_collection_age_cutoff"
  echo -e "\t\tBLOB_GC_FORCE_THRESHOLD\t\t\tvalue for blog_garbage_collection_force_threshold"
}

function dump_env {
  echo "Base args" > "$odir"/args
  echo "${base_args[@]}" | tr ' ' '\n' >> "$odir"/args

  echo -e "\nOther args" >> "$odir"/args
  echo -e "dbdir\t$dbdir" >> "$odir"/args
  echo -e "duration_rw\t$duration_rw" >> "$odir"/args
  echo -e "duration_ro\t$duration_ro" >> "$odir"/args
  echo -e "per_level_fanout\t$per_level_fanout" >> "$odir"/args

  echo -e "\nargs_load:" >> "$odir"/args
  echo "${args_load[@]}" | tr ' ' '\n' >> "$odir"/args
  echo -e "\nargs_nolim:" >> "$odir"/args
  echo "${args_nolim[@]}" | tr ' ' '\n' >> "$odir"/args
  echo -e "\nargs_lim:" >> "$odir"/args
  echo "${args_lim[@]}" | tr ' ' '\n' >> "$odir"/args
}

if [ $# -lt 3 ]; then
  usage
  echo
  echo "Need at least 3 arguments"
  exit 1
fi

shift 2

mkdir -p "$odir"

echo Test versions: "$@"
echo Test versions: "$@" >> "$odir"/args

for v in "$@" ; do
  my_odir="$odir"/"$v"

  if [ -d "$my_odir" ]; then
    echo Exiting because the output directory exists: "$my_odir"
    exit 1
  fi

  args_common=("${base_args[@]}")

  args_common+=( OUTPUT_DIR="$my_odir" DB_DIR="$dbdir" WAL_DIR="$dbdir" DB_BENCH_NO_SYNC=1 )

  if [ "$compaction_style" == "leveled" ]; then
    args_common+=( MIN_LEVEL_TO_COMPRESS="$min_level_to_compress" )
  elif [ "$compaction_style" == "universal" ]; then
    args_common+=( UNIVERSAL=1 COMPRESSION_SIZE_PERCENT="$universal_compression_size_percent" )
  else
    args_common+=( MIN_LEVEL_TO_COMPRESS="$min_level_to_compress" )
  fi

  args_load=("${args_common[@]}")

  args_nolim=("${args_common[@]}")

  args_lim=("${args_nolim[@]}")
  args_lim+=( MB_WRITE_PER_SEC="$mb_write_per_sec" )

  dump_env

  echo Run benchmark for "$v" at "$( date )" with results at "$my_odir"
  rm -f db_bench
  echo ln -s db_bench."$v" db_bench
  ln -s db_bench."$v" db_bench

  find "$dbdir" -type f -exec rm \{\} \;

  # Load in key order
  echo env "${args_load[@]}" bash ./benchmark.sh fillseq_disable_wal
  env -i "${args_load[@]}" bash ./benchmark.sh fillseq_disable_wal

  # Read-only tests. The LSM tree shape is in a deterministic state if trivial move
  # was used during the load.

  # Add revrange with a fixed duration and hardwired number of keys and threads to give
  # compaction debt leftover from fillseq a chance at being removed. Not using waitforcompaction
  # here because it isn't supported on older db_bench versions.
  env -i "${args_nolim[@]}" DURATION=300 NUM_KEYS=100 NUM_THREADS=1 bash ./benchmark.sh revrange
  env -i "${args_nolim[@]}" DURATION="$duration_ro" bash ./benchmark.sh readrandom

  # Skipped for CI - a single essentail readrandom is enough to set up for other tests
  if [ "$ci_tests_only" != "true" ]; then
    env -i "${args_nolim[@]}" DURATION="$duration_ro" bash ./benchmark.sh fwdrange
    env -i "${args_lim[@]}"   DURATION="$duration_ro" bash ./benchmark.sh multireadrandom --multiread_batched
  else
    echo "CI_TESTS_ONLY is set, skipping optional read steps."
  fi

  # Write 10% of the keys. The goal is to randomize keys prior to Lmax
  p10=$( echo "$num_keys" "$num_threads" | awk '{ printf "%.0f", $1 / $2 / 10.0 }' )
  env -i "${args_nolim[@]}" WRITES="$p10"        bash ./benchmark.sh overwritesome

  if [ "$compaction_style" == "leveled" ]; then
    # These are not supported by older versions
    # Flush memtable & L0 to get LSM tree into deterministic state
    env -i "${args_nolim[@]}"                  bash ./benchmark.sh flush_mt_l0
  elif [ "$compaction_style" == "universal" ]; then
    # For universal don't compact L0 as can have too many sorted runs
    # waitforcompaction can hang, see https://github.com/facebook/rocksdb/issues/9275
    # While this is disabled the test that follows will have more variance from compaction debt.
    # env -i "${args_nolim[@]}"                    bash ./benchmark.sh waitforcompaction
    echo TODO enable when waitforcompaction hang is fixed
  else
    # These are not supported by older versions
    # Flush memtable & L0 to get LSM tree into deterministic state
    env -i "${args_nolim[@]}"                  bash ./benchmark.sh flush_mt_l0
  fi

  # Read-mostly tests with a rate-limited writer
  env -i "${args_lim[@]}" DURATION="$duration_rw" bash ./benchmark.sh revrangewhilewriting
  env -i "${args_lim[@]}" DURATION="$duration_rw" bash ./benchmark.sh fwdrangewhilewriting
  env -i "${args_lim[@]}" DURATION="$duration_rw" bash ./benchmark.sh readwhilewriting

  # Write-only tests

  # This creates much compaction debt which will be a problem for tests added after it.
  # Also, the compaction stats measured at test end can underestimate write-amp depending
  # on how much compaction debt is allowed.
  if [ "$compaction_style" == "leveled" ] && ./db_bench --benchmarks=waitforcompaction ; then
    # Use waitforcompaction to get more accurate write-amp measurement
    env -i "${args_nolim[@]}" DURATION="$duration_rw" bash ./benchmark.sh overwriteandwait
  else
    # waitforcompaction hangs with universal, see https://github.com/facebook/rocksdb/issues/9275
    env -i "${args_nolim[@]}" DURATION="$duration_rw" bash ./benchmark.sh overwrite
  fi

  cp "$dbdir"/LOG* "$my_odir"
  gzip -9 "$my_odir"/LOG*

done

# Generate a file that groups lines from the same test for all versions
basev=$1
nlines=$( awk '/^ops_sec/,/END/' "$odir"/"$basev"/report.tsv | grep -v ops_sec | wc -l )
hline=$( awk '/^ops_sec/ { print NR }' "$odir"/"$basev"/report.tsv )
sline=$(( hline + 1 ))
eline=$(( sline + nlines - 1 ))

sum_file="$odir"/summary.tsv

for v in "$@" ; do
  echo "$odir"/"$v"/report.tsv
done >> "$sum_file"
echo >> "$sum_file"

for x in $( seq "$sline" "$eline" ); do
  awk '{ if (NR == lno) { print $0 } }' lno="$hline" "$odir"/"$basev"/report.tsv >> "$sum_file"
  for v in "$@" ; do
    r="$odir"/"$v"/report.tsv
    awk '{ if (NR == lno) { print $0 } }' lno="$x" "$r" >> "$sum_file"
  done
echo >> "$sum_file"
done


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
# REQUIRE: db_bench binary exists in the current directory
#
# This should be used with the LevelDB fork listed here to use additional test options.
# For more details on the changes see the blog post listed below.
#   https://github.com/mdcallag/leveldb-1
#   http://smalldatum.blogspot.com/2015/04/comparing-leveldb-and-rocksdb-take-2.html

if [ $# -ne 1 ]; then
  echo -n "./benchmark.sh [fillseq/overwrite/readrandom/readwhilewriting]"
  exit 0
fi

# size constants
K=1024
M=$((1024 * K))
G=$((1024 * M))

if [ -z $DB_DIR ]; then
  echo "DB_DIR is not defined"
  exit 0
fi

output_dir=${OUTPUT_DIR:-/tmp/}
if [ ! -d $output_dir ]; then
  mkdir -p $output_dir
fi

# all multithreaded tests run with sync=1 unless
# $DB_BENCH_NO_SYNC is defined
syncval="1"
if [ ! -z $DB_BENCH_NO_SYNC ]; then
  echo "Turning sync off for all multithreaded tests"
  syncval="0";
fi

num_threads=${NUM_THREADS:-16}
# Only for *whilewriting, *whilemerging
writes_per_second=${WRITES_PER_SECOND:-$((10 * K))}
cache_size=${CACHE_SIZE:-$((1 * G))}

num_keys=${NUM_KEYS:-$((1 * G))}
key_size=20
value_size=${VALUE_SIZE:-400}
block_size=${BLOCK_SIZE:-4096}

const_params="
  --db=$DB_DIR \
  \
  --num=$num_keys \
  --value_size=$value_size \
  --cache_size=$cache_size \
  --compression_ratio=0.5 \
  \
  --write_buffer_size=$((2 * M)) \
  \
  --histogram=1 \
  \
  --bloom_bits=10 \
  --open_files=$((20 * K))"

params_w="$const_params "

function summarize_result {
  test_out=$1
  test_name=$2
  bench_name=$3
  nthr=$4

  usecs_op=$( grep ^${bench_name} $test_out | awk '{ printf "%.1f", $3 }' )
  mb_sec=$( grep ^${bench_name} $test_out | awk '{ printf "%.1f", $5 }' )
  ops=$( grep "^Count:" $test_out | awk '{ print $2 }' )
  ops_sec=$( echo "scale=0; (1000000.0 * $nthr) / $usecs_op" | bc )
  avg=$( grep "^Count:" $test_out | awk '{ printf "%.1f", $4 }' )
  p50=$( grep "^Min:" $test_out | awk '{ printf "%.1f", $4 }' )
  echo -e "$ops_sec\t$mb_sec\t$usecs_op\t$avg\t$p50\t$test_name" \
    >> $output_dir/report.txt
}

function run_fillseq {
  # This runs with a vector memtable and the WAL disabled to load faster. It is still crash safe and the
  # client can discover where to restart a load after a crash. I think this is a good way to load.
  echo "Loading $num_keys keys sequentially"
  cmd="./db_bench --benchmarks=fillseq \
       --use_existing_db=0 \
       --sync=0 \
       $params_w \
       --threads=1 \
       --seed=$( date +%s ) \
       2>&1 | tee -a $output_dir/benchmark_fillseq.v${value_size}.log"
  echo $cmd | tee $output_dir/benchmark_fillseq.v${value_size}.log
  eval $cmd
  summarize_result $output_dir/benchmark_fillseq.v${value_size}.log fillseq.v${value_size} fillseq 1
}

function run_change {
  operation=$1
  echo "Do $num_keys random $operation"
  out_name="benchmark_${operation}.t${num_threads}.s${syncval}.log"
  cmd="./db_bench --benchmarks=$operation \
       --use_existing_db=1 \
       --sync=$syncval \
       $params_w \
       --threads=$num_threads \
       --seed=$( date +%s ) \
       2>&1 | tee -a $output_dir/${out_name}"
  echo $cmd | tee $output_dir/${out_name}
  eval $cmd
  summarize_result $output_dir/${out_name} ${operation}.t${num_threads}.s${syncval} $operation $num_threads
}

function run_readrandom {
  echo "Reading $num_keys random keys"
  out_name="benchmark_readrandom.t${num_threads}.log"
  cmd="./db_bench --benchmarks=readrandom \
       --use_existing_db=1 \
       $params_w \
       --threads=$num_threads \
       --seed=$( date +%s ) \
       2>&1 | tee -a $output_dir/${out_name}"
  echo $cmd | tee $output_dir/${out_name}
  eval $cmd
  summarize_result $output_dir/${out_name} readrandom.t${num_threads} readrandom $num_threads
}

function run_readwhile {
  operation=$1
  echo "Reading $num_keys random keys while $operation"
  out_name="benchmark_readwhile${operation}.t${num_threads}.log"
  cmd="./db_bench --benchmarks=readwhile${operation} \
       --use_existing_db=1 \
       --sync=$syncval \
       $params_w \
       --threads=$num_threads \
       --writes_per_second=$writes_per_second \
       --seed=$( date +%s ) \
       2>&1 | tee -a $output_dir/${out_name}"
  echo $cmd | tee $output_dir/${out_name}
  eval $cmd
  summarize_result $output_dir/${out_name} readwhile${operation}.t${num_threads} readwhile${operation} $num_threads
}

function now() {
  echo `date +"%s"`
}

report="$output_dir/report.txt"
schedule="$output_dir/schedule.txt"

echo "===== Benchmark ====="

# Run!!!
IFS=',' read -a jobs <<< $1
# shellcheck disable=SC2068
for job in ${jobs[@]}; do

  if [ $job != debug ]; then
    echo "Start $job at `date`" | tee -a $schedule
  fi

  start=$(now)
  if [ $job = fillseq ]; then
    run_fillseq
  elif [ $job = overwrite ]; then
    run_change overwrite
  elif [ $job = readrandom ]; then
    run_readrandom
  elif [ $job = readwhilewriting ]; then
    run_readwhile writing
  elif [ $job = debug ]; then
    num_keys=1000; # debug
    echo "Setting num_keys to $num_keys"
  else
    echo "unknown job $job"
    exit
  fi
  end=$(now)

  if [ $job != debug ]; then
    echo "Complete $job in $((end-start)) seconds" | tee -a $schedule
  fi

  echo -e "ops/sec\tmb/sec\tusec/op\tavg\tp50\tTest"
  tail -1 $output_dir/report.txt

done


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# A shell script to build and run different versions of ldb to check for
# expected forward and backward compatibility with "current" version. The
# working copy must have no uncommitted changes.
#
# Usage: <SCRIPT> [ref_for_current]
# `ref_for_current` can be a revision, tag, commit or branch name. Default is HEAD.
#
# Return value 0 means all regression tests pass. 1 if not pass.
#
# Environment options:
#  SHORT_TEST=1 - Test only the oldest branch for each kind of test. This is
#    a good choice for PR validation as it is relatively fast and will find
#    most issues.
#  USE_SSH=1 - Connect to GitHub with ssh instead of https

if ! git diff-index --quiet HEAD; then
  echo "You have uncommitted changes. Aborting."
  exit 1
fi

current_checkout_name=${1:-HEAD}
# This allows the script to work even if with transient refs like "HEAD"
current_checkout_hash="$(git rev-parse --quiet --verify $current_checkout_name)"

if [ "$current_checkout_hash" == "" ]; then
  echo "Not a recognized ref: $current_checkout_name"
  exit 1
fi

# To restore to prior branch at the end
orig_branch="$(git rev-parse --abbrev-ref HEAD)"
tmp_branch=_tmp_format_compatible
tmp_origin=_tmp_origin

# Don't depend on what current "origin" might be
set -e
git remote remove $tmp_origin 2>/dev/null || true
if [ "$USE_SSH" ]; then
  git remote add $tmp_origin "git@github.com:facebook/rocksdb.git"
else
  git remote add $tmp_origin "https://github.com/facebook/rocksdb.git"
fi
git fetch $tmp_origin

# Used in building some ancient RocksDB versions where by default it tries to
# use a precompiled libsnappy.a checked in to the repo.
export SNAPPY_LDFLAGS=-lsnappy

cleanup() {
  echo "== Cleaning up"
  git reset --hard || true
  git checkout "$orig_branch" || true
  git branch -D $tmp_branch || true
  git remote remove $tmp_origin || true
}
trap cleanup EXIT # Always clean up, even on failure or Ctrl+C

scriptpath=`dirname ${BASH_SOURCE[0]}`

test_dir=${TEST_TMPDIR:-"/tmp"}"/rocksdb_format_compatible_$USER"
rm -rf ${test_dir:?}

# Prevent 'make clean' etc. from wiping out test_dir
export TEST_TMPDIR=$test_dir"/misc"

# For saving current version of scripts as we checkout different versions to test
script_copy_dir=$test_dir"/script_copy"
mkdir -p $script_copy_dir
cp -f $scriptpath/*.sh $script_copy_dir

# For shared raw input data
input_data_path=$test_dir"/test_data_input"
mkdir -p $input_data_path
# For external sst ingestion test
ext_test_dir=$test_dir"/ext"
mkdir -p $ext_test_dir
# For DB dump test
db_test_dir=$test_dir"/db"
mkdir -p $db_test_dir
# For backup/restore test (uses DB test)
bak_test_dir=$test_dir"/bak"
mkdir -p $bak_test_dir

python_bin=$(which python3 || which python || echo python3)

# Generate random files.
for i in {1..6}
do
  input_data[$i]=$input_data_path/data$i
  echo == Generating random input file ${input_data[$i]}
  $python_bin - <<EOF
import random
random.seed($i)
symbols=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
with open('${input_data[$i]}', 'w') as f:
  for i in range(1,1024):
    k = ""
    for j in range(1, random.randint(1,32)):
      k=k + symbols[random.randint(0, len(symbols) - 1)]
    vb = ""
    for j in range(1, random.randint(0,128)):
      vb = vb + symbols[random.randint(0, len(symbols) - 1)]
    v = ""
    for j in range(1, random.randint(1, 5)):
      v = v + vb
    print(k + " ==> " + v, file=f)
EOF
done

# Generate file(s) with sorted keys.
sorted_input_data=$input_data_path/sorted_data
echo == Generating file with sorted keys ${sorted_input_data}
$python_bin - <<EOF
with open('${sorted_input_data}', 'w') as f:
  for i in range(0,10):
    k = str(i)
    v = "value" + k
    print(k + " ==> " + v, file=f)
EOF

# db_backward_only_refs defined below the rest

# To check for DB forward compatibility with loading options (old version
# reading data from new), as well as backward compatibility
declare -a db_forward_with_options_refs=("6.27.fb" "6.28.fb" "6.29.fb" "7.0.fb" "7.1.fb" "7.2.fb" "7.3.fb" "7.4.fb" "7.5.fb" "7.6.fb" "7.7.fb" "7.8.fb" "7.9.fb" "7.10.fb" "8.0.fb" "8.1.fb" "8.2.fb" "8.3.fb" "8.4.fb")
# To check for DB forward compatibility without loading options (in addition
# to the "with loading options" set), as well as backward compatibility
declare -a db_forward_no_options_refs=() # N/A at the moment

# To check for SST ingestion backward compatibility (new version reading
# data from old) (ldb ingest_extern_sst added in 5.16.x, back-ported to
# 5.14.x, 5.15.x)
declare -a ext_backward_only_refs=("5.14.fb" "5.15.fb" "5.16.fb" "5.17.fb" "5.18.fb" "6.0.fb" "6.1.fb" "6.2.fb" "6.3.fb" "6.4.fb" "6.5.fb" "6.6.fb" "6.7.fb" "6.8.fb" "6.9.fb" "6.10.fb" "6.11.fb" "6.12.fb" "6.13.fb" "6.14.fb" "6.15.fb" "6.16.fb" "6.17.fb" "6.18.fb" "6.19.fb" "6.20.fb" "6.21.fb" "6.22.fb" "6.23.fb" "6.24.fb" "6.25.fb" "6.26.fb")
# To check for SST ingestion forward compatibility (old version reading
# data from new) as well as backward compatibility
declare -a ext_forward_refs=("${db_forward_no_options_refs[@]}" "${db_forward_with_options_refs[@]}")

# To check for backup backward compatibility (new version reading data
# from old) (ldb backup/restore added in 4.11.x)
declare -a bak_backward_only_refs=("4.11.fb" "4.12.fb" "4.13.fb" "5.0.fb" "5.1.fb" "5.2.fb" "5.3.fb" "5.4.fb" "5.5.fb" "5.6.fb" "5.7.fb" "5.8.fb" "5.9.fb" "5.10.fb" "5.11.fb" "5.12.fb" "5.13.fb" "${ext_backward_only_refs[@]}")
# To check for backup forward compatibility (old version reading data
# from new) as well as backward compatibility
declare -a bak_forward_refs=("${db_forward_no_options_refs[@]}" "${db_forward_with_options_refs[@]}")

# Branches (git refs) to check for DB backward compatibility (new version
# reading data from old) (in addition to the "forward compatible" list)
# NOTE: 2.7.fb.branch shows assertion violation in some configurations
declare -a db_backward_only_refs=("2.2.fb.branch" "2.3.fb.branch" "2.4.fb.branch" "2.5.fb.branch" "2.6.fb.branch" "2.7.fb.branch" "2.8.1.fb" "3.0.fb.branch" "3.1.fb" "3.2.fb" "3.3.fb" "3.4.fb" "3.5.fb" "3.6.fb" "3.7.fb" "3.8.fb" "3.9.fb" "4.2.fb" "4.3.fb" "4.4.fb" "4.5.fb" "4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "${bak_backward_only_refs[@]}")

if [ "$SHORT_TEST" ]; then
  # Use only the first (if exists) of each list
  db_backward_only_refs=(${db_backward_only_refs[0]})
  db_forward_no_options_refs=(${db_forward_no_options_refs[0]})
  db_forward_with_options_refs=(${db_forward_with_options_refs[0]})
  ext_backward_only_refs=(${ext_backward_only_refs[0]})
  ext_forward_refs=(${ext_forward_refs[0]})
  bak_backward_only_refs=(${bak_backward_only_refs[0]})
  bak_forward_refs=(${bak_forward_refs[0]})
fi

# De-duplicate & accumulate
declare -a checkout_refs=()
for checkout_ref in "${db_backward_only_refs[@]}" "${db_forward_no_options_refs[@]}" "${db_forward_with_options_refs[@]}" "${ext_backward_only_refs[@]}" "${ext_forward_refs[@]}" "${bak_backward_only_refs[@]}" "${bak_forward_refs[@]}"
do
  if [ ! -e $db_test_dir/$checkout_ref ]; then
    mkdir -p $db_test_dir/$checkout_ref
    checkout_refs+=($checkout_ref)
  fi
done

generate_db()
{
    set +e
    bash "$script_copy_dir"/generate_random_db.sh "$1" "$2"
    if [ $? -ne 0 ]; then
        echo ==== Error loading data from $2 to $1 ====
        exit 1
    fi
    set -e
}

compare_db()
{
    set +e
    bash "$script_copy_dir"/verify_random_db.sh "$1" "$2" "$3" "$4" "$5"
    if [ $? -ne 0 ]; then
        echo ==== Read different content from $1 and $2 or error happened. ====
        exit 1
    fi
    set -e
}

write_external_sst()
{
    set +e
    bash "$script_copy_dir"/write_external_sst.sh "$1" "$2" "$3"
    if [ $? -ne 0 ]; then
        echo ==== Error writing external SST file using data from $1 to $3 ====
        exit 1
    fi
    set -e
}

ingest_external_sst()
{
    set +e
    bash "$script_copy_dir"/ingest_external_sst.sh "$1" "$2"
    if [ $? -ne 0 ]; then
        echo ==== Error ingesting external SST in $2 to DB at $1 ====
        exit 1
    fi
    set -e
}

backup_db()
{
    set +e
    bash "$script_copy_dir"/backup_db.sh "$1" "$2"
    if [ $? -ne 0 ]; then
        echo ==== Error backing up DB $1 to $2 ====
        exit 1
    fi
    set -e
}

restore_db()
{
    set +e
    bash "$script_copy_dir"/restore_db.sh "$1" "$2"
    if [ $? -ne 0 ]; then
        echo ==== Error restoring from $1 to $2 ====
        exit 1
    fi
    set -e
}

member_of_array()
{
  local e match="$1"
  shift
  for e; do [[ "$e" == "$match" ]] && return 0; done
  return 1
}

force_no_fbcode()
{
  # Not all branches recognize ROCKSDB_NO_FBCODE and we should not need
  # to patch old branches for changes to available FB compilers.
  sed -i -e 's|-d /mnt/gvfs/third-party|"$ROCKSDB_FORCE_FBCODE"|' build_tools/build_detect_platform
}

# General structure from here:
# * Check out, build, and do stuff with the "current" branch.
# * For each older branch under consideration,
#   * Check out, build, and do stuff with it, potentially using data
#     generated from "current" branch.
# * (Again) check out, build, and do (other) stuff with the "current"
#    branch, potentially using data from older branches.
#
# This way, we only do at most n+1 checkout+build steps, without the
# need to stash away executables.

# Decorate name
current_checkout_name="$current_checkout_name ($current_checkout_hash)"

echo "== Building $current_checkout_name debug"
git checkout -B $tmp_branch $current_checkout_hash
force_no_fbcode
make clean
DISABLE_WARNING_AS_ERROR=1 make ldb -j32

echo "== Using $current_checkout_name, generate DB with extern SST and ingest"
current_ext_test_dir=$ext_test_dir"/current"
write_external_sst $input_data_path ${current_ext_test_dir}_pointless $current_ext_test_dir
ingest_external_sst ${current_ext_test_dir}_ingest $current_ext_test_dir

echo "== Generating DB from $current_checkout_name ..."
current_db_test_dir=$db_test_dir"/current"
generate_db $input_data_path $current_db_test_dir

echo "== Creating backup of DB from $current_checkout_name ..."
current_bak_test_dir=$bak_test_dir"/current"
backup_db $current_db_test_dir $current_bak_test_dir

for checkout_ref in "${checkout_refs[@]}"
do
  echo "== Building $checkout_ref debug"
  git reset --hard $tmp_origin/$checkout_ref
  force_no_fbcode
  make clean
  DISABLE_WARNING_AS_ERROR=1 make ldb -j32

  # We currently assume DB backward compatibility for every branch listed
  echo "== Use $checkout_ref to generate a DB ..."
  generate_db $input_data_path $db_test_dir/$checkout_ref

  if member_of_array "$checkout_ref" "${ext_backward_only_refs[@]}" ||
    member_of_array "$checkout_ref" "${ext_forward_refs[@]}"
  then
    echo "== Use $checkout_ref to generate DB with extern SST file"
    write_external_sst $input_data_path $ext_test_dir/${checkout_ref}_pointless $ext_test_dir/$checkout_ref
  fi

  if member_of_array "$checkout_ref" "${ext_forward_refs[@]}"
  then
    echo "== Use $checkout_ref to ingest extern SST file and compare vs. $current_checkout_name"
    ingest_external_sst $ext_test_dir/${checkout_ref}_ingest $ext_test_dir/$checkout_ref
    compare_db $ext_test_dir/${checkout_ref}_ingest ${current_ext_test_dir}_ingest db_dump.txt 1 1

    rm -rf ${ext_test_dir:?}/${checkout_ref}_ingest
    echo "== Use $checkout_ref to ingest extern SST file from $current_checkout_name"
    ingest_external_sst $ext_test_dir/${checkout_ref}_ingest $current_ext_test_dir
    compare_db $ext_test_dir/${checkout_ref}_ingest ${current_ext_test_dir}_ingest db_dump.txt 1 1
  fi

  if member_of_array "$checkout_ref" "${db_forward_no_options_refs[@]}" ||
    member_of_array "$checkout_ref" "${db_forward_with_options_refs[@]}"
  then
    echo "== Use $checkout_ref to open DB generated using $current_checkout_name..."
    compare_db $db_test_dir/$checkout_ref $current_db_test_dir forward_${checkout_ref}_dump.txt 0
  fi

  if member_of_array "$checkout_ref" "${db_forward_with_options_refs[@]}"
  then
    echo "== Use $checkout_ref to open DB generated using $current_checkout_name with its options..."
    compare_db $db_test_dir/$checkout_ref $current_db_test_dir forward_${checkout_ref}_dump.txt 1 1
  fi

  if member_of_array "$checkout_ref" "${bak_backward_only_refs[@]}" ||
    member_of_array "$checkout_ref" "${bak_forward_refs[@]}"
  then
    echo "== Use $checkout_ref to backup DB"
    backup_db $db_test_dir/$checkout_ref $bak_test_dir/$checkout_ref
  fi

  if member_of_array "$checkout_ref" "${bak_forward_refs[@]}"
  then
    echo "== Use $checkout_ref to restore DB from $current_checkout_name"
    rm -rf ${db_test_dir:?}/$checkout_ref
    restore_db $current_bak_test_dir $db_test_dir/$checkout_ref
    compare_db $db_test_dir/$checkout_ref $current_db_test_dir forward_${checkout_ref}_dump.txt 0
  fi
done

echo "== Building $current_checkout_name debug (again, final)"
git reset --hard $current_checkout_hash
force_no_fbcode
make clean
DISABLE_WARNING_AS_ERROR=1 make ldb -j32

for checkout_ref in "${checkout_refs[@]}"
do
  # We currently assume DB backward compatibility for every branch listed
  echo "== Use $current_checkout_name to open DB generated using $checkout_ref..."
  compare_db $db_test_dir/$checkout_ref $current_db_test_dir db_dump.txt 1 0

  if member_of_array "$checkout_ref" "${ext_backward_only_refs[@]}" ||
    member_of_array "$checkout_ref" "${ext_forward_refs[@]}"
  then
    rm -rf ${ext_test_dir:?}/${checkout_ref}_ingest
    echo "== Use $current_checkout_name to ingest extern SST file from $checkout_ref"
    ingest_external_sst $ext_test_dir/${checkout_ref}_ingest $current_ext_test_dir
    compare_db $ext_test_dir/${checkout_ref}_ingest ${current_ext_test_dir}_ingest db_dump.txt 1 1
  fi

  if member_of_array "$checkout_ref" "${bak_backward_only_refs[@]}" ||
    member_of_array "$checkout_ref" "${bak_forward_refs[@]}"
  then
    echo "== Use $current_checkout_name to restore DB from $checkout_ref"
    rm -rf ${db_test_dir:?}/$checkout_ref
    restore_db $bak_test_dir/$checkout_ref $db_test_dir/$checkout_ref
    compare_db $db_test_dir/$checkout_ref $current_db_test_dir db_dump.txt 1 0
  fi
done

echo ==== Compatibility Test PASSED ====


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# A shell script to load some pre generated data file to a DB using ldb tool
# ./ldb needs to be avaible to be executed.
#
# Usage: <SCRIPT> <input_data_path> <DB Path>

if [ "$#" -lt 2 ]; then
  echo "usage: $BASH_SOURCE <input_data_path> <DB Path>"
  exit 1
fi

input_data_dir=$1
db_dir=$2
rm -rf $db_dir

echo == Loading data from $input_data_dir to $db_dir

declare -a compression_opts=("no" "snappy" "zlib" "bzip2")

set -e

n=0

for f in `ls -1 $input_data_dir`
do
  echo == Loading $f with compression ${compression_opts[n % 4]}
  ./ldb load --db=$db_dir --compression_type=${compression_opts[n % 4]} --bloom_bits=10 --auto_compaction=false --create_if_missing < $input_data_dir/$f
  let "n = n + 1"
done


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
#

if [ "$#" -lt 2 ]; then
  echo "usage: $BASH_SOURCE <DB Path> <External SST Dir>"
  exit 1
fi

db_dir=$1
external_sst_dir=$2

for f in `find $external_sst_dir -name extern_sst*`
do
  echo == Ingesting external SST file $f to DB at $db_dir
  ./ldb --db=$db_dir --create_if_missing ingest_extern_sst $f
done


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
# The RocksDB regression test script.
# REQUIREMENT: must be able to run make db_bench in the current directory
#
# This script will do the following things in order:
#
# 1. check out the specified rocksdb commit.
# 2. build db_bench using the specified commit
# 3. setup test directory $TEST_PATH.  If not specified, then the test directory
#    will be "/tmp/rocksdb/regression_test"
# 4. run set of benchmarks on the specified host
#    (can be either locally or remotely)
# 5. generate report in the $RESULT_PATH.  If RESULT_PATH is not specified,
#    RESULT_PATH will be set to $TEST_PATH/current_time
#
# = Examples =
# * Run the regression test using rocksdb commit abcdef that outputs results
#   and temp files in "/my/output/dir"
#r
#   TEST_PATH=/my/output/dir COMMIT_ID=abcdef ./tools/regression_test.sh
#
# * Run the regression test on a remost host under "/my/output/dir" directory
#   and stores the result locally in "/my/benchmark/results" using commit
#   abcdef and with the rocksdb options specified in /my/path/to/OPTIONS-012345
#   with 1000000000 keys in each benchmark in the regression test where each
#   key and value are 100 and 900 bytes respectively:
#
#   REMOTE_USER_AT_HOST=yhchiang@my.remote.host \
#       TEST_PATH=/my/output/dir \
#       RESULT_PATH=/my/benchmark/results \
#       COMMIT_ID=abcdef \
#       OPTIONS_FILE=/my/path/to/OPTIONS-012345 \
#       NUM_KEYS=1000000000 \
#       KEY_SIZE=100 \
#       VALUE_SIZE=900 \
#       ./tools/regression_test.sh
#
# = Regression test environmental parameters =
#   DEBUG: If true, then the script will not build db_bench if db_bench already
#       exists
#       Default: 0
#   TEST_MODE: If 1, run fillseqdeterminstic and benchmarks both
#       if 0, only run fillseqdeterministc
#       if 2, only run benchmarks
#       Default: 1
#   TEST_PATH: the root directory of the regression test.
#       Default: "/tmp/rocksdb/regression_test"
#       !!! NOTE !!! - a DB will also be saved in $TEST_PATH/../db
#   RESULT_PATH: the directory where the regression results will be generated.
#       Default: "$TEST_PATH/current_time"
#   REMOTE_USER_AT_HOST: If set, then test will run on the specified host under
#       TEST_PATH directory and outputs test results locally in RESULT_PATH
#       The REMOTE_USER_AT_HOST should follow the format user-id@host.name
#   DB_PATH: the path where the rocksdb database will be created during the
#       regression test.  Default:  $TEST_PATH/db
#   WAL_PATH: the path where the rocksdb WAL will be outputed.
#       Default:  $TEST_PATH/wal
#   OPTIONS_FILE:  If specified, then the regression test will use the specified
#       file to initialize the RocksDB options in its benchmarks.  Note that
#       this feature only work for commits after 88acd93 or rocksdb version
#       later than 4.9.
#   DELETE_TEST_PATH: If true, then the test directory will be deleted
#       after the script ends.
#       Default: 0
#
# = db_bench parameters =
#   NUM_THREADS:  The number of concurrent foreground threads that will issue
#       database operations in the benchmark.  Default: 16.
#   NUM_KEYS:  The key range that will be used in the entire regression test.
#       Default: 1G.
#   NUM_OPS:  The number of operations (reads, writes, or deletes) that will
#       be issued in EACH thread.
#       Default: $NUM_KEYS / $NUM_THREADS
#   KEY_SIZE:  The size of each key in bytes in db_bench.  Default: 100.
#   VALUE_SIZE:  The size of each value in bytes in db_bench.  Default: 900.
#   CACHE_SIZE:  The size of RocksDB block cache used in db_bench.  Default: 1G
#   STATISTICS:  If 1, then statistics is on in db_bench.  Default: 0.
#   COMPRESSION_RATIO:  The compression ratio of the key generated in db_bench.
#       Default: 0.5.
#   HISTOGRAM:  If 1, then the histogram feature on performance feature is on.
#   STATS_PER_INTERVAL:  If 1, then the statistics will be reported for every
#       STATS_INTERVAL_SECONDS seconds.  Default 1.
#   STATS_INTERVAL_SECONDS:  If STATS_PER_INTERVAL is set to 1, then statistics
#       will be reported for every STATS_INTERVAL_SECONDS.  Default 60.
#   MAX_BACKGROUND_FLUSHES:  The maxinum number of concurrent flushes in
#       db_bench.  Default: 4.
#   MAX_BACKGROUND_COMPACTIONS:  The maximum number of concurrent compactions
#       in db_bench.  Default: 16.
#   NUM_HIGH_PRI_THREADS:  The number of high-pri threads available for
#       concurrent flushes in db_bench.  Default: 4.
#   NUM_LOW_PRI_THREADS:  The number of low-pri threads available for
#       concurrent compactions in db_bench.  Default: 16.
#   SEEK_NEXTS:  Controls how many Next() will be called after seek.
#       Default: 10.
#   SEED:  random seed that controls the randomness of the benchmark.
#       Default: $( date +%s )

#==============================================================================
#  CONSTANT
#==============================================================================
TITLE_FORMAT="%40s,%25s,%30s,%7s,%9s,%8s,"
TITLE_FORMAT+="%10s,%13s,%14s,%11s,%12s,"
TITLE_FORMAT+="%7s,%11s,"
TITLE_FORMAT+="%9s,%10s,%10s,%10s,%10s,%10s,%5s,"
TITLE_FORMAT+="%5s,%5s,%5s" # time
TITLE_FORMAT+="\n"

DATA_FORMAT="%40s,%25s,%30s,%7s,%9s,%8s,"
DATA_FORMAT+="%10s,%13.0f,%14s,%11s,%12s,"
DATA_FORMAT+="%7s,%11s,"
DATA_FORMAT+="%9.0f,%10.0f,%10.0f,%10.0f,%10.0f,%10.0f,%5.0f,"
DATA_FORMAT+="%5.0f,%5.0f,%5.0f" # time
DATA_FORMAT+="\n"

# In case of async_io, $1 is benchmark_asyncio
MAIN_PATTERN="${1%%_*}""[[:blank:]]+:.*[[:blank:]]+([0-9\.]+)[[:blank:]]+ops/sec"
PERC_PATTERN="Percentiles: P50: ([0-9\.]+) P75: ([0-9\.]+) "
PERC_PATTERN+="P99: ([0-9\.]+) P99.9: ([0-9\.]+) P99.99: ([0-9\.]+)"
#==============================================================================

function main {
  TEST_ROOT_DIR=${TEST_PATH:-"/tmp/rocksdb/regression_test"}
  init_arguments $TEST_ROOT_DIR

  build_db_bench_and_ldb

  setup_test_directory
  if [ $TEST_MODE -le 1 ]; then
      test_remote "test -d $ORIGIN_PATH"
      if [[ $? -ne 0 ]]; then
          echo "Building DB..."
          # compactall alone will not print ops or threads, which will fail update_report
          run_db_bench "fillseq,compactall" $NUM_KEYS 1 0 0
          # only save for future use on success
          test_remote "mv $DB_PATH $ORIGIN_PATH"
      fi
  fi
  if [ $TEST_MODE -ge 1 ]; then
      build_checkpoint
      # run_db_bench benchmark_name NUM_OPS NUM_THREADS USED_EXISTING_DB UPDATE_REPORT ASYNC_IO
      run_db_bench "seekrandom_asyncio" $NUM_OPS $NUM_THREADS  1 1 true
      run_db_bench "multireadrandom_asyncio" $NUM_OPS $NUM_THREADS  1 1 true
      run_db_bench "readrandom"
      run_db_bench "readwhilewriting"
      run_db_bench "deleterandom"
      run_db_bench "seekrandom"
      run_db_bench "seekrandomwhilewriting"
      run_db_bench "multireadrandom"
  fi

  cleanup_test_directory $TEST_ROOT_DIR
  echo ""
  echo "Benchmark completed!  Results are available in $RESULT_PATH"
}

############################################################################
function init_arguments {
  K=1024
  M=$((1024 * K))
  G=$((1024 * M))

  current_time=$(date +"%F-%H:%M:%S")
  RESULT_PATH=${RESULT_PATH:-"$1/results/$current_time"}
  COMMIT_ID=`hg id -i 2>/dev/null || git rev-parse HEAD 2>/dev/null || echo 'unknown'`
  SUMMARY_FILE="$RESULT_PATH/SUMMARY.csv"

  DB_PATH=${3:-"$1/db"}
  ORIGIN_PATH=${ORIGIN_PATH:-"$(dirname $(dirname $DB_PATH))/db"}
  WAL_PATH=${4:-""}
  if [ -z "$REMOTE_USER_AT_HOST" ]; then
    DB_BENCH_DIR=${5:-"."}
  else
    DB_BENCH_DIR=${5:-"$1/db_bench"}
  fi

  DEBUG=${DEBUG:-0}
  TEST_MODE=${TEST_MODE:-1}
  SCP=${SCP:-"scp"}
  SSH=${SSH:-"ssh"}
  NUM_THREADS=${NUM_THREADS:-16}
  NUM_KEYS=${NUM_KEYS:-$((1 * G))}  # key range
  NUM_OPS=${NUM_OPS:-$(($NUM_KEYS / $NUM_THREADS))}
  KEY_SIZE=${KEY_SIZE:-100}
  VALUE_SIZE=${VALUE_SIZE:-900}
  CACHE_SIZE=${CACHE_SIZE:-$((1 * G))}
  STATISTICS=${STATISTICS:-0}
  COMPRESSION_RATIO=${COMPRESSION_RATIO:-0.5}
  HISTOGRAM=${HISTOGRAM:-1}
  NUM_MULTI_DB=${NUM_MULTI_DB:-1}
  STATS_PER_INTERVAL=${STATS_PER_INTERVAL:-1}
  STATS_INTERVAL_SECONDS=${STATS_INTERVAL_SECONDS:-600}
  MAX_BACKGROUND_FLUSHES=${MAX_BACKGROUND_FLUSHES:-4}
  MAX_BACKGROUND_COMPACTIONS=${MAX_BACKGROUND_COMPACTIONS:-16}
  NUM_HIGH_PRI_THREADS=${NUM_HIGH_PRI_THREADS:-4}
  NUM_LOW_PRI_THREADS=${NUM_LOW_PRI_THREADS:-16}
  DELETE_TEST_PATH=${DELETE_TEST_PATH:-0}
  SEEK_NEXTS=${SEEK_NEXTS:-10}
  SEED=${SEED:-$( date +%s )}
  MULTIREAD_BATCH_SIZE=${MULTIREAD_BATCH_SIZE:-128}
  MULTIREAD_STRIDE=${MULTIREAD_STRIDE:-12}
  PERF_LEVEL=${PERF_LEVEL:-1}
}

# $1 --- benchmark name
# $2 --- number of operations.  Default: $NUM_OPS
# $3 --- number of threads.  Default $NUM_THREADS
# $4 --- use_existing_db.  Default: 1
# $5 --- update_report. Default: 1
# $6 --- async_io. Default: False
function run_db_bench {
  # Make sure no other db_bench is running. (Make sure command succeeds if pidof
  # command exists but finds nothing.)
  pids_cmd='pidof db_bench || pidof --version > /dev/null'
  # But first, make best effort to kill any db_bench that have run for more
  # than 12 hours, as that indicates a hung or runaway process.
  kill_old_cmd='for PID in $(pidof db_bench); do [ "$(($(stat -c %Y /proc/$PID) + 43200))" -lt "$(date +%s)" ] && echo "Killing old db_bench $PID" && kill $PID && sleep 5 && kill -9 $PID && sleep 5; done; pidof --version > /dev/null'
  if ! [ -z "$REMOTE_USER_AT_HOST" ]; then
    pids_cmd="$SSH $REMOTE_USER_AT_HOST '$pids_cmd'"
    kill_old_cmd="$SSH $REMOTE_USER_AT_HOST '$kill_old_cmd'"
  fi

  eval $kill_old_cmd
  exit_on_error $? "$kill_old_cmd"

  pids_output="$(eval $pids_cmd)"
  exit_on_error $? "$pids_cmd"

  if [ "$pids_output" != "" ]; then
    echo "Stopped regression_test.sh as there're still recent db_bench "
    echo "processes running: $pids_output"
    echo "Clean up test directory"
    cleanup_test_directory $TEST_ROOT_DIR
    exit 2
  fi

  # Build db_bench command
  ops=${2:-$NUM_OPS}
  threads=${3:-$NUM_THREADS}
  USE_EXISTING_DB=${4:-1}
  UPDATE_REPORT=${5:-1}
  async_io=${6:-false}
  seek_nexts=$SEEK_NEXTS

  if [ "$async_io" == "true" ]; then
    if ! [ -z "$SEEK_NEXTS_ASYNC_IO" ]; then
      seek_nexts=$SEEK_NEXTS_ASYNC_IO
    fi
  fi

  echo ""
  echo "======================================================================="
  echo "Benchmark $1"
  echo "======================================================================="
  echo ""
  db_bench_error=0
  options_file_arg=$(setup_options_file)
  echo "$options_file_arg"

  # In case of async_io, benchmark is benchmark_asyncio
  db_bench_type=${1%%_*}

  # use `which time` to avoid using bash's internal time command
  db_bench_cmd="\$(which time) -p $DB_BENCH_DIR/db_bench \
      --benchmarks=$db_bench_type --db=$DB_PATH --wal_dir=$WAL_PATH \
      --use_existing_db=$USE_EXISTING_DB \
      --perf_level=$PERF_LEVEL \
      --disable_auto_compactions \
      --threads=$threads \
      --num=$NUM_KEYS \
      --reads=$ops \
      --writes=$ops \
      --deletes=$ops \
      --key_size=$KEY_SIZE \
      --value_size=$VALUE_SIZE \
      --cache_size=$CACHE_SIZE \
      --statistics=$STATISTICS \
      $options_file_arg \
      --compression_ratio=$COMPRESSION_RATIO \
      --histogram=$HISTOGRAM \
      --seek_nexts=$seek_nexts \
      --stats_per_interval=$STATS_PER_INTERVAL \
      --stats_interval_seconds=$STATS_INTERVAL_SECONDS \
      --max_background_flushes=$MAX_BACKGROUND_FLUSHES \
      --num_multi_db=$NUM_MULTI_DB \
      --max_background_compactions=$MAX_BACKGROUND_COMPACTIONS \
      --num_high_pri_threads=$NUM_HIGH_PRI_THREADS \
      --num_low_pri_threads=$NUM_LOW_PRI_THREADS \
      --seed=$SEED \
      --multiread_batched=true \
      --batch_size=$MULTIREAD_BATCH_SIZE \
      --multiread_stride=$MULTIREAD_STRIDE \
      --async_io=$async_io"

  if [ "$async_io" == "true" ]; then
    db_bench_cmd="$db_bench_cmd $(set_async_io_parameters) "
  fi

  db_bench_cmd=" $db_bench_cmd 2>&1"

  if ! [ -z "$REMOTE_USER_AT_HOST" ]; then
    echo "Running benchmark remotely on $REMOTE_USER_AT_HOST"
    db_bench_cmd="$SSH $REMOTE_USER_AT_HOST '$db_bench_cmd'"
  fi
  echo db_bench_cmd="$db_bench_cmd"

  # Run the db_bench command
  eval $db_bench_cmd | tee -a "$RESULT_PATH/$1"
  exit_on_error ${PIPESTATUS[0]} db_bench
  if [ $UPDATE_REPORT -ne 0 ]; then
    update_report "$1" "$RESULT_PATH/$1" $ops $threads
  fi
}

function set_async_io_parameters {
  options=" --duration=500"
  # Below parameters are used in case of async_io only.
  # 1. If you want to run below parameters for all benchmarks, it should be
  #    specify in OPTIONS_FILE instead of exporting them.
  # 2. Below exported var takes precedence over OPTIONS_FILE.
  if ! [ -z "$MAX_READAHEAD_SIZE" ]; then
    options="$options --max_auto_readahead_size=$MAX_READAHEAD_SIZE "
  fi
  if ! [ -z "$INITIAL_READAHEAD_SIZE" ]; then
    options="$options --initial_auto_readahead_size=$INITIAL_READAHEAD_SIZE "
  fi
  if ! [ -z "$NUM_READS_FOR_READAHEAD_SIZE" ]; then
    options="$options --num_file_reads_for_auto_readahead=$NUM_READS_FOR_READAHEAD_SIZE "
  fi
  echo $options
}

function build_checkpoint {
    cmd_prefix=""
    if ! [ -z "$REMOTE_USER_AT_HOST" ]; then
        cmd_prefix="$SSH $REMOTE_USER_AT_HOST "
    fi
    if [ $NUM_MULTI_DB -gt 1 ]; then
        dirs=$($cmd_prefix find $ORIGIN_PATH -type d -links 2)
        for dir in $dirs; do
            db_index=$(basename $dir)
            echo "Building checkpoints: $ORIGIN_PATH/$db_index -> $DB_PATH/$db_index ..."
            $cmd_prefix $DB_BENCH_DIR/ldb checkpoint --checkpoint_dir=$DB_PATH/$db_index \
                        --db=$ORIGIN_PATH/$db_index --try_load_options 2>&1
            exit_on_error $?
        done
    else
        # checkpoint cannot build in directory already exists
        $cmd_prefix rm -rf $DB_PATH
        echo "Building checkpoint: $ORIGIN_PATH -> $DB_PATH ..."
        $cmd_prefix $DB_BENCH_DIR/ldb checkpoint --checkpoint_dir=$DB_PATH \
                    --db=$ORIGIN_PATH --try_load_options 2>&1
        exit_on_error $?
    fi
}

function multiply {
  echo "$1 * $2" | bc
}

# $1 --- name of the benchmark
# $2 --- the filename of the output log of db_bench
function update_report {
  # In case of async_io, benchmark is benchmark_asyncio
  db_bench_type=${1%%_*}

  main_result=`cat $2 | grep $db_bench_type`
  exit_on_error $?
  perc_statement=`cat $2 | grep Percentile`
  exit_on_error $?

  # Obtain micros / op

  [[ $main_result =~ $MAIN_PATTERN ]]
  ops_per_s=${BASH_REMATCH[1]}

  # Obtain percentile information
  [[ $perc_statement =~ $PERC_PATTERN ]]
  perc[0]=${BASH_REMATCH[1]}  # p50
  perc[1]=${BASH_REMATCH[2]}  # p75
  perc[2]=${BASH_REMATCH[3]}  # p99
  perc[3]=${BASH_REMATCH[4]}  # p99.9
  perc[4]=${BASH_REMATCH[5]}  # p99.99

  # Parse the output of the time command
  real_sec=`tail -3 $2 | grep real | awk '{print $2}'`
  user_sec=`tail -3 $2 | grep user | awk '{print $2}'`
  sys_sec=`tail -3 $2 | grep sys | awk '{print $2}'`

  (printf "$DATA_FORMAT" \
    $COMMIT_ID $1 $REMOTE_USER_AT_HOST $NUM_MULTI_DB $NUM_KEYS $KEY_SIZE $VALUE_SIZE \
       $(multiply $COMPRESSION_RATIO 100) \
       $3 $4 $CACHE_SIZE \
       $MAX_BACKGROUND_FLUSHES $MAX_BACKGROUND_COMPACTIONS \
       $ops_per_s \
       $(multiply ${perc[0]} 1000) \
       $(multiply ${perc[1]} 1000) \
       $(multiply ${perc[2]} 1000) \
       $(multiply ${perc[3]} 1000) \
       $(multiply ${perc[4]} 1000) \
       $DEBUG \
       $real_sec \
       $user_sec \
       $sys_sec \
       >> $SUMMARY_FILE)
  exit_on_error $?
}

function exit_on_error {
  if [ $1 -ne 0 ]; then
    echo ""
    echo "ERROR: Benchmark did not complete successfully."
    if ! [ -z "$2" ]; then
      echo "Failure command: $2"
    fi
    echo "Partial results are output to $RESULT_PATH"
    echo "ERROR" >> $SUMMARY_FILE
    exit $1
  fi
}

function build_db_bench_and_ldb {
  echo "Building db_bench & ldb ..."

  make clean
  exit_on_error $?

  DEBUG_LEVEL=0 make db_bench ldb -j32
  exit_on_error $?
}

function run_remote {
  test_remote "$1"
  exit_on_error $? "$1"
}

function test_remote {
  if ! [ -z "$REMOTE_USER_AT_HOST" ]; then
      cmd="$SSH $REMOTE_USER_AT_HOST '$1'"
  else
      cmd="$1"
  fi
  eval "$cmd"
}

function run_local {
  eval "$1"
  exit_on_error $? "$1"
}

function setup_options_file {
 if ! [ -z "$OPTIONS_FILE" ]; then
    if ! [ -z "$REMOTE_USER_AT_HOST" ]; then
      options_file="$DB_BENCH_DIR/OPTIONS_FILE"
      run_local "$SCP $OPTIONS_FILE $REMOTE_USER_AT_HOST:$options_file"
    else
      options_file="$OPTIONS_FILE"
    fi
    echo "--options_file=$options_file"
  fi
  echo ""
}

function setup_test_directory {
  echo "Deleting old regression test directories and creating new ones"

  run_local 'test "$DB_PATH" != "."'
  run_remote "rm -rf $DB_PATH"

  if [ "$DB_BENCH_DIR" != "." ]; then
    run_remote "rm -rf $DB_BENCH_DIR"
  fi

  run_local 'test "$RESULT_PATH" != "."'
  run_local "rm -rf $RESULT_PATH"

  if ! [ -z "$WAL_PATH" ]; then
    run_remote "rm -rf $WAL_PATH"
    run_remote "mkdir -p $WAL_PATH"
  fi

  run_remote "mkdir -p $DB_PATH"

  run_remote "mkdir -p $DB_BENCH_DIR"
  run_remote "ls -l $DB_BENCH_DIR"

  if ! [ -z "$REMOTE_USER_AT_HOST" ]; then
      shopt -s nullglob # allow missing librocksdb*.so* for static lib build
      run_local "tar cz db_bench ldb librocksdb*.so* | $SSH $REMOTE_USER_AT_HOST 'cd $DB_BENCH_DIR/ && tar xzv'"
      shopt -u nullglob
  fi

  run_local "mkdir -p $RESULT_PATH"

  (printf $TITLE_FORMAT \
      "commit id" "benchmark" "user@host" "num-dbs" "key-range" "key-size" \
      "value-size" "compress-rate" "ops-per-thread" "num-threads" "cache-size" \
      "flushes" "compactions" \
      "ops-per-s" "p50" "p75" "p99" "p99.9" "p99.99" "debug" \
      "real-sec" "user-sec" "sys-sec" \
      >> $SUMMARY_FILE)
  exit_on_error $?
}

function cleanup_test_directory {

  if [ $DELETE_TEST_PATH -ne 0 ]; then
    echo "Clear old regression test directories and creating new ones"
    run_remote "rm -rf $DB_PATH"
    run_remote "rm -rf $WAL_PATH"
    if ! [ -z "$REMOTE_USER_AT_HOST" ]; then
      run_remote "rm -rf $DB_BENCH_DIR"
    fi
    run_remote "rm -rf $1"
  else
    echo "------------ DEBUG MODE ------------"
    echo "DB  PATH: $DB_PATH"
    echo "WAL PATH: $WAL_PATH"
  fi
}

############################################################################

# shellcheck disable=SC2068
main $@


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
#

if [ "$#" -lt 2 ]; then
  echo "usage: ${BASH_SOURCE[0]} <Backup Dir> <DB Path>"
  exit 1
fi

backup_dir="$1"
db_dir="$2"

echo "== Restoring latest from $backup_dir to $db_dir"
./ldb restore --db="$db_dir" --backup_dir="$backup_dir"


# shellcheck disable=SC2148
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
TESTDIR=`mktemp -d ${TMPDIR:-/tmp}/rocksdb-dump-test.XXXXX`
DUMPFILE="tools/sample-dump.dmp"

# Verify that the sample dump file is undumpable and then redumpable.
./rocksdb_undump --dump_location=$DUMPFILE --db_path=$TESTDIR/db
./rocksdb_dump --anonymous --db_path=$TESTDIR/db --dump_location=$TESTDIR/dump
cmp $DUMPFILE $TESTDIR/dump