#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# BlobDB benchmark script
#
# REQUIRES: benchmark.sh is in the tools subdirectory
#
# After the execution of this script, log files are available in $output_dir.
# report.tsv provides high level statistics.
#
# Should be run from the parent of the tools directory. The command line is:
#   [$env_vars] tools/run_blob_bench.sh
#
# This runs the following sequence of BlobDB performance tests:
#   phase 1) write-only - bulkload+compact, overwrite+waitforcompaction
#   phase 2) read-write - readwhilewriting, fwdrangewhilewriting
#   phase 3) read-only - readrandom, fwdrange
#

# Exit Codes
EXIT_INVALID_ARGS=1

# Size constants
K=1024
M=$((1024 * K))
G=$((1024 * M))
T=$((1024 * G))

function display_usage() {
  echo "usage: run_blob_bench.sh [--help]"
  echo ""
  echo "Runs the following sequence of BlobDB benchmark tests using tools/benchmark.sh:"
  echo -e "\tPhase 1: write-only tests: bulkload+compact, overwrite+waitforcompaction"
  echo -e "\tPhase 2: read-write tests: readwhilewriting, fwdrangewhilewriting"
  echo -e "\tPhase 3: read-only tests: readrandom, fwdrange"
  echo ""
  echo "Environment Variables:"
  echo -e "\tJOB_ID\t\t\t\tIdentifier for the benchmark job, will appear in the results (default: empty)"
  echo -e "\tDB_DIR\t\t\t\tPath for the RocksDB data directory (mandatory)"
  echo -e "\tWAL_DIR\t\t\t\tPath for the RocksDB WAL directory (mandatory)"
  echo -e "\tOUTPUT_DIR\t\t\tPath for the benchmark results (mandatory)"
  echo -e "\tNUM_THREADS\t\t\tNumber of threads (default: 16)"
  echo -e "\tCOMPRESSION_TYPE\t\tCompression type for the SST files (default: lz4)"
  echo -e "\tDB_SIZE\t\t\t\tRaw (uncompressed) database size (default: 1 TB)"
  echo -e "\tVALUE_SIZE\t\t\tValue size (default: 1 KB)"
  echo -e "\tNUM_KEYS\t\t\tNumber of keys (default: raw database size divided by value size)"
  echo -e "\tDURATION\t\t\tIndividual duration for read-write/read-only tests in seconds (default: 1800)"
  echo -e "\tWRITE_BUFFER_SIZE\t\tWrite buffer (memtable) size (default: 1 GB)"
  echo -e "\tENABLE_BLOB_FILES\t\tEnable blob files (default: 1)"
  echo -e "\tMIN_BLOB_SIZE\t\t\tSize threshold for storing values in blob files (default: 0)"
  echo -e "\tBLOB_FILE_SIZE\t\t\tBlob file size (default: same as write buffer size)"
  echo -e "\tBLOB_COMPRESSION_TYPE\t\tCompression type for the blob files (default: lz4)"
  echo -e "\tENABLE_BLOB_GC\t\t\tEnable blob garbage collection (default: 1)"
  echo -e "\tBLOB_GC_AGE_CUTOFF\t\tBlob garbage collection age cutoff (default: 0.25)"
  echo -e "\tBLOB_GC_FORCE_THRESHOLD\t\tThreshold for forcing garbage collection of the oldest blob files (default: 1.0)"
  echo -e "\tBLOB_COMPACTION_READAHEAD_SIZE\tBlob compaction readahead size (default: 0)"
  echo -e "\tBLOB_FILE_STARTING_LEVEL\t\tBlob file starting level (default: 0)"
  echo -e "\tUSE_BLOB_CACHE\t\t\tEnable blob cache. (default: 1)"
  echo -e "\tUSE_SHARED_BLOCK_AND_BLOB_CACHE\t\t\tUse the same backing cache for block cache and blob cache. (default: 1)"
  echo -e "\tBLOB_CACHE_SIZE\t\t\tSize of the blob cache (default: 16GB)"
  echo -e "\tBLOB_CACHE_NUMSHARDBITS\t\t\tNumber of shards for the blob cache is 2 ** blob_cache_numshardbits (default: 6)"
  echo -e "\tPREPOPULATE_BLOB_CACHE\t\t\tPre-populate hot/warm blobs in blob cache (default: 0)"
  echo -e "\tTARGET_FILE_SIZE_BASE\t\tTarget SST file size for compactions (default: write buffer size, scaled down if blob files are enabled)"
  echo -e "\tMAX_BYTES_FOR_LEVEL_BASE\tMaximum size for the base level (default: 8 * target SST file size)"
}

if [ $# -ge 1 ]; then
  display_usage

  if [ "$1" == "--help" ]; then
    exit
  else
    exit $EXIT_INVALID_ARGS
  fi
fi

# shellcheck disable=SC2153
if [ -z "$DB_DIR" ]; then
  echo "DB_DIR is not defined"
  exit $EXIT_INVALID_ARGS
fi

# shellcheck disable=SC2153
if [ -z "$WAL_DIR" ]; then
  echo "WAL_DIR is not defined"
  exit $EXIT_INVALID_ARGS
fi

# shellcheck disable=SC2153
if [ -z "$OUTPUT_DIR" ]; then
  echo "OUTPUT_DIR is not defined"
  exit $EXIT_INVALID_ARGS
fi

# shellcheck disable=SC2153
job_id=$JOB_ID

db_dir=$DB_DIR
wal_dir=$WAL_DIR
output_dir=$OUTPUT_DIR

num_threads=${NUM_THREADS:-16}

compression_type=${COMPRESSION_TYPE:-lz4}

db_size=${DB_SIZE:-$((1 * T))}
value_size=${VALUE_SIZE:-$((1 * K))}
num_keys=${NUM_KEYS:-$((db_size / value_size))}

duration=${DURATION:-1800}

write_buffer_size=${WRITE_BUFFER_SIZE:-$((1 * G))}

enable_blob_files=${ENABLE_BLOB_FILES:-1}
min_blob_size=${MIN_BLOB_SIZE:-0}
blob_file_size=${BLOB_FILE_SIZE:-$write_buffer_size}
blob_compression_type=${BLOB_COMPRESSION_TYPE:-lz4}
enable_blob_garbage_collection=${ENABLE_BLOB_GC:-1}
blob_garbage_collection_age_cutoff=${BLOB_GC_AGE_CUTOFF:-0.25}
blob_garbage_collection_force_threshold=${BLOB_GC_FORCE_THRESHOLD:-1.0}
blob_compaction_readahead_size=${BLOB_COMPACTION_READAHEAD_SIZE:-0}
blob_file_starting_level=${BLOB_FILE_STARTING_LEVEL:-0}
use_blob_cache=${USE_BLOB_CACHE:-1}
use_shared_block_and_blob_cache=${USE_SHARED_BLOCK_AND_BLOB_CACHE:-1}
blob_cache_size=${BLOB_CACHE_SIZE:-$((16 * G))}
blob_cache_numshardbits=${BLOB_CACHE_NUMSHARDBITS:-6}
prepopulate_blob_cache=${PREPOPULATE_BLOB_CACHE:-0}

if [ "$enable_blob_files" == "1" ]; then
  target_file_size_base=${TARGET_FILE_SIZE_BASE:-$((32 * write_buffer_size / value_size))}
else
  target_file_size_base=${TARGET_FILE_SIZE_BASE:-$write_buffer_size}
fi

max_bytes_for_level_base=${MAX_BYTES_FOR_LEVEL_BASE:-$((8 * target_file_size_base))}

echo "======================== Benchmark setup ========================"
echo -e "Job ID:\t\t\t\t\t$job_id"
echo -e "Data directory:\t\t\t\t$db_dir"
echo -e "WAL directory:\t\t\t\t$wal_dir"
echo -e "Output directory:\t\t\t$output_dir"
echo -e "Number of threads:\t\t\t$num_threads"
echo -e "Compression type for SST files:\t\t$compression_type"
echo -e "Raw database size:\t\t\t$db_size"
echo -e "Value size:\t\t\t\t$value_size"
echo -e "Number of keys:\t\t\t\t$num_keys"
echo -e "Duration of read-write/read-only tests:\t$duration"
echo -e "Write buffer size:\t\t\t$write_buffer_size"
echo -e "Blob files enabled:\t\t\t$enable_blob_files"
echo -e "Blob size threshold:\t\t\t$min_blob_size"
echo -e "Blob file size:\t\t\t\t$blob_file_size"
echo -e "Compression type for blob files:\t$blob_compression_type"
echo -e "Blob GC enabled:\t\t\t$enable_blob_garbage_collection"
echo -e "Blob GC age cutoff:\t\t\t$blob_garbage_collection_age_cutoff"
echo -e "Blob GC force threshold:\t\t$blob_garbage_collection_force_threshold"
echo -e "Blob compaction readahead size:\t\t$blob_compaction_readahead_size"
echo -e "Blob file starting level:\t\t$blob_file_starting_level"
echo -e "Blob cache enabled:\t\t\t$use_blob_cache"
echo -e "Blob cache and block cache shared:\t\t\t$use_shared_block_and_blob_cache"
echo -e "Blob cache size:\t\t$blob_cache_size"
echo -e "Blob cache number of shard bits:\t\t$blob_cache_numshardbits"
echo -e "Blob cache prepopulated:\t\t\t$prepopulate_blob_cache"
echo -e "Target SST file size:\t\t\t$target_file_size_base"
echo -e "Maximum size of base level:\t\t$max_bytes_for_level_base"
echo "================================================================="

rm -rf "$db_dir"
rm -rf "$wal_dir"
rm -rf "$output_dir"

ENV_VARS="\
  JOB_ID=$job_id \
  DB_DIR=$db_dir \
  WAL_DIR=$wal_dir \
  OUTPUT_DIR=$output_dir \
  NUM_THREADS=$num_threads \
  COMPRESSION_TYPE=$compression_type \
  VALUE_SIZE=$value_size \
  NUM_KEYS=$num_keys"

ENV_VARS_D="$ENV_VARS DURATION=$duration"

PARAMS="\
  --enable_blob_files=$enable_blob_files \
  --min_blob_size=$min_blob_size \
  --blob_file_size=$blob_file_size \
  --blob_compression_type=$blob_compression_type \
  --blob_file_starting_level=$blob_file_starting_level \
  --use_blob_cache=$use_blob_cache \
  --use_shared_block_and_blob_cache=$use_shared_block_and_blob_cache \
  --blob_cache_size=$blob_cache_size \
  --blob_cache_numshardbits=$blob_cache_numshardbits \
  --prepopulate_blob_cache=$prepopulate_blob_cache \
  --write_buffer_size=$write_buffer_size \
  --target_file_size_base=$target_file_size_base \
  --max_bytes_for_level_base=$max_bytes_for_level_base"

PARAMS_GC="$PARAMS \
  --enable_blob_garbage_collection=$enable_blob_garbage_collection \
  --blob_garbage_collection_age_cutoff=$blob_garbage_collection_age_cutoff \
  --blob_garbage_collection_force_threshold=$blob_garbage_collection_force_threshold \
  --blob_compaction_readahead_size=$blob_compaction_readahead_size"

# bulk load (using fillrandom) + compact
env -u DURATION -S "$ENV_VARS" ./tools/benchmark.sh bulkload "$PARAMS"

# overwrite + waitforcompaction
env -u DURATION -S "$ENV_VARS" ./tools/benchmark.sh overwrite "$PARAMS_GC"

# readwhilewriting
env -S "$ENV_VARS_D" ./tools/benchmark.sh readwhilewriting "$PARAMS_GC"

# fwdrangewhilewriting
env -S "$ENV_VARS_D" ./tools/benchmark.sh fwdrangewhilewriting "$PARAMS_GC"

# readrandom
env -S "$ENV_VARS_D" ./tools/benchmark.sh readrandom "$PARAMS_GC"

# fwdrange
env -S "$ENV_VARS_D" ./tools/benchmark.sh fwdrange "$PARAMS_GC"

# save logs to output directory
cp "$db_dir"/LOG* "$output_dir/"


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
# REQUIRE: benchmark.sh exists in the current directory
# After execution of this script, log files are generated in $output_dir.
# report.txt provides a high level statistics

# This should be run from the parent of the tools directory. The command line is:
#   [$env_vars] tools/run_flash_bench.sh [list-of-threads]
#
# This runs a sequence of tests in the following sequence:
#   step 1) load - bulkload, compact, fillseq, overwrite
#   step 2) read-only for each number of threads
#   step 3) read-write for each number of threads
#   step 4) merge for each number of threads
#
# The list of threads is optional and when not set is equivalent to "24".
# Were list-of-threads specified as "1 2 4" then the tests in steps 2, 3 and
# 4 above would be repeated for 1, 2 and 4 threads. The tests in step 1 are
# only run for 1 thread.

# Test output is written to $OUTPUT_DIR, currently /tmp/output. The performance
# summary is in $OUTPUT_DIR/report.txt. There is one file in $OUTPUT_DIR per
# test and the tests are listed below.
#
# The environment variables are also optional. The variables are:
#
#   NKEYS         - number of key/value pairs to load
#   BG_MBWRITEPERSEC - write rate limit in MB/second for tests in which
#                   there is one thread doing writes and stats are
#                   reported for read threads. "BG" stands for background.
#                   If this is too large then the non-writer threads can get
#                   starved. This is used for the "readwhile" tests.
#   FG_MBWRITEPERSEC - write rate limit in MB/second for tests like overwrite
#                   where stats are reported for the write threads.
#   NSECONDS      - number of seconds for which to run each test in steps 2,
#                   3 and 4. There are currently 15 tests in those steps and
#                   they are repeated for each entry in list-of-threads so
#                   this variable lets you control the total duration to
#                   finish the benchmark.
#   RANGE_LIMIT   - the number of rows to read per range query for tests that
#                   do range queries.
#   VAL_SIZE      - the length of the value in the key/value pairs loaded.
#                   You can estimate the size of the test database from this,
#                   NKEYS and the compression rate (--compression_ratio) set
#                   in tools/benchmark.sh
#   BLOCK_LENGTH  - value for db_bench --block_size
#   CACHE_BYTES   - the size of the RocksDB block cache in bytes
#   DATA_DIR      - directory in which to create database files
#   LOG_DIR       - directory in which to create WAL files, may be the same
#                   as DATA_DIR
#   DO_SETUP      - when set to 0 then a backup of the database is copied from
#                   $DATA_DIR.bak to $DATA_DIR and the load tests from step 1
#                   The WAL directory is also copied from a backup if
#                   DATA_DIR != LOG_DIR. This allows tests from steps 2, 3, 4
#                   to be repeated faster.
#   SAVE_SETUP    - saves a copy of the database at the end of step 1 to
#                   $DATA_DIR.bak. When LOG_DIR != DATA_DIR then it is copied
#                   to $LOG_DIR.bak.
#   SKIP_LOW_PRI_TESTS - skip some of the tests which aren't crucial for getting
#                   actionable benchmarking data (look for keywords "bulkload",
#                   "sync=1", and "while merging").
#

# Size constants
K=1024
M=$((1024 * K))
G=$((1024 * M))

num_keys=${NKEYS:-$((1 * G))}
# write rate for readwhile... tests
bg_mbwps=${BG_MBWRITEPERSEC:-4}
# write rate for tests other than readwhile, 0 means no limit
fg_mbwps=${FG_MBWRITEPERSEC:-0}
duration=${NSECONDS:-$((60 * 60))}
nps=${RANGE_LIMIT:-10}
vs=${VAL_SIZE:-400}
cs=${CACHE_BYTES:-$(( 1 * G ))}
bs=${BLOCK_LENGTH:-8192}

# If no command line arguments then run for 24 threads.
if [[ $# -eq 0 ]]; then
  nthreads=( 24 )
else
  nthreads=( "$@" )
fi

for num_thr in "${nthreads[@]}" ; do
  echo Will run for $num_thr threads
done

# Update these parameters before execution !!!
db_dir=${DATA_DIR:-"/tmp/rocksdb/"}
wal_dir=${LOG_DIR:-"/tmp/rocksdb/"}

do_setup=${DO_SETUP:-1}
save_setup=${SAVE_SETUP:-0}

# By default we'll run all the tests. Set this to skip a set of tests which
# aren't critical for getting key metrics.
skip_low_pri_tests=${SKIP_LOW_PRI_TESTS:-0}

if [[ $skip_low_pri_tests == 1 ]]; then
  echo "Skipping some non-critical tests because SKIP_LOW_PRI_TESTS is set."
fi

output_dir="${TMPDIR:-/tmp}/output"

ARGS="\
OUTPUT_DIR=$output_dir \
NUM_KEYS=$num_keys \
DB_DIR=$db_dir \
WAL_DIR=$wal_dir \
VALUE_SIZE=$vs \
BLOCK_SIZE=$bs \
CACHE_SIZE=$cs"

mkdir -p $output_dir
echo -e "ops/sec\tmb/sec\tSize-GB\tL0_GB\tSum_GB\tW-Amp\tW-MB/s\tusec/op\tp50\tp75\tp99\tp99.9\tp99.99\tUptime\tStall-time\tStall%\tTest" \
  > $output_dir/report.txt

# Notes on test sequence:
#   step 1) Setup database via sequential fill followed by overwrite to fragment it.
#           Done without setting DURATION to make sure that overwrite does $num_keys writes
#   step 2) read-only tests for all levels of concurrency requested
#   step 3) non read-only tests for all levels of concurrency requested
#   step 4) merge tests for all levels of concurrency requested. These must come last.

###### Setup the database

if [[ $do_setup != 0 ]]; then
  echo Doing setup

  if [[ $skip_low_pri_tests != 1 ]]; then
    # Test 1: bulk load
    env $ARGS ./tools/benchmark.sh bulkload
  fi

  # Test 2a: sequential fill with large values to get peak ingest
  #          adjust NUM_KEYS given the use of larger values
  env $ARGS BLOCK_SIZE=$((1 * M)) VALUE_SIZE=$((32 * K)) NUM_KEYS=$(( num_keys / 64 )) \
       ./tools/benchmark.sh fillseq_disable_wal

  # Test 2b: sequential fill with the configured value size
  env $ARGS ./tools/benchmark.sh fillseq_disable_wal

  # Test 2c: same as 2a, but with WAL being enabled.
  env $ARGS BLOCK_SIZE=$((1 * M)) VALUE_SIZE=$((32 * K)) NUM_KEYS=$(( num_keys / 64 )) \
       ./tools/benchmark.sh fillseq_enable_wal

  # Test 2d: same as 2b, but with WAL being enabled.
  env $ARGS ./tools/benchmark.sh fillseq_enable_wal

  # Test 3: single-threaded overwrite
  env $ARGS NUM_THREADS=1 DB_BENCH_NO_SYNC=1 ./tools/benchmark.sh overwrite

else
  echo Restoring from backup

  rm -rf $db_dir

  if [ ! -d ${db_dir}.bak ]; then
    echo Database backup does not exist at ${db_dir}.bak
    exit -1
  fi

  echo Restore database from ${db_dir}.bak
  cp -p -r ${db_dir}.bak $db_dir

  if [[ $db_dir != $wal_dir ]]; then
    rm -rf $wal_dir

    if [ ! -d ${wal_dir}.bak ]; then
      echo WAL backup does not exist at ${wal_dir}.bak
      exit -1
    fi

    echo Restore WAL from ${wal_dir}.bak
    cp -p -r ${wal_dir}.bak $wal_dir
  fi
fi

if [[ $save_setup != 0 ]]; then
  echo Save database to ${db_dir}.bak
  cp -p -r $db_dir ${db_dir}.bak

  if [[ $db_dir != $wal_dir ]]; then
    echo Save WAL to ${wal_dir}.bak
    cp -p -r $wal_dir ${wal_dir}.bak
  fi
fi

###### Read-only tests

for num_thr in "${nthreads[@]}" ; do
  # Test 4: random read
  env $ARGS DURATION=$duration NUM_THREADS=$num_thr ./tools/benchmark.sh readrandom

  # Test 5: random range scans
  env $ARGS DURATION=$duration NUM_THREADS=$num_thr NUM_NEXTS_PER_SEEK=$nps \
    ./tools/benchmark.sh fwdrange

  # Test 6: random reverse range scans
  env $ARGS DURATION=$duration NUM_THREADS=$num_thr NUM_NEXTS_PER_SEEK=$nps \
    ./tools/benchmark.sh revrange
done

###### Non read-only tests

for num_thr in "${nthreads[@]}" ; do
  # Test 7: overwrite with sync=0
  env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$fg_mbwps \
    DB_BENCH_NO_SYNC=1 ./tools/benchmark.sh overwrite

  if [[ $skip_low_pri_tests != 1 ]]; then
    # Test 8: overwrite with sync=1
    env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$fg_mbwps \
      ./tools/benchmark.sh overwrite
  fi

  # Test 9: random update with sync=0
  env $ARGS DURATION=$duration NUM_THREADS=$num_thr DB_BENCH_NO_SYNC=1 \
      ./tools/benchmark.sh updaterandom

  if [[ $skip_low_pri_tests != 1 ]]; then
    # Test 10: random update with sync=1
   env $ARGS DURATION=$duration NUM_THREADS=$num_thr ./tools/benchmark.sh updaterandom
  fi

  # Test 11: random read while writing
  env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$bg_mbwps \
    DB_BENCH_NO_SYNC=1 ./tools/benchmark.sh readwhilewriting

  # Test 12: range scan while writing
  env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$bg_mbwps \
    DB_BENCH_NO_SYNC=1 NUM_NEXTS_PER_SEEK=$nps ./tools/benchmark.sh fwdrangewhilewriting

  # Test 13: reverse range scan while writing
  env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$bg_mbwps \
    DB_BENCH_NO_SYNC=1 NUM_NEXTS_PER_SEEK=$nps ./tools/benchmark.sh revrangewhilewriting
done

###### Merge tests

for num_thr in "${nthreads[@]}" ; do
  # Test 14: random merge with sync=0
  env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$fg_mbwps \
    DB_BENCH_NO_SYNC=1 ./tools/benchmark.sh mergerandom

  if [[ $skip_low_pri_tests != 1 ]]; then
    # Test 15: random merge with sync=1
    env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$fg_mbwps \
      ./tools/benchmark.sh mergerandom

    # Test 16: random read while merging
    env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$bg_mbwps \
      DB_BENCH_NO_SYNC=1 ./tools/benchmark.sh readwhilemerging

    # Test 17: range scan while merging
    env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$bg_mbwps \
      DB_BENCH_NO_SYNC=1 NUM_NEXTS_PER_SEEK=$nps ./tools/benchmark.sh fwdrangewhilemerging

    # Test 18: reverse range scan while merging
    env $ARGS DURATION=$duration NUM_THREADS=$num_thr MB_WRITE_PER_SEC=$bg_mbwps \
      DB_BENCH_NO_SYNC=1 NUM_NEXTS_PER_SEEK=$nps ./tools/benchmark.sh revrangewhilemerging
  fi
done

###### Universal compaction tests.

# Use a single thread to reduce the variability in the benchmark.
env $ARGS COMPACTION_TEST=1 NUM_THREADS=1 ./tools/benchmark.sh universal_compaction

if [[ $skip_low_pri_tests != 1 ]]; then
  echo bulkload > $output_dir/report2.txt
  head -1 $output_dir/report.txt >> $output_dir/report2.txt
  grep bulkload $output_dir/report.txt >> $output_dir/report2.txt
fi

echo fillseq_wal_disabled >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep fillseq.wal_disabled $output_dir/report.txt >> $output_dir/report2.txt

echo fillseq_wal_enabled >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep fillseq.wal_enabled $output_dir/report.txt >> $output_dir/report2.txt

echo overwrite sync=0 >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep overwrite $output_dir/report.txt | grep \.s0  >> $output_dir/report2.txt

if [[ $skip_low_pri_tests != 1 ]]; then
  echo overwrite sync=1 >> $output_dir/report2.txt
  head -1 $output_dir/report.txt >> $output_dir/report2.txt
  grep overwrite $output_dir/report.txt | grep \.s1  >> $output_dir/report2.txt
fi

echo updaterandom sync=0 >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep updaterandom $output_dir/report.txt | grep \.s0 >> $output_dir/report2.txt

if [[ $skip_low_pri_tests != 1 ]]; then
  echo updaterandom sync=1 >> $output_dir/report2.txt
  head -1 $output_dir/report.txt >> $output_dir/report2.txt
  grep updaterandom $output_dir/report.txt | grep \.s1 >> $output_dir/report2.txt
fi

echo mergerandom sync=0 >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep mergerandom $output_dir/report.txt | grep \.s0 >> $output_dir/report2.txt

if [[ $skip_low_pri_tests != 1 ]]; then
  echo mergerandom sync=1 >> $output_dir/report2.txt
  head -1 $output_dir/report.txt >> $output_dir/report2.txt
  grep mergerandom $output_dir/report.txt | grep \.s1 >> $output_dir/report2.txt
fi

echo readrandom >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep readrandom $output_dir/report.txt  >> $output_dir/report2.txt

echo fwdrange >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep fwdrange\.t $output_dir/report.txt >> $output_dir/report2.txt

echo revrange >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep revrange\.t $output_dir/report.txt >> $output_dir/report2.txt

echo readwhile >> $output_dir/report2.txt >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep readwhilewriting $output_dir/report.txt >> $output_dir/report2.txt

if [[ $skip_low_pri_tests != 1 ]]; then
  echo readwhile >> $output_dir/report2.txt
  head -1 $output_dir/report.txt >> $output_dir/report2.txt
  grep readwhilemerging $output_dir/report.txt >> $output_dir/report2.txt
fi

echo fwdreadwhilewriting >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep fwdrangewhilewriting $output_dir/report.txt >> $output_dir/report2.txt

if [[ $skip_low_pri_tests != 1 ]]; then
  echo fwdreadwhilemerging >> $output_dir/report2.txt
  head -1 $output_dir/report.txt >> $output_dir/report2.txt
  grep fwdrangewhilemerg $output_dir/report.txt >> $output_dir/report2.txt
fi

echo revreadwhilewriting >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep revrangewhilewriting $output_dir/report.txt >> $output_dir/report2.txt

if [[ $skip_low_pri_tests != 1 ]]; then
  echo revreadwhilemerging >> $output_dir/report2.txt
  head -1 $output_dir/report.txt >> $output_dir/report2.txt
  grep revrangewhilemerg $output_dir/report.txt >> $output_dir/report2.txt
fi

cat $output_dir/report2.txt


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
# REQUIRE: benchmark_leveldb.sh exists in the current directory
# After execution of this script, log files are generated in $output_dir.
# report.txt provides a high level statistics
#
# This should be used with the LevelDB fork listed here to use additional test options.
# For more details on the changes see the blog post listed below.
#   https://github.com/mdcallag/leveldb-1
#   http://smalldatum.blogspot.com/2015/04/comparing-leveldb-and-rocksdb-take-2.html
#
# This should be run from the parent of the tools directory. The command line is:
#   [$env_vars] tools/run_flash_bench.sh [list-of-threads]
#
# This runs a sequence of tests in the following sequence:
#   step 1) load - bulkload, compact, fillseq, overwrite
#   step 2) read-only for each number of threads
#   step 3) read-write for each number of threads
#
# The list of threads is optional and when not set is equivalent to "24".
# Were list-of-threads specified as "1 2 4" then the tests in steps 2, 3 and
# 4 above would be repeated for 1, 2 and 4 threads. The tests in step 1 are
# only run for 1 thread.

# Test output is written to $OUTPUT_DIR, currently /tmp/output. The performance
# summary is in $OUTPUT_DIR/report.txt. There is one file in $OUTPUT_DIR per
# test and the tests are listed below.
#
# The environment variables are also optional. The variables are:
#   NKEYS         - number of key/value pairs to load
#   NWRITESPERSEC - the writes/second rate limit for the *whilewriting* tests.
#                   If this is too large then the non-writer threads can get
#                   starved.
#   VAL_SIZE      - the length of the value in the key/value pairs loaded.
#                   You can estimate the size of the test database from this,
#                   NKEYS and the compression rate (--compression_ratio) set
#                   in tools/benchmark_leveldb.sh
#   BLOCK_LENGTH  - value for db_bench --block_size
#   CACHE_BYTES   - the size of the RocksDB block cache in bytes
#   DATA_DIR      - directory in which to create database files
#   DO_SETUP      - when set to 0 then a backup of the database is copied from
#                   $DATA_DIR.bak to $DATA_DIR and the load tests from step 1
#                   This allows tests from steps 2, 3 to be repeated faster.
#   SAVE_SETUP    - saves a copy of the database at the end of step 1 to
#                   $DATA_DIR.bak.

# Size constants
K=1024
M=$((1024 * K))
G=$((1024 * M))

num_keys=${NKEYS:-$((1 * G))}
wps=${NWRITESPERSEC:-$((10 * K))}
vs=${VAL_SIZE:-400}
cs=${CACHE_BYTES:-$(( 1 * G ))}
bs=${BLOCK_LENGTH:-4096}

# If no command line arguments then run for 24 threads.
if [[ $# -eq 0 ]]; then
  nthreads=( 24 )
else
  nthreads=( "$@" )
fi

for num_thr in "${nthreads[@]}" ; do
  echo Will run for $num_thr threads
done

# Update these parameters before execution !!!
db_dir=${DATA_DIR:-"/tmp/rocksdb/"}

do_setup=${DO_SETUP:-1}
save_setup=${SAVE_SETUP:-0}

output_dir="${TMPDIR:-/tmp}/output"

ARGS="\
OUTPUT_DIR=$output_dir \
NUM_KEYS=$num_keys \
DB_DIR=$db_dir \
VALUE_SIZE=$vs \
BLOCK_SIZE=$bs \
CACHE_SIZE=$cs"

mkdir -p $output_dir
echo -e "ops/sec\tmb/sec\tusec/op\tavg\tp50\tTest" \
  > $output_dir/report.txt

# Notes on test sequence:
#   step 1) Setup database via sequential fill followed by overwrite to fragment it.
#           Done without setting DURATION to make sure that overwrite does $num_keys writes
#   step 2) read-only tests for all levels of concurrency requested
#   step 3) non read-only tests for all levels of concurrency requested

###### Setup the database

if [[ $do_setup != 0 ]]; then
  echo Doing setup

  # Test 2a: sequential fill with large values to get peak ingest
  #          adjust NUM_KEYS given the use of larger values
  env $ARGS BLOCK_SIZE=$((1 * M)) VALUE_SIZE=$((32 * K)) NUM_KEYS=$(( num_keys / 64 )) \
       ./tools/benchmark_leveldb.sh fillseq

  # Test 2b: sequential fill with the configured value size
  env $ARGS ./tools/benchmark_leveldb.sh fillseq

  # Test 3: single-threaded overwrite
  env $ARGS NUM_THREADS=1 DB_BENCH_NO_SYNC=1 ./tools/benchmark_leveldb.sh overwrite

else
  echo Restoring from backup

  rm -rf $db_dir

  if [ ! -d ${db_dir}.bak ]; then
    echo Database backup does not exist at ${db_dir}.bak
    exit -1
  fi

  echo Restore database from ${db_dir}.bak
  cp -p -r ${db_dir}.bak $db_dir
fi

if [[ $save_setup != 0 ]]; then
  echo Save database to ${db_dir}.bak
  cp -p -r $db_dir ${db_dir}.bak
fi

###### Read-only tests

for num_thr in "${nthreads[@]}" ; do
  # Test 4: random read
  env $ARGS NUM_THREADS=$num_thr ./tools/benchmark_leveldb.sh readrandom

done

###### Non read-only tests

for num_thr in "${nthreads[@]}" ; do
  # Test 7: overwrite with sync=0
  env $ARGS NUM_THREADS=$num_thr DB_BENCH_NO_SYNC=1 \
    ./tools/benchmark_leveldb.sh overwrite

  # Test 8: overwrite with sync=1
  # Not run for now because LevelDB db_bench doesn't have an option to limit the
  # test run to X seconds and doing sync-per-commit for --num can take too long.
  # env $ARGS NUM_THREADS=$num_thr ./tools/benchmark_leveldb.sh overwrite

  # Test 11: random read while writing
  env $ARGS NUM_THREADS=$num_thr WRITES_PER_SECOND=$wps \
    ./tools/benchmark_leveldb.sh readwhilewriting

done

echo bulkload > $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep bulkload $output_dir/report.txt >> $output_dir/report2.txt
echo fillseq >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep fillseq $output_dir/report.txt >> $output_dir/report2.txt
echo overwrite sync=0 >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep overwrite $output_dir/report.txt | grep \.s0  >> $output_dir/report2.txt
echo overwrite sync=1 >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep overwrite $output_dir/report.txt | grep \.s1  >> $output_dir/report2.txt
echo readrandom >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep readrandom $output_dir/report.txt  >> $output_dir/report2.txt
echo readwhile >> $output_dir/report2.txt >> $output_dir/report2.txt
head -1 $output_dir/report.txt >> $output_dir/report2.txt
grep readwhilewriting $output_dir/report.txt >> $output_dir/report2.txt

cat $output_dir/report2.txt


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# A shell script to verify DB generated by generate_random_db.sh cannot opened and read correct data.
# ./ldb needs to be avaible to be executed.
#
# Usage: <SCRIPT> <DB Path>

scriptpath=`dirname $BASH_SOURCE`
if [ "$#" -lt 2 ]; then
  echo "usage: $BASH_SOURCE <db_directory> <compare_base_db_directory> [dump_file_name] [if_try_load_options] [if_ignore_unknown_options]"
  exit 1
fi

db_dir=$1
base_db_dir=$2
dump_file_name=${3:-"dump_file.txt"}
try_load_options=${4:-"1"}
ignore_unknown_options=${5:-"0"}
db_dump=$db_dir"/"$dump_file_name
base_db_dump=$base_db_dir"/"$dump_file_name
extra_params=

if [ "$try_load_options" = "0" ]; then
 extra_params=" --try_load_options=false"
elif [ "$try_load_options" = "1" ]; then
 extra_params=" --try_load_options=true"
fi

if [ "$ignore_unknown_options" = "1" ]; then
 extra_params="$extra_params --ignore_unknown_options"
fi

set -e
echo == Dumping data from $db_dir to $db_dump
./ldb dump --db=$db_dir $extra_params > $db_dump

echo == Dumping data from $base_db_dir to $base_db_dump
./ldb dump --db=$base_db_dir $extra_params > $base_db_dump

diff $db_dump $base_db_dump


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
#

if [ "$#" -lt 3 ]; then
  echo "usage: $BASH_SOURCE <input_data_path> <DB Path> <extern SST dir>"
  exit 1
fi

input_data_dir=$1
db_dir=$2
extern_sst_dir=$3
rm -rf $db_dir
mkdir -p $extern_sst_dir

set -e

n=0

for f in `find $input_data_dir -name sorted_data*`
do
  echo == Writing external SST file $f to $extern_sst_dir/extern_sst${n}
  ./ldb --db=$db_dir --create_if_missing write_extern_sst $extern_sst_dir/extern_sst${n} < $f
  let "n = n + 1"
done


#!/usr/bin/env bash
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# A shell script to run a batch of pysims and combine individual pysim output files.
#
# Usage: bash block_cache_pysim.sh trace_file_path result_dir downsample_size warmup_seconds max_jobs
# trace_file_path: The file path that stores the traces.
# result_dir: The directory to store pysim results. The output files from a pysim is stores in result_dir/ml
# downsample_size: The downsample size used to collect the trace.
# warmup_seconds: The number of seconds used for warmup.
# max_jobs: The max number of concurrent pysims to run.

# Install required packages to run simulations.
# sudo dnf install -y numpy scipy python-matplotlib ipython python-pandas sympy python-nose atlas-devel
ulimit -c 0

if [ $# -ne 5 ]; then
  echo "Usage: ./block_cache_pysim.sh trace_file_path result_dir downsample_size warmup_seconds max_jobs"
  exit 0
fi

trace_file="$1"
result_dir="$2"
downsample_size="$3"
warmup_seconds="$4"
max_jobs="$5"
max_num_accesses=100000000
current_jobs=1

ml_tmp_result_dir="$result_dir/ml"
rm -rf "$ml_tmp_result_dir"
mkdir -p "$result_dir"
mkdir -p "$ml_tmp_result_dir"

# Report miss ratio in the trace.
current_jobs=$(ps aux | grep pysim | grep python | grep -cv grep)
for cf_name in "all"
do
for cache_size in "1G" "2G" "4G" "8G" "16G" #"12G" "16G" "1T"
do
for cache_type in "opt" "lru" "pylru" "pycctbbt" "pyhb" "ts" "trace" "lru_hybrid"  #"pycctblevelbt" #"lru_hybridn" "opt" #"pylru" "pylru_hybrid" "pycctbbt" "pycccfbt" "trace"
do
    if [[ $cache_type == "trace" && $cache_size != "16G" ]]; then
      # We only need to collect miss ratios observed in the trace once.
      continue
    fi
    while [ "$current_jobs" -ge "$max_jobs" ]
    do
      sleep 10
      echo "Waiting jobs to complete. Number of running jobs: $current_jobs"
      current_jobs=$(ps aux | grep pysim | grep python | grep -cv grep)
      echo "Waiting jobs to complete. Number of running jobs: $current_jobs"
    done
    output="log-ml-$cache_type-$cache_size-$cf_name"
    echo "Running simulation for $cache_type, cache size $cache_size, and cf_name $cf_name. Number of running jobs: $current_jobs. "
    nohup python block_cache_pysim.py "$cache_type" "$cache_size" "$downsample_size" "$warmup_seconds" "$trace_file" "$ml_tmp_result_dir" "$max_num_accesses" "$cf_name" >& "$ml_tmp_result_dir/$output" &
    current_jobs=$((current_jobs+1))
done
done
done

# Wait for all jobs to complete.
while [ $current_jobs -gt 0 ]
do
  sleep 10
  echo "Waiting jobs to complete. Number of running jobs: $current_jobs"
  current_jobs=$(ps aux | grep pysim | grep python | grep -cv grep)
  echo "Waiting jobs to complete. Number of running jobs: $current_jobs"
done

echo "Combine individual pysim output files"

rm -rf "$result_dir/ml_*"
for header in "header-" "data-"
do
for fn in "$ml_tmp_result_dir"/*
do
  sum_file=""
  time_unit=""
  capacity=""
  target_cf_name=""
  if [[ $fn == *"timeline"* ]]; then
    tmpfn="$fn"
    IFS='-' read -ra elements <<< "$tmpfn"
    time_unit_index=0
    capacity_index=0
    for i in "${elements[@]}"
    do
       if [[ $i == "timeline" ]]; then
         break
       fi
       time_unit_index=$((time_unit_index+1))
    done
    time_unit_index=$((time_unit_index+1))
    capacity_index=$((time_unit_index+2))
    target_cf_name_index=$((time_unit_index+3))
    time_unit="${elements[$time_unit_index]}_"
    capacity="${elements[$capacity_index]}_"
    target_cf_name="${elements[$target_cf_name_index]}_"
  fi

  if [[ $fn == *"${header}ml-policy-timeline"* ]]; then
    sum_file="$result_dir/ml_${target_cf_name}${capacity}${time_unit}policy_timeline"
  fi
  if [[ $fn == *"${header}ml-policy-ratio-timeline"* ]]; then
    sum_file="$result_dir/ml_${target_cf_name}${capacity}${time_unit}policy_ratio_timeline"
  fi
  if [[ $fn == *"${header}ml-miss-timeline"* ]]; then
    sum_file="$result_dir/ml_${target_cf_name}${capacity}${time_unit}miss_timeline"
  fi
  if [[ $fn == *"${header}ml-miss-ratio-timeline"* ]]; then
    sum_file="$result_dir/ml_${target_cf_name}${capacity}${time_unit}miss_ratio_timeline"
  fi
  if [[ $fn == *"${header}ml-mrc"* ]]; then
    tmpfn="$fn"
    IFS='-' read -ra elements <<< "$tmpfn"
    target_cf_name=${elements[-1]}
    sum_file="${result_dir}/ml_${target_cf_name}_mrc"
  fi
  if [[ $fn == *"${header}ml-avgmb"* ]]; then
    tmpfn="$fn"
    IFS='-' read -ra elements <<< "$tmpfn"
    time_unit=${elements[3]}
    target_cf_name=${elements[-1]}
    sum_file="${result_dir}/ml_${time_unit}_${target_cf_name}_avgmb"
  fi
  if [[ $fn == *"${header}ml-p95mb"* ]]; then
    tmpfn="$fn"
    IFS='-' read -ra elements <<< "$tmpfn"
    time_unit=${elements[3]}
    target_cf_name=${elements[-1]}
    sum_file="${result_dir}/ml_${time_unit}_${target_cf_name}_p95mb"
  fi
  if [[ $sum_file == "" ]]; then
    continue
  fi
  if [[ $header == "header-" ]]; then
    if [ -e "$sum_file" ]; then
      continue
    fi
  fi
  cat "$fn" >> "$sum_file"
done
done

echo "Done"
for fn in $result_dir/*
do
  if [[ $fn == *"_mrc" || $fn == *"_avgmb" || $fn == *"_p95mb" ]]; then
    # Sort MRC file by cache_type and cache_size.
    tmp_file="$result_dir/tmp_mrc"
    cat "$fn" | sort -t ',' -k1,1 -k4,4n > "$tmp_file"
    cat "$tmp_file" > "$fn"
    rm -rf "$tmp_file"
  fi
done


#! /usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.

set -e
set -o pipefail

if [ "$1" ]; then
  # Target file specified on command line
  TARGET="$1"
else
  # Interactively choose a group and file name
  DIRS="`find unreleased_history/ -mindepth 1 -maxdepth 1 -type d`"
  echo "Choose a group for new release note:"
  echo "$DIRS" | grep -nEo '[^/]+$'
  echo -n "Enter a number: "
  while [ ! "$DIRNUM" ]; do read -r DIRNUM; done
  DIR="$(echo "$DIRS" | head -n "$DIRNUM" | tail -1)"
  echo "Choose a file name for new release note (e.g. improved_whatever.md):"
  while [ ! "$FNAME" ]; do read -re FNAME; done
  # Replace spaces with underscores
  TARGET="$(echo "$DIR/$FNAME" | tr ' ' '_')"
fi

# Edit/create the file
${EDITOR:-nano} "$TARGET"
# Add to version control (easy to forget!)
git add "$TARGET"


#! /usr/bin/env bash
# Copyright (c) Meta Platforms, Inc. and affiliates.

set -e

if [ ! -d unreleased_history ]; then
  echo "Can't find unreleased_history/ directory"
  exit 1
fi

GIT_PATHS="unreleased_history/ HISTORY.md"
if [ ! "$DRY_RUN" ]; then
  # Check for uncommitted changes
  UNCOMMITTED="$(git diff -- $GIT_PATHS)"
  if [ "$UNCOMMITTED" ]; then
    echo 'Uncommitted changes to files to be modified. Please commit first to'
    echo 'ensure a clean revert path. You can always `git commit -a --amend`'
    echo 'to add more changes to your commit.'
    exit 2
  fi
fi

# Add first part of existing HISTORY file to new version
awk '{ print } /NOTE/ { exit(0) }' < HISTORY.md > HISTORY.new

# And a blank line separator
echo >> HISTORY.new

# Add new version header
awk '/#define ROCKSDB_MAJOR/ { major = $3 }
     /#define ROCKSDB_MINOR/ { minor = $3 }
     /#define ROCKSDB_PATCH/ { patch = $3 }
     END { printf "## " major "." minor "." patch }' < include/rocksdb/version.h >> HISTORY.new
echo " (`date +%x`)" >> HISTORY.new

function process_file () {
  # use awk to correct extra or missing newlines, missing '* ' on first line
  awk '/./ { if (notfirstline || $1 == "*") print;
             else print "* " $0;
             notfirstline=1; }' < $1 >> HISTORY.new
  echo git rm $1
  if [ ! "$DRY_RUN" ]; then
    git rm $1
  fi
}

PROCESSED_DIRECTORIES=""

function process_dir () {
  PROCESSED_DIRECTORIES="$PROCESSED_DIRECTORIES $1"
  # ls will sort the files, including the permanent header file
  FILES="$(ls unreleased_history/$1/)"
  if [ "$FILES" ]; then
    echo "### $2" >> HISTORY.new
    for FILE in $FILES; do
      process_file "unreleased_history/$1/$FILE"
    done
    echo >> HISTORY.new
    echo "Saved entries from $1"
  else
    echo "Nothing new in $1"
  fi
}

# Process dirs and files
process_dir new_features "New Features"
process_dir public_api_changes "Public API Changes"
process_dir behavior_changes "Behavior Changes"
process_dir bug_fixes "Bug Fixes"
process_dir performance_improvements "Performance Improvements"

# Check for unexpected files or dirs at top level. process_dir/process_file
# will deal with contents of these directories
EXPECTED_REGEX="[^/]*[.]sh|README[.]txt|$(echo $PROCESSED_DIRECTORIES | tr ' ' '|')"
platform=`uname`
if [ $platform = 'Darwin' ]; then
  UNEXPECTED="$(find -E unreleased_history -mindepth 1 -maxdepth 1 -not -regex "[^/]*/($EXPECTED_REGEX)")"
else
  UNEXPECTED="$(find unreleased_history/ -mindepth 1 -maxdepth 1 -regextype egrep -not -regex "[^/]*/($EXPECTED_REGEX)")"
fi
if [ "$UNEXPECTED" ]; then
  echo "Unexpected files I don't know how to process:"
  echo "$UNEXPECTED"
  rm HISTORY.new
  exit 3
fi

# Add rest of existing HISTORY file to new version (collapsing newlines)
awk '/./ { if (note) pr=1 }
     /NOTE/ { note=1 }
     { if (pr) print }' < HISTORY.md >> HISTORY.new

if [ "$DRY_RUN" ]; then
  echo '==========================================='
  diff -U3 HISTORY.md HISTORY.new || true
  rm HISTORY.new
else
  mv HISTORY.new HISTORY.md
  echo "Done. Revert command: git checkout HEAD -- $GIT_PATHS"
fi


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2.

TOOLCHAIN_DIR=/opt/rh/devtoolset-8/root/usr/bin
if [[ -d "$TOOLCHAIN_DIR" ]]; then
    PATH="$TOOLCHAIN_DIR:$PATH"
fi

SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
GETDEPS_PATHS=(
    "$SCRIPT_DIR/build/fbcode_builder/getdeps.py"
    "$SCRIPT_DIR/../../opensource/fbcode_builder/getdeps.py"
)
for getdeps in "${GETDEPS_PATHS[@]}"; do
    if [[ -x "$getdeps" ]]; then
        exec "$getdeps" build eden "$@"
    fi
done
echo "Could not find getdeps.py" >&2
exit 1


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# This file removes all internal-only files and deletes @fb-only commented lines from code
# This is only used in our internal CI tests, the actual removing when syncing to GitHub is done by shipit.

cd "$(dirname "$0")" || exit

function findAllSourceFiles() {
  find . \
    \( -name '*.ts' -o -name '*.tsx' -o -name '*.js' \) \
    -type f \
    \( -not -path '*/node_modules/*' -not -path '*/dist/*' -not -path '*/build/*' -not -path '.vscode-build' -not -path '*/coverage/*' \) \
    -print0
}

echo "replacing all fb-only comments..."

fbOnlyRegex='s/^([[:space:]]*).* \/\/ @fb-only/\1\/\/ @fb-only/g'
OS=$(uname)
if [ "$OS" = 'Darwin' ]; then
  # macOS ships with BSD version of sed, where -i '' is required to not create backup files,
  # but -i '' doesn't work on linux.
  findAllSourceFiles | xargs -0 sed -i '' -r "$fbOnlyRegex"
else
  findAllSourceFiles | xargs -0 sed -i'' -r "$fbOnlyRegex"
fi

echo "deleting all facebook/ directories..."
find . -path '*/facebook' -type d -print0 | xargs -0 rm -rf


#!/bin/sh
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

SCRIPT_DIR=$( cd -- "$( dirname -- "$0" )" &> /dev/null && pwd )
# Facilitate redefining NODE in terms of SCRIPT_DIR via a regex.
NODE=node

if [ ! -x "$(command -v $NODE)" ]; then
  # shellcheck disable=SC2016
  echo 'ERROR: `node` is required to run Interactive Smartlog, but it was not'
  # shellcheck disable=SC2016
  echo 'found on the $PATH. For information on installing Node.js, see:'
  echo 'https://nodejs.dev/en/learn/how-to-install-nodejs/'
  exit 1
fi

"$NODE" "$SCRIPT_DIR/isl-server/dist/run-proxy.js" "$@"


#!/bin/sh
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

cd "$(dirname "$0")" || exit

# Tauri doesn't like being run with the node configured by our repo's .yarnrc
# Instead of yarn tauri dev, we have to run it from node_modules.
# Additionally, in dev mode, we also want to use a dev run of the ISL server.
# We run this by invoking run-proxy directly rather than `yarn serve`, since
# we're not in the isl-server directory.
# You can pass additional arguments to run-proxy, but not all of them make sense,
# such as --foreground.

./node_modules/.bin/tauri dev -- -- node ../../isl-server/dist/run-proxy.js --dev "$@"


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -x
name="$1"
gh release view "$name" || gh release create --prerelease --title "$name" "$name"


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

set -x
set -e

max_attempts=4
attempt=1

until "$@"
do
  if (( attempt == max_attempts ))
  then
    echo "Failed after $max_attempts attempts"
    exit 1
  fi

  sleep 5
  : $(( attempt++ ))
done


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

DIR=$(dirname -- "$0")
VERSION=$(<"$DIR"/../SAPLING_VERSION)

# Create the commit info using either sl or git, whichever way we cloned this repo
if ! command -v sl &> /dev/null; then
  COMMIT_INFO=$(git -c "core.abbrev=8" show -s "--format=%cd+%h" "--date=format:%Y%m%d-%H%M%S")
else
  COMMIT_INFO=$(sl log --rev . --template '{date(date, "%Y%m%d-%H%M%S")}+{shortest(node, 8)}')
fi

echo "$VERSION"."$COMMIT_INFO"


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# Verifies a Sapling .deb on Ubuntu 20.04 or 22.04.
#
# This script takes the URL to a .deb from the Releases page such as:
#
#     https://github.com/bolinfest/eden/releases/download/20220831-124726-b53cb8ea/sapling_0.0-20220831.124726.b53cb8ea_amd64.Ubuntu20.04.deb
#
# and creates a barebones Docker container to install and run it.
#
# If the file has already been download locally (perhaps from a private
# GitHub repo?), then the local path can also be used as an argument:
#
#     verify_release.sh ~/Downloads/sapling_0.0-20220831.124726.b53cb8ea_amd64.Ubuntu20.04.deb
#
# Note that a sapling.deb should not assume the user has `apt install`'d
# anything else.

set -e
set -x

FILE_OR_URL="$1"
echo "$FILE_OR_URL"

tmp_dir=$(mktemp -d -t ci-XXXXXXXXXX)
echo "$tmp_dir"
cd "$tmp_dir"

if [[ "$FILE_OR_URL" == *"Ubuntu20.04.deb" ]]; then
  UBUNTU_VERSION=20.04
else
  if [[ "$FILE_OR_URL" == *"Ubuntu22.04.deb" ]]; then
    UBUNTU_VERSION=22.04
  else
    echo "could not determine Ubuntu version from arg '${FILE_OR_URL}'" >> /dev/stderr
    exit 1
  fi
fi

if [ ! -f "$FILE_OR_URL" ]; then
  curl --location -O "$FILE_OR_URL"
else
  cp "$FILE_OR_URL" .
fi

cat > Dockerfile <<EOF
FROM ubuntu:${UBUNTU_VERSION}
COPY *.deb /root/sapling.deb

# https://serverfault.com/a/1016972 to ensure installing tzdata does not
# result in a prompt that hangs forever.
ARG DEBIAN_FRONTEND=noninteractive
ENV TZ=Etc/UTC

RUN apt update -y
RUN apt install -y /root/sapling.deb
RUN which sl
RUN sl clone https://github.com/bolinfest/opensnoop-native
RUN cd /opensnoop-native && sl isl --no-open --json
EOF

IMAGE_ID=$(echo $RANDOM | shasum | head -c 8)
IMAGE_NAME="sapling-test-${IMAGE_ID}"
docker build --progress=plain -t "$IMAGE_NAME" .
docker run -it "$IMAGE_NAME" /bin/bash


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2.

set -e

NAME=$1
if [[ "$NAME" == facebook/* ]]; then
    SUBDIR="/facebook"
    # Strip facebook prefix
    NAME="${NAME#*/}"
else
    SUBDIR=""
fi
shift 1
fbsource=$(hg root)

set -x
"$fbsource/buck-out/v2/gen/fbcode/eden/mononoke/tests/integration/integration_runner_real.par" "$fbsource/buck-out/v2/gen/fbcode/eden/mononoke/tests/integration$SUBDIR/$NAME-manifest.json" "$@"


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2.

set -e

BUCK=buck2

if [[ "$1" == facebook/* ]]; then
    # Strip facebook prefix
    TARGET="/facebook:${1#*/}-manifest"
else
    TARGET=":$1-manifest"
fi

shift

set -x

$BUCK build "$@" //eden/mononoke/tests/integration:integration_runner_real "//eden/mononoke/tests/integration$TARGET" "//eden/mononoke/tests/integration${TARGET}[deps]"


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2.

# Library routines for generating or interacting with commits

# shellcheck source=fbcode/eden/mononoke/tests/integration/library.sh
. "${TEST_FIXTURES}/library.sh"

function whereami() {
  hg log -r . -T "{node}"
}

function add_triangle_merge_commits_and_push() {
  # Add multiple merge commits in a triangle like shape:
  # o
  # |\
  # o o
  # | |\
  # o o o
  # | |/
  # o o
  # |/
  # o
  # The first argument represents how many merges to add.
  # The second argument is optional and represents a unique identifier for the
  # current invocation of the function.
  count="$1"
  shift
  local seed tomerge topush st commit i j
  seed=${1:-$(wherami)}
  for i in $(seq 1 "$count"); do
    echo "i=${i}" >> "${seed}_${i}"
    hg ci -Am "${seed}: base branch ${i}"
  done
  tomerge="$(whereami)"
  topush=()
  for i in $(seq 1 "$count"); do
    hg prev
    st="$(whereami)"
    for j in $(seq 1 $((2 * i - 1))); do
      echo "j=$j" >> "${seed}_${i}_${j}"
      hg ci -Am "${seed}: commit ${j} for branch ${i}"
      if [ "${i}" == "${count}" ]; then
        topush+=( "$(whereami)" )
      fi
    done
    hg merge -r "${tomerge}"
    hg commit -Am "${seed}: merge branch ${i}"
    tomerge="$(whereami)"
    hg co -r "${st}"
  done
  topush+=( "$tomerge" )

  # simulate updating master with several single parents commits then a merge
  for commit in "${topush[@]}"; do
    hg co "${commit}"
    hgmn push -r . --to master_bookmark --pushvar NON_FAST_FORWARD=true
  done
}


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2.

# shellcheck source=fbcode/eden/mononoke/tests/integration/library.sh
. "${TEST_FIXTURES}/library.sh"

function verify_wc() {
   local large_repo_commit
   large_repo_commit="$1"
   "$MONONOKE_ADMIN" \
     "${CACHE_ARGS[@]}" \
     "${COMMON_ARGS[@]}" --log-level ERROR \
     --mononoke-config-path  "$TESTTMP"/mononoke-config \
     --source-repo-id="$REPOIDLARGE" --target-repo-id="$REPOIDSMALL" \
     crossrepo verify-wc "$large_repo_commit"
}

function validate_commit_sync() {
  local entry_id
  entry_id="$1"
  shift
  "$COMMIT_VALIDATOR" \
    "${CACHE_ARGS[@]}" \
    "${COMMON_ARGS[@]}" --debug --repo-id "$REPOIDLARGE" \
    --mononoke-config-path "$TESTTMP/mononoke-config" \
    --master-bookmark=master_bookmark \
    once --entry-id "$entry_id" "$@"
}

function large_small_megarepo_config() {
cat >> "$TESTTMP/mononoke-config/common/commitsyncmap.toml" <<EOF
[megarepo_test]
large_repo_id = 0
common_pushrebase_bookmarks = ["master_bookmark"]
 [[megarepo_test.small_repos]]
 repoid = 1
 bookmark_prefix = "bookprefix/"
 default_action = "prepend_prefix"
 default_prefix = "smallrepofolder"
 direction = "large_to_small"
    [megarepo_test.small_repos.mapping]
    "non_path_shifting" = "non_path_shifting"
EOF

  if [ -n "$COMMIT_SYNC_CONF" ]; then
    cat > "$COMMIT_SYNC_CONF/current" << EOF
{
  "repos": {
    "megarepo_test": {
      "large_repo_id": 0,
      "common_pushrebase_bookmarks": ["master_bookmark"],
      "small_repos": [
        {
          "repoid": 1,
          "bookmark_prefix": "bookprefix/",
          "default_action": "prepend_prefix",
          "default_prefix": "smallrepofolder",
          "direction": "large_to_small",
          "mapping": {
            "non_path_shifting": "non_path_shifting"
          }
        }
      ],
      "version_name": "test_version"
    }
  }
}
EOF

    cat > "$COMMIT_SYNC_CONF/all" << EOF
{
  "repos": {
    "megarepo_test": {
      "versions": [
        {
          "large_repo_id": 0,
          "common_pushrebase_bookmarks": ["master_bookmark"],
          "small_repos": [
            {
              "repoid": 1,
              "bookmark_prefix": "bookprefix/",
              "default_action": "prepend_prefix",
              "default_prefix": "smallrepofolder",
              "direction": "large_to_small",
              "mapping": {
                "non_path_shifting": "non_path_shifting"
              }
            }
          ],
          "version_name": "test_version"
        }
      ],
      "common": {
        "common_pushrebase_bookmarks": ["master_bookmark"],
        "large_repo_id": 0,
        "small_repos": {
          1: {
            "bookmark_prefix": "bookprefix/"
          }
        }
      }
    }
  }
}
EOF

  fi

}

function large_small_config() {
  REPOTYPE="blob_files"
  INFINITEPUSH_ALLOW_WRITES=true REPOID=0 REPONAME=large-mon setup_common_config "$REPOTYPE"
  INFINITEPUSH_ALLOW_WRITES=true REPOID=1 REPONAME=small-mon setup_common_config "$REPOTYPE"
  large_small_megarepo_config
}

function large_small_setup() {
  quiet testtool_drawdag -R small-mon <<'EOF'
S_B
|
S_A
# message: S_A "pre-move commit"
# author: S_A test
# modify: S_A file.txt "1\n"
# forget: S_A S_A
# message: S_B "first post-move commit"
# author: S_B test
# modify: S_B filetoremove "1\n"
# bookmark: S_B master_bookmark
# forget: S_B S_B
EOF

  export SMALL_MASTER_BONSAI
  SMALL_MASTER_BONSAI=$S_B

  quiet testtool_drawdag -R large-mon <<'EOF'
L_C
|
L_B
|
L_A
# message: L_A "pre-move commit"
# author: L_A test
# modify: L_A file.txt "1\n"
# message: L_B "move commit"
# author: L_B test
# forget: L_A L_A
# copy: L_B smallrepofolder/file.txt "1\n" L_A file.txt
# delete: L_B file.txt
# forget: L_B L_B
# message: L_C "first post-move commit"
# author: L_C test
# modify: L_C smallrepofolder/filetoremove "1\n"
# bookmark: L_C master_bookmark
# forget: L_C L_C
EOF

  export LARGE_MASTER_BONSAI
  LARGE_MASTER_BONSAI=$L_C

  echo "Adding synced mapping entry"
  export REPOIDLARGE=0
  export REPOIDSMALL=1
  add_synced_commit_mapping_entry "$REPOIDSMALL" "$SMALL_MASTER_BONSAI" \
   "$REPOIDLARGE" "$LARGE_MASTER_BONSAI" "test_version"

}

function create_large_small_repo() {
  large_small_config
  large_small_setup
}

function update_mapping_version {
  local small_repo_id large_repo_id small_bcs_id large_bcs_id version
  small_repo_id="$1"
  small_bcs_id="$2"
  large_repo_id="$3"
  large_bcs_id="$4"
  version="$5"
  sqlite3 "$TESTTMP/monsql/sqlite_dbs" <<EOF
    REPLACE INTO synced_commit_mapping (small_repo_id, small_bcs_id, large_repo_id, large_bcs_id, sync_map_version_name)
    VALUES ('$small_repo_id', X'$small_bcs_id', '$large_repo_id', X'$large_bcs_id', '$version');
EOF

}

function update_commit_sync_map_first_option {
  cat > "$COMMIT_SYNC_CONF/current" << EOF
{
  "repos": {
    "megarepo_test": {
      "large_repo_id": 0,
      "common_pushrebase_bookmarks": ["master_bookmark"],
      "small_repos": [
        {
          "repoid": 1,
          "bookmark_prefix": "bookprefix/",
          "default_action": "prepend_prefix",
          "default_prefix": "smallrepofolder_after",
          "direction": "large_to_small",
          "mapping": {
            "non_path_shifting": "non_path_shifting"
          }
        }
      ],
      "version_name": "new_version"
    }
  }
}
EOF

  cat > "$COMMIT_SYNC_CONF/all" << EOF
{
  "repos": {
    "megarepo_test": {
      "versions": [
        {
          "large_repo_id": 0,
          "common_pushrebase_bookmarks": ["master_bookmark"],
          "small_repos": [
            {
              "repoid": 1,
              "bookmark_prefix": "bookprefix/",
              "default_action": "prepend_prefix",
              "default_prefix": "smallrepofolder",
              "direction": "large_to_small",
              "mapping": {
                "non_path_shifting": "non_path_shifting"
              }
            }
          ],
          "version_name": "test_version"
        },
      {
        "large_repo_id": 0,
        "common_pushrebase_bookmarks": ["master_bookmark"],
        "small_repos": [
          {
            "repoid": 1,
            "bookmark_prefix": "bookprefix/",
            "default_action": "prepend_prefix",
            "default_prefix": "smallrepofolder_after",
            "direction": "large_to_small",
            "mapping": {
              "non_path_shifting": "non_path_shifting"
            }
          }
        ],
        "version_name": "new_version"
      }
      ],
      "common": {
        "common_pushrebase_bookmarks": ["master_bookmark"],
        "large_repo_id": 0,
        "small_repos": {
          1: {
            "bookmark_prefix": "bookprefix/"
          }
        }
      }
    }
  }
}
EOF

}

function update_commit_sync_map_second_option {
  cat > "$COMMIT_SYNC_CONF/current" <<EOF
{
  "repos": {
    "megarepo_test": {
        "large_repo_id": 0,
        "common_pushrebase_bookmarks": [
          "master_bookmark"
        ],
        "small_repos": [
          {
            "repoid": 1,
            "default_action": "prepend_prefix",
            "default_prefix": "smallrepofolder1",
            "bookmark_prefix": "bookprefix1/",
            "mapping": {
              "special": "specialsmallrepofolder_after_change"
            },
            "direction": "large_to_small"
          }
        ],
        "version_name": "TEST_VERSION_NAME_LIVE_V2"
    }
  }
}
EOF

  cat > "$COMMIT_SYNC_CONF/all" << EOF
{
  "repos": {
    "megarepo_test": {
      "versions": [
        {
          "large_repo_id": 0,
          "common_pushrebase_bookmarks": ["master_bookmark"],
          "small_repos": [
            {
              "repoid": 1,
              "default_action": "prepend_prefix",
              "default_prefix": "smallrepofolder1",
              "bookmark_prefix": "bookprefix1/",
              "mapping": {
                "special": "specialsmallrepofolder1"
              },
              "direction": "large_to_small"
            }
          ],
          "version_name": "TEST_VERSION_NAME_LIVE_V1"
        },
        {
          "large_repo_id": 0,
          "common_pushrebase_bookmarks": [
            "master_bookmark"
          ],
          "small_repos": [
            {
              "repoid": 1,
              "default_action": "prepend_prefix",
              "default_prefix": "smallrepofolder1",
              "bookmark_prefix": "bookprefix1/",
              "mapping": {
                "special": "specialsmallrepofolder_after_change"
              },
              "direction": "large_to_small"
            }
          ],
          "version_name": "TEST_VERSION_NAME_LIVE_V2"
        }
      ],
      "common": {
        "common_pushrebase_bookmarks": ["master_bookmark"],
        "large_repo_id": 0,
        "small_repos": {
          1: {
            "bookmark_prefix": "bookprefix1/"
          }
        }
      }
    }
  }
}
EOF

}

function init_large_small_repo() {
  create_large_small_repo
  start_large_small_repo "$@"
  init_local_large_small_clones
}

function init_local_large_small_clones {
  cd "$TESTTMP" || exit 1
  REPONAME=small-mon hgmn_clone "mononoke://$(mononoke_address)/small-mon" small-hg-client --config extensions.remotenames=
  cat >> small-hg-client/.hg/hgrc <<EOF
[extensions]
pushrebase =
EOF
  REPONAME=large-mon hgmn_clone "mononoke://$(mononoke_address)/large-mon" large-hg-client --config extensions.remotenames=
  cat >> large-hg-client/.hg/hgrc <<EOF
[extensions]
pushrebase =
EOF
}

function start_large_small_repo {
  echo "Starting Mononoke server"
  mononoke "$@"
  wait_for_mononoke
  # Setting XREPOSYNC allows the user of this fn to set x-repo sync mutable counter instead of the backsync one
  # This is useful, if the intention is to use x-repo sync instead of backsync after the setup
  if [[ -v XREPOSYNC ]]; then
    sqlite3 "$TESTTMP/monsql/sqlite_dbs" "INSERT INTO mutable_counters (repo_id, name, value) VALUES ($REPOIDLARGE, 'xreposync_from_$REPOIDSMALL', 1)";
  else
    sqlite3 "$TESTTMP/monsql/sqlite_dbs" "INSERT INTO mutable_counters (repo_id, name, value) VALUES ($REPOIDSMALL, 'backsync_from_$REPOIDLARGE', 1)";
  fi
}

function createfile {
  mkdir -p "$(dirname "$1")" && echo "$1" > "$1" && hg add -q "$1";
}

function create_first_post_move_commit {
 echo 1 > "$1/filetoremove" && hg add "$1/filetoremove" && hg ci -m 'first post-move commit'
 hg revert -r .^ "$1/filetoremove"
}

function init_client() {
  cd "$TESTTMP" || exit 1
  hgclone_treemanifest ssh://user@dummy/"$1" "$2" --noupdate --config extensions.remotenames=
  cd "$TESTTMP/$2" || exit 1
  cat >> .hg/hgrc <<EOF
[extensions]
pushrebase =
remotenames =
EOF
}

function backsync_large_to_small() {
  if [[ ! -d "$SCRIBE_LOGS_DIR" ]]; then
    mkdir "$SCRIBE_LOGS_DIR"
  fi

  "$BACKSYNCER" \
    "${CACHE_ARGS[@]}" \
    "${COMMON_ARGS[@]}" --debug --source-repo-id "$REPOIDLARGE" \
    --mononoke-config-path "$TESTTMP/mononoke-config" \
    --scribe-logging-directory "$TESTTMP/scribe_logs" \
    --target-repo-id "$REPOIDSMALL" \
    backsync-all
}

function backsync_large_to_small_forever {
  if [[ ! -d "$SCRIBE_LOGS_DIR" ]]; then
    mkdir "$SCRIBE_LOGS_DIR"
  fi

  "$BACKSYNCER" \
    "${CACHE_ARGS[@]}" \
    "${COMMON_ARGS[@]}" --debug \
    --mononoke-config-path "$TESTTMP/mononoke-config" \
    --scribe-logging-directory "$TESTTMP/scribe_logs" \
    --source-repo-id "$REPOIDLARGE" \
    --target-repo-id "$REPOIDSMALL" \
    "$@" \
    backsync-forever >> "$TESTTMP/backsyncer.out" 2>&1 &

  export BACKSYNCER_PID=$!
  echo "$BACKSYNCER_PID" >> "$DAEMON_PIDS"
}

function mononoke_x_repo_sync_forever() {
  source_repo_id=$1
  target_repo_id=$2
  shift
  shift
  GLOG_minloglevel=5 "$MONONOKE_X_REPO_SYNC" \
    "${CACHE_ARGS[@]}" \
    "${COMMON_ARGS[@]}" \
    --mononoke-config-path "$TESTTMP/mononoke-config" \
    --scribe-logging-directory "$TESTTMP/scribe_logs" \
    --source-repo-id "$source_repo_id" \
    --target-repo-id "$target_repo_id" \
    "$@" \
    tail --sleep-secs=1 >> "$TESTTMP/xreposync.out" 2>&1 &

  export XREPOSYNC_PID=$!
  echo "$XREPOSYNC_PID" >> "$DAEMON_PIDS"
}

# Wait for xrepo sync (15s at most)
function wait_for_xrepo_sync {
  bookmark_update_log_entry="$1"
  local attempts=150
  for _ in $(seq 1 $attempts); do
    grep -q "successful sync bookmark update log #$bookmark_update_log_entry" "$TESTTMP/xreposync.out" && break
    sleep 0.1
  done

  if ! grep -q "successful sync bookmark update log #$bookmark_update_log_entry" "$TESTTMP/xreposync.out"; then
    echo "xrepo sync of bookmark update log entry $1 did not finish" >&2
    cat "$TESTTMP/xreposync.out"
    exit 1
  fi
}

function init_two_small_one_large_repo() {
  # setup configuration
  # Disable bookmarks cache because bookmarks are modified by two separate processes
  REPOTYPE="blob_files"
  REPOID=0 REPONAME=meg-mon setup_common_config $REPOTYPE
  REPOID=1 REPONAME=fbs-mon setup_common_config $REPOTYPE
  REPOID=2 REPONAME=ovr-mon setup_common_config $REPOTYPE

  cat >> "$HGRCPATH" <<EOF
[ui]
ssh="$DUMMYSSH"
[extensions]
amend=
pushrebase=
remotenames=
EOF

  setup_commitsyncmap
  setup_configerator_configs

  # setup hg server repos

  function createfile { mkdir -p "$(dirname  "$1")" && echo "$1" > "$1" && hg add -q "$1"; }
  function createfile_with_content { mkdir -p "$(dirname  "$1")" && echo "$2" > "$1" && hg add -q "$1"; }

  # init fbsource
  cd "$TESTTMP" || exit 1
  hginit_treemanifest fbs-hg-srv
  cd fbs-hg-srv || exit 1
  # create an initial commit, which will be the last_synced_commit
  createfile fbcode/fbcodefile_fbsource
  createfile arvr/arvrfile_fbsource
  createfile otherfile_fbsource
  hg -q ci -m "fbsource commit 1" && hg book -ir . master_bookmark

  # init ovrsource
  cd "$TESTTMP" || exit 1
  hginit_treemanifest ovr-hg-srv
  cd ovr-hg-srv || exit 1
  createfile fbcode/fbcodefile_ovrsource
  createfile arvr/arvrfile_ovrsource
  createfile otherfile_ovrsource
  createfile Research/researchfile_ovrsource
  hg -q ci -m "ovrsource commit 1" && hg book -r . master_bookmark

  # init megarepo - note that some paths are shifted, but content stays the same
  cd "$TESTTMP" || exit 1
  hginit_treemanifest meg-hg-srv
  cd meg-hg-srv || exit 1
  createfile fbcode/fbcodefile_fbsource
  createfile_with_content .fbsource-rest/arvr/arvrfile_fbsource arvr/arvrfile_fbsource
  createfile otherfile_fbsource
  createfile_with_content .ovrsource-rest/fbcode/fbcodefile_ovrsource fbcode/fbcodefile_ovrsource
  createfile arvr/arvrfile_ovrsource
  createfile_with_content arvr-legacy/otherfile_ovrsource otherfile_ovrsource
  createfile_with_content arvr-legacy/Research/researchfile_ovrsource Research/researchfile_ovrsource
  hg -q ci -m "megarepo commit 1"
  hg book -r . master_bookmark

  # blobimport hg servers repos into Mononoke repos
  cd "$TESTTMP"
  REPOID=0 blobimport meg-hg-srv/.hg meg-mon
  REPOID=1 blobimport fbs-hg-srv/.hg fbs-mon
  REPOID=2 blobimport ovr-hg-srv/.hg ovr-mon
}

function enable_pushredirect {
  repo_id=$1
  shift

  cat >"$PUSHREDIRECT_CONF/enable" << EOF
{"per_repo": {"$repo_id": {"public_push": true, "draft_push": false}}}
EOF
}


#!/bin/bash
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2.

# shellcheck source=fbcode/eden/mononoke/tests/integration/library.sh
. "${TEST_FIXTURES}/library.sh"

function base_snapshot_repo_setup {
    if [ $# -eq  0 ]; then
        echo "Must provide at least one clone name"
        exit 1
    fi
    ENABLE_API_WRITES=true INFINITEPUSH_ALLOW_WRITES=true setup_common_config
    cd "$TESTTMP"
    cat >> "$HGRCPATH" <<EOF
[extensions]
snapshot =
commitcloud =
infinitepush =
amend =
EOF

    hginit_treemanifest repo
    cd repo
    mkcommit "base_commit"

    cd "$TESTTMP"
    for clone in "$@"; do
        hgclone_treemanifest ssh://user@dummy/repo "$clone"
    done
    blobimport repo/.hg repo

    # start mononoke
    mononoke
    wait_for_mononoke
}

function base_commit_and_snapshot {
    export BASE_SNAPSHOT_COMMIT
    export BASE_SNAPSHOT
    # Make an interesting commit and snapshot that tests all types of file changes
    echo a > modified_file
    echo a > missing_file
    echo a > untouched_file
    echo a > deleted_file
    echo a > deleted_file_then_untracked_modify
    mkdir dir
    echo a > dir/file_in_dir
    hg addremove -q
    hg commit -m "Add base files"
    BASE_SNAPSHOT_COMMIT=$(hg log -T "{node}" -r .)
    EDENSCM_LOG=edenapi::client=error hgedenapi cloud upload -q
    # Create snapshot
    echo b > modified_file
    echo b > untracked_file
    echo b > added_file
    hg add added_file
    echo b > added_file_then_missing
    hg add added_file_then_missing
    rm added_file_then_missing
    rm missing_file
    hg rm deleted_file
    hg rm deleted_file_then_untracked_modify
    echo b > deleted_file_then_untracked_modify
    ln -s symlink_target symlink_file
    BASE_SNAPSHOT=$(HGPLAIN=1 hgedenapi snapshot create --labels testing,labels)
}

function assert_on_base_snapshot {
    # Using subshell to set pipefail
    (
    set -e -o pipefail
    [[ "$(hg log -T "{node}" -r .)" = "$BASE_SNAPSHOT_COMMIT" ]] || ( echo wrong parent && hg log -T "{node}" -r . )
    [[ "$(cat untouched_file)" = "a" ]] || echo wrong untouched_file
    [[ "$(cat dir/file_in_dir)" = "a" ]] || echo wrong file_in_dir
    [[ "$(cat modified_file)" = "b" ]] || echo wrong modified_file
    [[ "$(cat added_file)" = "b" ]] || echo wrong added_file
    [[ "$(cat deleted_file_then_untracked_modify)" = "b" ]] || wrong deleted_file_then_untracked_modify
    [[ "$(cat untracked_file)" = "b" ]] || echo wrong untracked_file
    [[ "$(readlink symlink_file)" = "symlink_target" ]] || echo wrong symlink target
    [[ ! -f deleted_file ]] || echo Existing deleted_file
    [[ ! -f added_file_then_missing ]] || echo Existing added_file_then_missing
    [[ ! -f missing_file ]] || echo Existing missing_file
    )
}