#!/usr/bin/env bash
#
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

set -e

if [ -z "$IOS_CMAKE_TOOLCHAIN_FILE" ]
then
  echo "IOS_CMAKE_TOOLCHAIN_FILE not set; please set it to path of CMake toolchain file for iOS"
  exit 1
fi

if [ ! -f "$IOS_CMAKE_TOOLCHAIN_FILE" ]
then
  echo "IOS_CMAKE_TOOLCHAIN_FILE not a file path; did you properly setup ${IOS_CMAKE_TOOLCHAIN_FILE}?"
  exit 1
fi

mkdir -p build/ios/x86_64

CMAKE_ARGS=()

# CMake-level configuration
CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=$IOS_CMAKE_TOOLCHAIN_FILE")
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
CMAKE_ARGS+=("-DCMAKE_POSITION_INDEPENDENT_CODE=ON")

# QNNPACK-specific options
CMAKE_ARGS+=("-DPYTORCH_QNNPACK_LIBRARY_TYPE=static")
CMAKE_ARGS+=("-DPYTORCH_QNNPACK_BUILD_BENCHMARKS=ON")
CMAKE_ARGS+=("-DPYTORCH_QNNPACK_BUILD_TESTS=ON")

# Cross-compilation options for Google Benchmark
CMAKE_ARGS+=("-DHAVE_POSIX_REGEX=0")
CMAKE_ARGS+=("-DHAVE_STEADY_CLOCK=0")
CMAKE_ARGS+=("-DHAVE_STD_REGEX=0")

# iOS-specific options
CMAKE_ARGS+=("-DIOS_PLATFORM=SIMULATOR64")
CMAKE_ARGS+=("-DIOS_ARCH=x86_64")
CMAKE_ARGS+=("-DENABLE_BITCODE=OFF")
CMAKE_ARGS+=("-DENABLE_ARC=OFF")

# Use-specified CMake arguments go last to allow overridding defaults
CMAKE_ARGS+=($@)

cd build/ios/x86_64 && cmake ../../.. \
    "${CMAKE_ARGS[@]}"

# Cross-platform parallel build
if [ "$(uname)" == "Darwin" ]
then
  cmake --build . -- "-j$(sysctl -n hw.ncpu)"
else
  cmake --build . -- "-j$(nproc)"
fi


#!/usr/bin/env bash
#
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

set -e

mkdir -p build/local

CMAKE_ARGS=()

# CMake-level configuration
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
CMAKE_ARGS+=("-DCMAKE_POSITION_INDEPENDENT_CODE=ON")

# If Ninja is installed, prefer it to Make
if [ -x "$(command -v ninja)" ]
then
  CMAKE_ARGS+=("-GNinja")
fi

CMAKE_ARGS+=("-DPYTORCH_QNNPACK_LIBRARY_TYPE=static")

CMAKE_ARGS+=("-DPYTORCH_QNNPACK_BUILD_BENCHMARKS=ON")
CMAKE_ARGS+=("-DPYTORCH_QNNPACK_BUILD_TESTS=ON")

# Use-specified CMake arguments go last to allow overridding defaults
CMAKE_ARGS+=($@)

cd build/local && cmake ../.. \
    "${CMAKE_ARGS[@]}"

# Cross-platform parallel build
if [ "$(uname)" == "Darwin" ]
then
  cmake --build . -- "-j$(sysctl -n hw.ncpu)"
else
  cmake --build . -- "-j$(nproc)"
fi


#!/usr/bin/env bash
#
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

set -e

adb push build/android/arm64-v8a/convolution-test /data/local/tmp/convolution-test
adb push build/android/arm64-v8a/deconvolution-test /data/local/tmp/deconvolution-test
adb push build/android/arm64-v8a/q8gemm-test /data/local/tmp/q8gemm-test
adb push build/android/arm64-v8a/q8conv-test /data/local/tmp/q8conv-test
adb push build/android/arm64-v8a/q8dw-test /data/local/tmp/q8dw-test
adb push build/android/arm64-v8a/hgemm-test /data/local/tmp/hgemm-test
adb push build/android/arm64-v8a/sgemm-test /data/local/tmp/sgemm-test

adb shell /data/local/tmp/convolution-test --gtest_color=yes
adb shell /data/local/tmp/deconvolution-test --gtest_color=yes
adb shell /data/local/tmp/q8gemm-test --gtest_color=yes
adb shell /data/local/tmp/q8conv-test --gtest_color=yes
adb shell /data/local/tmp/q8dw-test --gtest_color=yes
adb shell /data/local/tmp/hgemm-test --gtest_color=yes
adb shell /data/local/tmp/sgemm-test --gtest_color=yes


#!/usr/bin/env bash
#
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

set -e

adb push build/android/armeabi-v7a/convolution-test /data/local/tmp/convolution-test
adb push build/android/armeabi-v7a/deconvolution-test /data/local/tmp/deconvolution-test
adb push build/android/armeabi-v7a/q8gemm-test /data/local/tmp/q8gemm-test
adb push build/android/armeabi-v7a/q8conv-test /data/local/tmp/q8conv-test
adb push build/android/armeabi-v7a/q8dw-test /data/local/tmp/q8dw-test
adb push build/android/armeabi-v7a/hgemm-test /data/local/tmp/hgemm-test
adb push build/android/armeabi-v7a/sgemm-test /data/local/tmp/sgemm-test

adb shell /data/local/tmp/convolution-test --gtest_color=yes
adb shell /data/local/tmp/deconvolution-test --gtest_color=yes
adb shell /data/local/tmp/q8gemm-test --gtest_color=yes
adb shell /data/local/tmp/q8conv-test --gtest_color=yes
adb shell /data/local/tmp/q8dw-test --gtest_color=yes
adb shell /data/local/tmp/hgemm-test --gtest_color=yes
adb shell /data/local/tmp/sgemm-test --gtest_color=yes


#!/usr/bin/env bash
#
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

set -e

adb push build/android/x86/convolution-test /data/local/tmp/convolution-test
adb push build/android/x86/deconvolution-test /data/local/tmp/deconvolution-test
adb push build/android/x86/q8gemm-test /data/local/tmp/q8gemm-test
adb push build/android/x86/q8conv-test /data/local/tmp/q8conv-test
adb push build/android/x86/q8dw-test /data/local/tmp/q8dw-test
adb push build/android/x86/hgemm-test /data/local/tmp/hgemm-test
adb push build/android/x86/sgemm-test /data/local/tmp/sgemm-test

adb shell /data/local/tmp/convolution-test --gtest_color=yes
adb shell /data/local/tmp/deconvolution-test --gtest_color=yes
adb shell /data/local/tmp/q8gemm-test --gtest_color=yes
adb shell /data/local/tmp/q8conv-test --gtest_color=yes
adb shell /data/local/tmp/q8dw-test --gtest_color=yes
adb shell /data/local/tmp/hgemm-test --gtest_color=yes
adb shell /data/local/tmp/sgemm-test --gtest_color=yes


#!/bin/bash
set -x
set -e

VALGRIND_SUP="${PWD}/`dirname $0`/valgrind.sup"
pushd $1

VALGRIND=${VALGRIND:=ON}
./basic
./atest
./scalar_test
./broadcast_test
./wrapdim_test
./apply_utils_test
./dlconvertor_test
./native_test
./scalar_tensor_test
./tensor_interop_test
./undefined_tensor_test
./extension_backend_test
./lazy_tensor_test
./tensor_iterator_test
./Dimname_test
./Dict_test
./NamedTensor_test
./cpu_generator_test
./vmap_test
./operators_test
if [[ -x ./cudnn_test ]]; then
  ./cudnn_test
fi
if [[ -x ./cuda_generator_test ]]; then
  ./cuda_generator_test
fi
if [[ -x ./apply_test ]]; then
  ./apply_test
fi
if [[ -x ./stream_test ]]; then
  ./stream_test
fi
if [[ -x ./cuda_half_test ]]; then
  ./cuda_half_test
fi
if [[ -x ./cuda_vectorized_test ]]; then
  ./cuda_vectorized_test
fi
if [[ -x ./cuda_distributions_test ]]; then
  ./cuda_distributions_test
fi
if [[ -x ./cuda_optional_test ]]; then
  ./cuda_optional_test
fi
if [[ -x ./cuda_tensor_interop_test ]]; then
  ./cuda_tensor_interop_test
fi
if [[ -x ./cuda_complex_test ]]; then
  ./cuda_complex_test
fi
if [[ -x ./cuda_complex_math_test ]]; then
  ./cuda_complex_math_test
fi
if [[ -x ./cuda_cub_test ]]; then
  ./cuda_cub_test
fi
if [ "$VALGRIND" == "ON" ]
then
  valgrind --suppressions="$VALGRIND_SUP" --error-exitcode=1 ./basic --gtest_filter='-*CUDA'
  valgrind --suppressions="$VALGRIND_SUP" --error-exitcode=1 ./tensor_interop_test
fi

popd


#!/bin/bash
set -x
set -e
INSTALL_ROOT=$1
SRC_ROOT=$2
rm -rf test_build
mkdir test_build
cd test_build
cmake -DCMAKE_PREFIX_PATH=$INSTALL_ROOT $SRC_ROOT/src/ATen/test/test_install
make
./main


#!/bin/bash
python -m fastrnns.bench --fuser=old --group=rnns --print-json oss > old.json
python -m fastrnns.bench --fuser=te --group=rnns  --print-json oss > te.json
python compare-fastrnn-results.py old.json te.json --format md


OUTFILE=spmm-no-mkl-test.txt
PYTORCH_HOME=$1

cd $PYTORCH_HOME

echo "" >> $OUTFILE
echo "----- USE_MKL=1 -----" >> $OUTFILE
rm -rf build

export USE_MKL=1
export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
python setup.py build --cmake-only
ccmake build  # or cmake-gui build

python setup.py install

cd benchmarks
echo "!! SPARSE SPMM TIME BENCHMARK!! " >> $OUTFILE
for dim0 in 1000 5000 10000; do
    for nnzr in 0.01 0.05 0.1 0.3; do
        python -m sparse.spmm --format csr --m $dim0 --n $dim0 --k $dim0 --nnz_ratio $nnzr --outfile $OUTFILE
        # python -m sparse.spmm --format coo --m $dim0 --n $dim0 --k $dim0 --nnz_ratio $nnzr --outfile $OUTFILE
    done
done
echo "----------------------" >> $OUTFILE

cd $PYTORCH_HOME
echo "----- USE_MKL=0 ------" >> $OUTFILE
rm -rf build

export USE_MKL=0
python setup.py install

cd benchmarks
for dim0 in 1000 5000 10000; do
    for nnzr in 0.01 0.05 0.1 0.3; do
        python -m sparse.spmv --format csr --m $dim0 --nnz_ratio $nnzr --outfile $OUTFILE
        python -m sparse.spmv --format coo --m $dim0 --nnz_ratio $nnzr --outfile $OUTFILE
    done
done
echo "----------------------" >> $OUTFILE


#!/bin/bash

DATASET_ROOT_DIR=$HOME/datasets/

# wget https://storage.googleapis.com/sgk-sc2020/dlmc.tar.gz -P $DATASET_ROOT_DIR
# tar -xvf $DATASET_ROOT_DIR/dlmc.tar.gz

echo "!! SPARSE SPMS TIME BENCHMARK!! "

# cpu
python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@sparse
python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@sparse --backward_test

python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@dense
python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@dense --backward_test

python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@vector

# cuda
python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@sparse --with_cuda
python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@sparse --with_cuda--backward_test

python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@dense --with_cuda
python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@dense --with_cuda --backward_test

python -m dlmc.matmul_bench  --path $DATASET_ROOT_DIR/dlmc/rn50 --dataset magnitude_pruning --operation sparse@vector --with_cuda


#!/usr/bin/env bash

# Generate type stubs for .proto definition files.

# This should be run from as
# ./gen_proto_typestubs.sh
# (i.e., from inside the proto/ directory)

# assumes mypy-protobuf installed to ~/.local; i.e. via
# pip3 install mypy-protobuf --user

set -euxo pipefail

MYPY_PROTOBUF_HOME="${1:-${HOME}/.local/bin}"

pushd ../../
buck run fbsource//third-party/protobuf:protoc -- --plugin=protoc-gen-mypy="${MYPY_PROTOBUF_HOME}"/protoc-gen-mypy --mypy_out=./ caffe2/proto/*.proto
popd

# get rid of 'builtins.' prefix, which pyre does not like
sed -E -i 's/builtins\.//g' ./*.pyi

# mypy-protobuf references types from other mypy-protobuf-generated stubs as
# 'type.V', but it should just be 'type', so we get rid of the '.V' suffix
# when it's not followed by parens to indicate a particular enum value.
sed -E -i 's/\.V([^(_[:alnum:]])/\1/g' ./*.pyi

# ---------------------------
# Freedom-patched DeviceTypes
# ---------------------------
#
# In order to make DeviceTypes like CPU, CUDA, etc. directly accessible from
# the caffe2_pb2 module, they are currently freedom-patched into it in
# caffe2/python/__init__.py. This is not ideal: it would be better if these
# were autogenerated when the protobuf definitions were created by using
# allow_alias = true in the DeviceTypeProto definition in caffe2.proto.
#
# However, it is impossible to do this currently without significant effort.
# The issue is that the generated proto constants would conflict with various
# constants defined in the C++ caffe2 codebase (`caffe2_pb2.h`). We cannot
# simply remove these constants and replace them with the caffe2
# DeviceTypeProto constants, because a huge portion of code expects
# at::DeviceType constants defined in `core/DeviceType.h` (apparently
# duplicated to avoid having to figure out how to autogenerate the protobuf
# definitions using cmake for ATen).
#
# Instead, we make a best-effort to add additional definitions in
# `caffe2_pb2.py` by looking for any freedom-patched constants in
# `caffe2/python/__init__.py` and making sure they have corresponding stubs in
# the pyi (see `gen_proto_typestubs_helper.py`).

python3 ./gen_proto_typestubs_helper.py >> caffe2_pb2.pyi


#!/bin/bash

set -ex

ignore_warning() {
  # Invert match to filter out $1.
  set +e
  grep -v "$1" doxygen-log.txt > temp.txt
  set -e
  mv temp.txt doxygen-log.txt
}

command -v doxygen >/dev/null 2>&1 || { echo >&2 "doxygen is not supported. Aborting."; exit 1; }

pushd "$(dirname "$0")/../../.."

cp torch/_utils_internal.py tools/shared

python -m tools.codegen.gen

python tools/setup_helpers/generate_code.py                 \
  --declarations-path build/aten/src/ATen/Declarations.yaml \
  --native-functions-path aten/src/ATen/native/native_functions.yaml \
  --nn-path aten/src

popd

# Run doxygen and log all output.
doxygen "$(dirname $0)" 2> original-doxygen-log.txt
cp original-doxygen-log.txt doxygen-log.txt

# Uncomment this if you need it for debugging; we're not printing this
# by default because it is confusing.
# echo "Original output"
# cat original-doxygen-log.txt

# Filter out some warnings.
ignore_warning "warning: no uniquely matching class member found for"
ignore_warning "warning: explicit link request to 'Item' could not be resolved"
ignore_warning "warning: Included by graph for 'types.h' not generated, too many nodes"

# Count the number of remaining warnings.
warnings="$(grep 'warning:' doxygen-log.txt | wc -l)"

echo "Treating all remaining warnings as errors"

if [[ "$warnings" -ne "0" ]]; then
  echo "Failing Doxygen test because the following warnings were treated fatally:"
  cat doxygen-log.txt
  echo "Please fix these warnings.  To run this test locally, use docs/cpp/source/check-doxygen.sh"
  rm -f doxygen-log.txt original-doxygen-log.txt
  exit 1
fi

rm -f doxygen-log.txt original-doxygen-log.txt


cat  apache_header.txt $1 > _add_apache_header.txt && mv _add_apache_header.txt $1


#!/bin/bash
##############################################################################
# Example command to build the android target.
##############################################################################
#
# This script shows how one can build a Caffe2 binary for the Android platform
# using android-cmake. A few notes:
#
# (1) This build also does a host build for protobuf. You will need autoconf
#     to carry out this. If autoconf is not possible, you will need to provide
#     a pre-built protoc binary that is the same version as the protobuf
#     version under third_party.
#     If you are building on Mac, you might need to install autotool and
#     libtool. The easiest way is via homebrew:
#         brew install automake
#         brew install libtool
# (2) You will need to have android ndk installed. The current script assumes
#     that you set ANDROID_NDK to the location of ndk.
# (3) The toolchain and the build target platform can be specified with the
#     cmake arguments below. For more details, check out android-cmake's doc.

set -e

# Android specific flags
if [ -z "$ANDROID_ABI" ]; then
  ANDROID_ABI="armeabi-v7a with NEON"
fi
ANDROID_NATIVE_API_LEVEL="21"
echo "Build with ANDROID_ABI[$ANDROID_ABI], ANDROID_NATIVE_API_LEVEL[$ANDROID_NATIVE_API_LEVEL]"

CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"
if [ -z "$ANDROID_NDK" ]; then
  echo "ANDROID_NDK not set; please set it to the Android NDK directory"
  exit 1
fi

if [ ! -d "$ANDROID_NDK" ]; then
  echo "ANDROID_NDK not a directory; did you install it under $ANDROID_NDK?"
  exit 1
fi

if [ -z "$PYTHON" ]; then
  PYTHON=python
  PYTHON_VERSION_MAJOR=$($PYTHON -c 'import sys; print(sys.version_info[0])')
  if [ "${PYTHON_VERSION_MAJOR}" -le 2 ]; then
    echo "Default python executable is Python-2, trying to use python3 alias"
    PYTHON=python3
  fi
fi

ANDROID_NDK_PROPERTIES="$ANDROID_NDK/source.properties"
[ -f "$ANDROID_NDK_PROPERTIES" ] && ANDROID_NDK_VERSION=$(sed -n 's/^Pkg.Revision[^=]*= *\([0-9]*\)\..*$/\1/p' "$ANDROID_NDK_PROPERTIES")

echo "Bash: $(/bin/bash --version | head -1)"
echo "Python: $($PYTHON -c 'import sys; print(sys.version)')"
echo "Caffe2 path: $CAFFE2_ROOT"
echo "Using Android NDK at $ANDROID_NDK"
echo "Android NDK version: $ANDROID_NDK_VERSION"

CMAKE_ARGS=()

if [ -z "${BUILD_CAFFE2_MOBILE:-}" ]; then
  # Build PyTorch mobile
  CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$($PYTHON -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
  CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$($PYTHON -c 'import sys; print(sys.executable)')")
  CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
  # custom build with selected ops
  if [ -n "${SELECTED_OP_LIST}" ]; then
    SELECTED_OP_LIST="$(cd $(dirname $SELECTED_OP_LIST); pwd -P)/$(basename $SELECTED_OP_LIST)"
    echo "Choose SELECTED_OP_LIST file: $SELECTED_OP_LIST"
    if [ ! -r ${SELECTED_OP_LIST} ]; then
      echo "Error: SELECTED_OP_LIST file ${SELECTED_OP_LIST} not found."
      exit 1
    fi
    CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}")
  fi
else
  # Build Caffe2 mobile
  CMAKE_ARGS+=("-DBUILD_CAFFE2_MOBILE=ON")
  # Build protobuf from third_party so we have a host protoc binary.
  echo "Building protoc"
  $CAFFE2_ROOT/scripts/build_host_protoc.sh
  # Use locally built protoc because we'll build libprotobuf for the
  # target architecture and need an exact version match.
  CMAKE_ARGS+=("-DCAFFE2_CUSTOM_PROTOC_EXECUTABLE=$CAFFE2_ROOT/build_host_protoc/bin/protoc")
fi

# If Ninja is installed, prefer it to Make
if [ -x "$(command -v ninja)" ]; then
  CMAKE_ARGS+=("-GNinja")
fi

# Use android-cmake to build Android project from CMake.
CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake")

if [ -z "$BUILD_MOBILE_BENCHMARK" ]; then
  BUILD_MOBILE_BENCHMARK=0
fi

if [ -z "$BUILD_MOBILE_TEST" ]; then
  BUILD_MOBILE_TEST=0
fi
# Don't build artifacts we don't need
CMAKE_ARGS+=("-DBUILD_TEST=OFF")
CMAKE_ARGS+=("-DBUILD_BINARY=OFF")

# If there exists env variable and it equals to 0, build full jit interpreter.
# Default behavior is to build lite interpreter
# cmd:  BUILD_LITE_INTERPRETER=0 ./scripts/build_android.sh
if [ "${BUILD_LITE_INTERPRETER}" == 0 ]; then
  CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=OFF")
else
  CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=ON")
fi
CMAKE_ARGS+=("-DBUILD_MOBILE_BENCHMARK=$BUILD_MOBILE_BENCHMARK")
CMAKE_ARGS+=("-DBUILD_MOBILE_TEST=$BUILD_MOBILE_TEST")
CMAKE_ARGS+=("-DBUILD_PYTHON=OFF")
CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF")
if (( "${ANDROID_NDK_VERSION:-0}" < 18 )); then
  CMAKE_ARGS+=("-DANDROID_TOOLCHAIN=gcc")
else
  CMAKE_ARGS+=("-DANDROID_TOOLCHAIN=clang")
fi
# Disable unused dependencies
CMAKE_ARGS+=("-DUSE_CUDA=OFF")
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
CMAKE_ARGS+=("-DUSE_LMDB=OFF")
CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
CMAKE_ARGS+=("-DUSE_MPI=OFF")
CMAKE_ARGS+=("-DUSE_OPENMP=OFF")
# Only toggle if VERBOSE=1
if [ "${VERBOSE:-}" == '1' ]; then
  CMAKE_ARGS+=("-DCMAKE_VERBOSE_MAKEFILE=1")
fi

# Android specific flags
CMAKE_ARGS+=("-DANDROID_NDK=$ANDROID_NDK")
CMAKE_ARGS+=("-DANDROID_ABI=$ANDROID_ABI")
CMAKE_ARGS+=("-DANDROID_NATIVE_API_LEVEL=$ANDROID_NATIVE_API_LEVEL")
CMAKE_ARGS+=("-DANDROID_CPP_FEATURES=rtti exceptions")
if [ "${ANDROID_STL_SHARED:-}" == '1' ]; then
  CMAKE_ARGS+=("-DANDROID_STL=c++_shared")
fi
if [ "${ANDROID_DEBUG_SYMBOLS:-}" == '1' ]; then
  CMAKE_ARGS+=("-DANDROID_DEBUG_SYMBOLS=1")
fi

if [ -n "${USE_VULKAN}" ]; then
  CMAKE_ARGS+=("-DUSE_VULKAN=ON")
fi

# Use-specified CMake arguments go last to allow overridding defaults
CMAKE_ARGS+=($@)

# Now, actually build the Android target.
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_android"}
INSTALL_PREFIX=${BUILD_ROOT}/install
mkdir -p $BUILD_ROOT
cd $BUILD_ROOT
cmake "$CAFFE2_ROOT" \
    -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \
    -DCMAKE_BUILD_TYPE=Release \
    "${CMAKE_ARGS[@]}"

# Cross-platform parallel build
if [ -z "$MAX_JOBS" ]; then
  if [ "$(uname)" == 'Darwin' ]; then
    MAX_JOBS=$(sysctl -n hw.ncpu)
  else
    MAX_JOBS=$(nproc)
  fi
fi

echo "Will install headers and libs to $INSTALL_PREFIX for further Android project usage."
cmake --build . --target install -- "-j${MAX_JOBS}"
echo "Installation completed, now you can copy the headers/libs from $INSTALL_PREFIX to your Android project directory."


#!/bin/bash
##############################################################################
# Build script to build the protoc compiler for the host platform.
##############################################################################
# This script builds the protoc compiler for the host platform, which is needed
# for any cross-compilation as we will need to convert the protobuf source
# files to cc files.
#
# --other-flags accepts flags that should be passed to cmake. Optional.
#
# After the execution of the file, one should be able to find the host protoc
# binary at build_host_protoc/bin/protoc.

set -e

CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)"
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_host_protoc"}
mkdir -p $BUILD_ROOT/build
cd $BUILD_ROOT/build

CMAKE_ARGS=()
CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$BUILD_ROOT")
CMAKE_ARGS+=("-Dprotobuf_BUILD_TESTS=OFF")

# If Ninja is installed, prefer it to Make
if [ -x "$(command -v ninja)" ]; then
  CMAKE_ARGS+=("-GNinja")
fi

while true; do
    case "$1" in
        --other-flags)
            shift;
            CMAKE_ARGS+=("$@")
            break ;;
        "")
            break ;;
        *)
            echo "Unknown option passed as argument: $1"
            break ;;
    esac
done

# Use ccache if available (this path is where Homebrew installs ccache symlinks)
if [ "$(uname)" == 'Darwin' ] && [ -d /usr/local/opt/ccache/libexec ]; then
  CMAKE_ARGS+=("-DCMAKE_C_COMPILER=/usr/local/opt/ccache/libexec/gcc")
  CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=/usr/local/opt/ccache/libexec/g++")
fi

cmake "$CAFFE2_ROOT/third_party/protobuf/cmake" ${CMAKE_ARGS[@]}

if [ -z "$MAX_JOBS" ]; then
  if [ "$(uname)" == 'Darwin' ]; then
    MAX_JOBS=$(sysctl -n hw.ncpu)
  else
    MAX_JOBS=$(nproc)
  fi
fi
cmake --build . -- "-j${MAX_JOBS}" install


#!/bin/bash -xe
##############################################################################
# Example command to build the iOS target.
##############################################################################
#
# This script shows how one can build a Caffe2 binary for the iOS platform
# using ios-cmake. This is very similar to the android-cmake - see
# build_android.sh for more details.

CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"

CMAKE_ARGS=()

if [ -z "${BUILD_CAFFE2_MOBILE:-}" ]; then
  # Build PyTorch mobile
  CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
  CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
  CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
  # custom build with selected ops
  if [ -n "${SELECTED_OP_LIST}" ]; then
    SELECTED_OP_LIST="$(cd $(dirname $SELECTED_OP_LIST); pwd -P)/$(basename $SELECTED_OP_LIST)"
    echo "Choose SELECTED_OP_LIST file: $SELECTED_OP_LIST"
    if [ ! -r ${SELECTED_OP_LIST} ]; then
      echo "Error: SELECTED_OP_LIST file ${SELECTED_OP_LIST} not found."
      exit 1
    fi
    CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}")
  fi
  # bitcode
  if [ "${ENABLE_BITCODE:-}" == '1' ]; then
    CMAKE_ARGS+=("-DCMAKE_C_FLAGS=-fembed-bitcode")
    CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fembed-bitcode")
  fi
else
  # Build Caffe2 mobile
  CMAKE_ARGS+=("-DBUILD_CAFFE2_MOBILE=ON")
  # Build protobuf from third_party so we have a host protoc binary.
  echo "Building protoc"
  BITCODE_FLAGS="-DCMAKE_C_FLAGS=-fembed-bitcode -DCMAKE_CXX_FLAGS=-fembed-bitcode "
  $CAFFE2_ROOT/scripts/build_host_protoc.sh --other-flags $BITCODE_FLAGS
  # Use locally built protoc because we'll build libprotobuf for the
  # target architecture and need an exact version match.
  CMAKE_ARGS+=("-DCAFFE2_CUSTOM_PROTOC_EXECUTABLE=$CAFFE2_ROOT/build_host_protoc/bin/protoc")
  # Bitcode is enabled by default for caffe2
  CMAKE_ARGS+=("-DCMAKE_C_FLAGS=-fembed-bitcode")
  CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fembed-bitcode")
fi

# Use ios-cmake to build iOS project from CMake.
# This projects sets CMAKE_C_COMPILER to /usr/bin/gcc and
# CMAKE_CXX_COMPILER to /usr/bin/g++. In order to use ccache (if it is available) we
# must override these variables via CMake arguments.
CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=$CAFFE2_ROOT/cmake/iOS.cmake")
if [ -n "${CCACHE_WRAPPER_PATH:-}"]; then
  CCACHE_WRAPPER_PATH=/usr/local/opt/ccache/libexec
fi
if [ -d "$CCACHE_WRAPPER_PATH" ]; then
  CMAKE_ARGS+=("-DCMAKE_C_COMPILER=$CCACHE_WRAPPER_PATH/gcc")
  CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=$CCACHE_WRAPPER_PATH/g++")
fi

# IOS_PLATFORM controls type of iOS platform (see ios-cmake)
if [ -n "${IOS_PLATFORM:-}" ]; then
  CMAKE_ARGS+=("-DIOS_PLATFORM=${IOS_PLATFORM}")
  if [ "${IOS_PLATFORM}" == "WATCHOS" ]; then
      # enable bitcode by default for watchos
      CMAKE_ARGS+=("-DCMAKE_C_FLAGS=-fembed-bitcode")
      CMAKE_ARGS+=("-DCMAKE_CXX_FLAGS=-fembed-bitcode")
      # disable the QNNPACK
      CMAKE_ARGS+=("-DUSE_PYTORCH_QNNPACK=OFF")
  fi
else
  # IOS_PLATFORM is not set, default to OS, which builds iOS.
  CMAKE_ARGS+=("-DIOS_PLATFORM=OS")
fi

if [ -n "${IOS_ARCH:-}" ]; then
  CMAKE_ARGS+=("-DIOS_ARCH=${IOS_ARCH}")
fi

if [ "${BUILD_LITE_INTERPRETER}" == 0 ]; then
  CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=OFF")
else
  CMAKE_ARGS+=("-DBUILD_LITE_INTERPRETER=ON")
fi
CMAKE_ARGS+=("-DUSE_LITE_INTERPRETER_PROFILER=OFF")

# Don't build binaries or tests (only the library)
CMAKE_ARGS+=("-DBUILD_TEST=OFF")
CMAKE_ARGS+=("-DBUILD_BINARY=OFF")
CMAKE_ARGS+=("-DBUILD_PYTHON=OFF")

# Disable unused dependencies
CMAKE_ARGS+=("-DUSE_CUDA=OFF")
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
CMAKE_ARGS+=("-DUSE_LMDB=OFF")
CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
CMAKE_ARGS+=("-DUSE_MPI=OFF")
CMAKE_ARGS+=("-DUSE_NUMPY=OFF")
CMAKE_ARGS+=("-DUSE_NNPACK=OFF")
CMAKE_ARGS+=("-DUSE_MKLDNN=OFF")

# Metal
if [ "${USE_PYTORCH_METAL:-}" == "1" ]; then
  CMAKE_ARGS+=("-DUSE_PYTORCH_METAL=ON")
fi

# Core ML
if [ "${USE_COREML_DELEGATE}" == "1" ]; then
  CMAKE_ARGS+=("-DUSE_COREML_DELEGATE=ON")
fi

# pthreads
CMAKE_ARGS+=("-DCMAKE_THREAD_LIBS_INIT=-lpthread")
CMAKE_ARGS+=("-DCMAKE_HAVE_THREADS_LIBRARY=1")
CMAKE_ARGS+=("-DCMAKE_USE_PTHREADS_INIT=1")

# Only toggle if VERBOSE=1
if [ "${VERBOSE:-}" == '1' ]; then
  CMAKE_ARGS+=("-DCMAKE_VERBOSE_MAKEFILE=1")
fi

# Now, actually build the iOS target.
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_ios"}
INSTALL_PREFIX=${BUILD_ROOT}/install
mkdir -p $BUILD_ROOT
cd $BUILD_ROOT
cmake "$CAFFE2_ROOT" \
    -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \
    -DCMAKE_BUILD_TYPE=MinSizeRel \
    -DBUILD_SHARED_LIBS=OFF \
    ${CMAKE_ARGS[@]} \
    $@

cmake --build . -- "-j$(sysctl -n hw.ncpu)"

# copy headers and libs to install directory
echo "Will install headers and libs to $INSTALL_PREFIX for further Xcode project usage."
make install
echo "Installation completed, now you can copy the headers/libs from $INSTALL_PREFIX to your Xcode project directory."


#!/bin/bash
#
##############################################################################
# Example command to build Caffe2
##############################################################################
#

set -ex

CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"

CMAKE_ARGS=()

# If Ninja is installed, prefer it to Make
if [ -x "$(command -v ninja)" ]; then
  CMAKE_ARGS+=("-GNinja")
fi

# Use ccache if available (this path is where Homebrew installs ccache symlinks)
if [ "$(uname)" == 'Darwin' ]; then
  if [ -n "${CCACHE_WRAPPER_PATH:-}"]; then
    CCACHE_WRAPPER_PATH=/usr/local/opt/ccache/libexec
  fi
  if [ -d "$CCACHE_WRAPPER_PATH" ]; then
    CMAKE_ARGS+=("-DCMAKE_C_COMPILER=$CCACHE_WRAPPER_PATH/gcc")
    CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER=$CCACHE_WRAPPER_PATH/g++")
  fi
fi

# Use special install script with Anaconda
if [ -n "${USE_ANACONDA}" ]; then
  export SKIP_CONDA_TESTS=1
  export CONDA_INSTALL_LOCALLY=1
  "${ROOT_DIR}/scripts/build_anaconda.sh" "$@"
else
  # Make sure that pyyaml is installed for the codegen of building Aten to work
  if [[ -n "$(python -c 'import yaml' 2>&1)" ]]; then
    echo "Installing pyyaml with pip at $(which pip)"
    pip install --user pyyaml
  fi

  # Make sure that typing is installed for the codegen of building Aten to work
  if [[ -n "$(python -c 'import typing' 2>&1)" ]]; then
    echo "Installing typing with pip at $(which pip)"
    pip install --user typing
  fi

  # Build protobuf compiler from third_party if configured to do so
  if [ -n "${USE_HOST_PROTOC:-}" ]; then
    echo "USE_HOST_PROTOC is set; building protoc before building Caffe2..."
    "$CAFFE2_ROOT/scripts/build_host_protoc.sh"
    CUSTOM_PROTOC_EXECUTABLE="$CAFFE2_ROOT/build_host_protoc/bin/protoc"
    echo "Built protoc $("$CUSTOM_PROTOC_EXECUTABLE" --version)"
    CMAKE_ARGS+=("-DCAFFE2_CUSTOM_PROTOC_EXECUTABLE=$CUSTOM_PROTOC_EXECUTABLE")
  fi

  # We are going to build the target into build.
  BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"}
  mkdir -p "$BUILD_ROOT"
  cd "$BUILD_ROOT"
  echo "Building Caffe2 in: $BUILD_ROOT"

  cmake "$CAFFE2_ROOT" \
        -DCMAKE_BUILD_TYPE=Release \
        "${CMAKE_ARGS[@]}" \
        "$@"

  # Determine the number of CPUs to build with.
  # If the `CAFFE_MAKE_NCPUS` variable is not specified, use them all.
  if [ -n "${MAX_JOBS}" ]; then
      CAFFE_MAKE_NCPUS="$MAX_JOBS"
  elif [ -n "${CAFFE_MAKE_NCPUS}" ]; then
      CAFFE_MAKE_NCPUS="$CAFFE_MAKE_NCPUS"
  elif [ "$(uname)" == 'Darwin' ]; then
      CAFFE_MAKE_NCPUS="$(sysctl -n hw.ncpu)"
  else
      CAFFE_MAKE_NCPUS="$(nproc)"
  fi

  # Now, actually build the target.
  cmake --build . -- "-j$CAFFE_MAKE_NCPUS"
fi


#!/bin/bash
##############################################################################
# Example command to build the mobile target.
##############################################################################
#
# This script shows how one can build a libtorch library optimized for mobile
# devices using host toolchain.

set -e

export BUILD_PYTORCH_MOBILE_WITH_HOST_TOOLCHAIN=1
CAFFE2_ROOT="$( cd "$(dirname "$0")"/.. ; pwd -P)"

echo "Bash: $(/bin/bash --version | head -1)"
echo "Caffe2 path: $CAFFE2_ROOT"

CMAKE_ARGS=()
CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$(python -c 'import sysconfig; print(sysconfig.get_path("purelib"))')")
CMAKE_ARGS+=("-DPYTHON_EXECUTABLE=$(python -c 'import sys; print(sys.executable)')")
CMAKE_ARGS+=("-DBUILD_CUSTOM_PROTOBUF=OFF")
CMAKE_ARGS+=("-DBUILD_SHARED_LIBS=OFF")
# custom build with selected ops
if [ -n "${SELECTED_OP_LIST}" ]; then
  SELECTED_OP_LIST="$(cd $(dirname $SELECTED_OP_LIST); pwd -P)/$(basename $SELECTED_OP_LIST)"
  echo "Choose SELECTED_OP_LIST file: $SELECTED_OP_LIST"
  if [ ! -r ${SELECTED_OP_LIST} ]; then
    echo "Error: SELECTED_OP_LIST file ${SELECTED_OP_LIST} not found."
    exit 1
  fi
  CMAKE_ARGS+=("-DSELECTED_OP_LIST=${SELECTED_OP_LIST}")
fi

# If Ninja is installed, prefer it to Make
if [ -x "$(command -v ninja)" ]; then
  CMAKE_ARGS+=("-GNinja")
fi

# Disable unused dependencies
CMAKE_ARGS+=("-DUSE_ROCM=OFF")
CMAKE_ARGS+=("-DUSE_CUDA=OFF")
CMAKE_ARGS+=("-DUSE_GFLAGS=OFF")
CMAKE_ARGS+=("-DUSE_OPENCV=OFF")
CMAKE_ARGS+=("-DUSE_LMDB=OFF")
CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
CMAKE_ARGS+=("-DUSE_MPI=OFF")
CMAKE_ARGS+=("-DUSE_OPENMP=OFF")

# Only toggle if VERBOSE=1
if [ "${VERBOSE:-}" == '1' ]; then
  CMAKE_ARGS+=("-DCMAKE_VERBOSE_MAKEFILE=1")
fi

# Use-specified CMake arguments go last to allow overridding defaults
CMAKE_ARGS+=("$@")

# Now, actually build the Android target.
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build_mobile"}
INSTALL_PREFIX=${BUILD_ROOT}/install
mkdir -p $BUILD_ROOT
cd $BUILD_ROOT
cmake "$CAFFE2_ROOT" \
    -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX \
    -DCMAKE_BUILD_TYPE=Release \
    "${CMAKE_ARGS[@]}"

# Cross-platform parallel build
if [ -z "$MAX_JOBS" ]; then
  if [ "$(uname)" == 'Darwin' ]; then
    MAX_JOBS=$(sysctl -n hw.ncpu)
  else
    MAX_JOBS=$(nproc)
  fi
fi

echo "Will install headers and libs to $INSTALL_PREFIX for further project usage."
cmake --build . --target install -- "-j${MAX_JOBS}"
echo "Installation completed, now you can copy the headers/libs from $INSTALL_PREFIX to your project directory."


#!/bin/bash
set -eux

##############################################################################
# Master script to build PyTorch Android library with Java bindings.
##############################################################################
# Example usage:
# - Build default AARs:
#   scripts/build_pytorch_android.sh
#
# - Build for specific ABI(s):
#   scripts/build_pytorch_android.sh armeabi-v7a
#   scripts/build_pytorch_android.sh arm64-v8a,x86,x86_64
#
# Script's workflow:
# 1. Builds libtorch for android for specified android abisi (by default for all 4).
# Custom list of android abis can be specified as a bash argument as comma separated list.
# For example just for testing on android x86 emulator we need only x86 build.
# ./scripts/build_pytorch_android.sh x86
# 2. Creates symbolic links to android/pytorch_android/src/main/jniLibs/${abi} for libtorch build output,
# android/pytorch_android/src/main/cpp/libtorch_include/${abi} for headers.
# 3. Runs pyotrch_android gradle build:
# gradle assembleRelease

PYTORCH_DIR="$(cd $(dirname $0)/..; pwd -P)"
PYTORCH_ANDROID_DIR=$PYTORCH_DIR/android

echo "PYTORCH_DIR:$PYTORCH_DIR"

source "$PYTORCH_ANDROID_DIR/common.sh"

check_android_sdk
check_gradle
parse_abis_list "$@"
build_android

# To set proxy for gradle add following lines to ./gradle/gradle.properties:
# systemProp.http.proxyHost=...
# systemProp.http.proxyPort=8080
# systemProp.https.proxyHost=...
# systemProp.https.proxyPort=8080

if [ "$CUSTOM_ABIS_LIST" = true ]; then
  # Skipping clean task here as android gradle plugin 3.3.2 exteralNativeBuild has problems
  # with it when abiFilters are specified.
  $GRADLE_PATH -PABI_FILTERS=$ABIS_LIST -p $PYTORCH_ANDROID_DIR assembleRelease
else
  $GRADLE_PATH -p $PYTORCH_ANDROID_DIR clean assembleRelease
fi

find $PYTORCH_ANDROID_DIR -type f -name *aar | xargs ls -lah


#!/bin/bash
##############################################################################
# Example command to build the Raspbian target.
##############################################################################
#
# This script shows how one can build a Caffe2 binary for raspbian. The build
# is essentially much similar to a host build, with one additional change
# which is to specify -mfpu=neon for optimized speed.

CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)"
echo "Caffe2 codebase root is: $CAFFE2_ROOT"
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"}
mkdir -p $BUILD_ROOT
echo "Build Caffe2 raspbian into: $BUILD_ROOT"

# obtain dependencies.
echo "Installing dependencies."
sudo apt-get install \
  cmake \
  libgflags-dev \
  libgoogle-glog-dev \
  libprotobuf-dev \
  libpython-dev \
  python-pip \
  python-numpy \
  protobuf-compiler \
  python-protobuf
# python dependencies
sudo pip install hypothesis

# Now, actually build the raspbian target.
echo "Building caffe2"
cd $BUILD_ROOT

# Note: you can add more dependencies above if you need libraries such as
# leveldb, lmdb, etc.
cmake "$CAFFE2_ROOT" \
    -DCMAKE_VERBOSE_MAKEFILE=1 \
    -DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=hard" \
    || exit 1

# Note: while Raspberry pi has 4 cores, running too many builds in parallel may
# cause out of memory errors so we will simply run -j 2 only.
make -j 2 || exit 1


#!/bin/bash
##############################################################################
# Example command to build Caffe2 on Tegra X1.
##############################################################################
#
# This script shows how one can build a Caffe2 binary for NVidia's TX1.
# The build script assumes that you have the most recent libraries installed
# via the JetPack toolkit available at
#     https://developer.nvidia.com/embedded/jetpack
# and it assumes that we are starting from a fresh system after the jetpack
# installation. If you have already installed some of the dependencies, you
# may be able to skip quite a few of the apt-get installs.

CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)"
echo "Caffe2 codebase root is: $CAFFE2_ROOT"
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"}
mkdir -p $BUILD_ROOT
echo "Build Caffe2 raspbian into: $BUILD_ROOT"

# obtain necessary dependencies
echo "Installing dependencies."
sudo apt-get install \
  cmake \
  libgflags-dev \
  libgoogle-glog-dev \
  libprotobuf-dev \
  protobuf-compiler

# obtain optional dependencies that are usually useful to have.
echo "Installing optional dependencies."
sudo apt-get install \
  libleveldb-dev \
  liblmdb-dev \
  libpython-dev \
  libsnappy-dev \
  python-numpy \
  python-pip \
  python-protobuf

# Obtain python hypothesis, which Caffe2 uses for unit testing. Note that
# the one provided by apt-get is quite old so we install it via pip
sudo pip install hypothesis

# Install the six module, which includes Python 2 and 3 compatibility utilities,
# and is required for Caffe2
sudo pip install six

# Now, actually build the android target.
echo "Building caffe2"
cd $BUILD_ROOT

# CUDA_USE_STATIC_CUDA_RUNTIME needs to be set to off so that opencv can be
# properly used. Otherwise, opencv will complain that opencv_dep_cudart cannot
# be found.
cmake "$CAFFE2_ROOT" -DCUDA_USE_STATIC_CUDA_RUNTIME=OFF \
    || exit 1

make -j 4 || exit 1


#!/usr/bin/env bash
##############################################################################
#  Example command to build the Tizen target (RPi3).
##############################################################################
#
# This script shows how one can build a Caffe2 binary for a Tizen device (RPi3).
# The build is essentially much similar to a host build, with one additional change
# which is to specify -mfpu=neon for optimized speed.

setup_environment(){
# The rootfs image for a Tizen target (RPi3)is located at the below webpage:
# http://download.tizen.org/releases/milestone/tizen/4.0.m1/tizen-unified_20170529.1/images/
# If you do not have a Tizen device, Please, run qemu-arm-static and chroot command.
# $ sudo chroot ~/tizen-rootfs qemu-arm-static /usr/bin/bash

CAFFE2_ROOT="$( cd "$(dirname -- "$0")"/.. ; pwd -P)"
echo "Caffe2 codebase root is: $CAFFE2_ROOT"
BUILD_ROOT=${BUILD_ROOT:-"$CAFFE2_ROOT/build"}
mkdir -p $BUILD_ROOT
echo "Build Caffe2 Tizen into: $BUILD_ROOT"
}

caffe2_lite_dep_packages(){
# Obtain necessary dependencies
# You can set-up a rpm repository with zypper, yum, and dnf because Tizen
# software platform officially support rpm format such as Fedora, OpenSUSE.
# The official Tizen repository is as following:
# http://download.tizen.org/releases/milestone/tizen/4.0.m1/
echo "Installing dependencies."
sudo zypper install \
  make \
  strace \
  cmake \
  gcc* \
  binutils \
  glibc* \
  cpp \
  protobuf-devel \
  libstdc++*
}

caffe2_lite_build(){
# Now, actually build the android target.
echo "Building caffe2"
cd $BUILD_ROOT

# Note: add more dependencies above if you need libraries such as leveldb, lmdb, etc.
# If you have to disable a specific package due to a package absence
# from https://git.tizen.org/cgit/, append -Dxxx_xxx=OFF option before executing cmake.
cmake .. \
    -DCMAKE_VERBOSE_MAKEFILE=1 \
    -DUSE_GFLAGS=OFF  \
    -DUSE_GLOG=OFF -DUSE_NNPACK=OFF \
    -DRUN_HAVE_STD_REGEX=0 \
    -DRUN_HAVE_POSIX_REGEX=0 \
    -DHAVE_GNU_POSIX_REGEX=0 \
    -DUSE_MPI=OFF -DUSE_OPENMP=OFF \
    -DUSE_ROCKSDB=OFF \
    -DUSE_LEVELDB=OFF \
    -DUSE_LMDB=OFF \
    -DBUILD_PYTHON=OFF \
    -DUSE_GLOO=OFF \
    -DUSE_OPENCV=OFF \
    -DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=soft" \
    || exit 1

make -j`nproc` || exit 1
}

caffe2_full_dep_packages(){
# Obtain necessary dependencies
# You can set-up a rpm repository with zypper, yum, and dnf because Tizen
# software platform officially support rpm format such as Fedora, OpenSUSE.
# The official Tizen repository is as following:
# http://download.tizen.org/releases/milestone/tizen/4.0.m1/
echo "Installing dependencies."
sudo zypper install \
  cmake \
  libgflags-dev \
  libgoogle-glog-dev \
  libprotobuf-dev \
  protobuf-compiler

# Obtain optional dependencies that are usually useful to have.
echo "Installing optional dependencies."
sudo zypper install \
  libleveldb-dev \
  liblmdb-dev \
  libpython-dev \
  libsnappy-dev \
  python-numpy \
  python-pip \
  python-protobuf

# Obtain python hypothesis, which Caffe2 uses for unit testing. Note that
# the one provided by zypper is quite old so we install it via pip
sudo pip install hypothesis

# Install the six module, which includes Python 2 and 3 compatibility utilities,
# and is required for Caffe2
sudo pip install six
}

caffe2_full_build(){
# Now, actually build the android target.
echo "Building caffe2"
cd $BUILD_ROOT

# Note: add more dependencies above if you need libraries such as leveldb, lmdb, etc.
# If you have to disable a specific package due to a package absence
# from https://git.tizen.org/cgit/, append -Dxxx_xxx=OFF option before executing cmake.
cmake "$CAFFE2_ROOT" \
    -DCMAKE_VERBOSE_MAKEFILE=1 \
    -DUSE_CUDA=OFF \
    -DUSE_OPENCV=OFF \
    -DUSE_LMDB=OFF \
    -DCAFFE2_CPU_FLAGS="-mfpu=neon -mfloat-abi=soft" \
    || exit 1

make -j`nproc` || exit 1
}

#### Main
# Setup a build environment to compile Caffe2 deeplearning framework in Tizen platform.
setup_environment
# There are two build options to support 'full' version and 'lite' version (by default).
caffe2_lite_dep_packages
caffe2_lite_build


# Simple script used to easily search all packages in conda for their
# dependency requirements
# TODO also search through output of ldd
# TODO update conda info syntax for different channels

if [ -z "$CONDA_ROOT" ]; then
  # TODO create our own environment
  echo "Please set CONDA_ROOT so that I know where to search for conda libraries"
  echo "I expect CONDA_ROOT to be the path to the current conda environment."
  echo "Also FYI I will probably mess up the current conda environment."
  exit 1
fi

if [ -z "$1" ]; then
  echo "Please give me a package name to search for"
  exit 1
fi
PKG_NAME="$1"

if [ -n "$2" ]; then
  echo "Searching in channel $2"
  CONDA_CHANNEL="$2"
fi

# These are the packages of interest to search the dependencies for
# TODO use this
PACKAGES_OF_INTEREST=( libgcc-ng libprotobuf numpy )

# We will run `conda install` and `conda uninstall` a lot, but we don't want
# this very noisy output to clutter the user experience
VERBOSE_LOG='read_conda_versions.log'
echo "Conda install/uninstall log for $PKG_NAME" > $VERBOSE_LOG

#
# Build up the name of the installed library to call `nm` on
#
PKG_INSTALLED_LIB="$PKG_NAME"

# opencv installs a bunch of libraries. We'll just check libopencv_core
if [[ $PKG_NAME == opencv ]]; then
  PKG_INSTALLED_LIB="${PKG_INSTALLED_LIB}_core"
fi

# Most packages prepend a 'lib' to the package name, but libprotobuf is an
# exception
if [[ $PKG_NAME != lib* ]]; then
  PKG_INSTALLED_LIB="lib${PKG_INSTALLED_LIB}"
fi

# The shared library suffix differs on macOS an Linux
if [[ "$(uname)" == Darwin ]]; then
  PKG_INSTALLED_LIB="${PKG_INSTALLED_LIB}.dylib"
else
  PKG_INSTALLED_LIB="${PKG_INSTALLED_LIB}.so"
fi
echo "Determined the library name of $PKG_NAME to be $PKG_INSTALLED_LIB"
echo "Determined the library name of $PKG_NAME to be $PKG_INSTALLED_LIB" >> $VERBOSE_LOG

#
# Get all available packages with conda-search
#

# Split the output from conda search into an array, one line per package (plus
# the header)
conda_search_packages=()
while read -r line; do conda_search_packages+=("$line"); done <<< "$(conda search $PKG_NAME $CONDA_CHANNEL)"

### Typical `conda search` output looks like
###   Loading channels: done
###   Name                       Version                   Build  Channel
###   protobuf                   2.6.1                    py27_0  defaults
###                              2.6.1                    py27_1  defaults
###                              3.2.0                    py27_0  defaults
###                              3.2.0                    py35_0  defaults
###                              3.2.0                    py36_0  defaults
###                              3.4.1            py27h66c1d77_0  defaults
###                              3.4.1            py35h9d33684_0  defaults
###                              3.4.1            py36h314970b_0  defaults
###                              3.5.1            py27h0a44026_0  defaults
###                              3.5.1            py35h0a44026_0  defaults
###                              3.5.1            py36h0a44026_0  defaults
##
### Typical `conda info` output looks like
###   protobuf 3.5.1 py36h0a44026_0
###     -----------------------------
###   file name   : protobuf-3.5.1-py36h0a44026_0.tar.bz2
###   name        : protobuf
###   version     : 3.5.1
###   build string: py36h0a44026_0
###   build number: 0
###   channel     : https://repo.continuum.io/pkgs/main/osx-64
###   size        : 589 KB
###   arch        : None
###   constrains  : ()
###   license     : New BSD License
###   license_family: BSD
###   md5         : 7dbdb06612e21c42fbb8a62354e13e10
###   platform    : None
###   subdir      : osx-64
###   timestamp   : 1519951502766
###   url         : https://repo.continuum.io/pkgs/main/osx-64/protobuf-3.5.1-py36h0a44026_0.tar.bz2
###   dependencies:
###       libcxx >=4.0.1
###       libprotobuf >=3.5.1,<3.6.0a0
###       python >=3.6,<3.7.0a0
###       six

# Echo what packages we'll look through.
echo "Processing these packages:"
for pkg in "${conda_search_packages[@]:2}"; do
  echo "  $pkg"
done

#
# Look up each package in conda info, then install it and search the exported
# symbols for signs of cxx11
#
for pkg in "${conda_search_packages[@]:2}"; do
  echo "Processing $pkg" >> $VERBOSE_LOG

  # Split each line into an array and build the package specification
  # <package_name (1st line only)>  maj.min.patch  build_string  channel_name
  line_parts=( $pkg )
  if [[ ${line_parts[0]} == $PKG_NAME ]]; then
    # First line of output
    PKG_VERSION="${line_parts[1]}"
    PKG_BUILD_STR="${line_parts[2]}"
  else
    PKG_VERSION="${line_parts[0]}"
    PKG_BUILD_STR="${line_parts[1]}"
  fi
  PKG_SPEC="$PKG_NAME=$PKG_VERSION=$PKG_BUILD_STR"

  # Output current pkg spec
  echo
  echo "${PKG_SPEC}:"
  echo "Determined that the package spec is $PKG_SPEC" >> $VERBOSE_LOG

  # Split the output of conda_info into an array of lines
  pkg_dependencies=()
  while read -r line; do pkg_dependencies+=("$line"); done <<< "$(conda info "$PKG_SPEC" $CONDA_CHANNEL)"

  # List all the listed dependencies in `conda info`
  if [ "${#pkg_dependencies[@]}" -gt 19 ]; then
    echo "  Listed dependencies:"
    echo "  Listed dependencies:" >> $VERBOSE_LOG
    for pkg_dependency in "${pkg_dependencies[@]:20}"; do
      echo "    $pkg_dependency"
      echo "    $pkg_dependency" >> $VERBOSE_LOG
    done
  else
    echo "  No listed dependencies in conda-info" >> $VERBOSE_LOG
  fi

  # But sometimes (a lot of the time) the gcc with which a package was built
  # against is not listed in dependencies. So we try to figure it out manually
  # We install this exact package, and then grep the exported symbols for signs
  # of cxx11
  echo "Calling conda-uninstall on $PKG_NAME" >> $VERBOSE_LOG
  echo "conda uninstall -y $PKG_NAME --quiet" >> $VERBOSE_LOG
  conda uninstall -y "$PKG_NAME" --quiet >> $VERBOSE_LOG 2>&1

  echo "Calling conda-install on $PKG_SPEC" >> $VERBOSE_LOG
  echo "conda install -y $PKG_SPEC --quiet --no-deps $CONDA_CHANNEL" >> $VERBOSE_LOG
  conda install -y "$PKG_SPEC" --quiet --no-deps $CONDA_CHANNEL >> $VERBOSE_LOG 2>&1
  if [ $? -eq 0 ]; then
    # Only grep the exported symbols if the library was installed correctly

    MENTIONS_CXX11="$(nm "$CONDA_ROOT/lib/$PKG_INSTALLED_LIB" | grep cxx11 | wc -l)"
    if [ $MENTIONS_CXX11 -gt 0 ]; then
      echo "  This package is built against the recent gcc ABI ($MENTIONS_CXX11 mentions of cxx11)"
      echo "$CONDA_ROOT/lib/$PKG_INSTALLED_LIB mentions cxx11 $MENTIONS_CXX11 times" >> $VERBOSE_LOG
    fi
  else
    echo "Error installing $PKG_SPEC , continuing"
    echo "Error installing $PKG_SPEC , continuing" >> $VERBOSE_LOG
  fi
done


if [[ "$1" == *.py ]]; then
  apache_header="apache_python.txt"
else
  apache_header="apache_header.txt"
fi
apache_lines=$(wc -l < "${apache_header}")
apache_md5=$(cat "${apache_header}" | md5)
header_md5=$(head -n ${apache_lines} $1 | md5)
if [ "${header_md5}" == "${apache_md5}" ]; then
  keep_lines=$(($(wc -l < $1) - ${apache_lines}))
  tail -n ${keep_lines} $1 > _remove_apache_header.txt
  mv _remove_apache_header.txt $1
fi


find ../caffe2 -name "*.py" -exec ./remove_apache_header.sh {} \;
find ../caffe2 -name "*.h" -exec ./remove_apache_header.sh {} \;
find ../caffe2 -name "*.cc" -exec ./remove_apache_header.sh {} \;
find ../caffe2 -name "*.cpp" -exec ./remove_apache_header.sh {} \;
find ../caffe2 -name "*.cu" -exec ./remove_apache_header.sh {} \;
find ../caffe2 -name "*.mm" -exec ./remove_apache_header.sh {} \;
find ../caffe2 -name "*.m" -exec ./remove_apache_header.sh {} \;


#!/bin/bash

# This script installs CCache with CUDA support.
# Example usage:
#     ./ccache_setup.sh --path /installed/folder

set -e
shopt -s expand_aliases

# Setup the proxy
alias with_proxy="HTTPS_PROXY=http://fwdproxy:8080 HTTP_PROXY=http://fwdproxy:8080 FTP_PROXY=http://fwdproxy:8080 https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 ftp_proxy=http://fwdproxy:8080 http_no_proxy='*.facebook.com|*.tfbnw.net|*.fb.com'"

# Parse options
path="$HOME/ccache"
force=false

while [[ $# -gt 0 ]]; do
  case "$1" in
    --path)
      shift
      path="$1"
      path=$(realpath "$path")
      ;;
    --force)  # Force install
      force=true
      ;;
    --help)
      echo 'usage: ./ccache_setup.py --path /installed/folder [--force]'
      exit 0
      ;;
    *)
      echo "Invalid option: $1"
      exit 1
      ;;
  esac
  shift
done

# Check whether you put nvcc in PATH
set +e
nvcc_path=$(which nvcc)
if [[ -z "$nvcc_path" ]]; then
  nvcc_path="/usr/local/cuda/bin/nvcc"
  export PATH="/usr/local/cuda/bin:$PATH"
fi
set -e
if [ ! -f "$nvcc_path" ] && ! $force; then
  # shellcheck disable=SC2016
  echo 'nvcc is not detected in $PATH'
  exit 1
fi
echo "nvcc is detected at $nvcc_path"

if [ -f "$CUDA_NVCC_EXECUTABLE" ] && [[ "$CUDA_NVCC_EXECUTABLE" == *"ccache"* ]]; then  # Heuristic rule
  if $CUDA_NVCC_EXECUTABLE --version; then
    if ! $force; then
      echo "CCache with nvcc support is already installed at $CUDA_NVCC_EXECUTABLE, please add --force"
      exit 0
    fi
  fi
fi

# Installing CCache
echo "CCache will be installed at $path"
if [ -e "$path" ]; then
  mv --backup=t -T "$path" "${path}.old"
fi

with_proxy git clone https://github.com/colesbury/ccache.git "$path" -b ccbin
cd "$path"
./autogen.sh
./configure
make install prefix="$path"

mkdir -p "$path/lib"
mkdir -p "$path/cuda"
ln -sf "$path/bin/ccache" "$path/lib/cc"
ln -sf "$path/bin/ccache" "$path/lib/c++"
ln -sf "$path/bin/ccache" "$path/lib/gcc"
ln -sf "$path/bin/ccache" "$path/lib/g++"
ln -sf "$path/bin/ccache" "$path/cuda/nvcc"
"$path/bin/ccache" -M 25Gi

# Make sure the nvcc wrapped in CCache is runnable
"$path/cuda/nvcc" --version
echo 'Congrats! The CCache with nvcc support is installed!'
echo -e "Please add the following lines to your bash init script:\\n"
echo "################ Env Var for CCache with CUDA support ################"
# shellcheck disable=SC2016
echo 'export PATH="'"$path"'/lib:$PATH"'
echo 'export CUDA_NVCC_EXECUTABLE="'"$path"'/cuda/nvcc"'
echo '######################################################################'


#!/bin/bash

set -e

python -c 'from caffe2.python import build; from pprint import pprint; pprint(build.build_options)'
python -c 'from caffe2.python import core, workspace; print("GPUs found: " + str(workspace.NumCudaDevices()))'
python -c "import onnx"
python -c "import torch"

echo "Caffe2, PyTorch and ONNX installed successfully!!"


#!/bin/bash

# This script helps developers set up the ONNX Caffe2 and PyTorch develop environment on devgpu.
# It creates an virtualenv instance, and installs all the dependencies in this environment.
# The script will creates a folder called onnx-dev folder under the $HOME directory.
# onnx, pytorch and caffe2 are installed separately.
# Please source $HOME/onnx-dev/.onnx_env_init to initialize the development before starting developing.

# TODO: support python 3.

# Set script configuration
set -e
shopt -s expand_aliases

# Proxy setup
alias with_proxy="HTTPS_PROXY=http://fwdproxy:8080 HTTP_PROXY=http://fwdproxy:8080 FTP_PROXY=http://fwdproxy:8080 https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 ftp_proxy=http://fwdproxy:8080 http_no_proxy='*.facebook.com|*.tfbnw.net|*.fb.com'"

# Set the variables
RED='\033[0;31m'
CYAN='\033[0;36m'
NC='\033[0m'
onnx_root="$HOME/local/onnx-dev"   # I think hardcoding the onnx root dir is fine, just like fbsource
onnx_root_link="$HOME/onnx-dev"
venv="$onnx_root/onnxvenv"
onnx_init_file="$onnx_root_link/.onnx_env_init"
ccache_root="$onnx_root/ccache"
ccache_script="$(pwd)/ccache_install.sh"
sanity_script="$onnx_root/sanity.sh"

# Check whether default CUDA exists
# TODO check the required header and lib files
default_cuda="/usr/local/cuda"
if [[ ! -e "$default_cuda" ]]; then
  echo "Default CUDA is not found at $default_cuda"
fi

# Checking to see if CuDNN is present, and install it if not exists
if [ -f /usr/local/cuda/include/cudnn.h ]; then
  echo "CuDNN header already exists!!"
else
  sudo cp -R /home/engshare/third-party2/cudnn/6.0.21/src/cuda/include/* /usr/local/cuda/include/
  sudo cp -R /home/engshare/third-party2/cudnn/6.0.21/src/cuda/lib64/* /usr/local/cuda/lib64/
fi

# TODO set the specific version for each package
# Install the dependencies for Caffe2
sudo yum install python-virtualenv freetype-devel libpng-devel glog gflags protobuf protobuf-devel protobuf-compiler -y
rpm -q protobuf  # check the version and if necessary update the value below
protoc --version  # check protoc
protoc_path=$(which protoc)
if [[ "$protoc_path" != "/bin/protoc" ]]; then
  echo "Warning: Non-default protoc is detected, the script may not work with non-default protobuf!!!"
  echo "Please try to remove the protoc at $protoc_path and rerun this script."
  exit 1
fi

# Upgrade Cmake to the right version (>3.0)
sudo yum remove cmake3 -y
sudo yum install cmake -y

# Install the dependencies for CCache
sudo yum install autoconf asciidoc -y

# Create the root folder
if [ -e "$onnx_root" ]; then
  timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
  mv --backup=t -T "$onnx_root" "${onnx_root}.old.$timestamp"
fi
mkdir -p "$onnx_root"
if [ -e "$onnx_root_link"]; then
  timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
  mv --backup=t -T "$onnx_root_link" "${onnx_root_link}.old.$timestamp"
fi
ln -s "$onnx_root" "$onnx_root_link"

# Set the name of virtualenv instance
with_proxy virtualenv "$venv"

# Creating a script that can be sourced in the future for the environmental variable
touch "$onnx_init_file"
{
  # shellcheck disable=SC2016
  echo 'if [ -z "$LD_LIBRARY_PATH" ]; then';
  echo '  export LD_LIBRARY_PATH=/usr/local/cuda/lib64';
  echo 'else'
  # shellcheck disable=SC2016
  echo '  export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH';
  echo "fi"
  # shellcheck disable=SC2016
  echo 'export PATH='"$ccache_root"'/lib:/usr/local/cuda/bin:$PATH';
  echo "source $venv/bin/activate";
  echo 'alias with_proxy="HTTPS_PROXY=http://fwdproxy:8080 HTTP_PROXY=http://fwdproxy:8080 FTP_PROXY=http://fwdproxy:8080 https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 ftp_proxy=http://fwdproxy:8080 http_no_proxy='"'"'*.facebook.com|*.tfbnw.net|*.fb.com'"'"'"'
} >> "$onnx_init_file"
chmod u+x "$onnx_init_file"

# Installing CCache
cd "$onnx_root"
if [ ! -f "$ccache_script" ]; then
  ccache_script="$onnx_root/ccache_install.sh"
  with_proxy wget https://raw.githubusercontent.com/pytorch/pytorch/master/scripts/fbcode-dev-setup/ccache_setup.sh -O "$ccache_script"
fi
chmod u+x "$ccache_script"
"$ccache_script" --path "$ccache_root"

# Test nvcc with CCache
own_ccache=true
if [ -f "$CUDA_NVCC_EXECUTABLE" ] && [[ "$ccache_root/cuda/nvcc" != "$CUDA_NVCC_EXECUTABLE" ]] && \
  [[ "$CUDA_NVCC_EXECUTABLE" == *"ccache"* ]]; then  # Heuristic rule
  if $CUDA_NVCC_EXECUTABLE --version; then
    own_ccache=false
  fi
fi
if [ "$own_ccache" = true ]; then
  echo "export CUDA_NVCC_EXECUTABLE=$ccache_root/cuda/nvcc" >> "$onnx_init_file"
fi

# Loading env vars
# shellcheck disable=SC1090
source "$onnx_init_file"

"$CUDA_NVCC_EXECUTABLE" --version

# Create a virtualenv, activate it, upgrade pip
if [ -f "$HOME/.pip/pip.conf" ]; then
  echo "${RED}Warning: $HOME/.pip/pip.conf is detected, pip install may fail!${NC}"
fi
with_proxy python -m pip install -U pip setuptools
with_proxy python -m pip install future numpy "protobuf>3.2" pytest-runner pyyaml typing ipython

# Cloning repos
cd "$onnx_root"
with_proxy git clone https://github.com/onnx/onnx --recursive
with_proxy git clone https://github.com/pytorch/pytorch --recursive

# Build ONNX
cd "$onnx_root/onnx"
with_proxy python setup.py develop

# Build PyTorch and Caffe2
cd "$onnx_root/pytorch"
with_proxy pip install -r "requirements.txt"
with_proxy python setup.py develop

# Sanity checks and useful info
cd "$onnx_root"
with_proxy wget https://raw.githubusercontent.com/pytorch/pytorch/master/scripts/fbcode-dev-setup/onnx_c2_sanity_check.sh -O "$sanity_script"
chmod u+x "$sanity_script"
$sanity_script

echo "Congrats, you are ready to rock!!"
echo "################ Please run the following command before development ################"
echo -e "${CYAN}source $onnx_init_file${NC}"
echo "#####################################################################################"


#!/bin/bash

set -ex

# realpath might not be available on MacOS
script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
top_dir=$(dirname $(dirname $(dirname "$script_path")))
tp2_dir="$top_dir/third_party"

pip install ninja

# Install onnx
pip install --no-use-pep517 -e "$tp2_dir/onnx"

# Install caffe2 and pytorch
pip install -r "$top_dir/caffe2/requirements.txt"
pip install -r "$top_dir/requirements.txt"
python setup.py develop


#!/bin/bash

set -ex

# realpath might not be available on MacOS
script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
top_dir=$(dirname $(dirname $(dirname "$script_path")))
tp2_dir="$top_dir/third_party"
BUILD_DIR="$top_dir/build"
mkdir -p "$BUILD_DIR"

_pip_install() {
    if [[ -n "$CI" ]]; then
        if [[ -z "${SCCACHE_BUCKET}" ]]; then
            ccache -z
        fi
    fi
    if [[ -n "$CI" ]]; then
        time pip install "$@"
    else
        pip install "$@"
    fi
    if [[ -n "$CI" ]]; then
        if [[ -n "${SCCACHE_BUCKET}" ]]; then
            sccache --show-stats
        else
            ccache -s
        fi
    fi
}

# Install onnx
_pip_install -b "$BUILD_DIR/onnx" "file://$tp2_dir/onnx#egg=onnx"

# Install caffe2 and pytorch
pip install -r "$top_dir/caffe2/requirements.txt"
pip install -r "$top_dir/requirements.txt"
python setup.py install


#!/bin/bash

set -ex

UNKNOWN=()

# defaults
PARALLEL=0

while [[ $# -gt 0 ]]
do
    arg="$1"
    case $arg in
        -p|--parallel)
            PARALLEL=1
            shift # past argument
            ;;
        *) # unknown option
            UNKNOWN+=("$1") # save it in an array for later
            shift # past argument
            ;;
    esac
done
set -- "${UNKNOWN[@]}" # leave UNKNOWN

if [[ $PARALLEL == 1 ]]; then
    pip install pytest-xdist
fi

pip install pytest scipy hypothesis # these may not be necessary
pip install pytest-cov # installing since `coverage run -m pytest ..` doesn't work
pip install -e tools/coverage_plugins_package # allows coverage to run w/o failing due to a missing plug-in

# realpath might not be available on MacOS
script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
top_dir=$(dirname $(dirname $(dirname "$script_path")))
test_paths=(
    "$top_dir/test/onnx"
)

args=()
args+=("-v")
args+=("--cov")
args+=("--cov-report")
args+=("xml:test/coverage.xml")
args+=("--cov-append")
if [[ $PARALLEL == 1 ]]; then
  args+=("-n")
  args+=("3")
fi

# onnxruntime only support py3
# "Python.h" not found in py2, needed by TorchScript custom op compilation.
if [[ "$BUILD_ENVIRONMENT" == *ort_test1* ]]; then
  # These exclusions are for tests that take a long time / a lot of GPU
  # memory to run; they should be passing (and you will test them if you
  # run them locally
  pytest "${args[@]}" \
    --ignore "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py" \
    --ignore "$top_dir/test/onnx/test_custom_ops.py" \
    --ignore "$top_dir/test/onnx/test_models_onnxruntime.py" \
    --ignore "$top_dir/test/onnx/test_utility_funs.py" \
    --ignore "$top_dir/test/onnx/test_pytorch_onnx_caffe2.py" \
    --ignore "$top_dir/test/onnx/test_pytorch_onnx_shape_inference.py" \
    --ignore "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime_cuda.py" \
    --ignore "$top_dir/test/onnx/test_pytorch_onnx_caffe2_quantized.py" \
    "${test_paths[@]}"

  pytest "${args[@]}" \
    "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py::TestONNXRuntime_opset7" \
    "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py::TestONNXRuntime_opset8" \
    "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py::TestONNXRuntime" \
    "$top_dir/test/onnx/test_custom_ops.py" \
    "$top_dir/test/onnx/test_models_onnxruntime.py" \
    "$top_dir/test/onnx/test_utility_funs.py" \
    "$top_dir/test/onnx/test_pytorch_onnx_caffe2.py" \
    "$top_dir/test/onnx/test_pytorch_onnx_caffe2_quantized.py" \
    "$top_dir/test/onnx/test_pytorch_onnx_shape_inference.py"
fi

if [[ "$BUILD_ENVIRONMENT" == *ort_test2* ]]; then
  # Update the loop for new opsets
  for i in $(seq 10 14); do
    pytest "${args[@]}" \
      "$top_dir/test/onnx/test_pytorch_onnx_onnxruntime.py::TestONNXRuntime_opset$i"
  done
fi

# Our CI expects both coverage.xml and .coverage to be within test/
mv .coverage test/.coverage


#!/usr/bin/env bash

set -eou pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "${DIR}/promote/common_utils.sh"

if [[ -z "${RESTORE_FROM:-}" ]]; then
    echo "ERROR: RESTORE_FROM environment variable must be specified"
    echo "       example: RESTORE_FROM=v1.6.0-rc3 ${0}"
    exit 1
fi

DRY_RUN=${DRY_RUN:-enabled}

PYTORCH_S3_BACKUP_BUCKET=${PYTORCH_S3_BACKUP_BUCKET:-s3://pytorch-backup/${RESTORE_FROM}}
PYTORCH_S3_TEST_BUCKET=${PYTORCH_S3_TEST_BUCKET:-s3://pytorch/}
PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BACKUP_BUCKET}}
PYTORCH_S3_TO=${PYTORCH_S3_TO:-s3://pytorch/}

restore_wheels() {
    aws_promote torch whl
}

restore_libtorch() {
    aws_promote libtorch-* libtorch
}

ANACONDA="true anaconda"
if [[ ${DRY_RUN} = "disabled" ]]; then
    ANACONDA="anaconda"
fi
PYTORCH_CONDA_TO=${PYTORCH_CONDA_TO:-pytorch-test}

upload_conda() {
    local pkg
    pkg=${1}
    (
        set -x
        ${ANACONDA} upload --skip -u "${PYTORCH_CONDA_TO}" "${pkg}"
    )
}

export -f upload_conda

restore_conda() {
    TMP_DIR="$(mktemp -d)"
    trap 'rm -rf ${TMP_DIR}' EXIT
    (
        set -x
        aws s3 cp --recursive "${PYTORCH_S3_BACKUP_BUCKET}/conda" "${TMP_DIR}/"
    )
    export ANACONDA
    export PYTORCH_CONDA_TO
    # Should upload all bz2 packages in parallel for quick restoration
    find "${TMP_DIR}" -name '*.bz2' -type f \
        | xargs -P 10 -I % bash -c "(declare -t upload_conda); upload_conda %"
}

restore_wheels
restore_libtorch
restore_conda


#!/usr/bin/env bash

grab_prune_version() {
    conda search -c "${CHANNEL}" --platform "${platform}" "${PKG}" 2>/dev/null | \
        grep "${CHANNEL}" | \
        awk -F '  *' '{print $2}' | \
        uniq | \
        head -n -1 | \
        xargs
}

grab_latest_version() {
    conda search -c "${CHANNEL}" --platform "${platform}" "${PKG}" 2>/dev/null | \
        grep "${CHANNEL}" | \
        awk -F '  *' '{print $2}' | \
        uniq | \
        tail -n 1 | \
        xargs
}

grab_specs_for_version() {
    conda search -c "${CHANNEL}" --platform "${platform}" "${PKG}" 2>/dev/null | \
        grep "${CHANNEL}" | \
        grep "$1" | \
        awk -F '  *' '{print $3}' | \
        uniq | \
        xargs
}

set -eou pipefail

CHANNEL=${CHANNEL:-pytorch-nightly}
PKG=${PKG:-pytorch}
PLATFORMS=${PLATFORMS:-noarch osx-64 linux-64 win-64}

for platform in ${PLATFORMS}; do
    latest_version="$(grab_latest_version || true)"
    specs_in_latest_version="$(grab_specs_for_version "${latest_version}" || true)"
    versions_to_prune="$(grab_prune_version || true)"
    for version in ${versions_to_prune}; do
        specs_in_prune_version="$(grab_specs_for_version "${version}" || true)"
        for spec in ${specs_in_prune_version}; do
        # If this spec is included in specs_in_latest_version, then remove it.
        if [[ "${specs_in_latest_version}" =~ ${spec} ]];then
            (
                set -x
                anaconda remove --force "${CHANNEL}/${PKG}/${version}/${platform}/${PKG}-${version}-${spec}.tar.bz2"
            )
        fi
        done
    done
done


#!/usr/bin/env bash

set -euo pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"

CHANNEL=${CHANNEL:-pytorch-nightly}
PACKAGES=${PACKAGES:-pytorch}

for pkg in ${PACKAGES}; do
    echo "+ Attempting to prune: ${CHANNEL}/${pkg}"
    CHANNEL="${CHANNEL}" PKG="${pkg}" "${DIR}/prune.sh"
    echo
done


#!/usr/bin/env bash

exit_if_not_on_git_tag() {
    # Have an override for debugging purposes
    if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]] ;then
        >&2 echo "+ WARN: Continuing without being on a git tag"
        exit 0
    fi
    # Exit if we're not currently on a git tag
    if ! git describe --tags --exact >/dev/null 2>/dev/null; then
        >&2 echo "- ERROR: Attempting to promote on a non-git tag, must have tagged current commit locally first"
        exit 1
    fi
    # Exit if we're currently on an RC
    if git describe --tags | grep "-rc" >/dev/null 2>/dev/null; then
        >&2 echo "- ERROR: Attempting to promote on a non GA git tag, current tag must be a GA tag"
        >&2 echo "         Example: v1.5.0"
        exit 1
    fi
}

get_pytorch_version() {
    if [[ -n "${TEST_WITHOUT_GIT_TAG-}" ]];then
        if  [[ -z "${TEST_PYTORCH_PROMOTE_VERSION-}" ]]; then
            >&2 echo "- ERROR: Specified TEST_WITHOUT_GIT_TAG without specifying TEST_PYTORCH_PROMOTE_VERSION"
            >&2 echo "-        TEST_PYTORCH_PROMOTE_VERSION must be specified"
            exit 1
        else
            echo "${TEST_PYTORCH_PROMOTE_VERSION}"
            exit 0
        fi
    fi
    exit_if_not_on_git_tag
    # Echo git tag, strip leading v
    git describe --tags | sed -e 's/^v//'
}

aws_promote() {
    package_name=$1
    pytorch_version=$(get_pytorch_version)
    # Dry run by default
    DRY_RUN=${DRY_RUN:-enabled}
    DRY_RUN_FLAG="--dryrun"
    if [[ $DRY_RUN = "disabled" ]]; then
        DRY_RUN_FLAG=""
    fi
    AWS=${AWS:-aws}
    (
        set -x
        ${AWS} s3 cp ${DRY_RUN_FLAG} \
            --only-show-errors \
            --acl public-read \
            --recursive \
            --exclude '*' \
            --include "*${package_name}-${pytorch_version}*" \
            "${PYTORCH_S3_FROM/\/$//}" \
            "${PYTORCH_S3_TO/\/$//}"
    )
    # ^ We grep for package_name-.*pytorch_version to avoid any situations where domain libraries have
    #   the same version on our S3 buckets
}


#!/usr/bin/env bash

set -eou pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "${DIR}/common_utils.sh"

# Allow for users to pass PACKAGE_NAME
# For use with other packages, i.e. torchvision, etc.
PACKAGE_NAME=${PACKAGE_NAME:-pytorch}
PYTORCH_CONDA_FROM=${PYTORCH_CONDA_FROM:-pytorch-test}
PYTORCH_CONDA_TO=${PYTORCH_CONDA_TO:-pytorch}
CONDA_PLATFORMS="linux-64 osx-64 win-64 noarch"

pytorch_version="$(get_pytorch_version)"

tmp_dir="$(mktemp -d)"
pushd "${tmp_dir}"
trap 'rm -rf ${tmp_dir}' EXIT

conda_search() {
    conda search -q "${PYTORCH_CONDA_FROM}::${PACKAGE_NAME}==${pytorch_version}" -c "${PYTORCH_CONDA_FROM}" --platform "${platform}" \
        | grep -e "^${PACKAGE_NAME}" \
        | awk -F ' *' '{print $3}' \
        | xargs -I % echo "https://anaconda.org/${PYTORCH_CONDA_FROM}/${PACKAGE_NAME}/${pytorch_version}/download/${platform}/${PACKAGE_NAME}-${pytorch_version}-%.tar.bz2"
}

pkgs_to_download=()
for platform in ${CONDA_PLATFORMS}; do
    pkgs_to_download+=($(\
        conda_search 2>/dev/null || true
    ))
    # Create directory where packages will eventually be downloaded
    mkdir -p "${platform}"
done

my_curl() {
    local dl_url=$1
    local start=$(date +%s)
    # downloads should be distinguished by platform which should be the second
    # to last field in the url, this is to avoid clobbering same named files
    # for different platforms
    dl_dir=$(echo "${dl_url}" | rev | cut -d'/' -f 2 | rev)
    dl_name=$(echo "${dl_url}" | rev | cut -d'/' -f 1 | rev)
    curl -fsSL -o "${dl_dir}/${dl_name}" "${dl_url}"
    local end=$(date +%s)
    local diff=$(( end - start ))
    echo "+ ${dl_url} took ${diff}s"
}
export -f my_curl

# Download all packages in parallel
printf '%s\n' "${pkgs_to_download[@]}" \
    | xargs -P 10 -I % bash -c '(declare -t my_curl); my_curl %'

# dry run by default
DRY_RUN=${DRY_RUN:-enabled}
ANACONDA="true anaconda"
if [[ $DRY_RUN = "disabled" ]]; then
    ANACONDA="anaconda"
fi
(
    # We use --skip here to avoid re-uploading files we've already uploaded
    set -x
    ${ANACONDA} upload --skip -u ${PYTORCH_CONDA_TO} $(find . -name '*.bz2')
)

popd


#!/usr/bin/env bash

# Preps binaries for publishing to pypi by removing the
# version suffix we normally add for all binaries
# (outside of default ones, CUDA 10.2 currently)

# Usage is:
# $ prep_binary_for_pypy.sh <path_to_whl_file> <path_to_multiple_whl_files>

# Will output a whl in your current directory

set -eou pipefail
shopt -s globstar

OUTPUT_DIR=${OUTPUT_DIR:-$(pwd)}

tmp_dir="$(mktemp -d)"
trap 'rm -rf ${tmp_dir}' EXIT

for whl_file in "$@"; do
    whl_file=$(realpath "${whl_file}")
    whl_dir="${tmp_dir}/$(basename "${whl_file}")_unzipped"
    mkdir -pv "${whl_dir}"
    (
        set -x
        unzip -q "${whl_file}" -d "${whl_dir}"
    )
    version_with_suffix=$(grep '^Version:' "${whl_dir}"/*/METADATA | cut -d' ' -f2)
    version_with_suffix_escaped=${version_with_suffix/+/%2B}
    # Remove all suffixed +bleh versions
    version_no_suffix=${version_with_suffix/+*/}
    new_whl_file=${OUTPUT_DIR}/$(basename "${whl_file/${version_with_suffix_escaped}/${version_no_suffix}}")
    dist_info_folder=$(find "${whl_dir}" -type d -name '*.dist-info' | head -1)
    basename_dist_info_folder=$(basename "${dist_info_folder}")
    dirname_dist_info_folder=$(dirname "${dist_info_folder}")
    (
        set -x
        find "${dist_info_folder}" -type f -exec sed -i "s!${version_with_suffix}!${version_no_suffix}!" {} \;
        # Moves distinfo from one with a version suffix to one without
        # Example: torch-1.8.0+cpu.dist-info => torch-1.8.0.dist-info
        mv "${dist_info_folder}" "${dirname_dist_info_folder}/${basename_dist_info_folder/${version_with_suffix}/${version_no_suffix}}"
        cd "${whl_dir}"
        zip -qr "${new_whl_file}" .
    )
done


#!/usr/bin/env bash

set -eou pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "${DIR}/common_utils.sh"

# Allow for users to pass PACKAGE_NAME
# For use with other packages, i.e. torchvision, etc.
PACKAGE_NAME=${PACKAGE_NAME:-torch}
PACKAGE_TYPE=${PACKAGE_TYPE:-whl}

PYTORCH_S3_BUCKET=${PYTORCH_S3_BUCKET:-s3://pytorch}
FROM=${FROM:-test}
PYTORCH_S3_FROM=${PYTORCH_S3_FROM:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${FROM}}
TO=${TO:-}
PYTORCH_S3_TO=${PYTORCH_S3_TO:-${PYTORCH_S3_BUCKET}/${PACKAGE_TYPE}/${TO}}

aws_promote "${PACKAGE_NAME}"


#!/usr/bin/env bash

set -eou pipefail

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "${DIR}/common_utils.sh"

# Allow for users to pass PACKAGE_NAME
# For use with other packages, i.e. torchvision, etc.
PACKAGE_NAME=${PACKAGE_NAME:-torch}

pytorch_version="$(get_pytorch_version)"
# Refers to the specific package we'd like to promote
# i.e. VERSION_SUFFIX='%2Bcu102'
#      torch-1.8.0+cu102 -> torch-1.8.0
VERSION_SUFFIX=${VERSION_SUFFIX:-}
# Refers to the specific platofmr we'd like to promote
# i.e. PLATFORM=linux_x86_64
# For domains like torchaudio / torchtext this is to be left blank
PLATFORM=${PLATFORM:-}

pkgs_to_promote=$(\
    curl -fsSL https://download.pytorch.org/whl/torch_stable.html \
        | grep "${PACKAGE_NAME}-${pytorch_version}${VERSION_SUFFIX}-" \
        | grep "${PLATFORM}" \
        | cut -d '"' -f2
)

tmp_dir="$(mktemp -d)"
output_tmp_dir="$(mktemp -d)"
trap 'rm -rf ${tmp_dir} ${output_tmp_dir}' EXIT
pushd "${output_tmp_dir}"

# Dry run by default
DRY_RUN=${DRY_RUN:-enabled}
# On dry run just echo the commands that are meant to be run
TWINE_UPLOAD="echo twine upload"
if [[ $DRY_RUN = "disabled" ]]; then
    TWINE_UPLOAD="twine upload"
fi

for pkg in ${pkgs_to_promote}; do
    pkg_basename="$(basename "${pkg//linux/manylinux1}")"
    orig_pkg="${tmp_dir}/${pkg_basename}"
    (
        set -x
        # Download package, sub out linux for manylinux1
        curl -fsSL -o "${orig_pkg}" "https://download.pytorch.org/whl/${pkg}"
    )

    if [[ -n "${VERSION_SUFFIX}" ]]; then
        OUTPUT_DIR="${output_tmp_dir}" ${DIR}/prep_binary_for_pypi.sh "${orig_pkg}"
    else
        mv "${orig_pkg}" "${output_tmp_dir}/"
    fi

    (
        set -x
        ${TWINE_UPLOAD} \
            --disable-progress-bar \
            --non-interactive \
            ./*.whl
        rm -rf ./*.whl
    )
done


#!/bin/bash

# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.

FILE="$1/$RANK"
echo "creating $FILE"
touch "$FILE"


#!/bin/bash
###############################################################################
# This script tests the flow to build libtorch locally with optimized binary
# size for mobile devices and the flow to integrate it with a simple predictor
# in c++.
#
# Supported custom build types:
#
# 1. `TEST_DEFAULT_BUILD=1 ./build.sh` - it is similar to the prebuilt libtorch
# libraries released for Android and iOS (same CMake build options + host
# toolchain), which doesn't contain autograd function nor backward ops thus is
# smaller than full LibTorch.
#
# 2. `TEST_CUSTOM_BUILD_DYNAMIC=1 ./build.sh` - it further optimizes libtorch
# size by only including ops used by a specific model.
# Note that LLVM_DIR environment variable should be set to the location of
# LLVM-dev toolchain.
#
# 3. `TEST_CUSTOM_BUILD_STATIC=1 ./build.sh` - similar as 2) except that it
# relies on the static dispatch + linker to prune code.
#
###############################################################################

set -ex -o pipefail

SRC_ROOT="$( cd "$(dirname "$0")"/../../.. ; pwd -P)"
TEST_SRC_ROOT="${SRC_ROOT}/test/mobile/custom_build"
BUILD_ROOT="${BUILD_ROOT:-${SRC_ROOT}/build_test_custom_build}"
mkdir -p "${BUILD_ROOT}"
cd "${BUILD_ROOT}"

prepare_model_and_dump_root_ops() {
  cd "${BUILD_ROOT}"
  MODEL="${BUILD_ROOT}/MobileNetV2.pt"
  ROOT_OPS="${BUILD_ROOT}/MobileNetV2.yaml"

  python "${TEST_SRC_ROOT}/prepare_model.py"
}

generate_op_dependency_graph() {
  # Regular users should get this graph from prebuilt package.
  ANALYZER_BUILD_ROOT="${BUILD_ROOT}/build_analyzer"
  OP_DEPENDENCY="${ANALYZER_BUILD_ROOT}/work/torch_result.yaml"

  if [ ! -f "${OP_DEPENDENCY}" ]; then
    BUILD_ROOT="${ANALYZER_BUILD_ROOT}" \
      ANALYZE_TORCH=1 \
      "${SRC_ROOT}/tools/code_analyzer/build.sh"
  fi
}

run_default_build() {
  LIBTORCH_BUILD_ROOT="${BUILD_ROOT}/build_default_libtorch"
  LIBTORCH_INSTALL_PREFIX="${LIBTORCH_BUILD_ROOT}/install"

  BUILD_ROOT="${LIBTORCH_BUILD_ROOT}" \
    "${SRC_ROOT}/scripts/build_mobile.sh"
}

run_custom_build_with_static_dispatch() {
  LIBTORCH_BUILD_ROOT="${BUILD_ROOT}/build_custom_libtorch_static"
  LIBTORCH_INSTALL_PREFIX="${LIBTORCH_BUILD_ROOT}/install"

  # Here we omitted the OP_DEPENDENCY flag so it generates registration
  # code for used ROOT ops only, whose unboxing kernels are still needed
  # by the JIT runtime. The intermediate ops will be automatically kepted
  # by the linker as they are statically referenced by the static dispatch
  # code, for which we can bypass the registration.
  # We don't set '-DSTATIC_DISPATCH_BACKEND=CPU' explicitly to test automatic
  # fallback to static dispatch when '-DOP_DEPENDENCY' is omitted.
  BUILD_ROOT="${LIBTORCH_BUILD_ROOT}" \
    "${SRC_ROOT}/scripts/build_mobile.sh" \
    -DCMAKE_CXX_FLAGS="-DSTRIP_ERROR_MESSAGES" \
    -DSELECTED_OP_LIST="${ROOT_OPS}"
}

run_custom_build_with_dynamic_dispatch() {
  LIBTORCH_BUILD_ROOT="${BUILD_ROOT}/build_custom_libtorch_dynamic"
  LIBTORCH_INSTALL_PREFIX="${LIBTORCH_BUILD_ROOT}/install"

  BUILD_ROOT="${LIBTORCH_BUILD_ROOT}" \
    "${SRC_ROOT}/scripts/build_mobile.sh" \
    -DCMAKE_CXX_FLAGS="-DSTRIP_ERROR_MESSAGES" \
    -DSELECTED_OP_LIST="${ROOT_OPS}" \
    -DOP_DEPENDENCY="${OP_DEPENDENCY}"
}

build_predictor() {
  PREDICTOR_BUILD_ROOT="${BUILD_ROOT}/predictor"

  rm -rf "${PREDICTOR_BUILD_ROOT}" && mkdir -p "${PREDICTOR_BUILD_ROOT}"
  cd "${PREDICTOR_BUILD_ROOT}"

  cmake "${TEST_SRC_ROOT}" \
    -DCMAKE_PREFIX_PATH="${LIBTORCH_INSTALL_PREFIX}" \
    -DCMAKE_BUILD_TYPE=Release

  make
}

run_predictor() {
  cd "${PREDICTOR_BUILD_ROOT}"
  ./Predictor "${MODEL}" > output.txt

  if cmp -s output.txt "${TEST_SRC_ROOT}/expected_output.txt"; then
    echo "Test result is the same as expected."
  else
    echo "Test result is DIFFERENT from expected!"
    diff output.txt "${TEST_SRC_ROOT}/expected_output.txt"
    exit 1
  fi
}

test_default_build() {
  prepare_model_and_dump_root_ops
  run_default_build
  build_predictor
  run_predictor
}

test_custom_build_with_static_dispatch() {
  prepare_model_and_dump_root_ops
  run_custom_build_with_static_dispatch
  build_predictor
  run_predictor
}

test_custom_build_with_dynamic_dispatch() {
  prepare_model_and_dump_root_ops
  generate_op_dependency_graph
  run_custom_build_with_dynamic_dispatch
  build_predictor
  run_predictor
}

if [ -n "${TEST_DEFAULT_BUILD}" ]; then
  test_default_build
fi

if [ -n "${TEST_CUSTOM_BUILD_STATIC}" ]; then
  test_custom_build_with_static_dispatch
fi

if [ -n "${TEST_CUSTOM_BUILD_DYNAMIC}" ]; then
  test_custom_build_with_dynamic_dispatch
fi


#!/bin/bash
##############################################################################
# A simple project that uses C10 op registration API to create a bunch of
# inter-dependent dummy ops in order to test op dependency analysis script for
# mobile custom build workflow.
##############################################################################

set -ex

SRC_ROOT="$( cd "$(dirname "$0")"/../../.. ; pwd -P)"
BUILD_ROOT="${BUILD_ROOT:-${SRC_ROOT}/build_test_op_deps}"
INSTALL_PREFIX="${BUILD_ROOT}/install"

mkdir -p "${BUILD_ROOT}"
cd "${BUILD_ROOT}"

if [ ! -d "${TORCH_INSTALL_PREFIX:=${SRC_ROOT}/build_mobile/install}" ]; then
  echo "Unable to find torch library in ${TORCH_INSTALL_PREFIX}"
  exit 1
fi

cmake "${SRC_ROOT}/test/mobile/op_deps" \
  -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" \
  -DCMAKE_BUILD_TYPE=Release \
  -DCMAKE_PREFIX_PATH="${TORCH_INSTALL_PREFIX}" \
  "$@" # Use-specified CMake arguments

cmake --build . --target install -- "-j${MAX_JOBS}"
echo "Installation completed: ${INSTALL_PREFIX}"


th test.lua > lua.out
python3 test.py > python.out

diff lua.out python.out >/dev/null 2>&1
RESULT=$?
if [[ RESULT -eq 0 ]]; then
    echo "PASS"
else
    echo "FAIL"
    echo "Press ENTER to open vimdiff"
    read
    vimdiff lua.out python.out
fi


#!/bin/sh

# HOW TO USE:
# 1) python setup.py ...
# 2) tools/git_add_generated_dirs
# 3) Edit codegen
# 4) python setup.py ...
# 5) git diff to see changes
# 6) If satisfied: tools/git_reset_generated_dirs, commit, etc.
#    If not satisfied: Go to 3)

BASEDIR=$(dirname "$0")
(< $BASEDIR/generated_dirs.txt xargs -i find {} -type f) | xargs git add -f


#!/bin/sh

# HOW TO USE:
# 1) python setup.py ...
# 2) tools/git_add_generated_dirs
# 3) Edit codegen
# 4) python setup.py ...
# 5) git diff to see changes
# 6) If satisfied: tools/git_reset_generated_dirs, commit, etc.
#    If not satisfied: Go to 3)

BASEDIR=$(dirname "$0")
(< $BASEDIR/generated_dirs.txt xargs -i find {} -type f) | xargs git reset HEAD


#!/bin/bash
##############################################################################
# Build LLVM code analyzer and analyze torch code dependency.
##############################################################################
#
# Example usage:
#
# 1. Analyze torch and generate yaml file of op dependency transitive closure:
# LLVM_DIR=${HOME}/src/llvm8/build/install \
# ANALYZE_TORCH=1 tools/code_analyzer/build.sh
#
# 2. Analyze test project and compare with expected result:
# LLVM_DIR=${HOME}/src/llvm8/build/install \
# ANALYZE_TEST=1 tools/code_analyzer/build.sh
#
# 3. Analyze torch and generate yaml file of op dependency with debug path:
# LLVM_DIR=${HOME}/src/llvm8/build/install \
# ANALYZE_TORCH=1 tools/code_analyzer/build.sh -debug_path=true
#
# If you're a Facebook employee, chances are you're running on CentOS 8.
# If that's the case, you can install all the dependencies you need with:
#
#   sudo dnf install llvm-devel llvm-static clang ncurses-devel
#
# and then set LLVM_DIR=/usr

set -ex

SRC_ROOT="$( cd "$(dirname "$0")"/../.. ; pwd -P)"
ANALYZER_SRC_HOME="${SRC_ROOT}/tools/code_analyzer"

# Clang/LLVM path
export LLVM_DIR="${LLVM_DIR:-/usr/lib/llvm-8}"
export CC="${LLVM_DIR}/bin/clang"
export CXX="${LLVM_DIR}/bin/clang++"
EXTRA_ANALYZER_FLAGS=$@

BUILD_ROOT="${BUILD_ROOT:-${SRC_ROOT}/build_code_analyzer}"
WORK_DIR="${BUILD_ROOT}/work"

rm -rf "${BUILD_ROOT}"
mkdir -p "${BUILD_ROOT}"
mkdir -p "${WORK_DIR}"
cd "${BUILD_ROOT}"

build_analyzer() {
  cmake "${ANALYZER_SRC_HOME}" -DCMAKE_BUILD_TYPE=Release

  if [ -z "${MAX_JOBS}" ]; then
    if [ "$(uname)" == 'Darwin' ]; then
      MAX_JOBS=$(sysctl -n hw.ncpu)
    else
      MAX_JOBS=$(nproc)
    fi
  fi

  make "-j${MAX_JOBS}"
}

build_torch_mobile() {
  TORCH_BUILD_ROOT="${BUILD_ROOT}/build_mobile"
  TORCH_INSTALL_PREFIX="${TORCH_BUILD_ROOT}/install"

  BUILD_ROOT="${TORCH_BUILD_ROOT}" "${SRC_ROOT}/scripts/build_mobile.sh" \
    -DCMAKE_CXX_FLAGS="-S -emit-llvm -DSTRIP_ERROR_MESSAGES" \
    ${MOBILE_BUILD_FLAGS}
}

build_test_project() {
  TEST_SRC_ROOT="${SRC_ROOT}/test/mobile/op_deps"
  TEST_BUILD_ROOT="${BUILD_ROOT}/build_test"
  TEST_INSTALL_PREFIX="${TEST_BUILD_ROOT}/install"

  BUILD_ROOT="${TEST_BUILD_ROOT}" \
    TORCH_INSTALL_PREFIX="${TORCH_INSTALL_PREFIX}" \
    "${TEST_SRC_ROOT}/build.sh" \
    -DCMAKE_CXX_FLAGS="-S -emit-llvm -DSTRIP_ERROR_MESSAGES"
}

call_analyzer() {
  ANALYZER_BIN="${BUILD_ROOT}/analyzer" \
    INPUT="${INPUT}" OUTPUT="${OUTPUT}" \
    EXTRA_ANALYZER_FLAGS="${EXTRA_ANALYZER_FLAGS}" \
    "${ANALYZER_SRC_HOME}/run_analyzer.sh"
}

analyze_torch_mobile() {
  INPUT="${WORK_DIR}/torch.ll"
  OUTPUT="${WORK_DIR}/torch_result.yaml"

  if [ ! -f "${INPUT}" ]; then
    # Link libtorch into a single module
    # TODO: invoke llvm-link from cmake directly to avoid this hack.
    # TODO: include *.c.o when there is meaningful fan-out from pure-c code.
    "${LLVM_DIR}/bin/llvm-link" -S \
    $(find "${TORCH_BUILD_ROOT}" -name '*.cpp.o' -o -name '*.cc.o') \
    -o "${INPUT}"
  fi

  # Analyze dependency
  call_analyzer
}

print_output_file_path() {
  echo "Deployed file at: ${OUTPUT}"
}

analyze_test_project() {
  INPUT="${WORK_DIR}/test.ll"
  OUTPUT="${WORK_DIR}/test_result.yaml"

  # Link into a single module (only need c10 and OpLib srcs)
  # TODO: invoke llvm-link from cmake directly to avoid this hack.
  "${LLVM_DIR}/bin/llvm-link" -S \
  $(find "${TORCH_BUILD_ROOT}" -path '*/c10*' \( -name '*.cpp.o' -o -name '*.cc.o' \)) \
  $(find "${TEST_BUILD_ROOT}" -path '*/OpLib*' \( -name '*.cpp.o' -o -name '*.cc.o' \)) \
  -o "${INPUT}"

  # Analyze dependency
  call_analyzer
}

check_test_result() {
  if cmp -s "${OUTPUT}" "${TEST_SRC_ROOT}/expected_deps.yaml"; then
    echo "Test result is the same as expected."
  else
    echo "Test result is DIFFERENT from expected!"
    diff -u "${TEST_SRC_ROOT}/expected_deps.yaml" "${OUTPUT}"
    exit 1
  fi
}

build_analyzer

if [ -n "${ANALYZE_TORCH}" ]; then
  build_torch_mobile
  analyze_torch_mobile
  if [ -n "${DEPLOY}" ]; then
    print_output_file_path
  fi
fi

if [ -n "${ANALYZE_TEST}" ]; then
  build_torch_mobile
  build_test_project
  analyze_test_project
  check_test_result
fi


#!/bin/bash
##############################################################################
# Invoke code analyzer binary with pre-defined parameters for LibTorch.
# This script should be called via build.sh. Do NOT use it directly.
##############################################################################

set -exu

echo "Analyze: ${INPUT}"

# NB: op_register_pattern actually contains "too" many entries.  We only
# need to regex for symbols which occur after inlining; and most of the
# public API for the registration API disappears after inlining (e.g.,
# only _def and _impl are retained).  But the inliner isn't guaranteed
# to operate, so for safety we match a more expansive set.
"${ANALYZER_BIN}" \
  -op_schema_pattern="^(_aten|_prim|aten|quantized|_quantized|prepacked|profiler|_test)::[a-zA-Z0-9_.]+(\(.*)?$" \
  -op_register_pattern="c10::RegisterOperators::(op|checkSchemaAndRegisterOp_)|c10::Module::(_?def|_?impl)|torch::Library::(_?def|_?impl)" \
  -op_invoke_pattern="c10::Dispatcher::findSchema" \
  -root_symbol_pattern="torch::jit::[^(]" \
  -torch_library_init_pattern="^.*TORCH_LIBRARY_init_([^(]+)(\(.*)?$" \
  -torch_library_init_pattern="^.*TORCH_LIBRARY_FRAGMENT_init_([_]*[^_]+)_[0-9]+(\(.*)?$" \
  -torch_library_init_pattern="^.*TORCH_LIBRARY_IMPL_init_([_]*[^_]+)_([^_]+)_[0-9]+(\(.*)?$" \
  ${EXTRA_ANALYZER_FLAGS} \
  "${INPUT}" \
  > "${OUTPUT}"

echo "Result: ${OUTPUT}"


#!/bin/sh
set -eux

# Runs clang-format on allowlisted files.
# Requires a single argument, which is the <commit> argument to git-clang-format

# If you edit this allowlist, please edit the one in clang_format_all.py as well
find . -type f \
  -path './c10/*' -or \
  -path './torch/csrc/jit/*' -or \
  -path './test/cpp/jit/*' -or \
  -path './test/cpp/tensorexpr/*' \
  | xargs tools/linter/git-clang-format --verbose "$1" --


#!/usr/bin/env bash
find "$@" -name '*.sh' -print0 | xargs -0 -n1 shellcheck --external-sources


#!/bin/bash
set -ex
wget https://www.openssl.org/source/openssl-1.1.1k.tar.gz
tar xf openssl-1.1.1k.tar.gz
(cd openssl-1.1.1k && ./config --prefix="$PYTHON_INSTALL_DIR" && make -j32 && make install)
CFLAGS=-fPIC CPPFLAGS=-fPIC ./configure --prefix "$PYTHON_INSTALL_DIR" --with-openssl="$PYTHON_INSTALL_DIR"


#! /bin/bash
set -e

if [ -z "${PYTORCH_REPO_PATH}" ]; then
  echo "Please specify \`PYTORCH_REPO_PATH\`"
  exit 1
fi

cd "${PYTORCH_REPO_PATH}"

# Helper function to manage conda environments.
make_clean_env(){
    ENV_NAME="$1"

    # Cleanup prior if it exists
    conda env remove --name "${ENV_NAME}" 2> /dev/null || true

    conda create -yn "${ENV_NAME}" python=3
    source activate "${ENV_NAME}"
    conda install -y numpy ninja pyyaml mkl mkl-include setuptools cmake cffi hypothesis
    conda install -y -c pytorch magma-cuda102
    conda deactivate
}

make_clean_env ref_39850
make_clean_env pr_39850
make_clean_env ref_39967
make_clean_env pr_39967
make_clean_env ref_39744
make_clean_env pr_39744

git submodule update --init --recursive --jobs 0
wget https://github.com/pytorch/pytorch/pull/39850.diff
wget https://github.com/pytorch/pytorch/pull/39744.diff

source activate ref_39850
git checkout 766889b6bfeb9802f892fa0782d629554b2c71b4
python setup.py install
conda deactivate

source activate pr_39850
git apply 39850.diff
python setup.py install
git checkout .
conda deactivate

source activate ref_39967
git checkout c4fc278fa8cbd0fb45b8130a679c62f673087484
python setup.py install
conda deactivate

source activate pr_39967
git checkout 7a3c223bbb711c7a93910ce406a0126b8000b43b
python setup.py install
conda deactivate

source activate ref_39744
git checkout 541814f2b7eacabacdc87ccb1b4495bf486f501a
python setup.py install
conda deactivate

source activate pr_39744
git apply 39744.diff
python setup.py install
git checkout .
conda deactivate


#! /bin/sh
#
# Run 'autoreconf' to build 'configure', 'Makefile.in' and other configure
# control files.
#
# The first time this is run on a GIT checkout the only files that exist are
# configure.ac and Makefile.am; all of the autotools support scripts are
# missing.  They are instantiated with autoreconf --force --install.
#
# For regular ("tarball") distributions all the files should exist.  We do not
# want them to be updated *under any circumstances*.  It should never be
# necessary to run autogen.sh because ./configure --enable-maintainer-mode says
# what to do if Makefile.am or configure.ac are changed.
#
# It is *probably* OK to update the files on a GIT checkout, because they have
# come from the local tools, but leave that to the user who is assumed to know
# whether it is ok or required.
#
# This script is intended to work without arguments, there are, however, hidden
# arguments (a) for use while testing the script and (b) to fix up systems that
# have been broken.  If (b) is required the script prompts for the correct
# options.  For this reason the options are *NOT* documented in the help; this
# is deliberate; UTSL.
#
clean=
maintainer=
while test $# -gt 0
do
   case "$1" in
      --maintainer)
         maintainer=1;;

      --clean)
         clean=1;;

      *)
         exec >&2
         echo "$0: usage: ./autogen.sh"
         if test -d .git
         then
            echo "  ./autogen.sh generates the configure script and"
            echo "  Makefile.in, or refreshes them after changes to Makefile.am"
            echo "  or configure.ac.  You may prefer to just run autoreconf."
         elif test -z "$maintainer"
         then
            echo "  DO NOT RUN THIS SCRIPT."
            echo "  If you need to change Makefile.am or configure.ac then you"
            echo "  also need to run ./configure --enable-maintainer-mode and"
            echo "  use the appropriate autotools, *NOT* this script, to update"
            echo "  everything, please check the documentation of autoreconf."
            echo "  WARNING: libpng is intentionally generated with a known,"
            echo "  fixed, set of autotools.  It is known *NOT* to work with"
            echo "  the collection of autotools distributed on highly reputable"
            echo "  operating systems."
            echo "  Remember: autotools is GNU software, you are expected to"
            echo "  pay for support."
         else
            echo "  You have run autogen.sh with --maintainer enabled and you"
            echo "  are not using a GIT distribution, then you have given an"
            echo "  unrecognized argument.  This is not good. --maintainer"
            echo "  switches off any assumptions that you might not know what"
            echo "  you are doing."
         fi
         exit 1;;
   esac

   shift
done
#
# First check for a set of the autotools files; if absent then this is assumed
# to be a GIT version and the local autotools must be used.  If present this
# is a tarball distribution and the script should not be used.  If partially
# present bad things are happening.
#
# The autotools generated files:
libpng_autotools_files="Makefile.in aclocal.m4 config.guess config.h.in\
   config.sub configure depcomp install-sh ltmain.sh missing test-driver"
#
# Files generated by versions of configue >2.68 or automake >1.13 (i.e. later
# versions than those required by configure.ac):
libpng_autotools_extra="compile"
#
# These are separate because 'maintainer-clean' does not remove them.
libpng_libtool_files="scripts/libtool.m4 scripts/ltoptions.m4\
   scripts/ltsugar.m4 scripts/ltversion.m4 scripts/lt~obsolete.m4"

libpng_autotools_dirs="autom4te.cache" # not required
#
# The configure generated files:
libpng_configure_files="Makefile config.h config.log config.status\
   libpng-config libpng.pc libtool stamp-h1"

libpng_configure_dirs=".deps"
#
# We must remove the configure generated files as well as the autotools
# generated files if autotools are regenerated because otherwise if configure
# has been run without "--enable-maintainer-mode" make can do a partial update
# of Makefile.  These functions do the two bits of cleaning.
clean_autotools(){
   rm -rf $libpng_autotools_files $libpng_libtool_files $libpng_autotools_dirs
   rm -rf $libpng_autotools_extra
}

clean_configure(){
   rm -rf $libpng_configure_files $libpng_configure_dirs
}
#
# Clean: remove everything (this is to help with testing)
if test -n "$clean"
then
   clean_configure
   if test -n "$maintainer"
   then
      clean_autotools
   fi

   exit 0
fi
#
# Validate the distribution.
libpng_autotools_file_found=
libpng_autotools_file_missing=
for file in $libpng_autotools_files
do
   if test -f  "$file"
   then
      libpng_autotools_file_found=1
   else
      libpng_autotools_file_missing=1
   fi
done
#
# Presence of one of these does not *invalidate* missing, but absence
# invalidates found.
for file in $libpng_libtool_files
do
   if test ! -f "$file"
   then
      libpng_autotools_file_missing=1
   fi
done
#
# The cache directory doesn't matter - it will be regenerated and does not exist
# anyway in a tarball.
#
# Either everything is missing or everything is there, the --maintainer option
# just changes this so that the mode is set to generate all the files.
mode=
if test -z "$libpng_autotools_file_found" -o -n "$maintainer"
then
   mode="autoreconf"
else
   if test -n "$libpng_autotools_file_missing"
   then
      mode="broken"
   else
      mode="configure"
   fi
fi
#
# So:
case "$mode" in
   autoreconf)
      # Clean in case configure files exist
      clean_configure
      clean_autotools
      # Everything must be initialized, so use --force
      if autoreconf --warnings=all --force --install
      then
         missing=
         for file in $libpng_autotools_files
         do
            test -f "$file" || missing=1
         done
         # ignore the cache directory
         test -z "$missing" || {
            exec >&2
            echo "autoreconf was run, but did not produce all the expected"
            echo "files.  It is likely that your autotools installation is"
            echo "not compatible with that expected by libpng."
            exit 1
         }
      else
         exec >&2
         echo "autoreconf failed: your version of autotools is incompatible"
         echo "with this libpng version.  Please use a distributed archive"
         echo "(which includes the autotools generated files) and run configure"
         echo "instead."
         exit 1
      fi;;

   configure)
      if test -d .git
      then
         exec >&2
         echo "ERROR: running autoreconf on an initialized sytem"
         echo "  This is not necessary; it is only necessary to remake the"
         echo "  autotools generated files if Makefile.am or configure.ac"
         echo "  change and make does the right thing with:"
         echo
         echo "     ./configure --enable-maintainer-mode."
         echo
         echo "  You can run autoreconf yourself if you don't like maintainer"
         echo "  mode and you can also just run autoreconf -f -i to initialize"
         echo "  everything in the first place; this script is only for"
         echo "  compatibility with prior releases."
         exit 1
      else
         exec >&2
         echo "autogen.sh is intended only to generate 'configure' on systems"
         echo "that do not have it.  You have a complete 'configure', if you"
         echo "need to change Makefile.am or configure.ac you also need to"
         echo "run configure with the --enable-maintainer-mode option."
         exit 1
      fi;;

   broken)
      exec >&2
      echo "Your system has a partial set of autotools generated files."
      echo "autogen.sh is unable to proceed.  The full set of files is"
      echo "contained in the libpng 'tar' distribution archive and you do"
      echo "not need to run autogen.sh if you use it."
      exit 1;;
esac