x86_64Ubuntu22上でARM64向けにPeer用Catapultをビルドする

## 環境

Windows 上の Docker で Ubuntu22 を動かしてます。

Windows 11 Home
Ryzen 5 3600
メモリ 16GB

Windows Docker 上の Ubuntu。

```
# cat /etc/os-release
PRETTY_NAME="Ubuntu 22.04.3 LTS"
NAME="Ubuntu"
VERSION_ID="22.04"
VERSION="22.04.3 LTS (Jammy Jellyfish)"
VERSION_CODENAME=jammy
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=jammy
```

## ARM64 クロスコンパイラインストール

```bash
apt install -y g++-aarch64-linux-gnu
```

ビルドで使用するツールチェインを作成しておく

```bash
cd /opt/symbol/client/catapult
vi arm-toolchain.cmake
```

```arm-toolchain.cmake
SET(CMAKE_C_COMPILER /usr/bin/aarch64-linux-gnu-gcc-11)
SET(CMAKE_CXX_COMPILER /usr/bin/aarch64-linux-gnu-g++-11)
SET(CMAKE_LINKER /usr/bin/aarch64-linux-gnu-ld.gold)
SET(CMAKE_NM /usr/bin/aarch64-linux-gnu-nm)
SET(CMAKE_OBJCOPY /usr/bin/aarch64-linux-gnu-objcopy)
SET(CMAKE_OBJDUMP /usr/bin/aarch64-linux-gnu-objdump)
SET(CMAKE_RANLIB /usr/bin/aarch64-linux-gnu-ranlib)
```

## 依存関係のダウンロード＆ビルド

ダウンロードのみ行う。

```bash
PYTHONPATH="../../jenkins/catapult/" python3 "../../jenkins/catapult/installDepsLocal.py" \
	--target "./_deps" \
	--versions "../../jenkins/catapult/versions.properties" \
	--download
```

### Boost

```bash
mkdir /opt/symbol/client/catapult/_deps/boost
cd /opt/symbol/client/catapult/_deps/source/boost
./bootstrap.sh --prefix=/opt/symbol/client/catapult/_deps/boost
```

`project-config.jam` の 12 行目を書き換える。

```bash
vi project-config.jam
```

```project-config.jam
# B2 Configuration
# Automatically generated by bootstrap.sh

import option ;
import feature ;

# Compiler configuration. This definition will be used unless
# you already have defined some toolsets in your user-config.jam
# file.
if ! gcc in [ feature.values <toolset> ]
{
    using gcc : arm : aarch64-linux-gnu-g++ ; #この行を書き換え
}

project : default-build <toolset>gcc ;
```

ビルド

```bash
./b2 \
    --prefix=/opt/symbol/client/catapult/_deps/boost \
    toolset=gcc-arm \
    runtime-link=shared \
    threading=multi \
    link=shared \
    --layout=system \
    variant=release \
    -j \
    8 \
    stage \
    release
```

インストール

```bash
./b2 \
    install \
    --prefix=/opt/symbol/client/catapult/_deps/boost \
    runtime-link=shared \
    threading=multi \
    link=shared \
    --layout=system \
    variant=release
```

### Google Test

作っておいたツールチェインを使用してビルドする。

```bash
cd /opt/symbol/client/catapult/_deps/source/googletest
mkdir _build && cd _build
cmake \
    -DCMAKE_TOOLCHAIN_FILE=/opt/symbol/client/catapult/arm-toolchain.cmake \
    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
    -DCMAKE_INSTALL_PREFIX=/opt/symbol/client/catapult/_deps/google \
    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
    -DBUILD_GMOCK=OFF \
    ..
make -j 8
make install
```

### Google Benchmark

```bash
cd /opt/symbol/client/catapult/_deps/source/benchmark
mkdir _build && cd _build
cmake \
    -DCMAKE_TOOLCHAIN_FILE=/opt/symbol/client/catapult/arm-toolchain.cmake \
    -DCMAKE_CROSSCOMPILING=0 \
    -DRUN_HAVE_STD_REGEX=0 \
    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
    -DCMAKE_INSTALL_PREFIX=/opt/symbol/client/catapult/_deps/google \
    -DBENCHMARK_ENABLE_GTEST_TESTS=OFF \
    ..
make -j 8
make install
```

### OpenSSL

```bash
cd /opt/symbol/client/catapult/_deps/source/openssl
perl \
    ./Configure \
    linux-aarch64 \
    --cross-compile-prefix=/usr/bin/aarch64-linux-gnu- \
    --prefix=/opt/symbol/client/catapult/_deps/openssl \
    --openssldir=/opt/symbol/client/catapult/_deps/openssl \
    --libdir=/opt/symbol/client/catapult/_deps/openssl
make -j 8
make install_sw install_ssldirs
```

Catapult ビルド用に一時的にパスを通しておく。

```bash
export OPENSSL_ROOT_DIR=/opt/symbol/client/catapult/_deps/openssl
```

### RocksDB

```bash
cd /opt/symbol/client/catapult/_deps/source/rocksdb
mkdir _build && cd _build
cmake \
    -DCMAKE_TOOLCHAIN_FILE=/opt/symbol/client/catapult/arm-toolchain.cmake \
    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
    -DCMAKE_INSTALL_PREFIX=/opt/symbol/client/catapult/_deps/facebook \
    -DPORTABLE=1 \
    -DWITH_TESTS=OFF \
    -DWITH_TOOLS=OFF \
    -DWITH_BENCHMARK_TOOLS=OFF \
    -DWITH_CORE_TOOLS=OFF \
    -DWITH_GFLAGS=OFF \
    ..
make -j 8
make install
```

### Mongo

Peer だから使わない

### ZeroMQ

Peer だから使わない

## Catapult

```bash
cd /opt/symbol/client/catapult
mkdir _build && cd _build
```

110 行目付近に`set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")`追加。

```bash
vi ../CMakeGlobalSettings.cmake
```

```
                endif()
        endif()

        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZATION_FLAGS}")
        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZATION_FLAGS}")
endif()

### set compiler settings
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
if(MSVC)
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /WX /EHsc /Zc:__cplusplus")
        # in debug disable "potentially uninitialized local variable" (FP)
        set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd /D_SCL_SECURE_NO_WARNINGS /wd4701")
```

mongo と zeromq をコメントアウト

```bash
vi ../extensions/CMakeLists.txt
```

```CMakeLists.txt
        # need to promote custom test compiler flags to parent scope
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" PARENT_SCOPE)
endfunction()

add_subdirectory(addressextraction)
add_subdirectory(diagnostics)
add_subdirectory(filespooling)
add_subdirectory(finalization)
add_subdirectory(harvesting)
add_subdirectory(hashcache)
#add_subdirectory(mongo)
add_subdirectory(networkheight)
add_subdirectory(nodediscovery)
add_subdirectory(packetserver)
add_subdirectory(partialtransaction)
add_subdirectory(pluginhandlers)
add_subdirectory(sync)
add_subdirectory(syncsource)
add_subdirectory(timesync)
add_subdirectory(transactionsink)
add_subdirectory(unbondedpruning)
#add_subdirectory(zeromq)
```

ビルド

```bash
BOOST_ROOT="$(realpath ../_deps/boost)" cmake .. \
    -DCMAKE_TOOLCHAIN_FILE=/opt/symbol/client/catapult/arm-toolchain.cmake \
	-DENABLE_TESTS=OFF \
	-DCMAKE_BUILD_TYPE=RelWithDebInfo \
	-DCMAKE_PREFIX_PATH="$(realpath ../_deps/facebook);$(realpath ../_deps/google)" \
	-GNinja
```

```bash
ninja publish
```

```bash
ninja
```

## 仮確認

x86_64 上で動かしてみる。

```bash
apt install -y qemu-user-binfmt
export LD_LIBRARY_PATH=/opt/symbol/client/catapult/_deps/openssl:/opt/symbol/client/catapult/_deps/facebook/lib:/opt/symbol/client/catapult/_deps/boost/lib:/opt/symbol/client/catapult/_build/lib
```

```bash
qemu-aarch64 -L /usr/aarch64-linux-gnu bin/catapult.server
```

## 確認

`_build`を ARM64 環境に移動して確認する。
symbol-bootstrap を参考に必要なファイルを tar に固める。

```bash
cd /opt/symbol/client/catapult
mkdir catapult
cd catapult
cp -r ../_build/bin .

mkdir lib
cd lib
cp ../bin/*.so .
cd ..

mkdir deps
cd deps
cp -d ../../_deps/boost/lib/*.so* .
cp -d ../../_deps/facebook/lib/*.so* .
cp -d ../../_deps/openssl/*.so* .
cp -r ../../_deps/openssl/engines-3 .
cp -r ../../_deps/openssl/ossl-modules .
cd ../..

tar -czvf catapult.tar.gz catapult
```

リソースファイルなどは以下を参考にしてください。
pluginsDirectory の場所は、設置場所で変わってくると思うので要変更。

https://qiita.com/ccHarvestasya/items/99cf1dad7eb21ed6a86e

ARM64 マシンに元から入っている OpenSSL の方が新しい場合、deps に入っている libssl.so、libssl.so.3 を削除

## 参考

https://mijinc0.github.io/blog/20190602_building_catapult_for_arm/