How to cross compile OPTEE official os & client & sample code

This aritical inteprets my experience when building secure OS OPTEE's offical os, client and sample code.

##
:::note info
optee_os code
:::

Reference
https://optee.readthedocs.io/en/latest/building/gits/optee_os.html

Below is an example

```optee 3.14.0 build instructions
TOOL_CHAIN=/ssd/chen/Source/prebuilts/gcc-linaro-7.2.1-2017.11-x86_64_aarch64-linux-gnu/bin:/ssd/chen/Source/prebuilts/linaro-4.9.3-2014.11-arm-linux-gnueabihf/bin
OUTPUT=./out/arm
export PATH=$TOOL_CHAIN:$PATH
 
mkdir -p $OUTPUT
 
make \
    CFG_ARM64_core=y \
    CFG_TEE_BENCHMARK=n \
    CFG_TEE_CORE_LOG_LEVEL=3 \
    CROSS_COMPILE=aarch64-linux-gnu- \
    CROSS_COMPILE_core=aarch64-linux-gnu- \
    CROSS_COMPILE_ta_arm32=arm-linux-gnueabihf- \
    CROSS_COMPILE_ta_arm64=aarch64-linux-gnu- \
    DEBUG=1 \
    O=out/arm \
    PLATFORM=vexpress-qemu_armv8a
 
# Output
chen@coverity-HP-Z840-Workstation:~/optee/optee_os$ ls -al out/arm/
-rw-rw-r--  1 chen chen 7130  3月  1 09:19 conf.cmake
-rw-rw-r--  1 chen chen 4435  3月  1 09:19 conf.mk
drwxrwxr-x 11 chen chen 4096  3月  1 09:19 core
drwxrwxr-x  4 chen chen 4096  3月  1 09:19 core-lib
drwxrwxr-x 10 chen chen 4096  3月  1 09:20 export-ta_arm32 # All files needed to build customer arm32 TA
drwxrwxr-x  9 chen chen 4096  3月  1 09:20 export-ta_arm64 # All files needed to build customer aarch64 TA
...
drwxrwxr-x  6 chen chen 4096  3月  1 09:20 ta_arm64-lib # Where libutee.a and libutils.a are
```

However, I met below build error.
```build error reported by python3.4
 File "scripts/gen_ldelf_hex.py", line 152, in <module>
    main()
  File "scripts/gen_ldelf_hex.py", line 143, in main
    emit_load_segments(elffile, outf)
  File "scripts/gen_ldelf_hex.py", line 116, in emit_load_segments
    outf.write(b'const unsigned int ldelf_code_size = %d;\n' % code_size)
TypeError: unsupported operand type(s) for %: 'bytes' and 'int'
make: *** [/home/chen/optee_os/output/core/ldelf_hex.c] Error 1
make: *** Deleting file `/home/chen/optee_os/output/core/ldelf_hex.c'
```

A patch working as my workaroud
```diff gen_ldelf_hex.py
diff --git a/scripts/gen_ldelf_hex.py b/scripts/gen_ldelf_hex.py
index a6123b29..e6dc6be9 100755
--- a/scripts/gen_ldelf_hex.py
+++ b/scripts/gen_ldelf_hex.py
@@ -92,7 +92,8 @@ def emit_load_segments(elffile, outf):
     n = 0
     i = 0
     # Output data to C file
-    outf.write(b'const uint8_t ldelf_data[%d]' % round_up(load_size, 4096))
+    #outf.write(b'const uint8_t ldelf_data[%d]' % round_up(load_size, 4096))
+    outf.write('const uint8_t ldelf_data[{}]'.format(round_up(load_size, 4096)).encode())
     outf.write(b' __aligned(4096) = {\n')
     for segment in load_segments:
         data = segment.data()
@@ -112,9 +113,10 @@ def emit_load_segments(elffile, outf):
         n = n + 1
     outf.write(b'};\n')
 
-    outf.write(b'const unsigned int ldelf_code_size = %d;\n' % code_size)
-    outf.write(b'const unsigned int ldelf_data_size = %d;\n' % data_size)
-
+#    outf.write(b'const unsigned int ldelf_code_size = %d;\n' % code_size)
+#    outf.write(b'const unsigned int ldelf_data_size = %d;\n' % data_size)
+    outf.write('const unsigned int ldelf_code_size = {};\n'.format(code_size).encode())
+    outf.write('const unsigned int ldelf_data_size = {};\n'.format(data_size).encode())
 
 def get_args():
     parser = argparse.ArgumentParser()
@@ -141,8 +143,7 @@ def main():
     outf.write(b'#include <compiler.h>\n')
     outf.write(b'#include <stdint.h>\n')
     emit_load_segments(elffile, outf)
-    outf.write(b'const unsigned long ldelf_entry = %lu;\n' %
-               elffile.header['e_entry'])
+    outf.write('const unsigned long ldelf_entry = {};\n'.format(elffile.header['e_entry']).encode())
 
     inf.close()
     outf.close()
```
Hereby, I could release whole out/arm/core/tee.elf and you can export all symbol tables by IDA Pro and then it will be very helpful to analyze and reverse vendor specific optee os images. 

##
:::note info
optee client
:::

Reference
https://optee.readthedocs.io/en/latest/building/gits/optee_client.html

Toolchain
https://snapshots.linaro.org/gnu-toolchain/13.0-2022.11-1/arm-linux-gnueabihf/gcc-linaro-13.0.0-2022.11-x86_64_arm-linux-gnueabihf.tar.xz

Build PC
Ubuntu 18.04

```Step 1. download optee_client code
git clone https://github.com/OP-TEE/optee_client
```

```Step 2. install uuid-dev
sudo apt-get install uuid-dev
```

```Step 3. cmake and build
mkdir -p build
cd build
export TOOL_CHAIN=/home/chen/Android/gcc-linaro-13.0.0-2022.11-x86_64_arm-linux-gnueabihf/bin
export PATH=$TOOL_CHAIN:$PATH
cmake .. -DCMAKE_C_COMPILER=arm-linux-gnueabihf-gcc -DCMAKE_SYSTEM_NAME=Linux -DCMAKE_SYSTEM_PROCESSOR=arm
make  
```
```Step 4. build
chen@Build-BootMaker001:~/optee_client/optee_client-master/build$ ./build.sh
-- The C compiler identification is GNU 13.0.0
-- Detecting C compiler ABI info
-- Detecting C compiler ABI info - done
-- Check for working C compiler: /home/chen/Android/gcc-linaro-13.0.0-2022.11-x86_64_arm-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc - skipped
-- Detecting C compile features
-- Detecting C compile features - done
-- Looking for pthread.h
-- Looking for pthread.h - found
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD
-- Performing Test CMAKE_HAVE_LIBC_PTHREAD - Success
-- Found Threads: TRUE
-- Found PkgConfig: /usr/bin/pkg-config (found version "0.29.1")
-- Checking for module 'uuid'
--   Found uuid, version 2.31.1
-- Configuring done
-- Generating done
-- Build files have been written to: /home/chen/optee_client/optee_client-master/build
[  3%] Building C object libteec/CMakeFiles/teec.dir/src/tee_client_api.c.o
[  7%] Building C object libteec/CMakeFiles/teec.dir/src/teec_trace.c.o
[ 10%] Linking C static library libteec.a
[ 10%] Built target teec
[ 14%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/handle.c.o
[ 17%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/hmac_sha2.c.o
[ 21%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/rpmb.c.o
[ 25%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/sha2.c.o
[ 28%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/tee_supp_fs.c.o
[ 32%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/tee_supplicant.c.o
[ 35%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/teec_ta_load.c.o
[ 39%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/tee_socket.c.o
[ 42%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/prof.c.o
[ 46%] Building C object tee-supplicant/CMakeFiles/tee-supplicant.dir/src/plugin.c.o
[ 50%] Linking C executable tee-supplicant
[ 50%] Built target tee-supplicant
[ 53%] Building C object libckteec/CMakeFiles/ckteec.dir/src/pkcs11_api.c.o
[ 57%] Building C object libckteec/CMakeFiles/ckteec.dir/src/ck_debug.c.o
[ 60%] Building C object libckteec/CMakeFiles/ckteec.dir/src/ck_helpers.c.o
[ 64%] Building C object libckteec/CMakeFiles/ckteec.dir/src/invoke_ta.c.o
[ 67%] Building C object libckteec/CMakeFiles/ckteec.dir/src/pkcs11_processing.c.o
[ 71%] Building C object libckteec/CMakeFiles/ckteec.dir/src/pkcs11_token.c.o
[ 75%] Building C object libckteec/CMakeFiles/ckteec.dir/src/serializer.c.o
[ 78%] Building C object libckteec/CMakeFiles/ckteec.dir/src/serialize_ck.c.o
[ 82%] Linking C static library libckteec.a
[ 82%] Built target ckteec
[ 85%] Building C object libteeacl/CMakeFiles/teeacl.dir/src/group.c.o
[ 89%] Building C object libteeacl/CMakeFiles/teeacl.dir/src/tee_uuid.c.o
[ 92%] Linking C static library libteeacl.a
[ 92%] Built target teeacl
[ 96%] Building C object libseteec/CMakeFiles/seteec.dir/src/se.c.o
[100%] Linking C static library libseteec.a
[100%] Built target seteec
chen@Build-BootMaker001:~/optee_client/optee_client-master/build$ make
Consolidate compiler generated dependencies of target teec
[ 10%] Built target teec
Consolidate compiler generated dependencies of target tee-supplicant
[ 50%] Built target tee-supplicant
Consolidate compiler generated dependencies of target ckteec
[ 82%] Built target ckteec
Consolidate compiler generated dependencies of target teeacl
[ 92%] Built target teeacl
Consolidate compiler generated dependencies of target seteec
[100%] Built target seteec
chen@Build-BootMaker001:~/optee_client/optee_client-master/build$ make install
[ 10%] Built target teec
[ 50%] Built target tee-supplicant
[ 82%] Built target ckteec
[ 92%] Built target teeacl
[100%] Built target seteec
Install the project...
-- Install configuration: ""
-- Installing: /home/chen/optee_client/optee_client-master/build/output/lib/libteec.a
-- Installing: /home/chen/optee_client/optee_client-master/build/output/sbin/tee-supplicant
-- Set runtime path of "/home/chen/optee_client/optee_client-master/build/output/sbin/tee-supplicant" to "/usr/lib/tee-supplicant/plugins/"
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/tee_bench.h
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/tee_client_api.h
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/tee_client_api_extensions.h
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/tee_plugin_method.h
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/teec_trace.h
-- Installing: /home/chen/optee_client/optee_client-master/build/output/lib/libckteec.a
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/ck_debug.h
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/pkcs11.h
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/pkcs11_ta.h
-- Installing: /home/chen/optee_client/optee_client-master/build/output/lib/libteeacl.a
-- Up-to-date: /home/chen/optee_client/optee_client-master/build/output/include
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/teeacl.h
-- Installing: /home/chen/optee_client/optee_client-master/build/output/lib/libseteec.a
-- Installing: /home/chen/optee_client/optee_client-master/build/output/include/se_tee.h
```
Hereby, all OPTEE client API libraries are released. And OPTEE CA can be linked to libctee.a

##

:::note info
optee ta sample code (hello world ta)
:::

Premise: Need to first build optee_os and optee_client

```Step 1. download sample code
git clone https://github.com/linaro-swg/optee_examples.git
```

Modify Makefile
```diff
diff --git a/hello_world/Makefile b/hello_world/Makefile
index b188683..b32457b 100644
--- a/hello_world/Makefile
+++ b/hello_world/Makefile
@@ -1,15 +1,17 @@
 export V?=0
 
 # If _HOST or _TA specific compilers are not specified, then use CROSS_COMPILE
-HOST_CROSS_COMPILE ?= $(CROSS_COMPILE)
-TA_CROSS_COMPILE ?= $(CROSS_COMPILE)
+HOST_CROSS_COMPILE ?= arm-linux-gnueabihf-
+TA_CROSS_COMPILE ?= aarch64-linux-gnu-
+TEEC_LIB := /home/chen/optee/optee_client/build/output
+TA_DEV_KIT := /home/chen/optee/optee_os/out/arm/export-ta_arm64
 
 .PHONY: all
 all:
-       $(MAKE) -C host CROSS_COMPILE="$(HOST_CROSS_COMPILE)" --no-builtin-variables
-       $(MAKE) -C ta CROSS_COMPILE="$(TA_CROSS_COMPILE)" LDFLAGS=""
+       $(MAKE) -C host CROSS_COMPILE="$(HOST_CROSS_COMPILE)" TEEC_EXPORT="$(TEEC_LIB)" --no-builtin-variables
+       $(MAKE) -C ta CROSS_COMPILE="$(TA_CROSS_COMPILE)" LDFLAGS="" PLATFORM=vexpress-qemu_virt TA_DEV_KIT_DIR="$(TA_DEV_KIT)"
 
 .PHONY: clean
 clean:
        $(MAKE) -C host clean
-       $(MAKE) -C ta clean
+       $(MAKE) -C ta clean TA_DEV_KIT_DIR="$(TA_DEV_KIT)"
```

Setup toolchain and run make
```
TOOL_CHAIN=/home/chen/Android/gcc-linaro-13.0.0-2022.11-x86_64_arm-linux-gnueabihf/bin:/home/chen/Android/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu/bin
export PATH=$TOOL_CHAIN:$PATH
make
```

Build Log
```
make -C host CROSS_COMPILE="arm-linux-gnueabihf-" TEEC_EXPORT="/home/chen/optee/optee_client/build/output" --no-builtin-variables
make[1]: Entering directory '/home/chen/optee/optee_examples/hello_world/host'
arm-linux-gnueabihf-gcc -Wall -I../ta/include -I/home/chen/optee/optee_client/build/output/include -I./include -c main.c -o main.o
arm-linux-gnueabihf-gcc  -o optee_example_hello_world main.o -lteec -L/home/chen/optee/optee_client/build/output/lib
make[1]: Leaving directory '/home/chen/optee/optee_examples/hello_world/host'
make -C ta CROSS_COMPILE="aarch64-linux-gnu-" LDFLAGS="" PLATFORM=vexpress-qemu_virt TA_DEV_KIT_DIR="/home/chen/optee/optee_os/out/arm/export-ta_arm64"
make[1]: Entering directory '/home/chen/optee/optee_examples/hello_world/ta'
  CC      hello_world_ta.o
  CC      user_ta_header.o
  CPP     ta.lds
  GEN     dyn_list
  LD      8aaaf200-2450-11e4-abe2-0002a5d5c51b.elf
  OBJDUMP 8aaaf200-2450-11e4-abe2-0002a5d5c51b.dmp
  OBJCOPY 8aaaf200-2450-11e4-abe2-0002a5d5c51b.stripped.elf
  SIGN    8aaaf200-2450-11e4-abe2-0002a5d5c51b.ta
make[1]: Leaving directory '/home/chen/optee/optee_examples/hello_world/ta'
```

Hereby, a hello world OPTEE TA is fully released.