Repository: randombit/botan
Pull request #3077 (2022-11-24 09:36:55)

CI: Allow building out-of-source
This allows the CI scripts and configuration/build scripts to work for (a) out-of-source builds and (b) independent of the current working directory.

Some of the CI jobs are adapted to make use of this and make sure that out-of-source builds are working on all tested platforms. The directory layout in those jobs is as follows:

```
<base_dir> (default working-directory created by GitHub Actions)
-> source (checkout location of the botan sources)
-> build (created by ci_build.py and used for all build artifacts)
-> [boringssl] (checkout of BoringSSL -- when build 'coverage')
```

Note that the "current working directory" stays at `<base_dir>` for the entire script runtime. Therefore, some helper scripts needed to become aware of their own location to reach repository files by relative paths without relying on the "current working directory".

The Makefile generation needed several fixes and workarounds to be able to function. Mainly, normalizing paths on windows (e.g. `/` vs `\`). And various CI script locations assumed that build artifacts are built in-source (i.e. relative to `root_dir`).

Note that the windows jobs using out-of-source builds currently time out. This will work once #3072 is merged.

reneme: Rebased to `master` after merging #3072.
randombit: Something here seems to have broken the coverage report. Can you try rebasing again? If that doesn't fix it then some further investigation seems required.

Patch:
diff --git a/.github/actions/setup-build-agent/action.yml b/.github/actions/setup-build-agent/action.yml
index 16201a0f4..752896973 100644
--- a/.github/actions/setup-build-agent/action.yml
+++ b/.github/actions/setup-build-agent/action.yml
@@ -23,11 +23,11 @@ runs:
   using: composite
   steps:
     - name: Setup Build Agent (Windows)
-      run: ./src/scripts/ci/setup_gh_actions.ps1 "${{ inputs.target }}" "${{ inputs.arch }}"
+      run: ${{ github.action_path }}/../../../src/scripts/ci/setup_gh_actions.ps1 "${{ inputs.target }}" "${{ inputs.arch }}"
       shell: pwsh
       if: runner.os == 'Windows'
     - name: Setup Build Agent (Unix-like)
-      run: ./src/scripts/ci/setup_gh_actions.sh "${{ inputs.target }}" "${{ inputs.arch }}"
+      run: ${{ github.action_path }}/../../../src/scripts/ci/setup_gh_actions.sh "${{ inputs.target }}" "${{ inputs.arch }}"
       shell: bash
       if: runner.os != 'Windows'
 
@@ -49,6 +49,6 @@ runs:
       if: runner.os == 'Windows'
 
     - name: Install Build Dependencies # after setting up Visual Studio Environment
-      run: ./src/scripts/ci/setup_gh_actions_after_vcvars.ps1 ${{ inputs.target }}
+      run: ${{ github.action_path }}/../../../src/scripts/ci/setup_gh_actions_after_vcvars.ps1 ${{ inputs.target }}
       shell: pwsh
       if: runner.os == 'Windows'
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 508ca6e65..be4e0b840 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -168,15 +168,25 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+        with:
+          path: ./source
+
+      - name: Fetch BoringSSL fork for BoGo tests
+        uses: actions/checkout@v3
+        with:
+          repository: reneme/boringssl
+          ref: rene/runner-20220322 # TODO: merge changes to Botan's boring fork
+          path: ./boringssl
+        if: matrix.target == 'coverage'
 
       - name: Setup Build Agent
-        uses: ./.github/actions/setup-build-agent
+        uses: ./source/.github/actions/setup-build-agent
         with:
           target: ${{ matrix.target }}
           cache-key: ${{ matrix.host_os }}-${{ matrix.compiler }}-x86_64-${{ matrix.target }}
 
       - name: Build and Test Botan
-        run: python3 ./src/scripts/ci_build.py --cc='${{ matrix.compiler }}' --make-tool='${{ matrix.make_tool }}' --test-results-dir=junit_results ${{ matrix.target }}
+        run: python3 ./source/src/scripts/ci_build.py --root-dir=${{ github.workspace }}/source --build-dir=${{ github.workspace }}/build --boringssl-dir=${{ github.workspace }}/boringssl --cc='${{ matrix.compiler }}' --make-tool='${{ matrix.make_tool }}' --test-results-dir=junit_results ${{ matrix.target }}
 
   x-compile:
     name: "Cross-Compilation"
diff --git a/configure.py b/configure.py
index d04ad5e1f..291722163 100755
--- a/configure.py
+++ b/configure.py
@@ -50,10 +50,13 @@ def flatten(l):
 
 def normalize_source_path(source):
     """
-    cmake needs this, and nothing else minds
+    cmake and some versions of make need this, and nothing else minds
     """
     return os.path.normpath(source).replace('\\', '/')
 
+def normalize_source_paths(sources):
+    return [normalize_source_path(p) for p in sources]
+
 def parse_version_file(version_path):
     version_file = open(version_path, encoding='utf8')
     key_and_val = re.compile(r"([a-z_]+) = ([a-zA-Z0-9:\-\']+)")
@@ -218,7 +221,7 @@ class BuildPaths: # pylint: disable=too-many-instance-attributes
         self.external_headers = sorted(flatten([m.external_headers() for m in modules]))
 
         # this is overwritten if amalgamation is used
-        self.lib_sources = [normalize_source_path(s) for s in sorted(flatten([mod.sources() for mod in modules]))]
+        self.lib_sources = normalize_source_paths(sorted(flatten([mod.sources() for mod in modules])))
 
         self.public_headers = sorted(flatten([m.public_headers() for m in modules]))
 
@@ -234,9 +237,9 @@ class BuildPaths: # pylint: disable=too-many-instance-attributes
                     if filename.endswith('.h') and not filename.startswith('.'):
                         yield os.path.join(dirpath, filename)
 
-        self.cli_sources = [normalize_source_path(s) for s in find_sources_in(source_paths.src_dir, 'cli')]
-        self.cli_headers = [normalize_source_path(s) for s in find_headers_in(source_paths.src_dir, 'cli')]
-        self.test_sources = [normalize_source_path(s) for s in find_sources_in(source_paths.src_dir, 'tests')]
+        self.cli_sources = normalize_source_paths(find_sources_in(source_paths.src_dir, 'cli'))
+        self.cli_headers = normalize_source_paths(find_headers_in(source_paths.src_dir, 'cli'))
+        self.test_sources = normalize_source_paths(find_sources_in(source_paths.src_dir, 'tests'))
 
         if options.build_fuzzers:
             self.fuzzer_sources = list(find_sources_in(source_paths.src_dir, 'fuzzer'))
@@ -267,11 +270,11 @@ class BuildPaths: # pylint: disable=too-many-instance-attributes
     def format_include_paths(self, cc, external_includes):
         dash_i = cc.add_include_dir_option
         dash_isystem = cc.add_system_include_dir_option
-        output = dash_i + ' ' + self.include_dir
+        output = dash_i + ' ' + normalize_source_path(self.include_dir)
         if self.external_headers:
-            output += ' ' + dash_isystem + ' ' + self.external_include_dir
+            output += ' ' + dash_isystem + ' ' + normalize_source_path(self.external_include_dir)
         for external_include in external_includes:
-            output += ' ' + dash_isystem + ' ' + external_include
+            output += ' ' + dash_isystem + ' ' + normalize_source_path(external_include)
         return output
 
     def src_info(self, typ):
@@ -1851,7 +1854,7 @@ def yield_objectfile_list(sources, obj_dir, obj_suffix, options):
             name = filename
 
         name = name.replace('.cpp', obj_suffix)
-        yield os.path.join(obj_dir, name)
+        yield normalize_source_path(os.path.join(obj_dir, name))
 
 def generate_build_info(build_paths, modules, cc, arch, osinfo, options):
     # pylint: disable=too-many-locals
@@ -2012,10 +2015,16 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
     program_suffix = options.program_suffix or osinfo.program_suffix
 
     def join_with_build_dir(path):
-        # For some unknown reason MinGW doesn't like ./foo
-        if build_dir == os.path.curdir and options.os == 'mingw':
-            return path
-        return os.path.join(build_dir, path)
+        # jom (and mingw32-make) seem to string-compare Makefile targets and
+        # requirements. For them, `./botan.lib` is NOT equal to `botan.lib` or
+        # `C:\botan\botan-test.exe` is NOT equal to `C:\botan/botan-test.exe`
+        #
+        # `normalize_source_path` will "fix" the path slashes but remove
+        # a redundant `./` for the "trivial" relative path.
+        normalized = normalize_source_path(os.path.join(build_dir, path))
+        if build_dir == '.':
+            normalized = './%s' % normalized
+        return normalized
 
     def all_targets(options):
         yield 'libs'
@@ -2043,12 +2052,7 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
         return os.path.join(options.prefix or osinfo.install_root, p)
 
     def choose_python_exe():
-        exe = sys.executable
-
-        if options.os == 'mingw':  # mingw doesn't handle the backslashes in the absolute path well
-            return exe.replace('\\', '/')
-
-        return exe
+        return normalize_source_path(sys.executable)
 
     def choose_cxx_exe():
         cxx = options.compiler_binary or cc.binary_name
@@ -2129,11 +2133,11 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
         'with_doxygen': options.with_doxygen,
         'maintainer_mode': options.maintainer_mode,
 
-        'out_dir': build_dir,
-        'build_dir': build_paths.build_dir,
+        'out_dir': normalize_source_path(build_dir),
+        'build_dir': normalize_source_path(build_paths.build_dir),
         'module_info_dir': build_paths.doc_module_info,
 
-        'doc_stamp_file': os.path.join(build_paths.build_dir, 'doc.stamp'),
+        'doc_stamp_file': normalize_source_path(os.path.join(build_paths.build_dir, 'doc.stamp')),
         'makefile_path': os.path.join(build_paths.build_dir, '..', 'Makefile'),
 
         'build_static_lib': options.build_static_lib,
@@ -2293,7 +2297,7 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
     if options.os == 'llvm' or options.compiler == 'msvc':
         # llvm-link and msvc require just naming the file directly
         variables['build_dir_link_path'] = ''
-        variables['link_to_botan'] = os.path.join(build_dir, variables['static_lib_name'])
+        variables['link_to_botan'] = normalize_source_path(os.path.join(build_dir, variables['static_lib_name']))
     else:
         variables['build_dir_link_path'] = '%s%s' % (cc.add_lib_dir_option, build_dir)
         variables['link_to_botan'] = cc.add_lib_option % variables['libname']
diff --git a/src/build-data/makefile.in b/src/build-data/makefile.in
index 9b1c1ecd8..27fe3164a 100644
--- a/src/build-data/makefile.in
+++ b/src/build-data/makefile.in
@@ -91,7 +91,7 @@ fuzzer_corpus:
 	git clone --depth=1 https://github.com/randombit/crypto-corpus.git fuzzer_corpus
 
 fuzzer_corpus_zip: fuzzer_corpus
-	./src/scripts/create_corpus_zip.py fuzzer_corpus %{fuzzobj_dir}
+	%{base_dir}/src/scripts/create_corpus_zip.py fuzzer_corpus %{fuzzobj_dir}
 
 %{endif}
 
diff --git a/src/lib/math/numbertheory/primality.cpp b/src/lib/math/numbertheory/primality.cpp
index bbb028ab0..c776d356f 100644
--- a/src/lib/math/numbertheory/primality.cpp
+++ b/src/lib/math/numbertheory/primality.cpp
@@ -88,9 +88,7 @@ bool is_lucas_probable_prime(const BigInt& C, const Modular_Reducer& mod_C)
 
 bool is_bailie_psw_probable_prime(const BigInt& n, const Modular_Reducer& mod_n)
    {
-   if(n == 2)
-      return true;
-   else if(n <= 1 || n.is_even())
+   if(n < 3 || n.is_even())
       return false;
 
    auto monty_n = std::make_shared<Montgomery_Params>(n, mod_n);
diff --git a/src/scripts/ci/setup_gh_actions.sh b/src/scripts/ci/setup_gh_actions.sh
index b1d71a2c9..1afa760b4 100755
--- a/src/scripts/ci/setup_gh_actions.sh
+++ b/src/scripts/ci/setup_gh_actions.sh
@@ -14,6 +14,8 @@ set -ex
 TARGET="$1"
 ARCH="$2"
 
+SCRIPT_LOCATION=$(cd "$(dirname "$0")"; pwd)
+
 if type -p "apt-get"; then
     sudo apt-get -qq update
     sudo apt-get -qq install ccache
@@ -52,8 +54,8 @@ if type -p "apt-get"; then
     elif [ "$TARGET" = "baremetal" ]; then
         sudo apt-get -qq install gcc-arm-none-eabi libstdc++-arm-none-eabi-newlib
 
-        echo 'extern "C" void __sync_synchronize() {}' >> src/tests/main.cpp
-        echo 'extern "C" void __sync_synchronize() {}' >> src/cli/main.cpp
+        echo 'extern "C" void __sync_synchronize() {}' >> "${SCRIPT_LOCATION}/../../tests/main.cpp"
+        echo 'extern "C" void __sync_synchronize() {}' >> "${SCRIPT_LOCATION}/../../cli/main.cpp"
 
     elif [ "$TARGET" = "lint" ]; then
         sudo apt-get -qq install pylint
@@ -63,9 +65,6 @@ if type -p "apt-get"; then
         pip install --user codecov
         echo "$HOME/.local/bin" >> "$GITHUB_PATH"
 
-        # TODO: merge changes to Botan's boring fork
-        git clone --depth 1 --branch rene/runner-20220322 https://github.com/reneme/boringssl.git
-
         sudo chgrp -R "$(id -g)" /var/lib/softhsm/ /etc/softhsm
         sudo chmod g+w /var/lib/softhsm/tokens
 
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
index b7e835a77..4d78ce967 100755
--- a/src/scripts/ci_build.py
+++ b/src/scripts/ci_build.py
@@ -76,9 +76,9 @@ def build_targets(target, target_os):
     if target in ['coverage']:
         yield 'bogo_shim'
 
-def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
-                    ccache, root_dir, test_results_dir, pkcs11_lib, use_gdb, disable_werror, extra_cxxflags,
-                    disabled_tests):
+def determine_flags(target, target_os, target_cpu, target_cc, cc_bin, ccache,
+                    root_dir, build_dir, test_results_dir, pkcs11_lib, use_gdb,
+                    disable_werror, extra_cxxflags, disabled_tests):
     # pylint: disable=too-many-branches,too-many-statements,too-many-arguments,too-many-locals
 
     """
@@ -109,7 +109,7 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
 
     make_prefix = []
     test_prefix = []
-    test_cmd = [os.path.join(root_dir, 'botan-test')]
+    test_cmd = [os.path.join(build_dir, 'botan-test'), '--data-dir=%s' % os.path.join(root_dir, 'src', 'tests', 'data')]
 
     # generate JUnit test report
     if test_results_dir:
@@ -130,7 +130,8 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
     flags = ['--prefix=%s' % (install_prefix),
              '--cc=%s' % (target_cc),
              '--os=%s' % (target_os),
-             '--build-targets=%s' % ','.join(build_targets(target, target_os))]
+             '--build-targets=%s' % ','.join(build_targets(target, target_os)),
+             '--with-build-dir=%s' % build_dir]
 
     if ccache is not None:
         flags += ['--no-store-vc-rev', '--compiler-cache=%s' % (ccache)]
@@ -351,7 +352,7 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
 
     return flags, run_test_command, make_prefix, install_prefix
 
-def run_cmd(cmd, root_dir):
+def run_cmd(cmd, root_dir, build_dir):
     """
     Execute a command, die if it failed
     """
@@ -362,8 +363,8 @@ def run_cmd(cmd, root_dir):
 
     cmd = [os.path.expandvars(elem) for elem in cmd]
     sub_env = os.environ.copy()
-    sub_env['LD_LIBRARY_PATH'] = os.path.abspath(root_dir)
-    sub_env['DYLD_LIBRARY_PATH'] = os.path.abspath(root_dir)
+    sub_env['LD_LIBRARY_PATH'] = os.path.abspath(build_dir)
+    sub_env['DYLD_LIBRARY_PATH'] = os.path.abspath(build_dir)
     sub_env['PYTHONPATH'] = os.path.abspath(os.path.join(root_dir, 'src/python'))
     cwd = None
 
@@ -415,6 +416,10 @@ def parse_args(args):
                       help='Set path to compiler')
     parser.add_option('--root-dir', metavar='D', default='.',
                       help='Set directory to execute from (default %default)')
+    parser.add_option('--build-dir', metavar='D', default='.',
+                      help='Set directory to place build artifacts into (default %default)')
+    parser.add_option('--boringssl-dir', metavar='D', default='boringssl',
+                      help='Set directory of BoringSSL checkout to use for BoGo tests')
 
     parser.add_option('--make-tool', metavar='TOOL', default='make',
                       help='Specify tool to run to build source (default %default)')
@@ -511,9 +516,14 @@ def main(args=None):
         raise Exception("Don't know about %s as a compiler cache" % (options.compiler_cache))
 
     root_dir = options.root_dir
+    build_dir = options.build_dir
 
     if not os.access(root_dir, os.R_OK):
         raise Exception('Bad root dir setting, dir %s not readable' % (root_dir))
+    if not os.path.exists(build_dir):
+        os.makedirs(build_dir)
+    elif not os.path.isdir(build_dir) or not os.access(build_dir, os.R_OK | os.W_OK):
+        raise Exception("Bad build dir setting %s is not a directory or not accessible" % (build_dir))
 
     cmds = []
 
@@ -552,8 +562,8 @@ def main(args=None):
             os.makedirs(options.test_results_dir)
 
         config_flags, run_test_command, make_prefix, install_prefix = determine_flags(
-            target, options.os, options.cpu, options.cc,
-            options.cc_bin, options.compiler_cache, root_dir, options.test_results_dir,
+            target, options.os, options.cpu, options.cc, options.cc_bin,
+            options.compiler_cache, root_dir, build_dir, options.test_results_dir,
             options.pkcs11_lib, options.use_gdb, options.disable_werror,
             options.extra_cxxflags, options.disabled_tests)
 
@@ -563,8 +573,8 @@ def main(args=None):
             options.make_tool = 'make'
 
         make_cmd = [options.make_tool]
-        if root_dir != '.':
-            make_cmd += ['-C', root_dir]
+        if build_dir != '.':
+            make_cmd = ['indir:%s' % build_dir] + make_cmd
         if options.build_jobs > 1 and options.make_tool != 'nmake':
             make_cmd += ['-j%d' % (options.build_jobs)]
 
@@ -593,38 +603,46 @@ def main(args=None):
             cmds.append(run_test_command)
 
         if target == 'coverage':
-            runner_dir = os.path.abspath(os.path.join(root_dir, 'boringssl', 'ssl', 'test', 'runner'))
+            if not options.boringssl_dir:
+                raise Exception('coverage build needs --boringssl-dir')
+
+            runner_dir = os.path.abspath(os.path.join(options.boringssl_dir, 'ssl', 'test', 'runner'))
 
             cmds.append(['indir:%s' % (runner_dir),
                          'go', 'test', '-pipe',
                          '-num-workers', str(4*get_concurrency()),
-                         '-shim-path', os.path.abspath(os.path.join(root_dir, 'botan_bogo_shim')),
+                         '-shim-path', os.path.abspath(os.path.join(build_dir, 'botan_bogo_shim')),
                          '-shim-config', os.path.abspath(os.path.join(root_dir, 'src', 'bogo_shim', 'config.json'))])
 
         if target in ['coverage', 'fuzzers']:
             cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/test_fuzzers.py'),
-                         os.path.join(root_dir, 'fuzzer_corpus'),
-                         os.path.join(root_dir, 'build/fuzzer')])
+                         os.path.join(build_dir, 'fuzzer_corpus'),
+                         os.path.join(build_dir, 'build/fuzzer')])
 
         if target in ['shared', 'coverage'] and options.os != 'windows':
-            botan_exe = os.path.join(root_dir, 'botan-cli.exe' if options.os == 'windows' else 'botan')
+            botan_exe = os.path.join(build_dir, 'botan-cli.exe' if options.os == 'windows' else 'botan')
 
             args = ['--threads=%d' % (options.build_jobs)]
             if target == 'coverage':
                 args.append('--run-slow-tests')
+            if root_dir != '.':
+                args.append('--test-data-dir=%s' % root_dir)
             test_scripts = ['test_cli.py', 'test_cli_crypt.py']
             for script in test_scripts:
+                test_data_arg = []
                 cmds.append([py_interp, os.path.join(root_dir, 'src/scripts', script)] +
-                            args + [botan_exe])
+                            args + test_data_arg + [botan_exe])
 
-        python_tests = os.path.join(root_dir, 'src/scripts/test_python.py')
+        python_tests = [os.path.join(root_dir, 'src/scripts/test_python.py')]
+        if root_dir != '.':
+            python_tests.append('--test-data-dir=%s' % root_dir)
 
         if target in ['shared', 'coverage']:
-            cmds.append([py_interp, '-b', python_tests])
+            cmds.append([py_interp, '-b'] + python_tests)
 
         if target in ['shared', 'static', 'bsi', 'nist']:
             cmds.append(make_cmd + ['install'])
-            build_config = os.path.join(root_dir, 'build', 'build_config.json')
+            build_config = os.path.join(build_dir, 'build', 'build_config.json')
             cmds.append([py_interp, os.path.join(root_dir, 'src/scripts/ci_check_install.py'), build_config])
 
         if target in ['examples']:
@@ -642,25 +660,25 @@ def main(args=None):
                 print('Error: gcov not found in PATH (%s)' % (os.getenv('PATH')))
                 return 1
 
-            cov_file = 'coverage.info'
-            raw_cov_file = 'coverage.info.raw'
+            cov_file = os.path.join(build_dir, 'coverage.info')
+            raw_cov_file = os.path.join(build_dir, 'coverage.info.raw')
 
-            cmds.append(['lcov', '--capture', '--directory', options.root_dir,
+            cmds.append(['lcov', '--capture', '--directory', build_dir,
                          '--output-file', raw_cov_file])
             cmds.append(['lcov', '--remove', raw_cov_file, '/usr/*', '--output-file', cov_file])
             cmds.append(['lcov', '--list', cov_file])
 
             if have_prog('coverage'):
                 cmds.append(['coverage', 'run', '--branch',
-                             '--rcfile', os.path.join(root_dir, 'src/configs/coverage.rc'),
-                             python_tests])
+                             '--rcfile', os.path.join(root_dir, 'src/configs/coverage.rc')] +
+                            python_tests)
 
             if have_prog('codecov'):
                 # If codecov exists assume we are in CI and report to codecov.io
-                cmds.append(['codecov', '>', 'codecov_stdout.log'])
+                cmds.append(['indir:%s' % root_dir, 'codecov', '--required', '--gcov-root', build_dir, '>', os.path.join(build_dir, 'codecov_stdout.log')])
             else:
                 # Otherwise generate a local HTML report
-                cmds.append(['genhtml', cov_file, '--output-directory', 'lcov-out'])
+                cmds.append(['genhtml', cov_file, '--output-directory', os.path.join(build_dir, 'lcov-out')])
 
         cmds.append(make_cmd + ['clean'])
         cmds.append(make_cmd + ['distclean'])
@@ -669,7 +687,7 @@ def main(args=None):
         if options.dry_run:
             print('$ ' + ' '.join(cmd))
         else:
-            run_cmd(cmd, root_dir)
+            run_cmd(cmd, root_dir, build_dir)
 
     return 0
 


Test patch:
diff --git a/src/scripts/test_cli.py b/src/scripts/test_cli.py
index ee3b4bb79..141a91bef 100755
--- a/src/scripts/test_cli.py
+++ b/src/scripts/test_cli.py
@@ -31,6 +31,7 @@ from http.server import HTTPServer, BaseHTTPRequestHandler
 # pylint: disable=global-statement,unused-argument
 
 CLI_PATH = None
+TEST_DATA_DIR = '.'
 TESTS_RUN = 0
 TESTS_FAILED = 0
 
@@ -807,7 +808,7 @@ def cli_timing_test_tests(_tmp_dir):
     output_re = re.compile('[0-9]+;[0-9];[0-9]+')
 
     for suite in timing_tests:
-        output = test_cli("timing_test", [suite, "--measurement-runs=16", "--warmup-runs=3"], None).split('\n')
+        output = test_cli("timing_test", [suite, "--measurement-runs=16", "--warmup-runs=3", "--test-data-dir=%s" % TEST_DATA_DIR], None).split('\n')
 
         for line in output:
             if output_re.match(line) is None:
@@ -1336,6 +1337,7 @@ def main(args=None):
     parser.add_option('--quiet', action='store_true', default=False)
     parser.add_option('--threads', action='store', type='int', default=0)
     parser.add_option('--run-slow-tests', action='store_true', default=False)
+    parser.add_option('--test-data-dir', default='.')
 
     (options, args) = parser.parse_args(args)
 
@@ -1356,6 +1358,9 @@ def main(args=None):
     global CLI_PATH
     CLI_PATH = args[1]
 
+    global TEST_DATA_DIR
+    TEST_DATA_DIR = os.path.join(options.test_data_dir, 'src/tests/data/timing/')
+
     test_regex = None
     if len(args) == 3:
         try:
diff --git a/src/scripts/test_cli_crypt.py b/src/scripts/test_cli_crypt.py
index c12ce9a90..20f0642b3 100755
--- a/src/scripts/test_cli_crypt.py
+++ b/src/scripts/test_cli_crypt.py
@@ -165,6 +165,7 @@ def main(args=None):
     parser.add_argument('--verbose', action='store_true', default=False)
     parser.add_argument('--quiet', action='store_true', default=False)
     parser.add_argument('--run-slow-tests', action='store_true', default=False)
+    parser.add_argument('--test-data-dir', default='.')
     args = parser.parse_args()
 
     setup_logging(args)
@@ -176,7 +177,7 @@ def main(args=None):
     if threads == 0:
         threads = multiprocessing.cpu_count()
 
-    test_data_dir = os.path.join('src', 'tests', 'data')
+    test_data_dir = os.path.join(args.test_data_dir, 'src', 'tests', 'data')
 
     mode_test_data = [
         os.path.join(test_data_dir, 'aead', 'ccm.vec'),
diff --git a/src/scripts/test_python.py b/src/scripts/test_python.py
index 776ddb338..309dbcd8f 100644
--- a/src/scripts/test_python.py
+++ b/src/scripts/test_python.py
@@ -10,6 +10,8 @@ import unittest
 import binascii
 import os
 import platform
+import argparse
+import sys
 
 # Starting with Python 3.8 DLL search locations are more restricted on Windows.
 # Hence, we need to explicitly add the current working directory before trying
@@ -26,6 +28,13 @@ def hex_encode(buf):
 def hex_decode(buf):
     return binascii.unhexlify(buf.encode('ascii'))
 
+# pylint: disable=global-statement
+
+TEST_DATA_DIR = '.'
+
+def test_data(relpath):
+    return os.path.join(TEST_DATA_DIR, relpath)
+
 class BotanPythonTests(unittest.TestCase):
     # pylint: disable=too-many-public-methods,too-many-locals
 
@@ -450,7 +459,7 @@ ofvkP1EDmpx50fHLawIDAQAB
 
     def test_certs(self):
         # pylint: disable=too-many-statements
-        cert = botan2.X509Cert(filename="src/tests/data/x509/ecc/CSCA.CSCA.csca-germany.1.crt")
+        cert = botan2.X509Cert(filename=test_data("src/tests/data/x509/ecc/CSCA.CSCA.csca-germany.1.crt"))
         pubkey = cert.subject_public_key()
 
         self.assertEqual(pubkey.algo_name(), 'ECDSA')
@@ -487,16 +496,16 @@ ofvkP1EDmpx50fHLawIDAQAB
         self.assertFalse(cert.allowed_usage(["DIGITAL_SIGNATURE"]))
         self.assertFalse(cert.allowed_usage(["DIGITAL_SIGNATURE", "CRL_SIGN"]))
 
-        root = botan2.X509Cert("src/tests/data/x509/nist/root.crt")
+        root = botan2.X509Cert(test_data("src/tests/data/x509/nist/root.crt"))
 
-        int09 = botan2.X509Cert("src/tests/data/x509/nist/test09/int.crt")
-        end09 = botan2.X509Cert("src/tests/data/x509/nist/test09/end.crt")
+        int09 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test09/int.crt"))
+        end09 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test09/end.crt"))
         self.assertEqual(end09.verify([int09], [root]), 2001)
 
-        end04 = botan2.X509Cert("src/tests/data/x509/nist/test04/end.crt")
-        int04_1 = botan2.X509Cert("src/tests/data/x509/nist/test04/int1.crt")
-        int04_2 = botan2.X509Cert("src/tests/data/x509/nist/test04/int2.crt")
-        self.assertEqual(end04.verify([int04_1, int04_2], [], "src/tests/data/x509/nist/", required_strength=80), 0)
+        end04 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test04/end.crt"))
+        int04_1 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test04/int1.crt"))
+        int04_2 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test04/int2.crt"))
+        self.assertEqual(end04.verify([int04_1, int04_2], [], test_data("src/tests/data/x509/nist/"), required_strength=80), 0)
         self.assertEqual(end04.verify([int04_1, int04_2], [], required_strength=80), 3000)
         self.assertEqual(end04.verify([int04_1, int04_2], [root], required_strength=80, hostname="User1-CP.02.01"), 0)
         self.assertEqual(end04.verify([int04_1, int04_2], [root], required_strength=80, hostname="invalid"), 4008)
@@ -506,21 +515,21 @@ ofvkP1EDmpx50fHLawIDAQAB
         self.assertEqual(botan2.X509Cert.validation_status(3000), 'Certificate issuer not found')
         self.assertEqual(botan2.X509Cert.validation_status(4008), 'Certificate does not match provided name')
 
-        rootcrl = botan2.X509CRL("src/tests/data/x509/nist/root.crl")
+        rootcrl = botan2.X509CRL(test_data("src/tests/data/x509/nist/root.crl"))
 
-        end01 = botan2.X509Cert("src/tests/data/x509/nist/test01/end.crt")
+        end01 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test01/end.crt"))
         self.assertEqual(end01.verify([], [root], required_strength=80, crls=[rootcrl]), 0)
 
-        int20 = botan2.X509Cert("src/tests/data/x509/nist/test20/int.crt")
-        end20 = botan2.X509Cert("src/tests/data/x509/nist/test20/end.crt")
-        int20crl = botan2.X509CRL("src/tests/data/x509/nist/test20/int.crl")
+        int20 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test20/int.crt"))
+        end20 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test20/end.crt"))
+        int20crl = botan2.X509CRL(test_data("src/tests/data/x509/nist/test20/int.crl"))
 
         self.assertEqual(end20.verify([int20], [root], required_strength=80, crls=[int20crl, rootcrl]), 5000)
         self.assertEqual(botan2.X509Cert.validation_status(5000), 'Certificate is revoked')
 
-        int21 = botan2.X509Cert("src/tests/data/x509/nist/test21/int.crt")
-        end21 = botan2.X509Cert("src/tests/data/x509/nist/test21/end.crt")
-        int21crl = botan2.X509CRL("src/tests/data/x509/nist/test21/int.crl")
+        int21 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test21/int.crt"))
+        end21 = botan2.X509Cert(test_data("src/tests/data/x509/nist/test21/end.crt"))
+        int21crl = botan2.X509CRL(test_data("src/tests/data/x509/nist/test21/int.crl"))
         self.assertEqual(end21.verify([int21], [root], required_strength=80, crls=[int21crl, rootcrl]), 5000)
 
         self.assertTrue(int20.is_revoked(rootcrl))
@@ -733,5 +742,17 @@ ofvkP1EDmpx50fHLawIDAQAB
         except botan2.BotanException as e:
             self.assertEqual(str(e), "botan_srp6_server_session_step2 failed: -32 (Bad parameter)")
 
-if __name__ == '__main__':
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--test-data-dir', default='.')
+    parser.add_argument('unittest_args', nargs='*')
+
+    args = parser.parse_args()
+    global TEST_DATA_DIR
+    TEST_DATA_DIR = args.test_data_dir
+
+    sys.argv[1:] = args.unittest_args
     unittest.main()
+
+if __name__ == '__main__':
+    main()