Repository: randombit/botan
Pull request #3011 (2022-07-20 16:49:54)

CI: Improvements and preparation for Windows on GitHub Actions
Changes in #3007 that prepare the CI setup for the integration of the new Windows build configs.
This contains #3010, i.e. probably #3010 should be reviewed/merged first.

* auto-detect `sccache` in `ci_build.py`
* automatically cancel builds on consecutive pushes to pull requests
* compiler cache directory is dynamically detected in `setup_gh_actions.sh`
* build agent setup is pulled into a custom action
* build configurations are sorted into several job matrices


Patch:
diff --git a/.github/actions/setup-build-agent/action.yml b/.github/actions/setup-build-agent/action.yml
new file mode 100644
index 000000000..4f0a8a082
--- /dev/null
+++ b/.github/actions/setup-build-agent/action.yml
@@ -0,0 +1,38 @@
+
+# (C) 2022 Jack Lloyd
+# (C) 2022 René Meusel, Rohde & Schwarz Cybersecurity
+#
+# Botan is released under the Simplified BSD License (see license.txt)
+
+name: Setup Botan Build Agent
+description: Set up a build agent for building and testing the Botan library
+
+inputs:
+  target:
+    description: The ci_build.py target going to be built on this agent
+    required: true
+  cache-key:
+    description: The actions/cache key to be used for this runs, caching will be disabled when no key is provided
+    required: false
+  arch:
+    description: Target CPU architecture
+    required: false
+    default: x64
+
+runs:
+  using: composite
+  steps:
+    - name: Setup Build Agent (Unix-like)
+      run: ./src/scripts/ci/setup_gh_actions.sh ${{ inputs.target }}
+      shell: bash
+
+    - name: Check Availability of Compiler Cache
+      run: print("::warning ::No compiler cache available, build times might suffer")
+      shell: python
+      if: env.COMPILER_CACHE_LOCATION == '' && inputs.cache-key != ''
+    - uses: actions/cache@v2
+      if: env.COMPILER_CACHE_LOCATION != '' && inputs.cache-key != ''
+      with:
+          path: ${{ env.COMPILER_CACHE_LOCATION }}
+          key: ${{ inputs.cache-key }}-${{ github.run_id }}
+          restore-keys: ${{ inputs.cache-key }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e57a4acc8..753656c91 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,5 +1,7 @@
 
-# (C) 2020 Jack Lloyd
+# (C) 2020,2022 Jack Lloyd
+# (C) 2022      René Meusel, Rohde & Schwarz Cybersecurity
+#
 # Botan is released under the Simplified BSD License (see license.txt)
 
 name: ci
@@ -14,13 +16,78 @@ on:
   pull_request:
     branches: [ master ]
 
+# cancel running workflows when new commits are being pushed in pull requests
+# but not on the master branch
+concurrency:
+  group: ${{ github.workflow }} @ ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
 jobs:
-  ci:
-    runs-on: ${{ matrix.host_os }}
+  linux:
+    name: "Linux (Basic)"
+    strategy:
+      fail-fast: false
 
-    env:
-      ANDROID_NDK: android-ndk-r21d
+      matrix:
+        # basic configuration combinations should run on all target platforms
+        target:   [ static, shared, amalgamation ]
+        compiler: [ clang, gcc ]
+
+        exclude:
+          # fails due to "abstract class marked final"
+          # see TODOs here: https://github.com/randombit/botan/pull/3007
+          - compiler: clang
+            target: amalgamation
 
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Build Agent
+        uses: ./.github/actions/setup-build-agent
+        with:
+          target: ${{ matrix.target }}
+          cache-key: linux-${{ matrix.compiler }}-x86_64-${{ matrix.target }}
+
+      - name: Build and Test Botan
+        run: python3 ./src/scripts/ci_build.py --cc='${{ matrix.compiler }}' ${{ matrix.target }}
+
+  macos:
+    name: "macOS (Basic)"
+    strategy:
+      fail-fast: false
+
+      matrix:
+        # basic configuration combinations should run on all target platforms
+        target:   [ static, shared, amalgamation ]
+        compiler: [ clang, gcc ]
+
+        exclude:
+          # fails due to "abstract class marked final"
+          # see TODOs here: https://github.com/randombit/botan/pull/3007
+          - compiler: clang
+            target: amalgamation
+          # currently failing
+          # see TODOs here: https://github.com/randombit/botan/pull/3007
+          - compiler: gcc
+
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Build Agent
+        uses: ./.github/actions/setup-build-agent
+        with:
+          target: ${{ matrix.target }}
+          cache-key: macos-${{ matrix.compiler }}-x86_64-${{ matrix.target }}
+
+      - name: Build and Test Botan
+        run: python3 ./src/scripts/ci_build.py --cc='${{ matrix.compiler }}' ${{ matrix.target }}
+
+  specials:
+    name: "Metrics and Specials"
     strategy:
       fail-fast: false
 
@@ -28,71 +95,85 @@ jobs:
         include:
           - target: coverage
             compiler: gcc
-            host_os: ubuntu-20.04
-          - target: shared
-            compiler: gcc
-            host_os: ubuntu-20.04
-          - target: shared
-            compiler: clang
-            host_os: ubuntu-20.04
+            host_os: ubuntu-latest
           - target: valgrind
             compiler: gcc
-            host_os: ubuntu-20.04
+            host_os: ubuntu-latest
           - target: fuzzers
             compiler: gcc
-            host_os: ubuntu-20.04
-          - target: cross-i386
-            compiler: gcc
-            host_os: ubuntu-20.04
-          - target: cross-arm64
-            compiler: gcc
-            host_os: ubuntu-20.04
-          - target: cross-ppc64
-            compiler: gcc
-            host_os: ubuntu-20.04
-          - target: cross-android-arm32
-            compiler: clang
-            host_os: ubuntu-20.04
-          - target: cross-android-arm64
-            compiler: clang
-            host_os: ubuntu-20.04
-          - target: cross-win64
-            compiler: gcc
-            host_os: ubuntu-20.04
-          - target: amalgamation
-            compiler: gcc
-            host_os: ubuntu-20.04
+            host_os: ubuntu-latest
           - target: emscripten
             compiler: emcc
             host_os: macos-latest
           - target: baremetal
             compiler: gcc
-            host_os: ubuntu-20.04
+            host_os: ubuntu-latest
           - target: minimized
             compiler: gcc
-            host_os: ubuntu-20.04
+            host_os: ubuntu-latest
           - target: bsi
             compiler: gcc
-            host_os: ubuntu-20.04
+            host_os: ubuntu-latest
           - target: lint
             compiler: gcc
-            host_os: ubuntu-20.04
-          - target: shared
+            host_os: ubuntu-latest
+
+    runs-on: ${{ matrix.host_os }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Build Agent
+        uses: ./.github/actions/setup-build-agent
+        with:
+          target: ${{ matrix.target }}
+          cache-key: ${{ matrix.host_os }}-${{ matrix.compiler }}-x86_64-${{ matrix.target }}
+
+      - name: Build and Test Botan
+        run: python3 ./src/scripts/ci_build.py --cc='${{ matrix.compiler }}' --make-tool='${{ matrix.make_tool }}' ${{ matrix.target }}
+
+  x-compile:
+    name: "Cross-Compilation"
+    strategy:
+      fail-fast: false
+
+      matrix:
+        include:
+          - target: cross-i386
+            compiler: gcc
+            host_os: ubuntu-latest
+          - target: cross-arm64
+            compiler: gcc
+            host_os: ubuntu-latest
+          - target: cross-ppc64
+            compiler: gcc
+            host_os: ubuntu-latest
+          - target: cross-android-arm32
             compiler: clang
-            host_os: macos-latest
+            host_os: ubuntu-latest
+          - target: cross-android-arm64
+            compiler: clang
+            host_os: ubuntu-latest
+          - target: cross-win64
+            compiler: gcc
+            host_os: ubuntu-latest
           - target: cross-ios-arm64
             compiler: clang
             host_os: macos-latest
 
+    runs-on: ${{ matrix.host_os }}
+
+    env:
+      ANDROID_NDK: android-ndk-r21d
+
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/cache@v2
+      - uses: actions/checkout@v3
+
+      - name: Setup Build Agent
+        uses: ./.github/actions/setup-build-agent
         with:
-            path: |
-              ~/.ccache
-              /Users/runner/Library/Caches/ccache
-            key: ${{ runner.os }}-${{ matrix.compiler }}-${{ matrix.target }}-${{ github.run_id }}
-            restore-keys: |
-               ${{ runner.os }}-${{ matrix.compiler }}-${{ matrix.target }}-
-      - run: ./src/scripts/ci/setup_gh_actions.sh ${{ matrix.target }}
-      - run: ./src/scripts/ci_build.py --cc '${{ matrix.compiler }}' ${{ matrix.target }}
+          target: ${{ matrix.target }}
+          cache-key: ${{ matrix.host_os }}-${{ matrix.compiler }}-xcompile-${{ matrix.target }}
+
+      - name: Build and Test Botan
+        run: python3 ./src/scripts/ci_build.py --cc='${{ matrix.compiler }}' ${{ matrix.target }}
diff --git a/configure.py b/configure.py
index beed9bbc6..fc9643210 100755
--- a/configure.py
+++ b/configure.py
@@ -267,11 +267,12 @@ class BuildPaths(object): # pylint: disable=too-many-instance-attributes
 
     def format_include_paths(self, cc, external_includes):
         dash_i = cc.add_include_dir_option
-        output = dash_i + self.include_dir
+        dash_isystem = cc.add_system_include_dir_option
+        output = dash_i + ' ' + self.include_dir
         if self.external_headers:
-            output += ' ' + dash_i + self.external_include_dir
+            output += ' ' + dash_isystem + ' ' + self.external_include_dir
         for external_include in external_includes:
-            output += ' ' + dash_i + external_include
+            output += ' ' + dash_isystem + ' ' + external_include
         return output
 
     def src_info(self, typ):
@@ -509,8 +510,6 @@ def process_command_line(args): # pylint: disable=too-many-locals,too-many-state
                            help=optparse.SUPPRESS_HELP)
     build_group.add_option('--without-pkg-config', dest='with_pkg_config', action='store_false',
                            help=optparse.SUPPRESS_HELP)
-    build_group.add_option('--boost-library-name', dest='boost_libnames', default=[],
-                           help="file name of some boost library to link", action='append')
 
     docs_group = optparse.OptionGroup(parser, 'Documentation Options')
 
@@ -1151,6 +1150,7 @@ class CompilerInfo(InfoObject): # pylint: disable=too-many-instance-attributes
                 'output_to_object': '-o ',
                 'output_to_exe': '-o ',
                 'add_include_dir_option': '-I',
+                'add_system_include_dir_option': '-I',
                 'add_lib_dir_option': '-L',
                 'add_compile_definition_option': '-D',
                 'add_sysroot_option': '',
@@ -1179,6 +1179,7 @@ class CompilerInfo(InfoObject): # pylint: disable=too-many-instance-attributes
 
         self.add_framework_option = lex.add_framework_option
         self.add_include_dir_option = lex.add_include_dir_option
+        self.add_system_include_dir_option = lex.add_system_include_dir_option
         self.add_lib_dir_option = lex.add_lib_dir_option
         self.add_lib_option = lex.add_lib_option
         self.add_compile_definition_option = lex.add_compile_definition_option
@@ -1880,33 +1881,6 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
     def external_link_cmd():
         return ' '.join([cc.add_lib_dir_option + libdir for libdir in options.with_external_libdir])
 
-    def adjust_library_name(info_txt_libname):
-        """
-        Apply custom library name mappings where necessary
-        """
-
-        # potentially map boost library names to the associated name provided
-        # via ./configure.py --boost-library-name <build/platform specific name>
-        #
-        # We assume that info.txt contains the library name's "stem", i.e.
-        # 'boost_system'. While the user-provided (actual) library will contain
-        # the same stem plus a set of prefixes and/or suffixes, e.g.
-        # libboost_system-vc140-mt-x64-1_69.lib. We use the stem for selecting
-        # the correct user-provided library name override.
-        if options.boost_libnames and 'boost_' in info_txt_libname:
-            adjusted_libnames = [chosen_libname for chosen_libname in options.boost_libnames \
-                                 if info_txt_libname in chosen_libname]
-
-            if len(adjusted_libnames) > 1:
-                logging.warning('Ambiguous boost library names: %s',
-                                ', '.join(adjusted_libnames))
-            if len(adjusted_libnames) == 1:
-                logging.debug('Replacing boost library name %s -> %s',
-                              info_txt_libname, adjusted_libnames[0])
-                return adjusted_libnames[0]
-
-        return info_txt_libname
-
     def link_to(module_member_name):
         """
         Figure out what external libraries/frameworks are needed based on selected modules
@@ -1926,7 +1900,7 @@ def create_template_vars(source_paths, build_paths, options, modules, cc, arch,
                         if osinfo.basename not in exceptions:
                             libs |= set(module_link_to)
 
-        return sorted([adjust_library_name(lib) for lib in libs])
+        return sorted(libs)
 
     def choose_mp_bits():
         mp_bits = arch.wordsize # allow command line override?
diff --git a/doc/building.rst b/doc/building.rst
index 9b74a3784..8b4fcf641 100644
--- a/doc/building.rst
+++ b/doc/building.rst
@@ -882,14 +882,6 @@ Specify an additional library that fuzzer binaries must link with.
 Build only the specific targets and tools
 (``static``, ``shared``, ``cli``, ``tests``, ``bogo_shim``).
 
-``--boost-library-name``
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Provide an alternative name for a boost library. Depending on the platform and
-boost's build configuration these library names differ significantly (see `Boost docs
-<https://www.boost.org/doc/libs/1_70_0/more/getting_started/unix-variants.html#library-naming>`_).
-The provided library name must be suitable as identifier in a linker parameter,
-e.g on unix: ``boost_system`` or windows: ``libboost_regex-vc71-x86-1_70``.
 
 ``--without-documentation``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/src/build-data/cc/clang.txt b/src/build-data/cc/clang.txt
index 1a6196c28..0c538306f 100644
--- a/src/build-data/cc/clang.txt
+++ b/src/build-data/cc/clang.txt
@@ -14,6 +14,7 @@ optimization_flags "-O3"
 sanitizer_optimization_flags "-O1 -fno-optimize-sibling-calls -fno-omit-frame-pointer"
 size_optimization_flags "-Os"
 
+add_system_include_dir_option "-isystem"
 add_sysroot_option "--sysroot="
 
 <sanitizers>
diff --git a/src/build-data/cc/gcc.txt b/src/build-data/cc/gcc.txt
index eb872e75e..6c50e8faf 100644
--- a/src/build-data/cc/gcc.txt
+++ b/src/build-data/cc/gcc.txt
@@ -20,6 +20,7 @@ shared_flags "-fPIC"
 coverage_flags "--coverage"
 stack_protector_flags "-fstack-protector"
 
+add_system_include_dir_option "-isystem"
 add_sysroot_option "--sysroot="
 
 <sanitizers>
diff --git a/src/build-data/cc/msvc.txt b/src/build-data/cc/msvc.txt
index c77a2fc73..cfe5c7b15 100644
--- a/src/build-data/cc/msvc.txt
+++ b/src/build-data/cc/msvc.txt
@@ -7,6 +7,7 @@ output_to_object "/Fo"
 output_to_exe "/OUT:"
 
 add_include_dir_option "/I"
+add_system_include_dir_option "/external:W0 /external:I"
 add_lib_dir_option "/LIBPATH:"
 add_compile_definition_option "/D"
 add_lib_option "%s.lib"
diff --git a/src/lib/tls/asio/asio_stream.h b/src/lib/tls/asio/asio_stream.h
index 2591408f6..49bc6f925 100644
--- a/src/lib/tls/asio/asio_stream.h
+++ b/src/lib/tls/asio/asio_stream.h
@@ -589,9 +589,6 @@ class Stream
        *
        * This class is provided to the stream's native_handle (Botan::TLS::Channel) and implements the callback
        * functions triggered by the native_handle.
-       *
-       * @param receive_buffer reference to the buffer where decrypted data should be placed
-       * @param send_buffer reference to the buffer where encrypted data should be placed
        */
       class StreamCore : public Botan::TLS::Callbacks
          {
diff --git a/src/lib/utils/boost/info.txt b/src/lib/utils/boost/info.txt
index 6330cbafd..64cfbe32b 100644
--- a/src/lib/utils/boost/info.txt
+++ b/src/lib/utils/boost/info.txt
@@ -5,5 +5,10 @@ BOOST_ASIO -> 20131228
 load_on vendor
 
 <libs>
-all -> boost_system
+linux -> rt
+mingw -> ws2_32
+windows -> ws2_32
+haiku -> network
+solaris -> socket,nsl
+qnx -> socket
 </libs>
diff --git a/src/scripts/ci/setup_gh_actions.sh b/src/scripts/ci/setup_gh_actions.sh
index 7b8135b24..e55887928 100755
--- a/src/scripts/ci/setup_gh_actions.sh
+++ b/src/scripts/ci/setup_gh_actions.sh
@@ -73,3 +73,9 @@ else
         brew install emscripten
     fi
 fi
+
+# find the ccache cache location and store it in the build job's environment
+if type -p "ccache"; then
+    cache_location="$( ccache --get-config cache_dir )"
+    echo "COMPILER_CACHE_LOCATION=${cache_location}" >> "${GITHUB_ENV}"
+fi
diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py
index c09303852..c3f7b9149 100755
--- a/src/scripts/ci_build.py
+++ b/src/scripts/ci_build.py
@@ -276,15 +276,13 @@ def determine_flags(target, target_os, target_cpu, target_cc, cc_bin,
 
         if target_os == 'windows' and target in ['shared', 'static']:
             # ./configure.py needs extra hand-holding for boost on windows
-            boost_root = os.environ.get('BOOST_ROOT')
-            boost_libs = os.environ.get('BOOST_LIBRARYDIR')
-            boost_system = os.environ.get('BOOST_SYSTEM_LIBRARY')
+            boost_root = os.environ.get('BOOST_ROOT') # remove this with appveyor
+            boost_incl = os.environ.get('BOOST_INCLUDEDIR')
 
-            if boost_root and boost_libs and boost_system:
-                flags += ['--with-boost',
-                          '--with-external-includedir', boost_root,
-                          '--with-external-libdir', boost_libs,
-                          '--boost-library-name', boost_system]
+            if boost_root:
+                flags += ['--with-external-includedir', boost_root]
+            elif boost_incl:
+                flags += ['--with-external-includedir', boost_incl]
 
         if target_os == 'linux':
             flags += ['--with-lzma']
@@ -430,8 +428,9 @@ def have_prog(prog):
     """
     for path in os.environ['PATH'].split(os.pathsep):
         exe_file = os.path.join(path, prog)
-        if os.path.exists(exe_file) and os.access(exe_file, os.X_OK):
-            return True
+        for ef in [exe_file, exe_file + ".exe"]:
+            if os.path.exists(ef) and os.access(ef, os.X_OK):
+                return True
     return False
 
 def main(args=None):
@@ -482,10 +481,14 @@ def main(args=None):
             print('Error unknown compiler %s' % (options.cc))
             return 1
 
-    if options.compiler_cache is None and options.cc != 'msvc':
-        # Autodetect ccache
-        if have_prog('ccache'):
+    if options.compiler_cache is None:
+        # Autodetect compiler cache
+        if have_prog('sccache'):
+            options.compiler_cache = 'sccache'
+        elif have_prog('ccache'):
             options.compiler_cache = 'ccache'
+        if options.compiler_cache:
+            print("Found '%s' installed, will use it..." % (options.compiler_cache))
 
     if options.compiler_cache not in [None, 'ccache', 'sccache']:
         raise Exception("Don't know about %s as a compiler cache" % (options.compiler_cache))
@@ -539,6 +542,9 @@ def main(args=None):
 
         cmds.append([py_interp, os.path.join(root_dir, 'configure.py')] + config_flags)
 
+        if options.make_tool == '':
+            options.make_tool = 'make'
+
         make_cmd = [options.make_tool]
         if root_dir != '.':
             make_cmd += ['-C', root_dir]


Test patch:
diff --git a/src/tests/data/x509/bsi/cert_path_ext_16/description.txt b/src/tests/data/x509/bsi/cert_path_ext_16/description.txt
index ab728d352..85544db80 100644
--- a/src/tests/data/x509/bsi/cert_path_ext_16/description.txt
+++ b/src/tests/data/x509/bsi/cert_path_ext_16/description.txt
@@ -1,4 +1,3 @@
 Test Case: CERT_PATH_EXT_16
 
 Purpose: Checks the behaviour of an application when the intermediate certificate contains a non-critical Name Constraints extension. This path is invalid because this extension must be critical.
-However, as of GH #2736 / GH #2738 (Botan 2.18.1) the library accepts this path to become compatible with other implementations (i.e OpenSSL) that also accept non-critical extensions.