Using GitHub Action to Build Python Wheel Package for Dynamsoft C++ Barcode SDK
Published: 2022-05-27 01:10:25

Implementing a Python extension in C or C++ is easy, but building the native module into a Python wheel package for different operating systems and different Python versions is a nightmare. A CPython wheel package is Python-version-dependent, therefore, to make a CPython module work universally for Python 3.x, you have to build wheels for Python 3.6, Python 3.7, Python 3.8, Python 3.9, and Python 3.10 per operating system. The maximum amount of packages equals to `Python versions (3.6, 3.7, 3.8, 3.9, 3.10) * operating systems (Windows, Linux, macOS) * CPU architectures (x64, aarch64)`. To simplify the process, we can utilize GitHub Actions. In this article, we take Dynamsoft C/C++ Barcode SDK as an example. You will see how to build a CPython extension that links to external C/C++ libraries (`*.dll, *.so, *.dylib`), and how to automate the process of building and publishing the Python wheel package with GitHub Actions.

## Requirements
[Dynamsoft C/C++ Barcode SDK v9.0](https://www.dynamsoft.com/barcode-reader/downloads//#desktop)

## Building CPython Extension Project with Scikit-build
If you have read Python development guide, you may know that [distutils.core.Extension](https://docs.python.org/3/distutils/apiref.html?highlight=extension#distutils.core.Extension) is the most widely used Python extension builder. However, `distutils` cannot sequentially build the extension and package the generated library with the package folder when running the `pip wheel` command for creating a wheel package. 

[Scikit-build](https://scikit-build.readthedocs.io/en/latest/#) is an alternative to `distutils`. It extends `distutils` functions with CMake build. To get started with scikit-build, we can visit [scikit-build-sample-projects](https://github.com/scikit-build/scikit-build-sample-projects/tree/master/projects).

For our Python barcode and QR code extension project, the `setup.py` file is as follows:

```python
from skbuild import setup
import io

packages = ['barcodeQrSDK']

setup (name = 'barcode-qr-code-sdk',
            version = '9.0.3',
            description = 'Barcode and QR code scanning SDK for Python',
            packages=packages,
            include_package_data=False,
          )
``` 

As you can see, the `setup.py` file is pretty simple comparing to the `setup.py` file used with `disutils`. It only contains the package folder. To trigger extension build, we create a `CMakeLists.txt` along with the `setup.py` file.

```cmake
cmake_minimum_required(VERSION 3.4...3.22)

project(barcodeQrSDK)

find_package(PythonExtensions REQUIRED)

if(CMAKE_HOST_UNIX)
    if(CMAKE_HOST_APPLE)
        SET(CMAKE_CXX_FLAGS "-std=c++11 -O3 -Wl,-rpath,@loader_path")
        SET(CMAKE_INSTALL_RPATH "@loader_path")
    else()
        SET(CMAKE_CXX_FLAGS "-std=c++11 -O3 -Wl,-rpath=$ORIGIN")
        SET(CMAKE_INSTALL_RPATH "$ORIGIN")
    endif()
    SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
endif()

MESSAGE( STATUS "CPU architecture ${CMAKE_SYSTEM_PROCESSOR}" )
if(CMAKE_HOST_WIN32)
    link_directories("${PROJECT_SOURCE_DIR}/lib/win/") 
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
    if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
        MESSAGE( STATUS "Link directory: ${PROJECT_SOURCE_DIR}/lib/linux/" )
        link_directories("${PROJECT_SOURCE_DIR}/lib/linux/")
    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l)
        MESSAGE( STATUS "Link directory: ${PROJECT_SOURCE_DIR}/lib/arm32/" )
        link_directories("${PROJECT_SOURCE_DIR}/lib/arm32/") 
    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) 
        MESSAGE( STATUS "Link directory: ${PROJECT_SOURCE_DIR}/lib/aarch64/" )
        link_directories("${PROJECT_SOURCE_DIR}/lib/aarch64/") 
    endif()
elseif(CMAKE_HOST_APPLE)
    MESSAGE( STATUS "Link directory: ${PROJECT_SOURCE_DIR}/lib/macos/" )
    link_directories("${PROJECT_SOURCE_DIR}/lib/macos/") 
endif()
include_directories("${PROJECT_BINARY_DIR}" "${PROJECT_SOURCE_DIR}/include/")

add_library(${PROJECT_NAME} MODULE src/barcodeQrSDK.cpp)
if(CMAKE_HOST_WIN32)
    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
        target_link_libraries (${PROJECT_NAME} "DynamsoftBarcodeReaderx64")
    else()
        target_link_libraries (${PROJECT_NAME} "DBRx64")
    endif()
else()
    target_link_libraries (${PROJECT_NAME} "DynamsoftBarcodeReader" pthread)
endif()

if(CMAKE_HOST_WIN32)
    add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD 
    COMMAND ${CMAKE_COMMAND} -E copy_directory
    "${PROJECT_SOURCE_DIR}/lib/win/"      
    $<TARGET_FILE_DIR:${PROJECT_NAME}>)
endif()

python_extension_module(barcodeQrSDK)
install(TARGETS barcodeQrSDK LIBRARY DESTINATION barcodeQrSDK)

if(CMAKE_HOST_WIN32)
    install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/win/" DESTINATION barcodeQrSDK)
elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
    if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
        install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/linux/" DESTINATION barcodeQrSDK)
    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL armv7l OR ARM32_BUILD)
        install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/arm32/" DESTINATION barcodeQrSDK)
    elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) 
        install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/aarch64/" DESTINATION barcodeQrSDK)
    endif()
elseif(CMAKE_HOST_APPLE)
    install (DIRECTORY  "${PROJECT_SOURCE_DIR}/lib/macos/" DESTINATION barcodeQrSDK)
endif()
```

What we do in the `CMakeLists.txt` file:

1. Set build arguments. The `rpath` is critical for finding dependent shared libraries on Linux and macOS.
2. Set the directories of header files and libraries.
3. Build the extension module.
4. Link external dynamic libraries.
5. Copy the Python module and dependent libraries to the `barcodeQrSDK` package folder.

A `__init__.py` file is also required in the `barcodeQrSDK` package folder.

```python
from .barcodeQrSDK import * 
__version__ = version
```

Run `python setup.py develop` command to test the extension module. If there is no error, we can create the wheel package:

```bash
pip wheel .
```

Once the wheel package is built successfully, its folder structure is like this: 

![Python wheel folder](https://www.dynamsoft.com/codepool/img/2022/05/python-wheel-folder-1.png)

![Python wheel folder](https://www.dynamsoft.com/codepool/img/2022/05/python-wheel-folder-2.png)

## Creating Multiple Wheel Packages with GitHub Actions
As we have mentioned above, it drives us crazy to create multiple wheel packages for each version of Python and different platforms. Fortunately, GitHub Actions can relieve us a lot of work.

Here are the steps to build and publish multiple wheel packages:
1. Go to the repository homepage and click `Actions` to create a new workflow. 
2. Click `set up a workflow yourself ` to create a custom workflow. We can refer to the examples provided by [cibuildwheel](https://github.com/pypa/cibuildwheel/tree/main/examples).
    
    ```yml
    name: Build and upload to PyPI

    on: [push, pull_request]

    jobs:
    build_wheels:
        name: Build wheels on ${{ matrix.os }}
        runs-on: ${{ matrix.os }}
        strategy:
        matrix:
            os: [windows-2019, macos-10.15]

        steps:
        - uses: actions/checkout@v2

        - name: Build wheels
            uses: pypa/cibuildwheel@v2.5.0
            env:
            CIBW_ARCHS_WINDOWS: AMD64
            CIBW_ARCHS_MACOS: x86_64
            CIBW_ARCHS_LINUX: "x86_64 aarch64"
            CIBW_SKIP: "pp* *-win32 *-manylinux_i686"

        - uses: actions/upload-artifact@v2
            with:
            path: ./wheelhouse/*.whl

    build_sdist:
        name: Build source distribution
        runs-on: ubuntu-latest
        steps:
        - uses: actions/checkout@v2

        - name: Build sdist
            run: pipx run build --sdist

        - uses: actions/upload-artifact@v2
            with:
            path: dist/*.tar.gz
            
    upload_pypi:
        needs: [build_wheels, build_sdist]
        runs-on: ubuntu-latest
        # upload to PyPI on every tag starting with 'v'
        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
        # alternatively, to publish when a GitHub Release is created, use the following rule:
        # if: github.event_name == 'release' && github.event.action == 'published'
        steps:
        - uses: actions/download-artifact@v2
            with:
            name: artifact
            path: dist

        - uses: pypa/gh-action-pypi-publish@v1.4.2
            with:
            user: __token__
            password: ${{ secrets.pypi_password }}
            skip_existing: true
    ```

    Since our extension is 64-bit only, we can skip 32-bit builds by setting `CIBW_SKIP: "pp* *-win32 *-manylinux_i686"`. We can also specify the OS architectures:
    ```yml
    CIBW_ARCHS_WINDOWS: AMD64
    CIBW_ARCHS_MACOS: x86_64
    CIBW_ARCHS_LINUX: "x86_64 aarch64"
    ```
3. Go to `Settings > Secrets > Actions` to create a repository secret for publishing the wheel packages to pypi.org.
    
    ![GitHub repository secret](https://www.dynamsoft.com/codepool/img/2022/05/github-repository-secret.png)

4. After finishing the workflows, we can download the artifact that contains the generate wheel packages.
    
    ![GitHub action for Python wheel](https://www.dynamsoft.com/codepool/img/2022/05/github-action-python-wheel.png)

    Besides, the wheel packages are available for download on pypi.org.

    ![Python barcode and QR code SDK](https://www.dynamsoft.com/codepool/img/2022/05/python-barcode-qrcode-sdk.png)

 
## Source Code
[https://github.com/yushulx/python-barcode-qrcode-sdk](https://github.com/yushulx/python-barcode-qrcode-sdk)