Repository: SPFlow/SPFlow (language: python)

Issue:
Set up CI for all backends
Similar to the tensorly test cases we should write the unit tests as backend-agnostic as possible. During CI we then run the tests with all available backends set via the `TENSORLY_BACKEND=...` environment variable. See also the GitHub test workflow of tensorly: https://github.com/tensorly/tensorly/blob/main/.github/workflows/test.yml


Discussion / hints:

Fix patch:
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yaml
similarity index 100%
rename from .github/ISSUE_TEMPLATE/bug_report.yml
rename to .github/ISSUE_TEMPLATE/bug_report.yaml
diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yaml
similarity index 100%
rename from .github/ISSUE_TEMPLATE/feature_request.yml
rename to .github/ISSUE_TEMPLATE/feature_request.yaml
diff --git a/.github/workflows/github-actions-build-python.yaml b/.github/workflows/github-actions-build-python.yaml
new file mode 100644
index 000000000..3eb5395a7
--- /dev/null
+++ b/.github/workflows/github-actions-build-python.yaml
@@ -0,0 +1,119 @@
+name: Development Build & Test
+
+on:
+  release:
+    types: [ published ]
+  pull_request:
+  push:
+
+jobs:
+
+  "Lint":
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+          cache: 'pip' # caching pip dependencies
+          cache-dependency-path: |
+            **/pyproject.toml
+      - name: Install library and dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[dev]
+      - name: Lint
+        run: ruff check src
+
+  "Check_Format":
+    name: "Check Format"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+          cache: 'pip' # caching pip dependencies
+          cache-dependency-path: |
+            **/pyproject.toml
+      - name: Install library and dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[dev]
+      - name: Checking formatting with black
+        run: black --check --diff src/
+
+  # We do not want to waste too many resources on testing all combinations. Thus we test:
+  # - on the oldest and the two most recent version of Python that we support (see README.md)
+  # - with the oldest version of the backend we support plus the two most recent major ones
+  #   (to catch if only the newest version breaks something)
+  # - A single test on Windows and macOS to catch if something breaks there
+  # - We only test on the most recent version of macOS and Windows to save resources
+  # - We test even fewer combinations for TensorFlow and Jax to save resources
+
+  "Test_OS":
+    name: "Test: Py ${{ matrix.version-python }} @ OS ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version-python: ['3.12']
+        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: "./.github/actions/run_tests"
+        with:
+          version-python: ${{ matrix.version-python }}
+
+  "Test_Torch":
+    name: "Test: Py ${{ matrix.version-python }} + Torch ${{ matrix.version-pytorch }}"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false 
+      matrix:
+        version-python: ['3.9', '3.11']  # 3.12 is not yet supported by torch
+        version-pytorch: ['1.11.0', '2.0.1', '2.1.1']
+        exclude: # Non-supported combinations
+          - version-python: '3.11'
+            version-pytorch: '1.11.0'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: "./.github/actions/run_tests"
+        with:
+          version-python: ${{ matrix.version-python }}
+          version-pytorch: ${{ matrix.version-pytorch }}
+
+  "Test_Tensorflow":
+    name: "Test: Py ${{ matrix.version-python }} + TF ${{ matrix.version-tensorflow }}"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false 
+      matrix:
+        include:
+          - version-python: '3.10'
+            version-tensorflow: '2.11.0'
+          - version-python: '3.11'
+            version-tensorflow: '2.15.0.post1'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: "./.github/actions/run_tests"
+        with:
+          version-python: ${{ matrix.version-python }}
+          version-tensorflow: ${{ matrix.version-tensorflow }}
+
+  "Test_Jax":
+    name: "Test: Py ${{ matrix.version-python }} + Jax ${{ matrix.version-jax }}"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false 
+      matrix:
+        version-python: ['3.12']
+        version-jax: ['0.4.23']
+    steps:
+      - uses: actions/checkout@v4
+      - uses: "./.github/actions/run_tests"
+        with:
+          version-python: ${{ matrix.version-python }}
+          version-jax: ${{ matrix.version-jax }}
diff --git a/.github/workflows/github-actions-build-python.yml b/.github/workflows/github-actions-build-python.yml
deleted file mode 100644
index bdfac1dbd..000000000
--- a/.github/workflows/github-actions-build-python.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-name: Development Build & Test
-
-on:
-  release:
-    types: [ published ]
-  pull_request:
-  push:
-
-jobs:
-
-  "Lint":
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-          cache: 'pip' # caching pip dependencies
-          cache-dependency-path: |
-            **/pyproject.toml
-      - name: Install library and dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install .[dev]
-      - name: Lint with flake8
-        run: ruff check src
-
-  "Check_Format":
-    name: "Check Format"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-          cache: 'pip' # caching pip dependencies
-          cache-dependency-path: |
-            **/pyproject.toml
-      - name: Install library and dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install .[dev]
-      - name: Checking formatting with black
-        run: black --check --diff src/
-
-  "Test":
-    name: "Test: Py ${{ matrix.python-version }} + Torch ${{ matrix.pytorch-version }}"
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false  # Don't fail fast to explore which configurations fail
-      matrix:
-        python-version: ['3.9', '3.10', '3.11']  # 3.12 is not yet supported by torch
-        pytorch-version: ['1.11.0', '1.12.1', '1.13.1', '2.0.1', '2.1.0']
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}, PyTorch ${{ matrix.pytorch-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip' # caching pip dependencies
-          cache-dependency-path: |
-            **/pyproject.toml
-      - name: Install library and dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install torch==${{ matrix.pytorch-version }}
-          pip install .[dev]
-      - name: Test with pytest
-        run: |
-          pytest
diff --git a/README.md b/README.md
index ea844de4a..3057eaaf2 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,23 @@ Cite using: TODO
   * [Custom Modules](#custom-modules)
   * [Implementing Dispatched Routines](#implementing-dispatched-routines)
 
+# Installation
+
+By default, only SPFlow with the NumPy backend is installed:
+```shell
+pip install spflow
+
+# Or for making local changes
+git clone git@github.com:SPFlow/SPFlow.git
+cd SPFlow
+pip install -e .[dev]
+```
+
+To install other backends, SPFlow will detect that you installed the respective packages according to their instructions:
+- [PyTorch](https://pytorch.org/get-started) (`>=1.11.0`)
+- [TensorFlow](https://www.tensorflow.org/install)
+- [Jax](https://jax.readthedocs.io/en/latest/installation.html)
+
 # User Guide
 ## Backends
 
diff --git a/pyproject.toml b/pyproject.toml
index 92eaa88a6..3d637abee 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,7 +34,6 @@ dependencies = [
     "plum-dispatch>=1.7.4",
     "numpy>=1.17.4",
     "scipy>=1.5.0",
-    "torch>=1.11.0",
     "scikit-learn>=1.0",
     "tensorly>=0.8.0",
 ]
@@ -51,15 +50,15 @@ version = { attr = "spflow.__version__" }
 
 [project.optional-dependencies]
 dev = [
-  "black~=23.10",
-  "ruff~=0.1.3",
-  "mypy~=1.5",
+  "black~=23.12",
+  "ruff~=0.1.8",
+  "mypy~=1.7",
   "pytest",
   "pytest-cov",
 ]
 
 [tool.pytest.ini_options]
-addopts = " --color=yes --doctest-modules" #--cov=spflow"
+addopts = " --color=yes --doctest-modules"
 testpaths = [
     # for the doctests:
     "src/spflow",
@@ -84,7 +83,7 @@ line-length = 110  # Same as Black
 target-version = "py39"  # The lowest supported version
 
 [tool.ruff.pydocstyle]
-convention = "numpy"
+convention = "google"
 
 [tool.isort]
 profile = "black"


Test patch:
diff --git a/.github/actions/run_tests/action.yaml b/.github/actions/run_tests/action.yaml
new file mode 100644
index 000000000..92d574cbc
--- /dev/null
+++ b/.github/actions/run_tests/action.yaml
@@ -0,0 +1,52 @@
+name: 'Setup Dependencies & Run Tests'
+description: 'Install the nessesary dependencies and run the tests in this repo'
+inputs:
+  version-python:
+    required: true
+
+  # One could also set multiple libraries to be tested in one workflow
+  version-pytorch:
+    required: false
+  version-tensorflow:
+    required: false
+  version-jax:
+    required: false
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Set up Python ${{ matrix.version-python }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.version-python }}
+        cache: 'pip' # caching pip dependencies
+        cache-dependency-path: |
+          **/pyproject.toml
+    - name: Install library and dependencies
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip install .[dev]
+
+    - name: Set up Torch ${{ matrix.version-pytorch }}
+      if: matrix.version-pytorch
+      shell: bash
+      run: |
+        pip install torch==${{ matrix.version-pytorch }} --index-url https://download.pytorch.org/whl/cpu
+
+    - name: Set up TensorFlow ${{ matrix.version-tensorflow }}
+      if: matrix.version-tensorflow
+      shell: bash
+      run: |
+        pip install tensorflow-cpu==${{ matrix.version-tensorflow }}
+
+    - name: Set up JAX ${{ matrix.version-jax }}
+      if: matrix.version-jax
+      shell: bash
+      run: |
+        pip install jax[cpu]==${{ matrix.version-jax }}
+
+    - name: Test with pytest
+      shell: bash
+      run: |
+        pytest --cov=spflow
diff --git a/src/tests/tensorly/conftest.py b/src/tests/tensorly/conftest.py
index 359c596cc..9137d0779 100644
--- a/src/tests/tensorly/conftest.py
+++ b/src/tests/tensorly/conftest.py
@@ -1,21 +1,72 @@
+import random
+from unittest import SkipTest
 from typing import Callable
 
-import tensorly as tl
-import torch
 import numpy as np
-import random
+import tensorly as tl
 from pytest import fixture
 
+try:
+    import torch
+except ImportError:
+    torch = None
+
+try:
+    import tensorflow
+except ImportError:
+    tensorflow = None
+
+try:
+    import jax
+except ImportError:
+    jax = None
+
+
+def make_all_deterministic(seed: int = 42) -> None:
+    """Set seeds and other things to ensure reproducibility."""
+    # Python's built-in random module
+    random.seed(seed)
+
+    # Numpy
+    np.random.seed(seed)
+
+    if torch is not None:
+        torch.manual_seed(seed)
+        torch.backends.cudnn.benchmark = False
+        torch.use_deterministic_algorithms(True, warn_only=True)
+
+    if tensorflow is not None:
+        tensorflow.random.set_seed(seed)
+        tensorflow.config.experimental.enable_op_determinism()
+
+    # Apparently, JAX does not have a global seed, so we cannot set it here.
+
+
 @fixture(
     scope="function",
     autouse=True,
-    params=["numpy", "pytorch"],
+    params=["numpy", "pytorch", "tensorflow", "jax"],
 )
 def do_for_all_backends(request) -> Callable:
-    # set seed
-    torch.manual_seed(0)
-    np.random.seed(0)
-    random.seed(0)
+    """Runs a test for all backends while ensuring reproducibility.
+
+    Tests for which this fixture is used should have a ``do_for_all_backends`` argument.
+    This suffices to run the test for all backends.
+
+    If a backend is not installed, the test is makred skipped.
+    """
+    make_all_deterministic()
+
     backend_name = request.param
+
+    if backend_name == "pytorch" and torch is None:
+        raise SkipTest("PyTorch is not installed")
+
+    if backend_name == "tensorflow" and tensorflow is None:
+        raise SkipTest("TensorFlow is not installed")
+
+    if backend_name == "jax" and jax is None:
+        raise SkipTest("JAX is not installed")
+
     with tl.backend_context(backend_name):
-        yield backend_name
\ No newline at end of file
+        yield backend_name