Repository: epignatelli/helx (language: python)

Issue:
Migrate to `.toml` standard and implement CD



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml
new file mode 100644
index 0000000..9a419ea
--- /dev/null
+++ b/.github/workflows/CD.yml
@@ -0,0 +1,38 @@
+name: CD
+
+on:
+  push:
+    branches:
+      - "main"
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Bump version and push tag
+        id: tag_version
+        uses: mathieudutour/github-tag-action@v6.1
+        with:
+          default_bump: "minor"
+      - name: Create GitHub release
+        uses: ncipollo/release-action@v1
+        with:
+          tag: ${{ steps.tag_version.outputs.new_tag }}
+          name: Release ${{ steps.tag_version.outputs.new_tag }}
+          body: ${{ steps.tag_version.outputs.changelog }}
+
+  publish:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+      matrix:
+        python_version: ["3.8", "3.9", "3.10", "3.11"]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build package
+        run: python3 -m pip install --upgrade build && python3 -m build
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_KEY }}
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index bdb3a15..be7f526 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -1,26 +1,68 @@
-# it is so stupid that needs does not share the workspace
-# but actually re-runs the job
-# or that there is no easy mechanism to share the workspace
-# This is currently on workflow_dispatch because jobs do not share the workspace
-# and the script actually does something different from what I'd like
-# Yet, I cannot find a better way to do this than the one in test.yml
 name: CI
 
 on:
-  workflow_dispatch:
+  pull_request:
+    types: [opened, synchronize, reopened]
 
 jobs:
-  setup:
-    uses: ./.github/workflows/setup.yml
+  Test:
+    runs-on: ${{ matrix.os }}-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+      matrix:
+        os: ["ubuntu"]
+    continue-on-error: false
+    steps:
+    - uses: actions/checkout@v3
+    - uses: conda-incubator/setup-miniconda@v2
+      with:
+        miniforge-version: latest
+        environment-file: environment.yml
+        activate-environment: helx
+        auto-activate-base: false
+    - name: Setup helx
+      run: |
+        pip install . -v
+        helx-download-extras
+        conda env config vars set LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.mujoco/mujoco210/bin
+        conda env config vars set MJLIB_PATH=$HOME/.mujoco/mujoco210/bin/libmujoco210.so
+        conda env config vars set MUJOCO_PY_MUJOCO_PATH=$HOME/.mujoco/mujoco210
+        conda env config vars set LD_PRELOAD=$CONDA_PREFIX/x86_64-conda-linux-gnu/sysroot/usr/lib64/libGLdispatch.so.0
+    - name: Check code quality
+      run: |
+        pip install pylint
+        MESSAGE=$(pylint -ry $(git ls-files '*.py') ||:)
+        echo "$MESSAGE"
+    - name: Run unit tests with pytest
+      run: |
+        pytest test/
+    - name: Run examples
+      run: |
+        wandb login ${{ secrets.WANDB_API_KEY }}
+        for f in examples/*.py; do python $f; done
 
-  linting:
-    needs: setup
-    uses: ./.github/workflows/linting.yml
-
-  pytest:
-    needs: setup
-    uses: ./.github/workflows/pytest.yml
-
-  examples:
-    needs: setup
-    uses: ./.github/workflows/examples.yml
\ No newline at end of file
+  Compliance:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: PEP8 Compliance
+      run: |
+        pip install pylint
+        PR_BRANCH=${{ github.event.pull_request.target.ref }}
+        MAIN_BRANCH=origin/${{ github.event.pull_request.base.ref }}
+        MESSAGE=$(pylint -ry --disable=E0401 `git diff --name-only --diff-filter=d $MAIN_BRANCH $PR_BRANCH | grep -E '\.py$' | tr '\n' ' '` ||:)
+        echo 'MESSAGE<<EOF' >> $GITHUB_ENV
+        echo "<pre><code>$MESSAGE</code></pre>" >> $GITHUB_ENV
+        echo 'EOF' >> $GITHUB_ENV
+        echo "Printing PR message: $MESSAGE"
+    - uses: mshick/add-pr-comment@v2
+      with:
+        issue: ${{ github.event.pull_request.number }}
+        message: ${{ env.MESSAGE }}
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
deleted file mode 100644
index 855f8b7..0000000
--- a/.github/workflows/examples.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-name: Examples
-
-on:
-  workflow_call:
-
-jobs:
-  examples:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Run examples
-      run: |
-        wandb login ${{ secrets.WANDB_API_KEY }}
-        for f in examples/*.py; do python $f; done
\ No newline at end of file
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
deleted file mode 100644
index 92c8429..0000000
--- a/.github/workflows/linting.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-name: Code quality
-
-on:
-  workflow_call:
-
-jobs:
-  linting:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check code quality
-      run: |
-        pip install pylint
-        MESSAGE=$(pylint -ry $(git ls-files '*.py') ||:)
-        echo "$MESSAGE"
\ No newline at end of file
diff --git a/.github/workflows/setup.yml b/.github/workflows/setup.yml
deleted file mode 100644
index 66bd759..0000000
--- a/.github/workflows/setup.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-name: Setup
-
-on:
-  workflow_call:
-
-jobs:
-  setup:
-    runs-on: "ubuntu-latest"
-    defaults:
-      run:
-        shell: bash -l {0}
-
-    steps:
-
-    - uses: actions/checkout@v3
-
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
-        miniforge-version: latest
-        environment-file: environment.yml
-        activate-environment: helx
-        auto-activate-base: false
-
-    - name: Setup helx
-      run: |
-        pip install . -v
-
-        conda env config vars set LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.mujoco/mujoco210/bin
-        conda env config vars set MJLIB_PATH=$HOME/.mujoco/mujoco210/bin/libmujoco210.so
-        conda env config vars set MUJOCO_PY_MUJOCO_PATH=$HOME/.mujoco/mujoco210
-        conda env config vars set LD_PRELOAD=$CONDA_PREFIX/x86_64-conda-linux-gnu/sysroot/usr/lib64/libGLdispatch.so.0
diff --git a/.gitignore b/.gitignore
index 42ee230..49750c9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,6 @@
+# setuptools_scm
+helx/version.py
+
 # vscode
 .vscode/
 wandb/
diff --git a/README.md b/README.md
index 908eeba..a23f062 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,12 @@ Why using `helx`? It allows to easily switch between different RL libraries, and
 pip install git+https://github.com/epignatelli/helx
 ```
 
-And tell the system where the mujoco binaries are:
+If you also want to download the binaries for `mujoco`, both `gym` and `dm_control`, and `atari`:
+```bash
+helx-download-extras
+```
+
+And then tell the system where the mujoco binaries are:
 ```bash
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/mujoco/lib
 export MJLIB_PATH=/path/to/home/.mujoco/mujoco210/bin/libmujoco210.so
diff --git a/helx/__init__.py b/helx/__init__.py
index dcc919d..f62d73e 100644
--- a/helx/__init__.py
+++ b/helx/__init__.py
@@ -12,25 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-__version__ = "0.2.0"
-
-from . import (
-    agents,
-    environment,
-    experiment,
-    flags,
-    image,
-    logging,
-    mdp,
-    memory,
-    networks,
-    preprocess,
-    random,
-    spaces,
-    stax,
-)
-
 # Make sure that the gym registry is populated
 # with exteral environments.
-import procgen, minigrid, gym_minigrid
+import gym_minigrid
+import minigrid
+import procgen
+
+from . import (agents, environment, experiment, flags, image, logging, mdp,
+               memory, networks, preprocess, random, spaces, stax)
diff --git a/setup.py b/helx/download.py
similarity index 55%
rename from setup.py
rename to helx/download.py
index c224903..474ea61 100644
--- a/setup.py
+++ b/helx/download.py
@@ -1,3 +1,4 @@
+#!python
 # Copyright [2023] The Helx Authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,60 +21,22 @@
 import tarfile
 
 import requests
-from setuptools import find_packages, setup
-from setuptools.command.install import install
-from setuptools.command.develop import develop
 
+from .logging import get_logger
 
 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
 REQUIREMENTS_PATH = "requirements.txt"
 MUJOCO_ROOT = os.path.join(os.path.expanduser("~"), ".mujoco")
 ATARI_ROOT = os.path.join(os.path.expanduser("~"), ".atari")
-BOLD = "\033[1m"
-DARK_GREY = "\033[1;30m"
-END = "\033[0m"
 
 
-logger = logging.getLogger("helx setup")
-ch = logging.StreamHandler()
-ch.setLevel(logging.INFO)
-ch.setFormatter(
-    logging.Formatter(
-        BOLD + DARK_GREY + "%(asctime)s | %(name)s | %(levelname)s | %(message)s" + END
-    )
-)
-logger.addHandler(ch)
-
-
-class PostInstallDeploy(install):
-    def run(self):
-        install.run(self)
-
-        # download engines
-        logger.info("Installing extra requirements: mujoco, atari.")
-        _download_mujoco210()
-        _download_mujoco_dm_control()
-        # TODO(epignatelli): switch to autorom when pipeline is fixed
-        _download_atari_roms()
-
-
-class PostInstallDevelop(develop):
-    def run(self):
-        develop.run(self)
-
-        # download engines
-        logger.info("Installing extra requirements: mujoco, atari.")
-        _download_mujoco210()
-        _download_mujoco_dm_control()
-        # TODO(epignatelli): switch to autorom when pipeline is fixed
-        _download_atari_roms()
-
+logging = get_logger()
 
 
 def _download_url(url, out_path, chunk_size=128):
     os.makedirs(os.path.dirname(out_path), exist_ok=True)
     r = requests.get(url, stream=True)
-    logger.info("Downloading {} into {}".format(url, out_path))
+    logging.info(f"Downloading {url} into {out_path}")
     with open(out_path, "wb") as fd:
         chunks = 0
         for chunk in r.iter_content(chunk_size=chunk_size):
@@ -81,7 +44,7 @@ def _download_url(url, out_path, chunk_size=128):
             fd.write(chunk)
 
 
-def _download_mujoco210():
+def download_mujoco210():
     # example_url: "https://mujoco.org/download/mujoco210-linux-x86_64.tar.gz"
     base_url = "https://mujoco.org/download"
     mujoco_version = "mujoco210"
@@ -94,17 +57,17 @@ def _download_mujoco210():
     }
     system = platform.system()
     if system not in mujoco_systems:
-        raise ValueError("Unsupported system: {}".format(system))
+        raise ValueError(f"Unsupported system: {system}")
     mujoco_system = mujoco_systems[system]
 
     # get architecture version
     machine = platform.machine().lower()
     if machine != "x86_64":
-        raise ValueError("Unsupported architecture: {}".format(machine))
+        raise ValueError(f"Unsupported architecture: {machine}")
 
     # download mujoco
     url = os.path.join(
-        base_url, "{}-{}-{}.tar.gz".format(mujoco_version, mujoco_system, machine)
+        base_url, f"{mujoco_version}-{mujoco_system}-{machine}.tar.gz"
     )
     out_filename = os.path.basename(url)
     out_path = os.path.join(MUJOCO_ROOT, out_filename)
@@ -115,7 +78,7 @@ def _download_mujoco210():
     _download_url(url, out_path)
 
     # unzip mujoco
-    logger.info("Extracting {} into {}".format(out_path, MUJOCO_ROOT))
+    logging.info(f"Extracting {out_path} into {MUJOCO_ROOT}")
     with tarfile.open(out_path, "r:gz") as tar_ref:
         tar_ref.extractall(MUJOCO_ROOT)
     os.remove(out_path)
@@ -126,7 +89,7 @@ def _download_mujoco210():
     _download_url(key_url, key_filepath)
 
 
-def _download_mujoco_dm_control():
+def download_mujoco_dm_control():
     # example url: https://github.com/deepmind/mujoco/releases/download/2.3.1/mujoco-2.3.1-linux-x86_64.tar.gz
 
     # get operating system
@@ -137,7 +100,7 @@ def _download_mujoco_dm_control():
     }
     system = platform.system()
     if system not in systems:
-        raise ValueError("Unsupported system: {}".format(system))
+        raise ValueError(f"Unsupported system: {system}")
 
     # get architecture
     architectures = {
@@ -149,7 +112,7 @@ def _download_mujoco_dm_control():
     }
     arch = platform.machine().lower()
     if arch not in architectures:
-        raise ValueError("Unsupported architecture: {}".format(arch))
+        raise ValueError(f"Unsupported architecture: {arch}")
 
     # download mujoco
     url = "https://github.com/deepmind/mujoco/releases/download/2.3.1/mujoco-2.3.1-{}-{}.tar.gz"
@@ -162,13 +125,13 @@ def _download_mujoco_dm_control():
     _download_url(url, out_path)
 
     # untar mujoco
-    logger.info("Extracting {} into {}".format(out_path, MUJOCO_ROOT))
+    logging.info(f"Extracting {out_path} into {MUJOCO_ROOT}")
     with tarfile.open(out_path, "r:gz") as tar_ref:
         tar_ref.extractall(MUJOCO_ROOT)
     os.remove(out_path)
 
 
-def _download_atari_roms():
+def download_atari_roms():
     out_path = os.path.join(ATARI_ROOT, "roms.rar")
     out_dir = os.path.dirname(out_path)
     if os.path.exists(out_dir) and len(os.listdir(out_dir)) > 0:
@@ -179,50 +142,21 @@ def _download_atari_roms():
     _download_url(url, out_path)
 
     # extract the file
-    logger.info("Extracting {} into {}".format(out_path, out_dir))
+    logging.info(f"Extracting {out_path} into {out_dir}")
     with tarfile.open(out_path, "r:gz") as tar_ref:
         tar_ref.extractall(out_dir)
 
     # try import the roms
     try:
-        subprocess.call("ale-import-roms {}".format(out_dir), shell=True)
+        subprocess.call(f"ale-import-roms {out_dir}", shell=True)
     except Exception as e:
-        msg = "ALE ROMs have been downloaded and extracted but there \
-        was an error with ale-py while importing roms: {}. \
-        Please install helx first and try download the extra requirements again".format(
-            e
-        )
-        logger.error(msg)
-
-
-def _get_version():
-    filepath = os.path.join(CURRENT_DIR, "helx", "__init__.py")
-    with open(filepath) as f:
-        for line in f:
-            if line.startswith("__version__") and "=" in line:
-                version = line[line.find("=") + 1 :].strip(" '\"\n")
-                if version:
-                    return version
-    raise ValueError("`__version__` not defined in {}".format(filepath))
-
-
-def _parse_requirements():
-    filepath = os.path.join(CURRENT_DIR, REQUIREMENTS_PATH)
-    with open(filepath) as f:
-        return [
-            line.rstrip() for line in f if not (line.isspace() or line.startswith("#"))
-        ]
-
-
-setup(
-    name="Helx",
-    version=_get_version(),
-    description="Helx is a helper library for Reinforcement Learning for JAX",
-    author="Eduardo Pignatelli",
-    author_email="edu.pignatelli@gmail.com",
-    url="https://github.com/epignatelli/helx",
-    packages=find_packages(exclude=["experiments", "test", "examples"]),
-    python_requires=">=3.9",
-    install_requires=_parse_requirements(),
-    cmdclass={"install": PostInstallDeploy, "develop": PostInstallDevelop},
-)
+        msg = f"ALE ROMs have been downloaded and extracted but there \
+        was an error with ale-py while importing roms: {e}. \
+        Please install helx first and try download the extra requirements again"
+        logging.error(msg)
+
+
+def download_all():
+    download_atari_roms()
+    download_mujoco210()
+    download_mujoco_dm_control()
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..942a5c8
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,90 @@
+# This toml file is in comliance with PEP 517, PEP 518 and PEP 621
+# However, setuptools support for these is only in Beta
+# For now, `setup.py` is still the first option for pip
+[build-system]
+build_backend = "setuptools.build_meta"
+requires = ["setuptools >= 50", "setuptools-scm[toml]>=6.2", "wheel"]
+
+[project]
+name = "Helx"
+dynamic = ["version", "dependencies"]
+description = "Interoperate among reinforcement learning libraries with jax, pytorch, gym and dm_env"
+requires-python = ">=3.9"
+readme = "README.md"
+license = {file = "LICENSE", name = "Apache-2.0"}
+authors = [
+  {name = "Eduardo Pignatelli", email = "edu.pignatelli@gmail.com"},
+]
+maintainers = [
+  {name = "Eduardo Pignatelli", email = "edu.pignatelli@gmail.com"},
+]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Intended Audience :: Science/Research",
+  "Intended Audience :: Financial and Insurance Industry",
+  "Intended Audience :: Healthcare Industry",
+  "Environment :: GPU",
+  "Environment :: GPU :: NVIDIA CUDA",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.0",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.1",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.2",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.3",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.4",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.5",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.6",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.7",
+  "Environment :: GPU :: NVIDIA CUDA :: 11.8",
+  "Framework :: Pytest",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Topic :: Scientific/Engineering",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+  "Topic :: Software Development",
+  "Topic :: Software Development :: Libraries",
+  "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+
+[project.urls]
+homepage = "https://github.com/epignatelli/helx"
+repository = "https://github.com/epignatelli/helx"
+bug_tracker = "https://github.com/epignatelli/helx/issues"
+
+
+[project.optional-dependencies]
+atari = [
+    "AutoROM.accept-rom-license==0.4.2"
+    ]
+mujoco = [
+
+  ]
+all = [
+    "AutoROM.accept-rom-license==0.4.2"
+    ]
+
+
+[project.scripts]
+helx-download-extras = "helx.download:download_all"
+
+
+[tool.setuptools.dynamic]
+dependencies = {file = "./requirements.txt"}
+
+
+[tool.setuptools.packages.find]
+include = ["helx*"]
+exclude = ["test", "examples"]
+
+
+[tool.setuptools_scm]
+write_to = "helx/version.py"
+
+
+[tool.black]
+line-length = 88


Test patch:
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
deleted file mode 100644
index 2de82d8..0000000
--- a/.github/workflows/pytest.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-name: Pytest
-
-on:
-  workflow_call:
-
-jobs:
-  pytest:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Run unit tests with pytest
-      run: pytest test/
\ No newline at end of file
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
deleted file mode 100644
index e203fde..0000000
--- a/.github/workflows/test.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-name: Linting
-
-on: [push]
-
-jobs:
-  Test:
-    runs-on: ${{ matrix.os }}-latest
-    defaults:
-      run:
-        shell: bash -l {0}
-    strategy:
-      fail-fast: false
-      max-parallel: 5
-      matrix:
-        os: ["ubuntu"]
-    continue-on-error: false
-
-
-    steps:
-
-    - uses: actions/checkout@v2
-
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
-        miniforge-version: latest
-        environment-file: environment.yml
-        activate-environment: helx
-        auto-activate-base: false
-
-    - name: Setup helx
-      run: |
-        pip install . -v
-
-        conda env config vars set LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.mujoco/mujoco210/bin
-        conda env config vars set MJLIB_PATH=$HOME/.mujoco/mujoco210/bin/libmujoco210.so
-        conda env config vars set MUJOCO_PY_MUJOCO_PATH=$HOME/.mujoco/mujoco210
-        conda env config vars set LD_PRELOAD=$CONDA_PREFIX/x86_64-conda-linux-gnu/sysroot/usr/lib64/libGLdispatch.so.0
-
-
-    - name: Check code quality
-      run: |
-        pip install pylint
-        MESSAGE=$(pylint -ry $(git ls-files '*.py') ||:)
-        echo "$MESSAGE"
-
-    - name: Run unit tests with pytest
-      run: |
-        pytest test/
-
-    - name: Run examples
-      run: |
-        wandb login ${{ secrets.WANDB_API_KEY }}
-        for f in examples/*.py; do python $f; done
\ No newline at end of file