Repository: catalystneuro/nwb-conversion-tools (language: python)

Issue:
Standardize releases pipeline
We should standardize our releases pipeline. Right now, Github releases are usually not made prior to PyPI publishing and the github action for automatic PiPY publishing is not working.
I'm opening a PR where we can discuss possible ways of doing that.



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/ci-development.yml b/.github/workflows/ci-development.yml
new file mode 100644
index 000000000..661a1d190
--- /dev/null
+++ b/.github/workflows/ci-development.yml
@@ -0,0 +1,36 @@
+name: Example workflow for Codecov
+on: [push]
+jobs:
+  run:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v2
+    - run: git fetch --prune --unshallow --tags
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.7
+    - name: Install pip
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install this package, on development mode
+      env:
+          NWB_CONVERSION_INSTALL_MODE: development
+      run: |
+        pip install .
+    - name: Generate coverage report
+      run: |
+        pip install pytest
+        pip install pytest-cov
+        pytest --cov=./ --cov-report=xml
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: ./coverage.xml
+        flags: unittests
+        name: codecov-umbrella
+        yml: ./codecov.yml
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci-production.yml
similarity index 80%
rename from .github/workflows/ci.yml
rename to .github/workflows/ci-production.yml
index 46307a3b4..d4034cf61 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci-production.yml
@@ -8,15 +8,17 @@ jobs:
         os: [ubuntu-latest]
     steps:
     - uses: actions/checkout@v2
+    - run: git fetch --prune --unshallow --tags
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
         python-version: 3.7
-    - name: Install dependencies
+    - name: Install pip
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements-dev.txt
-    - name: Install this package
+    - name: Install this package, on production mode
+      env:
+          NWB_CONVERSION_INSTALL_MODE: production
       run: |
         pip install .
     - name: Generate coverage report
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
index f3b9268cf..a8f2e51ba 100644
--- a/.github/workflows/python-publish.yml
+++ b/.github/workflows/python-publish.yml
@@ -1,47 +1,30 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
 name: Upload Python Package
 
 on:
-  push:
-    # Sequence of patterns matched against refs/tags
-    tags:
-      - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10
+  release:
+    types: [published]
 
 jobs:
-  release:
-    name: Create Release
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@master
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
-        with:
-          tag_name: ${{ github.ref }}
-          release_name: Release ${{ github.ref }}
-          body: |
-            Changes in this Release
-          draft: false
-          prerelease: false
   deploy:
-    needs: release
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+      - name: Build package
+        run: python -m build
+      - name: pypi-publish
+        uses: pypa/gh-action-pypi-publish@v1.4.2
+        with:
+          verbose: true
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
\ No newline at end of file
diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml
deleted file mode 100644
index 21f2f01de..000000000
--- a/.github/workflows/pythonpublish.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-name: Upload Python Package
-
-on:
-  release:
-    types: [created]
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v1
-    - name: Set up Python
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
diff --git a/requirements-dev.txt b/requirements-dev.txt
index a3ba55484..079caddf7 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,16 +1,18 @@
-pynwb==1.4.0
-tqdm==4.45.0
-natsort==7.0.1
-numpy==1.18.2
-pandas==1.0.3
+pynwb==1.5.1
+tqdm==4.60.0
+natsort==7.1.1
+numpy==1.19.3
+pandas==1.2.3
 scipy==1.4.1
 h5py==2.10.0
 PyYAML==5.4
-git+https://github.com/spikeinterface/spikeextractors@master
-spikesorters==0.4.4
-spiketoolkit==0.7.4
-psutil==5.7.0
-roiextractors==0.3.1
-git+https://github.com/neuralensemble/python-neo@master
+jsonschema==3.2.0
+psutil==5.8.0
+lxml==4.6.3
 opencv-python==4.5.1.48
-ndx-events==0.2.0
+spikeextractors @ git+https://github.com/SpikeInterface/spikeextractors.git@master
+spikesorters @ git+https://github.com/SpikeInterface/spikesorters.git@master
+spiketoolkit @ git+https://github.com/SpikeInterface/spiketoolkit.git@master
+neo @ git+https://github.com/NeuralEnsemble/python-neo.git@master
+roiextractors==0.3.1
+ndx-events==0.2.0
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 2e6219498..5a8fb047a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,14 +1,16 @@
 pynwb==1.4.0
-tqdm==4.45.0
-natsort==7.0.1
-numpy==1.18.2
-pandas==1.0.3
+tqdm==4.60.0
+natsort==7.1.1
+numpy==1.19.3
+pandas==1.2.3
 scipy==1.4.1
 h5py==2.10.0
 PyYAML==5.4
+jsonschema==3.2.0
+psutil==5.8.0
+lxml==4.6.3
 spikeextractors==0.9.5
 spikesorters==0.4.4
 spiketoolkit==0.7.4
-psutil==5.7.0
-roiextractors==0.3.1
 neo==0.9.0
+roiextractors==0.3.1
\ No newline at end of file
diff --git a/setup.py b/setup.py
index f84711da4..fb668a646 100644
--- a/setup.py
+++ b/setup.py
@@ -1,15 +1,34 @@
 from setuptools import setup, find_packages
+import subprocess
 # To use a consistent encoding
 from codecs import open
-from os import path
+import os
 
 
-here = path.abspath(path.dirname(__file__))
+here = os.path.abspath(os.path.dirname(__file__))
 
-
-with open(path.join(here, 'README.md')) as f:
+# Get the long description from the README file
+with open(os.path.join(here, 'README.md')) as f:
     long_description = f.read()
 
+# Get requirements
+if os.environ.get('NWB_CONVERSION_INSTALL_MODE', None) == 'development':
+    req_file = 'requirements-dev.txt'
+    print("installing nwb_conversion_tools on development mode")
+else:
+    req_file = 'requirements.txt'
+
+with open(os.path.join(here, req_file)) as f:
+    install_requires = f.read().strip().split('\n')
+
+# Get remote version
+remote_version = (
+    subprocess.run(["git", "describe", "--tags"], stdout=subprocess.PIPE)
+    .stdout.decode("utf-8")
+    .strip()
+)
+assert "." in remote_version
+
 setup(
     name='nwb-conversion-tools',
     version='0.8.8',
@@ -21,12 +40,10 @@
     url='https://github.com/catalystneuro/nwb-conversion-tools',
     keywords='nwb',
     packages=find_packages(),
-    package_data={'': ['template_metafile.yml']},
+    package_data={"": ['template_metafile.yml']},
     include_package_data=True,
-    install_requires=[
-        'pynwb', 'tqdm', 'natsort', 'numpy', 'scipy', 'pandas', 'h5py',
-        'pyyaml', 'spikeextractors', 'spikesorters', 'spiketoolkit',
-        'roiextractors', 'jsonschema', 'lxml', 'psutil'],
+    python_requires=">=3.7",
+    install_requires=install_requires,
     entry_points={
         'console_scripts': ['nwb-gui=nwb_conversion_tools.gui.command_line:main'],
     }


Test patch:
diff --git a/tests/test_converter.py b/tests/test_converter.py
index 819829db1..19f9a68db 100644
--- a/tests/test_converter.py
+++ b/tests/test_converter.py
@@ -3,32 +3,38 @@
 from pathlib import Path
 
 from pynwb import NWBFile
-from ndx_events import LabeledEvents
+
+try:
+    from ndx_events import LabeledEvents
+    HAVE_NDX_EVENTS = True
+except ImportError:
+    HAVE_NDX_EVENTS = False
 
 from nwb_conversion_tools.basedatainterface import BaseDataInterface
 from nwb_conversion_tools import NWBConverter, TutorialRecordingInterface
 
 
 def test_converter():
-    test_dir = Path(mkdtemp())
-    nwbfile_path = str(test_dir / "extension_test.nwb")
-
-    class NdxEventsInterface(BaseDataInterface):
-        def run_conversion(self, nwbfile: NWBFile, metadata: dict):
-            events = LabeledEvents(
-                name="LabeledEvents",
-                description="events from my experiment",
-                timestamps=[0., 0.5, 0.6, 2., 2.05, 3., 3.5, 3.6, 4.],
-                resolution=1e-5,
-                data=[0, 1, 2, 3, 5, 0, 1, 2, 4],
-                labels=["trial_start", "cue_onset", "cue_offset", "response_left", "response_right", "reward"]
-            )
-            nwbfile.add_acquisition(events)
-
-    class ExtensionTestNWBConverter(NWBConverter):
-        data_interface_classes = dict(NdxEvents=NdxEventsInterface)
-
-    converter = ExtensionTestNWBConverter(source_data=dict(NdxEvents=dict()))
-    converter.run_conversion(nwbfile_path=nwbfile_path, overwrite=True)
-
-    rmtree(test_dir)
+    if HAVE_NDX_EVENTS:
+        test_dir = Path(mkdtemp())
+        nwbfile_path = str(test_dir / "extension_test.nwb")
+
+        class NdxEventsInterface(BaseDataInterface):
+            def run_conversion(self, nwbfile: NWBFile, metadata: dict):
+                events = LabeledEvents(
+                    name="LabeledEvents",
+                    description="events from my experiment",
+                    timestamps=[0., 0.5, 0.6, 2., 2.05, 3., 3.5, 3.6, 4.],
+                    resolution=1e-5,
+                    data=[0, 1, 2, 3, 5, 0, 1, 2, 4],
+                    labels=["trial_start", "cue_onset", "cue_offset", "response_left", "response_right", "reward"]
+                )
+                nwbfile.add_acquisition(events)
+
+        class ExtensionTestNWBConverter(NWBConverter):
+            data_interface_classes = dict(NdxEvents=NdxEventsInterface)
+
+        converter = ExtensionTestNWBConverter(source_data=dict(NdxEvents=dict()))
+        converter.run_conversion(nwbfile_path=nwbfile_path, overwrite=True)
+
+        rmtree(test_dir)
diff --git a/tests/test_interfaces.py b/tests/test_interfaces.py
index de632327f..3782622cb 100644
--- a/tests/test_interfaces.py
+++ b/tests/test_interfaces.py
@@ -7,7 +7,11 @@
 
 import pytest
 
-import cv2
+try:
+    import cv2
+    HAVE_OPENCV = True
+except ImportError:
+    HAVE_OPENCV = False
 
 from nwb_conversion_tools import interface_list
 from nwb_conversion_tools import NWBConverter
@@ -27,69 +31,70 @@ def test_interface_conversion_options_schema(data_interface):
 
 
 def test_movie_interface():
-    test_dir = Path(mkdtemp())
-    movie_file = test_dir / "test1.avi"
-    nwbfile_path = str(test_dir / "test1.nwb")
-    (nf, nx, ny) = (50, 640, 480)
-    writer = cv2.VideoWriter(
-        filename=str(movie_file),
-        apiPreference=None,
-        fourcc=cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
-        fps=25,
-        frameSize=(ny, nx),
-        params=None
-    )
-    for k in range(nf):
-        writer.write(np.random.randint(0, 255, (nx, ny, 3)).astype('uint8'))
-    writer.release()
-
-    class MovieTestNWBConverter(NWBConverter):
-        data_interface_classes = dict(Movie=MovieInterface)
-    source_data = dict(Movie=dict(file_paths=[movie_file]))
-    converter = MovieTestNWBConverter(source_data)
-    metadata = converter.get_metadata()
+    if HAVE_OPENCV:
+        test_dir = Path(mkdtemp())
+        movie_file = test_dir / "test1.avi"
+        nwbfile_path = str(test_dir / "test1.nwb")
+        (nf, nx, ny) = (50, 640, 480)
+        writer = cv2.VideoWriter(
+            filename=str(movie_file),
+            apiPreference=None,
+            fourcc=cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'),
+            fps=25,
+            frameSize=(ny, nx),
+            params=None
+        )
+        for k in range(nf):
+            writer.write(np.random.randint(0, 255, (nx, ny, 3)).astype('uint8'))
+        writer.release()
 
-    # Default usage
-    converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, overwrite=True)
+        class MovieTestNWBConverter(NWBConverter):
+            data_interface_classes = dict(Movie=MovieInterface)
+        source_data = dict(Movie=dict(file_paths=[movie_file]))
+        converter = MovieTestNWBConverter(source_data)
+        metadata = converter.get_metadata()
 
-    # This conversion option operates independently of all others
-    converter.run_conversion(
-        metadata=metadata,
-        nwbfile_path=nwbfile_path,
-        overwrite=True,
-        conversion_options=dict(Movie=dict(starting_times=[123.]))
-    )
+        # Default usage
+        converter.run_conversion(metadata=metadata, nwbfile_path=nwbfile_path, overwrite=True)
 
-    # These conversion options do not operate independently, so test them jointly
-    conversion_options_testing_matrix = [
-        dict(Movie=dict(external_mode=False, stub_test=x, chunk_data=y)) for x, y in product([True, False], repeat=2)
-    ]
-    for conversion_options in conversion_options_testing_matrix:
+        # This conversion option operates independently of all others
         converter.run_conversion(
             metadata=metadata,
             nwbfile_path=nwbfile_path,
             overwrite=True,
-            conversion_options=conversion_options
+            conversion_options=dict(Movie=dict(starting_times=[123.]))
         )
 
-    module_name = "TestModule"
-    module_description = "This is a test module."
-    nwbfile = converter.run_conversion(
-        metadata=metadata,
-        save_to_file=False
-    )
-    assert f"Video: {Path(movie_file).stem}" in nwbfile.acquisition
-    nwbfile = converter.run_conversion(
-        metadata=metadata,
-        save_to_file=False,
-        nwbfile=nwbfile,
-        conversion_options=dict(Movie=dict(module_name=module_name)),
-    )
-    assert module_name in nwbfile.modules
-    nwbfile = converter.run_conversion(
-        metadata=metadata,
-        save_to_file=False,
-        conversion_options=dict(Movie=dict(module_name=module_name, module_description=module_description)),
-    )
-    assert module_name in nwbfile.modules and nwbfile.modules[module_name].description == module_description
-    rmtree(test_dir)
+        # These conversion options do not operate independently, so test them jointly
+        conversion_options_testing_matrix = [
+            dict(Movie=dict(external_mode=False, stub_test=x, chunk_data=y)) for x, y in product([True, False], repeat=2)
+        ]
+        for conversion_options in conversion_options_testing_matrix:
+            converter.run_conversion(
+                metadata=metadata,
+                nwbfile_path=nwbfile_path,
+                overwrite=True,
+                conversion_options=conversion_options
+            )
+
+        module_name = "TestModule"
+        module_description = "This is a test module."
+        nwbfile = converter.run_conversion(
+            metadata=metadata,
+            save_to_file=False
+        )
+        assert f"Video: {Path(movie_file).stem}" in nwbfile.acquisition
+        nwbfile = converter.run_conversion(
+            metadata=metadata,
+            save_to_file=False,
+            nwbfile=nwbfile,
+            conversion_options=dict(Movie=dict(module_name=module_name)),
+        )
+        assert module_name in nwbfile.modules
+        nwbfile = converter.run_conversion(
+            metadata=metadata,
+            save_to_file=False,
+            conversion_options=dict(Movie=dict(module_name=module_name, module_description=module_description)),
+        )
+        assert module_name in nwbfile.modules and nwbfile.modules[module_name].description == module_description
+        rmtree(test_dir)