Repository: databio/pypiper (language: python)

Issue:
Python 3.10 error on importing from collections
pypiper/manager.py:11: in <module>
    from collections import Iterable
E   ImportError: cannot import name 'Iterable' from 'collections' (/opt/hostedtoolcache/Python/3.10.2/x64/lib/python3.10/collections/__init__.py)
Python 3.10 error on importing from collections
pypiper/manager.py:11: in <module>
    from collections import Iterable
E   ImportError: cannot import name 'Iterable' from 'collections' (/opt/hostedtoolcache/Python/3.10.2/x64/lib/python3.10/collections/__init__.py)


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
new file mode 100644
index 00000000..9705deab
--- /dev/null
+++ b/.github/workflows/black.yml
@@ -0,0 +1,11 @@
+name: Lint
+
+on: [pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: psf/black@20.8b1
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
new file mode 100644
index 00000000..48c52e13
--- /dev/null
+++ b/.github/workflows/python-publish.yml
@@ -0,0 +1,30 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*
diff --git a/docs/changelog.md b/docs/changelog.md
index 139fffa4..ddecfdf5 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -1,6 +1,11 @@
 # Changelog
 
 
+## [0.12.3] -- 2022-01-25
+
+### Fixed
+- A few bugs with compatibility with Python version 3.9
+
 ## [0.12.2] -- 2021-12-20
 
 ### Fixed
diff --git a/pypiper/_version.py b/pypiper/_version.py
index 76da4a98..8e1395bd 100644
--- a/pypiper/_version.py
+++ b/pypiper/_version.py
@@ -1,1 +1,1 @@
-__version__ = "0.12.2"
+__version__ = "0.12.3"
diff --git a/pypiper/manager.py b/pypiper/manager.py
index 6c5b8467..aa43389a 100644
--- a/pypiper/manager.py
+++ b/pypiper/manager.py
@@ -8,7 +8,7 @@
 """
 
 import atexit
-from collections import Iterable
+from collections.abc import Iterable
 import datetime
 import errno
 import glob
diff --git a/pypiper/ngstk.py b/pypiper/ngstk.py
index fff2e554..dcc57e8e 100755
--- a/pypiper/ngstk.py
+++ b/pypiper/ngstk.py
@@ -301,7 +301,7 @@ class of inputs (which can in turn be a string or a list).
         if any(isinstance(i, list) for i in input_args):
             # We have a list of lists. Process each individually.
             local_input_files = list()
-            n_input_files = len(filter(bool, input_args))
+            n_input_files = len(list(filter(bool, input_args)))
             print("Number of input file sets: " + str(n_input_files))
 
             for input_i, input_arg in enumerate(input_args):
@@ -518,8 +518,8 @@ def temp_func(input_files=input_files, output_files=output_files,
             if type(output_files) != list:
                 output_files = [output_files]
 
-            n_input_files = len(filter(bool, input_files))
-            n_output_files = len(filter(bool, output_files))
+            n_input_files = len(list(filter(bool, input_files)))
+            n_output_files = len(list(filter(bool, output_files)))
 
             total_reads = sum([int(self.count_reads(input_file, paired_end))
                                for input_file in input_files])
@@ -1641,7 +1641,7 @@ def get_read_type(self, bam_file, n=10):
         :param int n: Number of lines to read from bam file.
         :return str, int: tuple of read type and read length
         """
-        from collections import Counter
+        from collections.abc import Counter
         try:
             p = subprocess.Popen([self.tools.samtools, 'view', bam_file],
                                  stdout=subprocess.PIPE)
diff --git a/pypiper/pipeline.py b/pypiper/pipeline.py
index 0a4dbc09..69474c22 100644
--- a/pypiper/pipeline.py
+++ b/pypiper/pipeline.py
@@ -1,14 +1,10 @@
 """ Pipeline base class """
 
 import abc
-from collections import OrderedDict
 import glob
 import os
 import sys
-if sys.version_info < (3, 3):
-    from collections import Iterable, Mapping
-else:
-    from collections.abc import Iterable, Mapping
+from collections.abc import Iterable, Mapping
 
 from .exceptions import \
     IllegalPipelineDefinitionError, IllegalPipelineExecutionError, \
@@ -396,9 +392,9 @@ def _is_unordered(collection):
     if not isinstance(collection, Iterable):
         raise TypeError("Non-iterable alleged collection: {}".
                         format(type(collection)))
-    return isinstance(collection, set) or \
-           (isinstance(collection, dict) and
-            not isinstance(collection, OrderedDict))
+
+
+    return isinstance(collection, set) or isinstance(collection, dict)
 
 
 
diff --git a/pypiper/utils.py b/pypiper/utils.py
index 67851aab..43eb789b 100644
--- a/pypiper/utils.py
+++ b/pypiper/utils.py
@@ -1,6 +1,6 @@
 """ Shared utilities """
 
-from collections import Iterable, Mapping, Sequence
+from collections.abc import Iterable, Mapping, Sequence
 import os
 import sys
 import re
@@ -799,24 +799,21 @@ def _determine_args(argument_groups, arguments, use_all_args=False):
     :return set[str]: Collection of (unique) argument names to add to a parser.
     """
 
-    if sys.version_info < (3, 3):
-        from collections import Iterable
-    else:
-        from collections.abc import Iterable
+    from collections.abc import Iterable
 
 
     from logmuse import LOGGING_CLI_OPTDATA
     # Define the argument groups.
     args_by_group = {
-        "pypiper": ["recover", "new-start", "dirty", "force-follow", "testmode"] +
-            LOGGING_CLI_OPTDATA.keys(),
+        "pypiper": ["recover", "new-start", "dirty", "force-follow", "testmode"]
+        + [*LOGGING_CLI_OPTDATA],
         "config": ["config"],
         "checkpoint": ["stop-before", "stop-after"],
         "resource": ["mem", "cores"],
         "looper": ["config", "output-parent", "mem", "cores"],
         "common": ["input", "sample-name"],
         "ngs": ["sample-name", "input", "input2", "genome", "single-or-paired"],
-        "logmuse": LOGGING_CLI_OPTDATA.keys()
+        "logmuse": [*LOGGING_CLI_OPTDATA]
     }
 
     # Handle various types of group specifications.
diff --git a/requirements/reqs-build-release.txt b/requirements/reqs-build-release.txt
deleted file mode 100644
index f82c9749..00000000
--- a/requirements/reqs-build-release.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-readme_renderer[md]
-twine>=1.12
diff --git a/requirements/reqs-docs.txt b/requirements/requirements-docs.txt
similarity index 100%
rename from requirements/reqs-docs.txt
rename to requirements/requirements-docs.txt
diff --git a/requirements/reqs-ngstk.txt b/requirements/requirements-ngstk.txt
similarity index 100%
rename from requirements/reqs-ngstk.txt
rename to requirements/requirements-ngstk.txt
diff --git a/requirements/reqs-plot.txt b/requirements/requirements-plot.txt
similarity index 100%
rename from requirements/reqs-plot.txt
rename to requirements/requirements-plot.txt
diff --git a/requirements/reqs-pypiper.txt b/requirements/requirements-pypiper.txt
similarity index 100%
rename from requirements/reqs-pypiper.txt
rename to requirements/requirements-pypiper.txt
diff --git a/setup.py b/setup.py
index d706171a..a689838b 100644
--- a/setup.py
+++ b/setup.py
@@ -18,7 +18,7 @@
 def read_reqs_file(reqs_name):
     """ Read requirements file for given requirements group. """
     path_reqs_file = os.path.join(
-            "requirements", "reqs-{}.txt".format(reqs_name))
+            "requirements", "requirements-{}.txt".format(reqs_name))
     with open(path_reqs_file, 'r') as reqs_file:
         return [pkg.rstrip() for pkg in reqs_file.readlines()
                 if not pkg.startswith("#")]


Test patch:
diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml
new file mode 100644
index 00000000..6bf573b1
--- /dev/null
+++ b/.github/workflows/run-pytest.yml
@@ -0,0 +1,35 @@
+name: Run pytests
+
+on:
+  pull_request:
+    branches: [master, dev]
+  workflow_dispatch:
+    inputs: null
+    
+jobs:
+  pytest:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dev dependencies
+        run: if [ -f requirements/requirements-dev.txt ]; then pip install -r requirements/requirements-dev.txt; fi
+
+      - name: Install test dependencies
+        run: if [ -f requirements/requirements-test.txt ]; then pip install -r requirements/requirements-test.txt; fi
+
+      - name: Install package
+        run: python -m pip install .
+
+      - name: Run pytest tests
+        run: pytest tests -x -vv --remote-data
diff --git a/requirements/reqs-test.txt b/requirements/requirements-test.txt
similarity index 82%
rename from requirements/reqs-test.txt
rename to requirements/requirements-test.txt
index 0d929f21..38cfaf32 100644
--- a/requirements/reqs-test.txt
+++ b/requirements/requirements-test.txt
@@ -4,3 +4,4 @@ hypothesis
 coveralls>=1.1
 pytest-cov==2.6.1
 veracitools
+pytest-remotedata