Repository: ESMValGroup/ESMValCore

Issue:
python setup.py test is deprecated
So our documentation on running tests as well as the test scripts that run on CircleCi need to be updated.

Discussion:
valeriupredoi: but what in the world is going on with the install tests? Are we asking for too many threads?
bouweandela: Looks like the CircleCI docker containers actually report the number of cores available on their host machine.
bouweandela: @valeriupredoi I fixed the problem with too many test runners on CircleCI and changed the way we run tests in the process. Could you please have another look?
valeriupredoi: @bouweandela all good for me with the pinning done in https://github.com/ESMValGroup/ESMValTool/pull/1804 - do you want to set the pinning here too or I can do it :beer: 
valeriupredoi: awesome! roll out @bouweandela :medal_military: 

Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 09e597e53..f0ea8afc7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1,19 +1,22 @@
 ---
-version: 2
+version: 2.1
 
-aliases:
-  check_changes: &check_changes
-    name: Check changes
-    command: |
-      if (test "$CIRCLE_BRANCH" = master ||
-          git --no-pager diff --name-only origin/master... |
-          grep -q -E -f .circleci/install_triggers)
-      then
-        echo Running installation tests
-      else
-        echo Skipping installation tests
-        circleci step halt
-      fi
+orbs:
+  coverage-reporter: codacy/coverage-reporter@11.1.1
+
+commands:
+  check_changes:
+    steps:
+      - run: |
+          if (test "$CIRCLE_BRANCH" = master ||
+            git --no-pager diff --name-only origin/master... |
+            grep -q -E -f .circleci/install_triggers)
+          then
+            echo Running installation tests
+          else
+            echo Skipping installation tests
+            circleci step halt
+          fi
 
 jobs:
   test:
@@ -25,7 +28,8 @@ jobs:
       - checkout
       - run:
           command: |
-            python setup.py test --addopts '-m "not installation"'
+            pip install .[test]
+            pytest -n 2 -m "not installation"
       - store_test_results:
           path: test-reports/
       - store_artifacts:
@@ -38,7 +42,7 @@ jobs:
       - image: continuumio/miniconda3
     steps:
       - checkout
-      - run: *check_changes
+      - check_changes
       - restore_cache:
           key: python3-install-{{ .Branch }}
       - run:
@@ -51,13 +55,13 @@ jobs:
             conda env update >> /logs/conda.txt 2>&1
             set +x; conda activate esmvaltool; set -x
             conda install -yS r-base r-yaml ncl -c conda-forge
-            pip install . > /logs/install.txt 2>&1
+            pip install .[test] > /logs/install.txt 2>&1
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
             pip freeze > /logs/requirements.txt
             # Test installation
-            python setup.py test
+            pytest -n 2
             esmvaltool version
       - save_cache:
           key: python3-install-{{ .Branch }}
@@ -70,17 +74,9 @@ jobs:
           path: test-reports/
       - store_test_results:
           path: test-reports/
-      - run:
-          when: always
-          command: |
-            if [[ -v CODACY_PROJECT_TOKEN ]]
-            then
-              echo Uploading coverage report
-              pip install codacy-coverage
-              python-codacy-coverage -r test-reports/coverage.xml
-            else
-              echo Not uploading coverage report
-            fi
+      - coverage-reporter/send_report:
+          coverage-reports: 'test-reports/coverage.xml'
+          project-token: $CODACY_PROJECT_TOKEN
 
   documentation:
     # Test building documentation
@@ -116,7 +112,7 @@ jobs:
       - image: continuumio/miniconda3
     steps:
       - checkout
-      - run: *check_changes
+      - check_changes
       - run:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
@@ -134,7 +130,7 @@ jobs:
             pip freeze > /logs/requirements.txt
             # Test installation
             esmvaltool version
-            python setup.py test
+            pytest -n 2
       - store_artifacts:
           path: /logs
 
@@ -145,7 +141,7 @@ jobs:
       - image: continuumio/miniconda3
     steps:
       - checkout
-      - run: *check_changes
+      - check_changes
       - run:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
@@ -184,10 +180,8 @@ jobs:
             conda env export > /logs/environment.yml
             # Test installation
             esmvaltool version
-            python -c 'from esmvalcore import *; print(__version__)'
 
 workflows:
-  version: 2
   commit:
     jobs:
       - test
diff --git a/doc/contributing.rst b/doc/contributing.rst
index 5fee80b1d..8baba0a9f 100644
--- a/doc/contributing.rst
+++ b/doc/contributing.rst
@@ -27,9 +27,9 @@ Running tests
 -------------
 
 Go to the directory where the repository is cloned and run
-``python setup.py test``. Optionally you can skip tests which require
+``pytest``. Optionally you can skip tests which require
 additional dependencies for supported diagnostic script languages by
-adding ``--addopts '-m "not installation"'`` to the previous command.
+adding ``-m 'not installation'`` to the previous command.
 Tests will also be run automatically by
 `CircleCI <https://circleci.com/gh/ESMValGroup/ESMValCore>`__.
 
diff --git a/package/meta.yaml b/package/meta.yaml
index 9b919a856..4996b4f77 100644
--- a/package/meta.yaml
+++ b/package/meta.yaml
@@ -58,14 +58,31 @@ requirements:
     - yamale==2.*  # in esmvalgroup channel
 
 test:
-  # TODO: add unit tests? This seems to require installing the tests
+  source_files:
+    - tests
+    - setup.cfg
+  requires:
+    - pytest>=3.9,!=6.0.0rc1,!=6.0.0
+    - pytest-cov
+    - pytest-env
+    - pytest-flake8
+    - pytest-html!=2.1.0
+    - pytest-metadata>=1.5.1
+    - pytest-mock
+    - pytest-xdist
+    - r-yaml
+    - ncl
+  commands:
+    - pytest -n 2 --ignore=run_test.py
+    - esmvaltool -- --help
+    - esmvaltool version
   imports:
     - esmvalcore
     - esmvalcore.cmor
+    - esmvalcore.cmor.check
+    - esmvalcore.cmor.fix
     - esmvalcore.preprocessor
 
-  commands:
-    - esmvaltool -h
 
 about:
   home: https://www.esmvaltool.org
diff --git a/setup.cfg b/setup.cfg
index ca856b2ae..dc7036dc6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,3 @@
-[aliases]
-test=pytest
-
 [build_sphinx]
 source-dir = doc/
 build-dir = doc/build
@@ -16,6 +13,7 @@ addopts =
     --cov-report=xml:test-reports/coverage.xml
     --cov-report=html:test-reports/coverage_html
     --html=test-reports/report.html
+    --numprocesses auto
 env =
     MPLBACKEND = Agg
 flake8-ignore =
diff --git a/setup.py b/setup.py
index b6783ef28..7a875b65c 100755
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,6 @@ PACKAGES = [
 REQUIREMENTS = {
     # Installation script (this file) dependencies
     'setup': [
-        'pytest-runner',
         'setuptools_scm',
     ],
     # Installation dependencies
@@ -50,12 +49,13 @@ REQUIREMENTS = {
     # Execute 'python setup.py test' to run tests
     'test': [
         'pytest>=3.9,!=6.0.0rc1,!=6.0.0',
-        'pytest-cov',
+        'pytest-cov>=2.10.1',
         'pytest-env',
-        'pytest-flake8',
+        'pytest-flake8>=1.0.6',
         'pytest-html!=2.1.0',
         'pytest-metadata>=1.5.1',
         'pytest-mock',
+        'pytest-xdist',
     ],
     # Development dependencies
     # Use pip install -e .[develop] to install in development mode


Test patch:
diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py
index 750e52f07..f716b518b 100644
--- a/tests/integration/cmor/test_table.py
+++ b/tests/integration/cmor/test_table.py
@@ -3,6 +3,7 @@
 import os
 import unittest
 
+import esmvalcore.cmor
 from esmvalcore.cmor.table import CMIP3Info, CMIP5Info, CMIP6Info, CustomInfo
 
 
@@ -25,9 +26,8 @@ class TestCMIP6Info(unittest.TestCase):
 
     def test_custom_tables_location(self):
         """Test constructor with custom tables location."""
-        cwd = os.path.dirname(os.path.realpath(__file__))
-        cmor_tables_path = os.path.join(cwd, '..', '..', '..', 'esmvalcore',
-                                        'cmor', 'tables', 'cmip6')
+        cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__))
+        cmor_tables_path = os.path.join(cmor_path, 'tables', 'cmip6')
         cmor_tables_path = os.path.abspath(cmor_tables_path)
         CMIP6Info(cmor_tables_path, default=None, strict=False)
 
@@ -122,9 +122,8 @@ class Testobs4mipsInfo(unittest.TestCase):
 
     def test_custom_tables_location(self):
         """Test constructor with custom tables location."""
-        cwd = os.path.dirname(os.path.realpath(__file__))
-        cmor_tables_path = os.path.join(cwd, '..', '..', '..', 'esmvalcore',
-                                        'cmor', 'tables', 'cmip6')
+        cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__))
+        cmor_tables_path = os.path.join(cmor_path, 'tables', 'cmip6')
         cmor_tables_path = os.path.abspath(cmor_tables_path)
         CMIP6Info(cmor_tables_path, None, True)
 
@@ -190,9 +189,8 @@ class TestCMIP5Info(unittest.TestCase):
 
     def test_custom_tables_location(self):
         """Test constructor with custom tables location."""
-        cwd = os.path.dirname(os.path.realpath(__file__))
-        cmor_tables_path = os.path.join(cwd, '..', '..', '..', 'esmvalcore',
-                                        'cmor', 'tables', 'cmip5')
+        cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__))
+        cmor_tables_path = os.path.join(cmor_path, 'tables', 'cmip5')
         cmor_tables_path = os.path.abspath(cmor_tables_path)
         CMIP5Info(cmor_tables_path, None, True)
 
@@ -265,9 +263,8 @@ class TestCMIP3Info(unittest.TestCase):
 
     def test_custom_tables_location(self):
         """Test constructor with custom tables location."""
-        cwd = os.path.dirname(os.path.realpath(__file__))
-        cmor_tables_path = os.path.join(cwd, '..', '..', '..', 'esmvalcore',
-                                        'cmor', 'tables', 'cmip3')
+        cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__))
+        cmor_tables_path = os.path.join(cmor_path, 'tables', 'cmip3')
         cmor_tables_path = os.path.abspath(cmor_tables_path)
         CMIP3Info(cmor_tables_path, None, True)
 
@@ -337,10 +334,8 @@ class TestCORDEXInfo(unittest.TestCase):
 
     def test_custom_tables_location(self):
         """Test constructor with custom tables location."""
-        cwd = os.path.dirname(os.path.realpath(__file__))
-        cmor_tables_path = os.path.join(cwd, '..', '..', '..', 'esmvalcore',
-                                        'cmor', 'tables', 'cordex')
-        cmor_tables_path = os.path.abspath(cmor_tables_path)
+        cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__))
+        cmor_tables_path = os.path.join(cmor_path, 'tables', 'cordex')
         CMIP5Info(cmor_tables_path)
 
     def test_get_variable_tas(self):
@@ -367,9 +362,8 @@ class TestCustomInfo(unittest.TestCase):
 
     def test_custom_tables_location(self):
         """Test constructor with custom tables location."""
-        cwd = os.path.dirname(os.path.realpath(__file__))
-        cmor_tables_path = os.path.join(cwd, '..', '..', '..', 'esmvalcore',
-                                        'cmor', 'tables', 'cmip5')
+        cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__))
+        cmor_tables_path = os.path.join(cmor_path, 'tables', 'cmip5')
         cmor_tables_path = os.path.abspath(cmor_tables_path)
         CustomInfo(cmor_tables_path)
 
diff --git a/tests/unit/preprocessor/_area/test_area.py b/tests/unit/preprocessor/_area/test_area.py
index f16ca8bf3..f1c67a1fb 100644
--- a/tests/unit/preprocessor/_area/test_area.py
+++ b/tests/unit/preprocessor/_area/test_area.py
@@ -1,6 +1,6 @@
 """Unit tests for the :func:`esmvalcore.preprocessor._area` module."""
-
 import unittest
+from pathlib import Path
 
 import fiona
 import iris
@@ -11,6 +11,7 @@ from iris.cube import Cube
 from shapely.geometry import Polygon, mapping
 
 import tests
+import esmvalcore.preprocessor
 from esmvalcore.preprocessor._area import (_crop_cube, area_statistics,
                                            extract_named_regions,
                                            extract_region, extract_shape)
@@ -569,10 +570,17 @@ def test_crop_cube_with_ne_file_imitation():
     np.testing.assert_allclose(result, expected)
 
 
-def test_crop_cube_with_ne_file():
+@pytest.fixture
+def ne_ocean_shapefile():
+    """Path to natural earth 50m ocean shapefile."""
+    preproc_path = Path(esmvalcore.preprocessor.__file__).parent
+    shapefile = preproc_path / "ne_masks" / "ne_50m_ocean.shp"
+    return str(shapefile)
+
+
+def test_crop_cube_with_ne_file(ne_ocean_shapefile):
     """Test for cropping a cube by shape bounds."""
-    shp_file = "esmvalcore/preprocessor/ne_masks/ne_50m_ocean.shp"
-    with fiona.open(shp_file) as geometries:
+    with fiona.open(ne_ocean_shapefile) as geometries:
         cube = _create_sample_full_cube()
         result = _crop_cube(cube, *geometries.bounds, cmor_coords=False)
         result = (result.coord("latitude").points[-1],
@@ -598,22 +606,22 @@ def test_extract_shape(make_testcube, square_shape, tmp_path, crop):
     np.testing.assert_array_equal(result.data.mask, expected.mask)
 
 
-def test_extract_shape_natural_earth(make_testcube):
+def test_extract_shape_natural_earth(make_testcube, ne_ocean_shapefile):
     """Test for extracting a shape from NE file."""
     expected = np.ones((5, 5))
     result = extract_shape(
         make_testcube,
-        "esmvalcore/preprocessor/ne_masks/ne_50m_ocean.shp",
+        ne_ocean_shapefile,
         crop=False)
     np.testing.assert_array_equal(result.data.data, expected)
 
 
-def test_extract_shape_ne_check_nans():
+def test_extract_shape_ne_check_nans(ne_ocean_shapefile):
     """Test shape from NE file with check for boundary NaN's."""
     cube = _create_sample_full_cube()
     result = extract_shape(
         cube,
-        "esmvalcore/preprocessor/ne_masks/ne_50m_ocean.shp",
+        ne_ocean_shapefile,
         crop=False)
     assert not result[:, 90, 180].data.mask.all()