Repository: sktime/sktime

Issue:
[DOC] reference to `pip install -e .[all_extras]` in installation instructions
The developer instructions should  make explicit reference to the command `pip install -e .[all_extras]`, for the option "with all extras" of the devlopment version install (similar to the stable version).

The file to edit is `docs/source/installation`.
[BUG] multiple tests related to `FRESH` fail on all platforms and python versions
**Describe the bug**
<!--
A clear and concise description of what the bug is.
-->

I found another test failing on all platforms and python versions.

<details>
<summary>test_fresh_prince_on_unit_test_data</summary>

```python
================================== FAILURES ===================================
_____________________ test_fresh_prince_on_unit_test_data _____________________
[gw0] win32 -- Python 3.8.5 C:\Miniconda\envs\test\python.exe

    def test_fresh_prince_on_unit_test_data():
        """Test of FreshPRINCE on unit test data."""
        # load unit test data
        X_train, y_train = load_unit_test(split="train")
        X_test, y_test = load_unit_test(split="test")
        indices = np.random.RandomState(0).choice(len(y_train), 10, replace=False)
    
        # train FreshPRINCE classifier
        fp = FreshPRINCE(
            random_state=0,
            default_fc_parameters="minimal",
            n_estimators=10,
            save_transformed_data=True,
        )
        fp.fit(X_train, y_train)
    
        # assert probabilities are the same
        probas = fp.predict_proba(X_test.iloc[indices])
>       testing.assert_array_almost_equal(probas, fp_classifier_unit_test_probas, decimal=2)
E       AssertionError: 
E       Arrays are not almost equal to 2 decimals
E       
E       Mismatched elements: 10 / 20 (50%)
E       Max absolute difference: 0.2
E       Max relative difference: 1.
E        x: array([[0.2, 0.8],
E              [1. , 0. ],
E              [0. , 1. ],...
E        y: array([[0.2, 0.8],
E              [1. , 0. ],
E              [0.1, 0.9],...

X_test     =                                                 dim_0
0   0      501.0
1      328.0
2      195.0
3      ...
1   0     ...
3      ...
20  0       95.0
1       63.0
2       45.0
3      ...
21  0      193.0
1      195.0
2       90.0
3      ...
X_train    =                                                 dim_0
0   0      573.0
1      375.0
2      301.0
3      ...
1   0     ...
3      ...
18  0       95.0
1       63.0
2       45.0
3      ...
19  0       82.0
1       93.0
2       51.0
3      ...
fp         = FreshPRINCE(chunksize=None, default_fc_parameters='minimal', n_estimators=10,
            n_jobs=1, random_state=0, save_transformed_data=True, verbose=0)
indices    = array([18,  1, 19,  8, 10, 17,  6, 13,  4,  2])
probas     = array([[0.2, 0.8],
       [1. , 0. ],
       [0. , 1. ],
       [1. , 0. ],
       [1. , 0. ],
       [1. , 0. ],
       [1. , 0. ],
       [1. , 0. ],
       [1. , 0. ],
       [1. , 0. ]])
y_test     = array(['1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2',
       '2', '2', '2', '2', '2', '2', '2', '2', '2'], dtype='<U1')
y_train    = <details>
<summary>test_fresh_prince_on_unit_test_data</summary>array(['1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2', '2', '2',
       '2', '2', '2', '2', '2', '2', '2'], dtype='<U1')

C:\Miniconda\envs\test\lib\site-packages\sktime\classification\feature_based\tests\test_fresh_prince.py:29: AssertionError
```

</details>

<details>
<summary>test_tsfresh_classifier_on_unit_test_data failure</summary>

```python
__________________ test_tsfresh_classifier_on_unit_test_data __________________
[gw0] win32 -- Python 3.8.5 C:\Miniconda\envs\test\python.exe

    def test_tsfresh_classifier_on_unit_test_data():
        """Test of TSFreshClassifier on unit test data."""
        # load unit test data
        X_train, y_train = load_unit_test(split="train")
        X_test, y_test = load_unit_test(split="test")
        indices = np.random.RandomState(0).choice(len(y_train), 10, replace=False)
    
        # train TSFresh classifier
        tsfc = TSFreshClassifier(
            random_state=0,
            default_fc_parameters="minimal",
            relevant_feature_extractor=False,
            estimator=RandomForestClassifier(n_estimators=10),
        )
        tsfc.fit(X_train, y_train)
    
        # assert probabilities are the same
        probas = tsfc.predict_proba(X_test.iloc[indices])
>       testing.assert_array_almost_equal(
            probas, tsfresh_classifier_unit_test_probas, decimal=2
        )
E       AssertionError: 
E       Arrays are not almost equal to 2 decimals
E       
E       Mismatched elements: 10 / 20 (50%)
E       Max absolute difference: 0.2
E       Max relative difference: 2.
E        x: array([[0.1, 0.9],
E              [0.7, 0.3],
E              [0. , 1. ],...
E        y: array([[0. , 1. ],
E              [0.9, 0.1],
E              [0. , 1. ],...

X_test     =                                                 dim_0
0   0      501.0
1      328.0
2      195.0
3      ...
1   0     ...
3      ...
20  0       95.0
1       63.0
2       45.0
3      ...
21  0      193.0
1      195.0
2       90.0
3      ...
X_train    =                                                 dim_0
0   0      573.0
1      375.0
2      301.0
3      ...
1   0     ...
3      ...
18  0       95.0
1       63.0
2       45.0
3      ...
19  0       82.0
1       93.0
2       51.0
3      ...
indices    = array([18,  1, 19,  8, 10, 17,  6, 13,  4,  2])
probas     = array([[0.1, 0.9],
       [0.7, 0.3],
       [0. , 1. ],
       [0.9, 0.1],
       [0.8, 0.2],
       [1. , 0. ],
       [0.6, 0.4],
       [0.9, 0.1],
       [1. , 0. ],
       [0.9, 0.1]])
tsfc       = TSFreshClassifier(chunksize=None, default_fc_parameters='minimal',
                  estimator=RandomForestClassifier(...tart=False),
                  n_jobs=1, random_state=0, relevant_feature_extractor=False,
                  verbose=0)
y_test     = array(['1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2',
       '2', '2', '2', '2', '2', '2', '2', '2', '2'], dtype='<U1')
y_train    = array(['1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2', '2', '2',
       '2', '2', '2', '2', '2', '2', '2'], dtype='<U1')

C:\Miniconda\envs\test\lib\site-packages\sktime\classification\feature_based\tests\test_tsfresh_classifier.py:29: AssertionError
```

</details>

<details>
<summary>test_tsfresh_classifier_on_basic_motions</summary>

```python
__________________ test_tsfresh_classifier_on_basic_motions ___________________
[gw1] win32 -- Python 3.8.5 C:\Miniconda\envs\test\python.exe

    def test_tsfresh_classifier_on_basic_motions():
        """Test of TSFreshClassifier on basic motions."""
        # load basic motions data
        X_train, y_train = load_basic_motions(split="train")
        X_test, y_test = load_basic_motions(split="test")
        indices = np.random.RandomState(4).choice(len(y_train), 10, replace=False)
    
        # train TSFresh classifier
        tsfc = TSFreshClassifier(
            random_state=0,
            default_fc_parameters="minimal",
            relevant_feature_extractor=False,
            estimator=RandomForestClassifier(n_estimators=10),
        )
        tsfc.fit(X_train.iloc[indices], y_train[indices])
    
        # assert probabilities are the same
        probas = tsfc.predict_proba(X_test.iloc[indices])
>       testing.assert_array_almost_equal(
            probas, tsfresh_classifier_basic_motions_probas, decimal=2
        )
E       AssertionError: 
E       Arrays are not almost equal to 2 decimals
E       
E       Mismatched elements: 14 / 40 (35%)
E       Max absolute difference: 0.2
E       Max relative difference: 1.
E        x: array([[0. , 0. , 0.2, 0.8],
E              [0.4, 0.3, 0. , 0.3],
E              [0. , 0. , 0.8, 0.2],...
E        y: array([[0. , 0. , 0.2, 0.8],
E              [0.4, 0.2, 0.1, 0.3],
E              [0. , 0. , 0.9, 0.1],...

X_test     =                                                 dim_0  ...                                              dim_5
0   0   ...901645
1      0.901645
2     -0.05469...  ...  0     0.245030
1     0.245030
2     0.005327
3...

[40 rows x 6 columns]
X_train    =                                                 dim_0  ...                                              dim_5
0   0   ...11973
1     1.211973
2    -0.605948
3...  ...  0    -0.042614
1    -0.042614
2    -0.095881
3...

[40 rows x 6 columns]
indices    = array([28, 39,  4, 15, 26, 20, 31,  7, 16, 11])
probas     = array([[0. , 0. , 0.2, 0.8],
       [0.4, 0.3, 0. , 0.3],
       [0. , 0. , 0.8, 0.2],
       [0. , 1. , 0. , 0. ],
  ...],
       [0.5, 0.3, 0. , 0.2],
       [0. , 0. , 0.9, 0.1],
       [0.1, 0.9, 0. , 0. ],
       [0. , 1. , 0. , 0. ]])
tsfc       = TSFreshClassifier(chunksize=None, default_fc_parameters='minimal',
                  estimator=RandomForestClassifier(...tart=False),
                  n_jobs=1, random_state=0, relevant_feature_extractor=False,
                  verbose=0)
y_test     = array(['standing', 'standing', 'standing', 'standing', 'standing',
       'standing', 'standing', 'standing', 'standin...
       'badminton', 'badminton', 'badminton', 'badminton', 'badminton',
       'badminton', 'badminton'], dtype='<U9')
y_train    = array(['standing', 'standing', 'standing', 'standing', 'standing',
       'standing', 'standing', 'standing', 'standin...
       'badminton', 'badminton', 'badminton', 'badminton', 'badminton',
       'badminton', 'badminton'], dtype='<U9')

C:\Miniconda\envs\test\lib\site-packages\sktime\classification\feature_based\tests\test_tsfresh_classifier.py:52: AssertionError
```

</details>

**To Reproduce**

In order to reproduce you need to bump statsmodels to `>=0.13` or use sktime version from #1620 PR.

To recreate the env you can use:

```bash
conda install -c anaconda libpython
conda install -c anaconda "pystan==2.19.1.1"
conda install -c conda-forge "fbprophet>=0.7.1"
conda install -c conda-forge scipy cython matplotlib
python -m pip install .[all_extras,dev]
```

then run the test with:

```bash
python -m pytest --showlocals --durations=10 --cov-report=xml -n 2 --cov=sktime -v --pyargs sktime
```

**Expected behavior**
<!--
A clear and concise description of what you expected to happen.
-->
Test passes
**Additional context**
<!--
Add any other context about the problem here.
-->

**Versions**
<details>

<!--
Please run the following code snippet and paste the output here:
 
from sktime import show_versions; show_versions()
-->

</details>

<!-- Thanks for contributing! -->



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -5,149 +5,179 @@
 # adapted from https://github.com/scikit-learn/scikit-learn/blob/master
 # /setup.py
 
-__author__ = ["Markus Löning"]
+####################
+
+# Helpers for OpenMP support during the build.
+
+# adapted fom https://github.com/scikit-learn/scikit-learn/blob/master
+# /sklearn/_build_utils/openmp_helpers.py
+# This code is adapted for a large part from the astropy openmp helpers, which
+# can be found at: https://github.com/astropy/astropy-helpers/blob/master
+# /astropy_helpers/openmp_helpers.py  # noqa
+
+
+__author__ = ["Markus Löning", "lmmentel"]
 
 import codecs
-import importlib
+import glob
 import os
-import platform
-import re
 import shutil
+import subprocess
 import sys
-import traceback
+import tempfile
+import textwrap
 from distutils.command.clean import clean as Clean  # noqa
+from distutils.errors import CompileError, LinkError
+from distutils.extension import Extension
+from distutils.sysconfig import customize_compiler
+
+import numpy
+import toml
+from Cython.Build import cythonize
+from numpy.distutils.ccompiler import new_compiler
+from setuptools import find_packages
+
+pyproject = toml.load("pyproject.toml")
+
+CCODE = textwrap.dedent(
+    """\
+    #include <omp.h>
+    #include <stdio.h>
+    int main(void) {
+    #pragma omp parallel
+    printf("nthreads=%d\\n", omp_get_num_threads());
+    return 0;
+    }
+    """
+)
 
-from pkg_resources import parse_version
-
-MIN_PYTHON_VERSION = "3.6"
-MIN_REQUIREMENTS = {
-    "numpy": "1.19.3",
-    "pandas": "1.1.0",
-    "scikit-learn": "0.24.0",
-    "statsmodels": "0.12.1",
-    "numba": "0.53",
-    "deprecated": "1.2.13",
-}
-MAX_REQUIREMENTS = {
-    "statsmodels": "0.12.1",
-    "numpy": "1.19.3",
-}
-EXTRAS_REQUIRE = {
-    "all_extras": [
-        "cython>=0.29.0",
-        "matplotlib>=3.3.2",
-        "pmdarima>=1.8.0,!=1.8.1",
-        "scikit_posthocs>= 0.6.5",
-        "seaborn>=0.11.0",
-        "tsfresh>=0.17.0",
-        "hcrystalball>=0.1.9",
-        "stumpy>=1.5.1,<=1.9.2",
-        "tbats>=1.1.0",
-        "fbprophet>=0.7.1",
-        "pyod>=0.8.0",
-        "dtw_python>=1.1.5",
-    ],
-}
-
-HERE = os.path.abspath(os.path.dirname(__file__))
 
+def get_openmp_flag(compiler):
+    """Return an compiler and platform specific flag for OpenMP."""
+    if hasattr(compiler, "compiler"):
+        compiler = compiler.compiler[0]
+    else:
+        compiler = compiler.__class__.__name__
+
+    if sys.platform == "win32" and ("icc" in compiler or "icl" in compiler):
+        return ["/Qopenmp"]
+    elif sys.platform == "win32":
+        return ["/openmp"]
+    elif sys.platform == "darwin" and ("icc" in compiler or "icl" in compiler):
+        return ["-openmp"]
+    elif sys.platform == "darwin" and "openmp" in os.getenv("CPPFLAGS", ""):
+        # -fopenmp can't be passed as compile flag when using Apple-clang.
+        # OpenMP support has to be enabled during preprocessing.
+        #
+        # For example, our macOS wheel build jobs use the following environment
+        # variables to build with Apple-clang and the brew installed "libomp":
+        #
+        # export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp"
+        # export CFLAGS="$CFLAGS -I/usr/local/opt/libomp/include"
+        # export CXXFLAGS="$CXXFLAGS -I/usr/local/opt/libomp/include"
+        # export LDFLAGS="$LDFLAGS -L/usr/local/opt/libomp/lib -lomp"
+        # export DYLD_LIBRARY_PATH=/usr/local/opt/libomp/lib
+        return []
+    # Default flag for GCC and clang:
+    return ["-fopenmp"]
+
+
+def check_openmp_support():
+    """Check whether OpenMP test code can be compiled and run."""
+    ccompiler = new_compiler()
+    customize_compiler(ccompiler)
+
+    if os.getenv("SKTIME_NO_OPENMP"):
+        # Build explicitly without OpenMP support
+        return False
+
+    start_dir = os.path.abspath(".")
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        try:
+            os.chdir(tmp_dir)
 
-def read(*parts):
-    """Read.
+            # Write test program
+            with open("test_openmp.c", "w") as f:
+                f.write(CCODE)
 
-    intentionally *not* adding an encoding option to open, See:
-    https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690
-    """
-    with codecs.open(os.path.join(HERE, *parts), "r") as fp:
-        return fp.read()
+            os.mkdir("objects")
 
+            # Compile, test program
+            openmp_flags = get_openmp_flag(ccompiler)
+            ccompiler.compile(
+                ["test_openmp.c"], output_dir="objects", extra_postargs=openmp_flags
+            )
 
-def find_version(*file_paths):
-    """Find the version."""
-    version_file = read(*file_paths)
-    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
-    if version_match:
-        return version_match.group(1)
-    else:
-        raise RuntimeError("Unable to find version string.")
-
-
-WEBSITE = "https://www.sktime.org"
-DISTNAME = "sktime"
-DESCRIPTION = "A unified Python toolbox for machine learning with time series"
-with codecs.open("README.md", encoding="utf-8-sig") as f:
-    LONG_DESCRIPTION = f.read()
-MAINTAINER = "F. Király"
-MAINTAINER_EMAIL = "f.kiraly@ucl.ac.uk"
-URL = "https://github.com/alan-turing-institute/sktime"
-LICENSE = "BSD-3-Clause"
-DOWNLOAD_URL = "https://pypi.org/project/sktime/#files"
-PROJECT_URLS = {
-    "Issue Tracker": "https://github.com/alan-turing-institute/sktime/issues",
-    "Documentation": WEBSITE,
-    "Source Code": "https://github.com/alan-turing-institute/sktime",
-}
-VERSION = find_version("sktime", "__init__.py")
-INSTALL_REQUIRES = [
-    *[
-        "{}>={}".format(package, version)
-        for package, version in MIN_REQUIREMENTS.items()
-    ],
-    *[
-        "{}<={}".format(package, version)
-        for package, version in MAX_REQUIREMENTS.items()
-    ],
-    "wheel",
-]
-CLASSIFIERS = [
-    "Intended Audience :: Science/Research",
-    "Intended Audience :: Developers",
-    "License :: OSI Approved",
-    "Programming Language :: Python",
-    "Topic :: Software Development",
-    "Topic :: Scientific/Engineering",
-    "Operating System :: Microsoft :: Windows",
-    "Operating System :: POSIX",
-    "Operating System :: Unix",
-    "Operating System :: MacOS",
-    "Programming Language :: Python :: 3.6",
-    "Programming Language :: Python :: 3.7",
-    "Programming Language :: Python :: 3.8",
-]
+            # Link test program
+            extra_preargs = os.getenv("LDFLAGS", None)
+            extra_preargs = (
+                extra_preargs.split(" ") if extra_preargs is not None else []
+            )
+            objects = glob.glob(os.path.join("objects", "*" + ccompiler.obj_extension))
+            ccompiler.link_executable(
+                objects,
+                "test_openmp",
+                extra_preargs=extra_preargs,
+                extra_postargs=openmp_flags,
+            )
 
-SETUP_REQUIRES = ["wheel"]
-
-# Optional setuptools features
-# For some commands, use setuptools
-SETUPTOOLS_COMMANDS = {
-    "install",
-    "develop",
-    "release",
-    "build_ext",
-    "bdist_egg",
-    "bdist_rpm",
-    "bdist_wininst",
-    "install_egg_info",
-    "build_sphinx",
-    "egg_info",
-    "easy_install",
-    "upload",
-    "bdist_wheel",
-    "--single-version-externally-managed",
-    "sdist",
-}
-if SETUPTOOLS_COMMANDS.intersection(sys.argv):
-    # We need to import setuptools early, if we want setuptools features,
-    # (e.g. "bdist_wheel") as it monkey-patches the 'setup' function
-    import setuptools  # noqa
-
-    extra_setuptools_args = dict(
-        zip_safe=False,  # the package can run out of an .egg file
-        include_package_data=True,
+            # Run test program
+            output = subprocess.check_output("./test_openmp")
+            output = output.decode(sys.stdout.encoding or "utf-8").splitlines()
+
+            # Check test program output
+            if "nthreads=" in output[0]:
+                nthreads = int(output[0].strip().split("=")[1])
+                openmp_supported = len(output) == nthreads
+            else:
+                openmp_supported = False
+
+        except (CompileError, LinkError, subprocess.CalledProcessError):
+            openmp_supported = False
+
+        finally:
+            os.chdir(start_dir)
+
+    err_message = textwrap.dedent(
+        """
+                            ***
+        It seems that sktime cannot be built with OpenMP support.
+
+        - If your compiler supports OpenMP but the build still fails, please
+          submit a bug report at:
+          'https://github.com/alan-turing-institute/sktime/issues'
+
+        - If you want to build sktime without OpenMP support, you can set
+          the environment variable SKTIME_NO_OPENMP and rerun the build
+          command. Note however that some estimators will run in sequential
+          mode and their `n_jobs` parameter will have no effect anymore.
+
+        - See sktime advanced installation instructions for more info:
+          'https://https://www.sktime.org/en/latest/installation.html'
+                            ***
+        """
     )
 
-else:
-    extra_setuptools_args = dict()
+    if not openmp_supported:
+        raise CompileError(err_message)
+
+    return True
+
+
+def long_description():
+    """Read and return README as long description."""
+    with codecs.open("README.md", encoding="utf-8-sig") as f:
+        return f.read()
+
+
+# ground truth package metadata is loaded from pyproject.toml
+# for context see:
+#   - [PEP 621 -- Storing project metadata in pyproject.toml]
+#     (https://www.python.org/dev/peps/pep-0621)
+pyproject = toml.load("pyproject.toml")
+
+HERE = os.path.abspath(os.path.dirname(__file__))
 
 
 # Custom clean command to remove build artifacts
@@ -187,35 +217,6 @@ def run(self):
 
 cmdclass = {"clean": CleanCommand}
 
-# custom build_ext command to set OpenMP compile flags depending on os and
-# compiler
-# build_ext has to be imported after setuptools
-try:
-    from numpy.distutils.command.build_ext import build_ext  # noqa
-
-    class build_ext_subclass(build_ext):
-        """Build extension subclass."""
-
-        def build_extensions(self):
-            """Build extensions."""
-            from sktime._build_utils.openmp_helpers import get_openmp_flag
-
-            if not os.getenv("SKTIME_NO_OPENMP"):
-                openmp_flag = get_openmp_flag(self.compiler)
-
-                for e in self.extensions:
-                    e.extra_compile_args += openmp_flag
-                    e.extra_link_args += openmp_flag
-
-            build_ext.build_extensions(self)
-
-    cmdclass["build_ext"] = build_ext_subclass
-
-except ImportError:
-    # Numpy should not be a dependency just to be able to introspect
-    # that python 3.6 is required.
-    pass
-
 
 def configuration(parent_package="", top_path=None):
     """Configure."""
@@ -239,108 +240,76 @@ def configuration(parent_package="", top_path=None):
     return config
 
 
-def check_package_status(package, min_version):
-    """
-    Return a dictionary.
-
-    Dictionary contains a boolean specifying whether given package
-    is up-to-date, along with the version string (empty string if
-    not installed).
-    """
-    if package == "scikit-learn":
-        package = "sklearn"
-
-    package_status = {}
-    try:
-        module = importlib.import_module(package)
-        package_version = module.__version__
-        package_status["up_to_date"] = parse_version(package_version) >= parse_version(
-            min_version
-        )
-        package_status["version"] = package_version
-    except ImportError:
-        traceback.print_exc()
-        package_status["up_to_date"] = False
-        package_status["version"] = ""
-
-    req_str = "sktime requires {} >= {}.\n".format(package, min_version)
-
-    instructions = (
-        "Installation instructions are available on the "
-        "sktime website: "
-        "https://www.sktime.org/en/latest"
-        "/installation.html\n"
-    )
-
-    if package_status["up_to_date"] is False:
-        if package_status["version"]:
-            raise ImportError(
-                "Your installation of {} "
-                "{} is out-of-date.\n{}{}".format(
-                    package, package_status["version"], req_str, instructions
-                )
-            )
-        else:
-            raise ImportError(
-                "{} is not " "installed.\n{}{}".format(package, req_str, instructions)
-            )
+extensions = [
+    Extension(
+        "sktime.__check_build._check_build",
+        ["sktime/__check_build/_check_build.pyx"],
+        extra_compile_args=["-O2"],
+    ),
+    Extension(
+        "sktime.distances.elastic_cython",
+        ["sktime/distances/elastic_cython.pyx"],
+        extra_compile_args=["/d2FH4-", "-O2"] if sys.platform == "win32" else ["-O2"],
+        language="c++",
+        include_dirs=[numpy.get_include()],
+    ),
+    Extension(
+        "sktime.classification.shapelet_based.mrseql.mrseql",
+        [
+            "sktime/classification/shapelet_based/mrseql/mrseql.pyx",
+            "sktime/classification/shapelet_based/mrseql/seql_learn.cpp",
+            "sktime/classification/shapelet_based/mrseql/SNode.cpp",
+        ],
+        extra_compile_args=["/d2FH4-", "-O2"] if sys.platform == "win32" else ["-O2"],
+        language="c++",
+        include_dirs=[numpy.get_include()],
+    ),
+]
 
 
 def setup_package():
     """Set up package."""
     metadata = dict(
-        name=DISTNAME,
-        maintainer=MAINTAINER,
-        maintainer_email=MAINTAINER_EMAIL,
-        description=DESCRIPTION,
-        license=LICENSE,
-        url=URL,
-        download_url=DOWNLOAD_URL,
-        project_urls=PROJECT_URLS,
-        version=VERSION,
-        long_description=LONG_DESCRIPTION,
-        classifiers=CLASSIFIERS,
+        author_email=pyproject["project"]["authors"][0]["email"],
+        author=pyproject["project"]["authors"][0]["name"],
+        classifiers=pyproject["project"]["classifiers"],
         cmdclass=cmdclass,
-        python_requires=">={}".format(MIN_PYTHON_VERSION),
-        setup_requires=SETUP_REQUIRES,
-        install_requires=INSTALL_REQUIRES,
-        extras_require=EXTRAS_REQUIRE,
-        **extra_setuptools_args
+        description=pyproject["project"]["description"],
+        download_url=pyproject["project"]["urls"]["download"],
+        ext_modules=cythonize(extensions, language_level="3"),
+        extras_require=pyproject["project"]["optional-dependencies"],
+        include_package_data=True,
+        install_requires=pyproject["project"]["dependencies"],
+        keywords=pyproject["project"]["keywords"],
+        license=pyproject["project"]["license"],
+        long_description=long_description(),
+        maintainer_email=pyproject["project"]["maintainers"][0]["email"],
+        maintainer=pyproject["project"]["maintainers"][0]["name"],
+        name=pyproject["project"]["name"],
+        package_data={
+            "sktime": [
+                "*.csv",
+                "*.csv.gz",
+                "*.arff",
+                "*.arff.gz",
+                "*.txt",
+                "*.ts",
+                "*.tsv",
+            ]
+        },
+        packages=find_packages(
+            where=".",
+            exclude=["tests", "tests.*"],
+        ),
+        project_urls=pyproject["project"]["urls"],
+        python_requires=pyproject["project"]["requires-python"],
+        setup_requires=pyproject["build-system"]["requires"],
+        url=pyproject["project"]["urls"]["repository"],
+        version=pyproject["project"]["version"],
+        zip_safe=False,
     )
 
-    # For these actions, NumPy is not required
-    # They are required to succeed without Numpy for example when
-    # pip is used to install sktime when Numpy is not yet
-    # present in the system.
-    if len(sys.argv) == 1 or (
-        len(sys.argv) >= 2
-        and (
-            "--help" in sys.argv[1:]
-            or sys.argv[1] in ("--help-commands", "egg_info", "--version", "clean")
-        )
-    ):
-        try:
-            from setuptools import setup
-        except ImportError:
-            from distutils.core import setup
-
-        metadata["version"] = VERSION
-
-    # otherwise check Python and required package versions
-    else:
-        if sys.version_info < tuple([int(i) for i in MIN_PYTHON_VERSION.split(".")]):
-            raise RuntimeError(
-                "sktime requires Python %s or later. The current"
-                " Python version is %s installed in %s."
-                % (MIN_PYTHON_VERSION, platform.python_version(), sys.executable)
-            )
-
-        for package, version in MIN_REQUIREMENTS.items():
-            check_package_status(package, version)
-
-        from numpy.distutils.core import setup
-
-        metadata["configuration"] = configuration
+    from setuptools import setup
 
     setup(**metadata)
 
diff --git a/sktime/benchmarking/experiments.py b/sktime/benchmarking/experiments.py
--- a/sktime/benchmarking/experiments.py
+++ b/sktime/benchmarking/experiments.py
@@ -111,10 +111,7 @@ def run_clustering_experiment(
         second = "Para info too long!"
     else:
 
-        if "Composite" in cls_name:
-            second = "Para info too long!"
-        else:
-            second = str(clusterer.get_params())
+        second = str(clusterer.get_params())
         second.replace("\n", " ")
         second.replace("\r", " ")
         third = "FORMAT NOT FINALISED"
@@ -289,10 +286,7 @@ def run_classification_experiment(
     y_train = le.fit_transform(y_train)
     y_test = le.transform(y_test)
 
-    encoder_dict = {}
-    for i, label in enumerate(le.classes_):
-        encoder_dict[label] = i
-
+    encoder_dict = {label: i for i, label in enumerate(le.classes_)}
     classifier_train_probs = train_file and callable(
         getattr(classifier, "_get_train_probs", None)
     )
@@ -303,50 +297,50 @@ def run_classification_experiment(
         classifier.fit(X_train, y_train)
         build_time = int(round(time.time() * 1000)) - start
 
-        if test_file:
-            start = int(round(time.time() * 1000))
-            probs = classifier.predict_proba(X_test)
-            test_time = int(round(time.time() * 1000)) - start
-
-            if "composite" in cls_name.lower():
-                second = "Para info too long!"
-            else:
-                second = str(classifier.get_params())
-            second.replace("\n", " ")
-            second.replace("\r", " ")
-
-            # Line 3 format:
-            preds = classifier.classes_[np.argmax(probs, axis=1)]
-            acc = accuracy_score(y_test, preds)
-            third = (
-                str(acc)  # 1. accuracy
-                + ","
-                + str(build_time)  # 2. fit time
-                + ","
-                + str(test_time)  # 3. predict time
-                + ",-1,-1,"  # 4. 5. benchmark time, memory (to do)
-                + str(len(classifier.classes_))  # 6. number of classes
-                + ",,-1,-1"  # 7. 8. 9.
-            )
+    if test_file:
+        start = int(round(time.time() * 1000))
+        probs = classifier.predict_proba(X_test)
+        test_time = int(round(time.time() * 1000)) - start
 
-            write_results_to_uea_format(
-                second_line=second,
-                third_line=third,
-                first_line_comment="PREDICTIONS,Generated by experiments.py on "
-                + datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
-                + ". Encoder dictionary: "
-                + str(encoder_dict),
-                timing_type="MILLISECONDS",
-                output_path=results_path,
-                estimator_name=cls_name,
-                resample_seed=resample_id,
-                y_pred=preds,
-                predicted_probs=probs,
-                dataset_name=dataset,
-                y_true=y_test,
-                split="TEST",
-                full_path=False,
-            )
+        if "composite" in cls_name.lower():
+            second = "Para info too long!"
+        else:
+            second = str(classifier.get_params())
+        second.replace("\n", " ")
+        second.replace("\r", " ")
+
+        # Line 3 format:
+        preds = classifier.classes_[np.argmax(probs, axis=1)]
+        acc = accuracy_score(y_test, preds)
+        third = (
+            str(acc)  # 1. accuracy
+            + ","
+            + str(build_time)  # 2. fit time
+            + ","
+            + str(test_time)  # 3. predict time
+            + ",-1,-1,"  # 4. 5. benchmark time, memory (to do)
+            + str(len(classifier.classes_))  # 6. number of classes
+            + ",,-1,-1"  # 7. 8. 9.
+        )
+
+        write_results_to_uea_format(
+            second_line=second,
+            third_line=third,
+            first_line_comment="PREDICTIONS,Generated by experiments.py on "
+            + datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
+            + ". Encoder dictionary: "
+            + str(encoder_dict),
+            timing_type="MILLISECONDS",
+            output_path=results_path,
+            estimator_name=cls_name,
+            resample_seed=resample_id,
+            y_pred=preds,
+            predicted_probs=probs,
+            dataset_name=dataset,
+            y_true=y_test,
+            split="TEST",
+            full_path=False,
+        )
 
     if train_file:
         start = int(round(time.time() * 1000))
@@ -485,7 +479,7 @@ def load_and_run_classification_experiment(
             if os.path.exists(full_path):
                 build_train = False
 
-        if build_train is False and build_test is False:
+        if build_train is False and not build_test:
             return
 
     if predefined_resample:
diff --git a/sktime/setup.py b/sktime/setup.py
deleted file mode 100644
--- a/sktime/setup.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python3 -u
-# -*- coding: utf-8 -*-
-# copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
-"""Set  up."""
-__author__ = "Markus Löning"
-
-# adapted from https://github.com/scikit-learn/scikit-learn/blob/master
-# /sklearn/setup.py
-
-import os
-
-from setuptools import find_packages
-from sktime._build_utils import maybe_cythonize_extensions
-
-
-def configuration(parent_package="", top_path=None):
-    """Configure."""
-    from numpy.distutils.misc_util import Configuration
-
-    libraries = []
-    if os.name == "posix":
-        libraries.append("m")
-
-    config = Configuration("sktime", parent_package, top_path)
-
-    for package in find_packages("sktime"):
-        config.add_subpackage(package)
-
-    maybe_cythonize_extensions(top_path, config)
-
-    return config
-
-
-if __name__ == "__main__":
-    from numpy.distutils.core import setup
-
-    setup(**configuration(top_path="").todict())


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,173 @@
+name: Install and test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  code-quality:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - id: file_changes
+        uses: trilom/file-changes-action@v1.2.4
+        with:
+          output: " "
+      - name: List changed files
+        run: echo '${{ steps.file_changes.outputs.files}}'
+      - uses: pre-commit/action@v2.0.0
+        with:
+          extra_args: --files ${{ steps.file_changes.outputs.files}}
+      - name: Check for missing init files
+        run: build_tools/fail_on_missing_init_files.sh
+        shell: bash
+
+  run-notebook-examples:
+    needs: code-quality
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install .[all_extras,binder]
+      - name: Run example notebooks
+        run: build_tools/run_examples.sh
+        shell: bash
+
+  test-windows:
+    needs: code-quality
+    runs-on: windows-2019
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8]
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: ${{ matrix.python-version }}
+          channels: anaconda, conda-forge,
+
+      - run: conda --version
+      - run: which python
+
+      - name: Fix windows paths
+        if: ${{ runner.os == 'Windows' }}
+        run: echo "C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
+      - name: Install conda libpython
+        run: conda install -c anaconda libpython
+
+      - name: Install conda dependencies
+        run: |
+          conda install -c anaconda "pystan==2.19.1.1"
+          conda install -c conda-forge "fbprophet>=0.7.1"
+          conda install -c conda-forge scipy cython matplotlib
+
+      - name: Install sktime and dependencies
+        run: python -m pip install .[all_extras,dev]
+
+      - name: Show dependecies
+        run: python -m pip list
+
+      - name: Run tests
+        run: |
+          mkdir -p testdir/
+          cp .coveragerc testdir/
+          cp setup.cfg testdir/
+          cd testdir/
+          python -m pytest --showlocals --durations=10 --cov-report=xml -n 2 --cov=sktime -v --pyargs sktime
+      - name: Display coverage report
+        run: ls -l ./testdir/
+      - name: Publish code coverage
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./testdir/coverage.xml
+
+  test-linux:
+    needs: code-quality
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+
+      - name: Install sktime and dependencies
+        run: |
+          python -m pip install .[all_extras,dev]
+
+      - name: Show dependecies
+        run: python -m pip list
+
+      - name: Run tests
+        run: make tests
+
+      - name: Display coverage report
+        run: ls -l ./testdir/
+      - name: Publish code coverage
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./testdir/coverage.xml
+
+  test-mac:
+    needs: code-quality
+    runs-on: macOS-10.15
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+
+      - run: brew install libomp
+
+      - name: Set env vars
+        run: |
+          echo "CC=/usr/bin/clang++" >> $GITHUB_ENV
+          echo "CXX=/usr/bin/clang++" >> $GITHUB_ENV
+          echo "DYLD_LIBRARY_PATH=/usr/local/opt/libomp/lib" >> $GITHUB_ENV
+          echo "CPPFLAGS=$CPPFLAGS -Xclang -fopenmp -std=c++11 -lstdc++" >> $GITHUB_ENV
+          echo "CFLAGS=$CFLAGS -I/opt/local/include/libomp -I/opt/local/include" >> $GITHUB_ENV
+          echo "CXXFLAGS=$CXXFLAGS -I/opt/local/include/libomp" >> $GITHUB_ENV
+          echo "LDFLAGS=$LDFLAGS -Wl,-rpath,/opt/local/lib/libomp -L/opt/local/lib/libomp -lomp" >> $GITHUB_ENV
+          echo "PATH=/usr/local/opt/ccache/libexec:$PATH" >> $GITHUB_ENV
+
+      - name: Install sktime and dependencies
+        run: |
+          python -m pip install .[all_extras,dev]
+
+      - name: Show dependecies
+        run: python -m pip list
+
+      - name: Run tests
+        run: make tests
+
+      - name: Display coverage report
+        run: ls -l ./testdir/
+      - name: Publish code coverage
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./testdir/coverage.xml
diff --git a/sktime/benchmarking/tests/test_experiments.py b/sktime/benchmarking/tests/test_experiments.py
--- a/sktime/benchmarking/tests/test_experiments.py
+++ b/sktime/benchmarking/tests/test_experiments.py
@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 """Functions to test the functions in experiments.py."""
 
-import os.path
-
 from sktime.benchmarking.experiments import (
     run_classification_experiment,
     run_clustering_experiment,
@@ -12,7 +10,7 @@
 from sktime.datasets import load_unit_test
 
 
-def test_run_clustering_experiment():
+def test_run_clustering_experiment(tmp_path):
     """Test running and saving results for clustering.
 
     Currently it just checks the files have been created, then deletes them.
@@ -23,7 +21,7 @@ def test_run_clustering_experiment():
     run_clustering_experiment(
         train_X,
         TimeSeriesKMeans(n_clusters=2),
-        results_path="../Temp/",
+        results_path=tmp_path,
         trainY=train_Y,
         testX=test_X,
         testY=test_Y,
@@ -31,15 +29,16 @@ def test_run_clustering_experiment():
         dataset_name=dataset,
         resample_id=0,
     )
-    test_path = f"../Temp/kmeans/Predictions/{dataset}/testResample0.csv"
-    train_path = f"../Temp/kmeans/Predictions/{dataset}/trainResample0.csv"
-    assert os.path.isfile(test_path)
-    assert os.path.isfile(train_path)
-    os.remove(test_path)
-    os.remove(train_path)
+    test_path = tmp_path.joinpath(f"kmeans/Predictions/{dataset}/testResample0.csv")
+    train_path = tmp_path.joinpath(f"kmeans/Predictions/{dataset}/trainResample0.csv")
+    assert test_path.is_file()
+    assert train_path.is_file()
+    # remove files
+    test_path.unlink()
+    train_path.unlink()
 
 
-def test_run_classification_experiment():
+def test_run_classification_experiment(tmp_path):
     """Test running and saving results for classifiers.
 
     Currently it just checks the files have been created, then deletes them.
@@ -53,18 +52,19 @@ def test_run_classification_experiment():
         test_X,
         test_Y,
         TimeSeriesForestClassifier(n_estimators=10),
-        "../Temp/",
+        str(tmp_path),
         cls_name="TSF",
         dataset="UnitTest",
         resample_id=0,
         train_file=True,
     )
-    test_path = f"../Temp/TSF/Predictions/{dataset}/testResample0.csv"
-    train_path = f"../Temp/TSF/Predictions/{dataset}/trainResample0.csv"
-    assert os.path.isfile(test_path)
-    assert os.path.isfile(train_path)
-    os.remove(test_path)
-    os.remove(train_path)
+    test_path = tmp_path.joinpath(f"TSF/Predictions/{dataset}/testResample0.csv")
+    train_path = tmp_path.joinpath(f"TSF/Predictions/{dataset}/trainResample0.csv")
+    assert test_path.is_file()
+    assert train_path.is_file()
+    # remove files
+    test_path.unlink()
+    train_path.unlink()
 
 
 # def test_load_and_run_clustering_experiment():
diff --git a/sktime/classification/compose/tests/test_column_ensemble.py b/sktime/classification/compose/tests/test_column_ensemble.py
--- a/sktime/classification/compose/tests/test_column_ensemble.py
+++ b/sktime/classification/compose/tests/test_column_ensemble.py
@@ -75,59 +75,30 @@ def test_col_ens_on_unit_test_data():
 
 col_ens_unit_test_probas = np.array(
     [
-        [
-            0.2,
-            0.8,
-        ],
-        [
-            1.0,
-            0.0,
-        ],
-        [
-            0.1,
-            0.9,
-        ],
-        [
-            1.0,
-            0.0,
-        ],
-        [
-            0.9,
-            0.1,
-        ],
-        [
-            1.0,
-            0.0,
-        ],
-        [
-            0.9,
-            0.1,
-        ],
-        [
-            0.8,
-            0.2,
-        ],
-        [
-            0.9,
-            0.1,
-        ],
-        [
-            1.0,
-            0.0,
-        ],
+        [0.2, 0.8],
+        [1.0, 0.0],
+        [0.0, 1.0],
+        [1.0, 0.0],
+        [1.0, 0.0],
+        [1.0, 0.0],
+        [1.0, 0.0],
+        [1.0, 0.0],
+        [1.0, 0.0],
+        [1.0, 0.0],
     ]
 )
+
 col_ens_basic_motions_probas = np.array(
     [
-        [0.00000, 0.07885, 0.03333, 0.88781],
-        [0.90585, 0.00000, 0.00000, 0.09415],
-        [0.00000, 0.09681, 0.82433, 0.07885],
-        [0.06667, 0.67615, 0.22385, 0.03333],
-        [0.00000, 0.00000, 0.06397, 0.93603],
-        [0.00000, 0.03063, 0.26667, 0.70270],
-        [0.72748, 0.03333, 0.06352, 0.17567],
-        [0.00000, 0.00000, 0.88781, 0.11219],
-        [0.00000, 0.76082, 0.16033, 0.07885],
-        [0.00000, 0.82429, 0.06352, 0.11219],
+        [0.0, 0.07885368, 0.0, 0.92114632],
+        [0.90585008, 0.0, 0.0, 0.09414992],
+        [0.0, 0.09681163, 0.82433469, 0.07885368],
+        [0.06666667, 0.67615215, 0.22384785, 0.03333333],
+        [0.0, 0.0, 0.0306318, 0.9693682],
+        [0.0, 0.0306318, 0.16666667, 0.80270153],
+        [0.72748325, 0.03333333, 0.06351811, 0.17566531],
+        [0.0, 0.0, 0.88781299, 0.11218701],
+        [0.0, 0.76081658, 0.16032974, 0.07885368],
+        [0.0, 0.85762821, 0.06351811, 0.07885368],
     ]
 )
diff --git a/sktime/classification/feature_based/tests/test_fresh_prince.py b/sktime/classification/feature_based/tests/test_fresh_prince.py
--- a/sktime/classification/feature_based/tests/test_fresh_prince.py
+++ b/sktime/classification/feature_based/tests/test_fresh_prince.py
@@ -45,32 +45,32 @@ def test_fresh_prince_on_unit_test_data():
             0.0,
         ],
         [
-            0.1,
-            0.9,
+            0.0,
+            1.0,
         ],
         [
             1.0,
             0.0,
         ],
         [
-            0.9,
-            0.1,
+            1.0,
+            0.0,
         ],
         [
             1.0,
             0.0,
         ],
         [
-            0.9,
-            0.1,
+            1.0,
+            0.0,
         ],
         [
-            0.8,
-            0.2,
+            1.0,
+            0.0,
         ],
         [
-            0.9,
-            0.1,
+            1.0,
+            0.0,
         ],
         [
             1.0,
diff --git a/sktime/classification/feature_based/tests/test_tsfresh_classifier.py b/sktime/classification/feature_based/tests/test_tsfresh_classifier.py
--- a/sktime/classification/feature_based/tests/test_tsfresh_classifier.py
+++ b/sktime/classification/feature_based/tests/test_tsfresh_classifier.py
@@ -56,109 +56,30 @@ def test_tsfresh_classifier_on_basic_motions():
 
 tsfresh_classifier_unit_test_probas = np.array(
     [
-        [
-            0.0,
-            1.0,
-        ],
-        [
-            0.9,
-            0.1,
-        ],
-        [
-            0.0,
-            1.0,
-        ],
-        [
-            0.9,
-            0.1,
-        ],
-        [
-            0.7,
-            0.3,
-        ],
-        [
-            1.0,
-            0.0,
-        ],
-        [
-            0.8,
-            0.2,
-        ],
-        [
-            0.9,
-            0.1,
-        ],
-        [
-            1.0,
-            0.0,
-        ],
-        [
-            1.0,
-            0.0,
-        ],
+        [0.1, 0.9],
+        [0.7, 0.3],
+        [0.0, 1.0],
+        [0.9, 0.1],
+        [0.8, 0.2],
+        [1.0, 0.0],
+        [0.6, 0.4],
+        [0.9, 0.1],
+        [1.0, 0.0],
+        [0.9, 0.1],
     ]
 )
+
 tsfresh_classifier_basic_motions_probas = np.array(
     [
-        [
-            0.0,
-            0.0,
-            0.2,
-            0.8,
-        ],
-        [
-            0.4,
-            0.2,
-            0.1,
-            0.3,
-        ],
-        [
-            0.0,
-            0.0,
-            0.9,
-            0.1,
-        ],
-        [
-            0.0,
-            0.9,
-            0.0,
-            0.1,
-        ],
-        [
-            0.0,
-            0.0,
-            0.2,
-            0.8,
-        ],
-        [
-            0.0,
-            0.0,
-            0.3,
-            0.7,
-        ],
-        [
-            0.3,
-            0.3,
-            0.0,
-            0.4,
-        ],
-        [
-            0.0,
-            0.0,
-            0.9,
-            0.1,
-        ],
-        [
-            0.0,
-            0.9,
-            0.0,
-            0.1,
-        ],
-        [
-            0.1,
-            0.9,
-            0.0,
-            0.0,
-        ],
+        [0.0, 0.0, 0.2, 0.8],
+        [0.4, 0.3, 0.0, 0.3],
+        [0.0, 0.0, 0.8, 0.2],
+        [0.0, 1.0, 0.0, 0.0],
+        [0.0, 0.2, 0.2, 0.6],
+        [0.0, 0.0, 0.3, 0.7],
+        [0.5, 0.3, 0.0, 0.2],
+        [0.0, 0.0, 0.9, 0.1],
+        [0.1, 0.9, 0.0, 0.0],
+        [0.0, 1.0, 0.0, 0.0],
     ]
 )
diff --git a/sktime/forecasting/tests/test_ets.py b/sktime/forecasting/tests/test_ets.py
--- a/sktime/forecasting/tests/test_ets.py
+++ b/sktime/forecasting/tests/test_ets.py
@@ -1,12 +1,15 @@
 # -*- coding: utf-8 -*-
+"""ETS tests."""
+
 __author__ = ["Hongyi Yang"]
 
-from numpy.testing import assert_array_equal
-from sktime.forecasting.ets import AutoETS
-from sktime.datasets import load_airline
-import pandas as pd
 import numpy as np
+import pandas as pd
+import pytest
+from numpy.testing import assert_array_equal
 
+from sktime.datasets import load_airline
+from sktime.forecasting.ets import AutoETS
 
 # test results against R implementation on airline dataset
 y = load_airline()
@@ -18,11 +21,14 @@
 )
 
 
-# Default condition
-# fit <- ets(AirPassengers, model = "ZZZ")
-# components: "M" "A" "M" "TRUE" (error, trend, season, damped)
-# discrepancy lies in damped (True in R but False in statsmodels)
 def test_airline_default():
+    """
+    Default condition.
+
+    fit <- ets(AirPassengers, model = "ZZZ")
+    components: "M" "A" "M" "TRUE" (error, trend, season, damped)
+    discrepancy lies in damped (True in R but False in statsmodels)
+    """
     fit_result_R = ["mul", "add", "mul"]
 
     forecaster = AutoETS(auto=True, sp=12, n_jobs=-1)
@@ -37,17 +43,17 @@ def test_airline_default():
     assert_array_equal(fit_result_R, fit_result)
 
 
-# Allow multiplicative trend
-# fit <- ets(AirPassengers, model = "ZZZ",
-# allow.multiplicative.trend = TRUE)
-# components: "M" "M" "M" "TRUE"
-# discrepancy lies in damped (True in R but False in statsmodels)
-# Test failed on linux environment, fixed by fixing pandas==1.1.5 in #581
-# @pytest.mark.skipif(
-#     sys.platform == "linux",
-#     reason="Skip test due to unknown error on Linux with Python 3.7 and 3.8",
-# )
+@pytest.mark.xfail(reason="flaky results on linux")
 def test_airline_allow_multiplicative_trend():
+    """
+    Allow multiplicative trend.
+
+    fit <- ets(AirPassengers, model = "ZZZ",
+    allow.multiplicative.trend = TRUE)
+    components: "M" "M" "M" "TRUE"
+    discrepancy lies in damped (True in R but False in statsmodels)
+    Test failed on linux environment, fixed by fixing pandas==1.1.5 in #581
+    """
     fit_result_R = ["mul", "mul", "mul"]
 
     forecaster = AutoETS(auto=True, sp=12, n_jobs=-1, allow_multiplicative_trend=True)
@@ -62,8 +68,8 @@ def test_airline_allow_multiplicative_trend():
     assert_array_equal(fit_result_R, fit_result)
 
 
-# Ignore infinite IC models when ignore_inf_ic is True
 def test_inf_ic_true():
+    """Ignore infinite IC models when ignore_inf_ic is `True`."""
     forecaster = AutoETS(auto=True, sp=52, n_jobs=-1, ignore_inf_ic=True)
     forecaster.fit(inf_ic_ts)
     fitted_forecaster = forecaster._fitted_forecaster
@@ -75,8 +81,9 @@ def test_inf_ic_true():
     )
 
 
-# Don't ignore infinite IC models when ignore_inf_ic is False
+@pytest.mark.xfail
 def test_inf_ic_false():
+    """Don't ignore infinite IC models when ignore_inf_ic is False."""
     forecaster = AutoETS(auto=True, sp=52, n_jobs=-1, ignore_inf_ic=False)
     forecaster.fit(inf_ic_ts)
     fitted_forecaster = forecaster._fitted_forecaster