Repository: aeon-toolkit/aeon
Pull request #1298 (2024-03-11 09:30:21)

[ENH] Replace mtype to scitype function
replaces the calls to mytpe_to_scitype with a a function called abstract_types in the utils.validation package.

Part of #42 #974 #1142 

this is a minimal change to get it done, I'm not fixed in this way of doing things but would like to isolate datatypes more. 



aeon-actions-bot[bot]: 
## Thank you for contributing to `aeon`

I have added the following labels to this PR based on the title: **[ $\\color{#FEF1BE}{\\textsf{enhancement}}$ ]**.
I would have added the following labels to this PR based on the changes made: **[ $\\color{#31FCCD}{\\textsf{forecasting}}$, $\\color{#2C2F20}{\\textsf{testing}}$, $\\color{#41A8F6}{\\textsf{transformations}}$ ]**, however some package labels are already present.

The [Checks](https://github.com/aeon-toolkit/aeon/pull/1298/checks) tab will show the status of our automated tests. You can click on individual test runs in the tab or "Details" in the panel below to see more information if there is a failure.

If our `pre-commit` code quality check fails, any trivial fixes will automatically be pushed to your PR unless it is a draft.

Don't hesitate to ask questions on the `aeon` [Slack](
https://join.slack.com/t/aeon-toolkit/shared_invite/zt-22vwvut29-HDpCu~7VBUozyfL_8j3dLA) channel if you have any.
    
review-notebook-app[bot]: Check out this pull request on&nbsp; <a href="https://app.reviewnb.com/aeon-toolkit/aeon/pull/1298"><img align="absmiddle"  alt="ReviewNB" height="28" class="BotMessageButtonImage" src="https://raw.githubusercontent.com/ReviewNB/support/master/images/button_reviewnb.png"/></a> 

 See visual diffs & provide feedback on Jupyter Notebooks. 

---

 <i>Powered by <a href='https://www.reviewnb.com/?utm_source=gh'>ReviewNB</a></i>

Patch:
diff --git a/aeon/forecasting/base/_base.py b/aeon/forecasting/base/_base.py
index 40a4af13..b7a59283 100644
--- a/aeon/forecasting/base/_base.py
+++ b/aeon/forecasting/base/_base.py
@@ -45,11 +45,11 @@ import pandas as pd
 from scipy.stats import norm
 
 from aeon.base import BaseEstimator
-from aeon.datatypes import VectorizedDF, convert_to, mtype_to_scitype
+from aeon.datatypes import VectorizedDF, convert_to
 from aeon.forecasting.base._fh import ForecastingHorizon
 from aeon.utils.datetime import _shift
 from aeon.utils.index_functions import get_cutoff, update_data
-from aeon.utils.validation import validate_input
+from aeon.utils.validation import abstract_types, validate_input
 from aeon.utils.validation._dependencies import _check_estimator_deps
 from aeon.utils.validation.forecasting import check_alpha, check_cv, check_fh, check_X
 from aeon.utils.validation.series import check_equal_time_index
@@ -1314,8 +1314,8 @@ class BaseForecaster(BaseEstimator):
         # retrieve supported mtypes
         y_inner_type = _coerce_to_list(self.get_tag("y_inner_type"))
         X_inner_type = _coerce_to_list(self.get_tag("X_inner_type"))
-        y_inner_abstract_type = mtype_to_scitype(y_inner_type, return_unique=True)
-        X_inner_abstract_type = mtype_to_scitype(X_inner_type, return_unique=True)
+        y_inner_abstract_type = abstract_types(y_inner_type)
+        X_inner_abstract_type = abstract_types(X_inner_type)
 
         # checking y
         if y is not None:
@@ -1567,7 +1567,7 @@ class BaseForecaster(BaseEstimator):
         y : aeon compatible time series data container
             must be of one of the following mtypes:
                 pd.Series, pd.DataFrame, np.ndarray, of Series abstract type
-                pd.multiindex, numpy3D, nested_univ, df-list, of Panel abstract type
+                pd.multiindex, numpy3D, nested_univ, of Panel abstract type
                 pd_multiindex_hier, of Hierarchical abstract type
 
         Notes
diff --git a/aeon/forecasting/compose/_grouped.py b/aeon/forecasting/compose/_grouped.py
index f95af108..65641847 100644
--- a/aeon/forecasting/compose/_grouped.py
+++ b/aeon/forecasting/compose/_grouped.py
@@ -1,8 +1,8 @@
 """Implements compositors for performing forecasting by group."""
 
-from aeon.datatypes import mtype_to_scitype
 from aeon.forecasting.base._delegate import _DelegatedForecaster
 from aeon.utils._data_types import ALL_TIME_SERIES_TYPES
+from aeon.utils.validation._input import _abstract_type
 
 __maintainer__ = []
 __all__ = ["ForecastByLevel"]
@@ -91,7 +91,7 @@ class ForecastByLevel(_DelegatedForecaster):
             )
 
         internal_types = [
-            x for x in ALL_TIME_SERIES_TYPES if mtype_to_scitype(x) in scitypes
+            x for x in ALL_TIME_SERIES_TYPES if _abstract_type(x) in scitypes
         ]
 
         # this ensures that we convert in the inner estimator
diff --git a/aeon/forecasting/model_selection/_tune.py b/aeon/forecasting/model_selection/_tune.py
index 574ee401..d376e188 100644
--- a/aeon/forecasting/model_selection/_tune.py
+++ b/aeon/forecasting/model_selection/_tune.py
@@ -9,10 +9,10 @@ import numpy as np
 import pandas as pd
 from sklearn.model_selection import ParameterGrid, ParameterSampler, check_cv
 
-from aeon.datatypes import mtype_to_scitype
 from aeon.exceptions import NotFittedError
 from aeon.forecasting.base._delegate import _DelegatedForecaster
 from aeon.forecasting.model_evaluation import evaluate
+from aeon.utils.validation import abstract_types
 from aeon.utils.validation.forecasting import check_scoring
 
 
@@ -90,12 +90,12 @@ class BaseGridSearch(_DelegatedForecaster):
         tagval = self.get_tag(tagname)
         if not isinstance(tagval, list):
             tagval = [tagval]
-        scitypes = mtype_to_scitype(tagval, return_unique=True)
-        if "Series" not in scitypes:
+        abs_types = abstract_types(tagval)
+        if "Series" not in abs_types:
             tagval = tagval + ["pd.DataFrame"]
-        if "Panel" not in scitypes:
+        if "Panel" not in abs_types:
             tagval = tagval + ["pd-multiindex"]
-        if "Hierarchical" not in scitypes:
+        if "Hierarchical" not in abs_types:
             tagval = tagval + ["pd_multiindex_hier"]
         self.set_tags(**{tagname: tagval})
 
diff --git a/aeon/transformations/base.py b/aeon/transformations/base.py
index 21056172..ad8263eb 100644
--- a/aeon/transformations/base.py
+++ b/aeon/transformations/base.py
@@ -50,7 +50,7 @@ import numpy as np
 import pandas as pd
 
 from aeon.base import BaseEstimator
-from aeon.datatypes import VectorizedDF, convert_to, mtype_to_scitype
+from aeon.datatypes import VectorizedDF, convert_to
 from aeon.datatypes._series_as_panel import convert_to_scitype
 from aeon.utils.index_functions import update_data
 from aeon.utils.sklearn import (
@@ -58,7 +58,7 @@ from aeon.utils.sklearn import (
     is_sklearn_regressor,
     is_sklearn_transformer,
 )
-from aeon.utils.validation import is_univariate_series, validate_input
+from aeon.utils.validation import abstract_types, is_univariate_series, validate_input
 from aeon.utils.validation._dependencies import _check_estimator_deps
 
 # single/multiple primitives
@@ -800,8 +800,8 @@ class BaseTransformer(BaseEstimator):
         # retrieve supported mtypes
         X_inner_type = _coerce_to_list(self.get_tag("X_inner_type"))
         y_inner_type = _coerce_to_list(self.get_tag("y_inner_type"))
-        X_inner_scitype = mtype_to_scitype(X_inner_type, return_unique=True)
-        y_inner_scitype = mtype_to_scitype(y_inner_type, return_unique=True)
+        X_inner_abstract_type = abstract_types(X_inner_type)
+        y_inner_abstract_type = abstract_types(y_inner_type)
 
         ALLOWED_MTYPES = self.ALLOWED_INPUT_TYPES
 
@@ -820,10 +820,10 @@ class BaseTransformer(BaseEstimator):
         if X_mtype not in ALLOWED_MTYPES:
             raise TypeError("X an invalid internal type")
 
-        if X_scitype in X_inner_scitype:
+        if X_scitype in X_inner_abstract_type:
             case = "case 1: scitype supported"
             req_vec_because_rows = False
-        elif any(_type_A_higher_B(x, X_scitype) for x in X_inner_scitype):
+        elif any(_type_A_higher_B(x, X_scitype) for x in X_inner_abstract_type):
             case = "case 2: higher scitype supported"
             req_vec_because_rows = False
         else:
@@ -863,7 +863,7 @@ class BaseTransformer(BaseEstimator):
         #       or inner is Hierarchical and X is Panel or Series
         #   then, consider X as one-instance Panel or Hierarchical
         if case == "case 2: higher scitype supported":
-            if X_scitype == "Series" and "Panel" in X_inner_scitype:
+            if X_scitype == "Series" and "Panel" in X_inner_abstract_type:
                 as_scitype = "Panel"
             else:
                 as_scitype = "Hierarchical"
@@ -897,7 +897,7 @@ class BaseTransformer(BaseEstimator):
         #   then apply vectorization, loop method execution over series/panels
         # elif case == "case 3: requires vectorization":
         else:  # if requires_vectorization
-            iterate_X = _most_complex_scitype(X_inner_scitype, X_scitype)
+            iterate_X = _most_complex_scitype(X_inner_abstract_type, X_scitype)
             X_inner = VectorizedDF(
                 X=X,
                 iterate_as=iterate_X,
@@ -914,7 +914,7 @@ class BaseTransformer(BaseEstimator):
                 #     "Consider extending _fit and _transform to handle the following "
                 #     "input types natively: Panel X and non-None y."
                 # )
-                iterate_y = _most_complex_scitype(y_inner_scitype, y_scitype)
+                iterate_y = _most_complex_scitype(y_inner_abstract_type, y_scitype)
                 y_inner = VectorizedDF(X=y, iterate_as=iterate_y, is_scitype=y_scitype)
             else:
                 y_inner = None
@@ -962,7 +962,7 @@ class BaseTransformer(BaseEstimator):
         if case == "case 2: higher scitype supported" and output_scitype == "Series":
             Xt = convert_to(
                 Xt,
-                to_type=["pd-multiindex", "numpy3D", "df-list", "pd_multiindex_hier"],
+                to_type=["pd-multiindex", "numpy3D", "np-list", "pd_multiindex_hier"],
             )
             Xt = convert_to_scitype(Xt, to_scitype=X_input_scitype)
 
diff --git a/aeon/utils/validation/__init__.py b/aeon/utils/validation/__init__.py
index 766f98b4..53533211 100644
--- a/aeon/utils/validation/__init__.py
+++ b/aeon/utils/validation/__init__.py
@@ -25,6 +25,7 @@ __all__ = [
     "is_hierarchical",
     "is_valid_input",
     "validate_input",
+    "abstract_types",
 ]
 
 import os
@@ -34,7 +35,7 @@ from typing import Union
 import numpy as np
 import pandas as pd
 
-from aeon.utils.validation._input import is_valid_input, validate_input
+from aeon.utils.validation._input import abstract_types, is_valid_input, validate_input
 from aeon.utils.validation.collection import (
     get_n_cases,
     get_type,
diff --git a/aeon/utils/validation/_input.py b/aeon/utils/validation/_input.py
index d2ed62a7..2d0ae257 100644
--- a/aeon/utils/validation/_input.py
+++ b/aeon/utils/validation/_input.py
@@ -18,6 +18,50 @@ from aeon.utils.validation.collection import (
 )
 from aeon.utils.validation.series import is_hierarchical, is_single_series
 
+# String identifiers for the different types of time series data
+COLLECTIONS = ["numpy3D", "nested_univ", "pd-multiindex"]
+SERIES = ["pd.Series", "pd.DataFrame", "np.ndarray"]
+HIERARCHICAL = ["pd_multiindex_hier"]
+# not needed PROBA = ["pred_interval", "pred_quantiles", "pred_var"]
+
+
+def _abstract_type(input_type: str) -> str:
+    """Return the abstract type based on the string identifier of the input.
+
+    Parameters
+    ----------
+    input_type : str
+        String representation of the input type.
+
+    Returns
+    -------
+    str
+        Abstract type of the input, one of Series, Panel, Hierarchical or Unknown.
+    """
+    if input_type in SERIES:
+        return "Series"
+    if input_type in COLLECTIONS:
+        return "Panel"
+    if input_type in HIERARCHICAL:
+        return "Hierarchical"
+    return "Unknown"
+
+
+def abstract_types(input_types: list) -> list:
+    """Return the abstract types based on the string identifier of the input.
+
+    Parameters
+    ----------
+    input_types : list of str
+        List of string representation of the input types.
+
+    Returns
+    -------
+    list of str
+        Abstract type of the input, one of Series, Panel, Hierarchical or Unknown.
+    """
+    return [_abstract_type(x) for x in input_types]
+
 
 def is_valid_input(X):
     """Test if input valid.
@@ -40,6 +84,12 @@ def is_valid_input(X):
 def validate_input(X):
     """Validate input.
 
+    This function checks if the input is a valid time series data structure for a
+    single series, collection of series or hierarchical series. If the input is valid
+    for one of these abstract types, it finds the metadata relating to the type,
+    whether series are univariate and whether it has nans or not, and returns this
+    information in a dictionary.
+
     Parameters
     ----------
     X : array-like


Test patch:
diff --git a/aeon/testing/utils/scenarios_transformers.py b/aeon/testing/utils/scenarios_transformers.py
index 93a85f7f..f969ee58 100644
--- a/aeon/testing/utils/scenarios_transformers.py
+++ b/aeon/testing/utils/scenarios_transformers.py
@@ -16,7 +16,6 @@ import pandas as pd
 from sklearn.utils import check_random_state
 
 from aeon.base import BaseObject
-from aeon.datatypes import mtype_to_scitype
 from aeon.testing.utils.data_gen import (
     _make_classification_y,
     _make_collection_X,
@@ -25,6 +24,7 @@ from aeon.testing.utils.data_gen import (
 )
 from aeon.testing.utils.scenarios import TestScenario
 from aeon.transformations.collection.base import BaseCollectionTransformer
+from aeon.utils.validation import abstract_types
 
 # random seed for generating data to keep scenarios exactly reproducible
 RAND_SEED = 42
@@ -109,9 +109,7 @@ class TransformerTestScenario(TestScenario, BaseObject):
 
         # the case that we would need to vectorize with y, skip
         X_inner_type = get_tag(obj, "X_inner_type")
-        X_inner_abstract_types = mtype_to_scitype(
-            X_inner_type, return_unique=True, coerce_to_list=True
-        )
+        X_inner_abstract_types = abstract_types(X_inner_type)
         # we require vectorization from of a Series trafo to Panel data ...
         if X_type == "Panel" and "Panel" not in X_inner_abstract_types:
             # ... but y is passed and y is not ignored internally ...
@@ -122,9 +120,7 @@ class TransformerTestScenario(TestScenario, BaseObject):
         # ensure scenario y matches type of inner y
         y_inner_type = get_tag(obj, "y_inner_type")
         if y_inner_type not in [None, "None"]:
-            y_inner_abstract_types = mtype_to_scitype(
-                y_inner_type, return_unique=True, coerce_to_list=True
-            )
+            y_inner_abstract_types = abstract_types(y_inner_type)
             if y_type not in y_inner_abstract_types:
                 return False
 
diff --git a/aeon/transformations/tests/test_base.py b/aeon/transformations/tests/test_base.py
index cc0f3fe2..0b35fa1f 100644
--- a/aeon/transformations/tests/test_base.py
+++ b/aeon/transformations/tests/test_base.py
@@ -3,7 +3,6 @@
 Each test covers a "decision path" in the base class boilerplate,
     with a focus on frequently breaking paths in base class refactor and bugfixing.
 The path taken depends on tags of a given transformer, and input data type.
-Concrete transformer classes from aeon are imported to cover
 Concrete transformer classes from aeon are imported to cover
     different combinations of transformer tags.
 Transformer scenarios cover different combinations of input data types.
diff --git a/aeon/utils/validation/tests/test_forecasting.py b/aeon/utils/validation/tests/test_forecasting.py
index e2c485e0..55f147eb 100644
--- a/aeon/utils/validation/tests/test_forecasting.py
+++ b/aeon/utils/validation/tests/test_forecasting.py
@@ -1,6 +1,6 @@
 """Test forecasting module."""
 
-__maintainer__ = []
+__maintainer__ = ["TonyBagnall"]
 
 from datetime import timedelta
 
diff --git a/aeon/utils/validation/tests/test_input.py b/aeon/utils/validation/tests/test_input.py
index e1e7f9f1..e6c04a53 100644
--- a/aeon/utils/validation/tests/test_input.py
+++ b/aeon/utils/validation/tests/test_input.py
@@ -4,17 +4,41 @@ import pytest
 
 from aeon.testing.utils.data_gen import get_examples
 from aeon.testing.utils.data_gen._collection import EQUAL_LENGTH_UNIVARIATE
-from aeon.utils.validation._input import is_hierarchical, is_valid_input, validate_input
+from aeon.utils.validation._input import (
+    COLLECTIONS,
+    HIERARCHICAL,
+    SERIES,
+    _abstract_type,
+    abstract_types,
+    is_hierarchical,
+    is_valid_input,
+    validate_input,
+)
 from aeon.utils.validation.collection import is_collection
 from aeon.utils.validation.series import is_single_series
 
-# Tests limited to these input, some  collection types are ambiguous
-COLLECTIONS = ["numpy3D", "nested_univ", "pd-multiindex"]
-SERIES = ["pd.Series", "pd.DataFrame", "np.ndarray"]
-HIERARCHICAL = ["pd_multiindex_hier"]
 PANDAS_TYPES = ["pd.Series", "pd.DataFrame", "pd-multiindex", "pd_multiindex_hier"]
 
 
+def test_abstract_types():
+    """Test abstract_types."""
+    for s in SERIES:
+        assert _abstract_type(s) == "Series"
+    for c in COLLECTIONS:
+        assert _abstract_type(c) == "Panel"
+    for h in HIERARCHICAL:
+        assert _abstract_type(h) == "Hierarchical"
+    assert _abstract_type("Arsenal") == "Unknown"
+    comb = ["pd.Series", "pd.DataFrame", "pd-multiindex", "pd_multiindex_hier", "Foo"]
+    assert abstract_types(comb) == [
+        "Series",
+        "Series",
+        "Panel",
+        "Hierarchical",
+        "Unknown",
+    ]
+
+
 @pytest.mark.parametrize("data", COLLECTIONS)
 def test_input_collections(data):
     """Test is_collection with correct input."""
diff --git a/aeon/utils/validation/tests/test_series.py b/aeon/utils/validation/tests/test_series.py
index 2bf97a8d..eba36e89 100644
--- a/aeon/utils/validation/tests/test_series.py
+++ b/aeon/utils/validation/tests/test_series.py
@@ -1,6 +1,6 @@
 """Test series module."""
 
-__maintainer__ = []
+__maintainer__ = ["TonyBagnall"]
 
 import numpy as np
 import pandas as pd
@@ -165,7 +165,7 @@ def test_df_checks():
 
 
 def test_is_pred_interval_proba():
-    """Test is pred interval proba."""
+    """Test is_pred_interval_proba."""
     # Create a correct MultiIndex DataFrame
     idx = pd.MultiIndex.from_tuples(
         [(1, 0.9, "upper"), (1, 0.9, "lower")], names=["level_0", "coverage", "bound"]