Repository: sktime/sktime
Pull request #7546 (2024-12-18 23:50:42)

[ENH] Detection metrics - coercion to detection type in boilerplate
Currently, detection metrics can accept segments or point event detections, as determined by the `"scitype:y"` tag.

Currently, if a metric is given the other type, it fails.

This PR adds conversion between the two types with, in my opinion, sensible choices to use in detection metrics - so all metrics can now be provided with point or segment detections.

Under the hood, this uses new `datatypes`-like functionality in the `detection` module - optimally, mid-term this gets refactored to deduplicate with the converters in `BaseDetector`, or moved to `sktime.datatypes` as an mtype.

However, for the time being, it seemed more sensible to add new code, as the conversions required for metrics are not exactly those in the base class.


Patch:
diff --git a/.all-contributorsrc b/.all-contributorsrc
index fa6c6a0a7..7db6a11b5 100644
--- a/.all-contributorsrc
+++ b/.all-contributorsrc
@@ -3295,6 +3295,15 @@
         "doc",
         "code"
       ]
+    },
+    {
+      "login": "VjayRam",
+      "name": "Vijay Ram Enaganti",
+      "avatar_url": "https://avatars.githubusercontent.com/u/76694566?v=4",
+      "profile": "https://github.com/VjayRam",
+      "contributions": [
+        "doc"
+      ]
     }
   ]
 }
diff --git a/sktime/datatypes/_table/_check.py b/sktime/datatypes/_table/_check.py
index 4517e7338..bfd24c776 100644
--- a/sktime/datatypes/_table/_check.py
+++ b/sktime/datatypes/_table/_check.py
@@ -45,7 +45,32 @@ PRIMITIVE_TYPES = (float, int, str)
 
 
 class TablePdDataFrame(ScitypeTable):
-    """Data type: pandas.DataFrame based specification of data frame table.
+    """Data type: pandas.DataFrame based specification of tabular data.
+
+    Name: ``"pd_DataFrame_Table"``
+
+    Short description:
+
+    a pandas ``DataFrame`` representing tabular data,
+    with rows = instances, cols = features
+
+    Long description:
+
+    The ``"pd_DataFrame_Table"`` :term:`mtype` is a concrete specification
+    that implements the ``Table`` :term:`scitype`, i.e., the abstract
+    type of tabular data.
+
+    An object ``obj: pandas.DataFrame`` follows the specification iff:
+
+    * structure convention: ``obj.index`` can be any valid pandas index.
+    * features: columns of ``obj`` correspond to different features
+    * feature names: column names ``obj.columns``
+    * instances: rows of ``obj`` correspond to different instances
+
+    Capabilities:
+
+    * can represent multivariate data
+    * can represent missing values
 
     Parameters are inferred by check.
 
@@ -139,7 +164,32 @@ def _check_pddataframe_table(obj, return_metadata=False, var_name="obj"):
 
 
 class TablePdSeries(ScitypeTable):
-    """Data type: pandas.Series based specification of data frame table.
+    """Data type: pandas.Series based specification of tabular data.
+
+    Name: ``"pd_Series_Table"``
+
+    Short description:
+
+    a pandas ``Series`` representing tabular data,
+    with rows = instances, single feature
+
+    Long description:
+
+    The ``"pd_Series_Table"`` :term:`mtype` is a concrete specification
+    that implements the ``Table`` :term:`scitype`, i.e., the abstract
+    type of tabular data.
+
+    An object ``obj: pandas.Series`` follows the specification iff:
+
+    * structure convention: ``obj.index`` can be any valid pandas index.
+    * feature: the series ``obj`` represents a single feature
+    * feature name: the ``name`` attribute of the ``pd.Series`` object
+    * instances: rows of ``obj`` correspond to different instances
+
+    Capabilities:
+
+    * cannot represent multivariate data
+    * can represent missing values
 
     Parameters are inferred by check.
 
diff --git a/sktime/detection/_datatypes/_convert.py b/sktime/detection/_datatypes/_convert.py
new file mode 100644
index 000000000..3d6f1c8ef
--- /dev/null
+++ b/sktime/detection/_datatypes/_convert.py
@@ -0,0 +1,80 @@
+"""Utilities to handle checks and conversions between output formats of detectors."""
+
+import numpy as np
+import pandas as pd
+
+
+def _convert_points_to_segments(points_df, len_X=None, include_labels=False):
+    """Convert points-like output to segments-like output.
+
+    Parameters
+    ----------
+    points_df : pd.DataFrame
+        Points-like output of a detector
+    len_X : int, optional, default=None
+        Length of the input data.
+        If None, the last segment is assumed to be length 1.
+    include_labels : bool, optional, default=False
+        Whether to include labels in the output.
+        If True, labels are RangeIndex of the same length as the segments.
+
+    Returns
+    -------
+    pd.DataFrame
+        Segments-like output of a detector
+    """
+    if len(points_df) == 0:
+        return pd.DataFrame(columns=["ilocs", "labels"], dtype="int64")
+
+    points = points_df.ilocs.values
+    if points[0] != 0:
+        points = np.insert(points, 0, 0)
+    if len_X is None:
+        points = np.append(points, points[-1] + 1)
+    elif points[-1] != len_X:
+        points = np.append(points, len_X)
+
+    ilocs = pd.IntervalIndex.from_breaks(points, closed="left")
+
+    df_dict = {"ilocs": ilocs}
+    if include_labels:
+        labels = pd.RangeIndex(len(ilocs))
+        df_dict["labels"] = labels
+
+    return pd.DataFrame(df_dict)
+
+
+def _convert_segments_to_points(seg_df, len_X=None):
+    """Convert points-like output to segments-like output.
+
+    Parameters
+    ----------
+    seg_df : pd.DataFrame
+        Points-like output of a detector
+    len_X : int, optional
+        Length of the input data.
+        If None, the (right point) end of the last segment is omitted.
+
+    Returns
+    -------
+    pd.DataFrame
+        Segments-like output of a detector
+    """
+    if len(seg_df) == 0:
+        return pd.DataFrame(columns=["ilocs"], dtype="int64")
+
+    ix = seg_df.set_index("ilocs").index
+    vals = np.array([ix.left.values, ix.right.values]).transpose().flatten()
+    vals = np.unique(vals)
+
+    if not ix.is_non_overlapping_monotonic:
+        vals = np.sort(vals)
+
+    if vals[0] == 0:
+        vals = vals[1:]
+
+    if len_X is None or vals[-1] == len_X:
+        vals = vals[:-1]
+
+    points_df = pd.DataFrame({"ilocs": vals})
+    return points_df
diff --git a/sktime/detection/_datatypes/_examples.py b/sktime/detection/_datatypes/_examples.py
new file mode 100644
index 000000000..7f74c22f1
--- /dev/null
+++ b/sktime/detection/_datatypes/_examples.py
@@ -0,0 +1,68 @@
+"""Utilities to handle checks and conversions between output formats of detectors."""
+
+import pandas as pd
+
+
+def _get_example_segments_0():
+    """Generate example 0 for segmentation output.
+
+    - non-overlapping
+    - non-exhaustive
+    - no labels
+    """
+    segs = pd.IntervalIndex.from_tuples([(1, 2), (3, 6), (7, 10)], "left")
+    return pd.DataFrame({"ilocs": segs})
+
+
+def _get_example_segments_1():
+    """Generate example 1 for segmentation output.
+
+    - non-overlapping
+    - non-exhaustive
+    - labeled
+    """
+    segs = pd.IntervalIndex.from_tuples([(1, 2), (3, 6), (7, 10), (12, 14)], "left")
+    labels = pd.Series([0, 1, 1, 0], index=segs)
+    return pd.DataFrame({"ilocs": segs, "labels": labels})
+
+
+def _get_example_segments_2():
+    """Generate example 2 for segmentation output.
+
+    - non-overlapping
+    - exhaustive
+    - labeled
+    """
+    segs = pd.IntervalIndex.from_tuples([(0, 2), (2, 3), (3, 7), (7, 10)], "left")
+    labels = pd.Series([0, 1, 2, 3])
+    return pd.DataFrame({"ilocs": segs, "labels": labels})
+
+
+def _get_example_points_2():
+    """Generate example 2 for point output.
+
+    Corresponds to example 2 for segmentation output.
+    """
+    points = pd.Series([2, 3, 7], name="ilocs")
+    return pd.DataFrame({"ilocs": points})
+
+
+def _get_example_segments_3():
+    """Generate example 3 for segmentation output.
+
+    - overlapping
+    - exhaustive
+    - labeled
+    """
+    segs = pd.IntervalIndex.from_tuples([(0, 3), (2, 5), (3, 7), (7, 10)], "left")
+    labels = pd.Series([0, 1, 2, 3])
+    return pd.DataFrame({"ilocs": segs, "labels": labels})
+
+
+def _get_example_points_3():
+    """Generate example 3 as points only.
+
+    Corresponds to example 3 for point output.
+    """
+    pts = [2, 3, 5, 7]
+    return pd.DataFrame({"ilocs": pts})
diff --git a/sktime/performance_metrics/detection/_base.py b/sktime/performance_metrics/detection/_base.py
index b802856bd..fdc1978ad 100644
--- a/sktime/performance_metrics/detection/_base.py
+++ b/sktime/performance_metrics/detection/_base.py
@@ -2,6 +2,14 @@
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
 
 from sktime.datatypes import check_is_scitype, convert_to
+from sktime.detection._datatypes._check import (
+    _is_points_dtype,
+    _is_segments_dtype,
+)
+from sktime.detection._datatypes._convert import (
+    _convert_points_to_segments,
+    _convert_segments_to_points,
+)
 from sktime.performance_metrics.base import BaseMetric
 
 
@@ -144,4 +152,25 @@ class BaseDetectionMetric(BaseMetric):
         y_pred = _coerce_to_df(y_pred, var_name="y_pred")
         X = _coerce_to_df(X, var_name="X", allow_none=allow_none_X)
 
+        # coerce to detection type
+        y_true = self._coerce_to_detection_type(y_true, X, allow_none=allow_none_y_true)
+        y_pred = self._coerce_to_detection_type(y_pred, X)
+
         return y_true, y_pred, X
+
+    def _coerce_to_detection_type(self, y, X, allow_none=False):
+        """Coerce input to detection type."""
+        if allow_none and y is None:
+            return None
+
+        detection_type = self.get_tag("scitype:y")
+        if X is None:
+            len_X = None
+        else:
+            len_X = len(X)
+
+        if _is_points_dtype(y) and detection_type == "segments":
+            y = _convert_points_to_segments(y, len_X=len_X)
+        elif _is_segments_dtype(y) and detection_type == "points":
+            y = _convert_segments_to_points(y, len_X=len_X)
+        return y
diff --git a/sktime/split/instance.py b/sktime/split/instance.py
index c36b72c80..678163538 100644
--- a/sktime/split/instance.py
+++ b/sktime/split/instance.py
@@ -2,7 +2,7 @@
 # copyright: sktime developers, BSD-3-Clause License (see LICENSE file)
 """Splitter that uses an sklearn splitter to split by instance."""
 
-__author__ = ["fkiraly"]
+__author__ = ["fkiraly", "ksharma6"]
 
 __all__ = [
     "InstanceSplitter",
@@ -10,11 +10,11 @@ __all__ = [
 
 from typing import Optional
 
-import numpy as np
 import pandas as pd
 
 from sktime.split.base import BaseSplitter
 from sktime.split.base._common import ACCEPTED_Y_TYPES, SPLIT_GENERATOR_TYPE
+from sktime.utils.multiindex import apply_split
 
 
 class InstanceSplitter(BaseSplitter):
@@ -62,15 +62,10 @@ class InstanceSplitter(BaseSplitter):
             y = pd.MultiIndex.from_arrays([zeros, y])
 
         inst_ix = y.droplevel(-1).unique()
-        iloc_ixer = pd.DataFrame(pd.RangeIndex(len(y)), index=y)
 
         for y_train_inst_iloc, y_test_inst_iloc in self.cv.split(inst_ix):
-            y_train_inst_loc = inst_ix[np.array(y_train_inst_iloc)]
-            y_test_inst_loc = inst_ix[np.array(y_test_inst_iloc)]
-            tr_np = [iloc_ixer.loc[x].to_numpy().flatten() for x in y_train_inst_loc]
-            tt_np = [iloc_ixer.loc[x].to_numpy().flatten() for x in y_test_inst_loc]
-            y_train_iloc = np.concatenate(tr_np)
-            y_test_iloc = np.concatenate(tt_np)
+            y_train_iloc = apply_split(y, y_train_inst_iloc)
+            y_test_iloc = apply_split(y, y_test_inst_iloc)
             yield y_train_iloc, y_test_iloc
 
     def get_n_splits(self, y: Optional[ACCEPTED_Y_TYPES] = None) -> int:
diff --git a/sktime/utils/multiindex.py b/sktime/utils/multiindex.py
index 38d23175b..a02acc0c0 100644
--- a/sktime/utils/multiindex.py
+++ b/sktime/utils/multiindex.py
@@ -4,6 +4,7 @@
 __author__ = ["fkiraly"]
 __all__ = []
 
+import numpy as np
 import pandas as pd
 
 
@@ -83,3 +84,46 @@ def rename_multiindex(idx, feature_names_out, idx_name="index"):
             'must be one of "flat", "multiindex", "original", "auto", '
             f"but found {feature_names_out}"
         )
+
+
+def apply_split(y, iloc_ix):
+    """Generate iloc indices to split according to first MultiIndex level.
+
+    Applies iloc_ix to the leftmost (0-th) level of a MultiIndex y.
+
+    Parameters
+    ----------
+    y : pd.MultiIndex or pd.Index
+        Index to split, coerced to MultiIndex if not already
+    iloc_ix: 1D np.ndarray of integer
+        iloc indices to apply to the first level of y
+
+    Returns
+    -------
+    y_iloc : ndarray
+        iloc indices for y after applying iloc_ix to the first level
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> import pandas as pd
+    >>> from sktime.utils.multiindex import apply_split
+    >>> y = pd.MultiIndex.from_tuples([(0, 0), (0, 1), (1, 0), (1, 1)])
+    >>> iloc_ix = np.array([1, 0])
+    >>> apply_split(y, iloc_ix)
+    array([2, 3, 0, 1])
+    """
+    if not isinstance(y, pd.MultiIndex):
+        zeros = [0] * len(y)
+        y = pd.MultiIndex.from_arrays([zeros, y])
+
+    if not isinstance(iloc_ix, np.ndarray):
+        iloc_ix = np.array(iloc_ix)
+
+    inst_ix = y.droplevel(-1).unique()
+    iloc_ixer = pd.DataFrame(pd.RangeIndex(len(y)), index=y)
+
+    y_loc = inst_ix[np.array(iloc_ix)]
+    y_np = [iloc_ixer.loc[x].to_numpy().flatten() for x in y_loc]
+    y_iloc = np.concatenate(y_np)
+    return y_iloc


Test patch:
diff --git a/sktime/detection/_datatypes/test/__init__.py b/sktime/detection/_datatypes/test/__init__.py
new file mode 100644
index 000000000..41ca60652
--- /dev/null
+++ b/sktime/detection/_datatypes/test/__init__.py
@@ -0,0 +1,1 @@
+"""Tests for utilities for checks and conversions for detection outputs."""
diff --git a/sktime/detection/_datatypes/test/test_convert.py b/sktime/detection/_datatypes/test/test_convert.py
new file mode 100644
index 000000000..348fcfe73
--- /dev/null
+++ b/sktime/detection/_datatypes/test/test_convert.py
@@ -0,0 +1,47 @@
+"""Tests for detection output converters."""
+
+import pandas as pd
+import pytest
+
+from sktime.detection._datatypes._convert import (
+    _convert_points_to_segments,
+    _convert_segments_to_points,
+)
+from sktime.detection._datatypes._examples import (
+    _get_example_points_2,
+    _get_example_points_3,
+    _get_example_segments_2,
+    _get_example_segments_3,
+)
+from sktime.tests.test_switch import run_test_module_changed
+
+
+@pytest.mark.skipif(
+    not run_test_module_changed("sktime.detection._datatypes"),
+    reason="Test only runs when module changed",
+)
+def test_convert_points_to_segments():
+    """Test _convert_points_to_segments on an example."""
+    points_df = _get_example_points_2()
+    segments_df_no_lab = _convert_points_to_segments(points_df, len_X=10)
+    segments_df_w_lab = _convert_points_to_segments(
+        points_df, len_X=10, include_labels=True
+    )
+    expected_segments_df_w_lab = _get_example_segments_2()
+    expected_segments_df_no_lab = expected_segments_df_w_lab.drop(columns=["labels"])
+
+    pd.testing.assert_frame_equal(segments_df_no_lab, expected_segments_df_no_lab)
+    pd.testing.assert_frame_equal(segments_df_w_lab, expected_segments_df_w_lab)
+
+
+@pytest.mark.skipif(
+    not run_test_module_changed("sktime.detection._datatypes"),
+    reason="Test only runs when module changed",
+)
+def test_convert_segments_to_points():
+    """Test _convert_points_to_segments on an example."""
+    seg_df = _get_example_segments_3()
+    points_df_expected = _get_example_points_3()
+    points_df = _convert_segments_to_points(seg_df, len_X=10)
+
+    pd.testing.assert_frame_equal(points_df, points_df_expected)
diff --git a/sktime/performance_metrics/detection/tests/test_base.py b/sktime/performance_metrics/detection/tests/test_base.py
new file mode 100644
index 000000000..1de740f8a
--- /dev/null
+++ b/sktime/performance_metrics/detection/tests/test_base.py
@@ -0,0 +1,38 @@
+"""Tests for base class boilerplate."""
+
+import pandas as pd
+import pytest
+
+from sktime.detection._datatypes._examples import (
+    _get_example_points_2,
+    _get_example_points_3,
+    _get_example_segments_2,
+    _get_example_segments_3,
+)
+from sktime.performance_metrics.detection._chamfer import DirectedChamfer
+from sktime.tests.test_switch import run_test_module_changed
+
+
+@pytest.mark.skipif(
+    not run_test_module_changed("sktime.performance_metrics.detection"),
+    reason="run test only if detection module changed",
+)
+def test_input_coercions():
+    """Test that segments are coerced for point metrics."""
+    y_pred_seg = _get_example_segments_2()
+    y_true_seg = _get_example_segments_3()
+
+    y_pred_pts = _get_example_points_2()
+    y_true_pts = _get_example_points_3()
+
+    metric = DirectedChamfer()
+
+    loss_seg = metric(y_true_seg, y_pred_seg)
+    loss_pts = metric(y_true_pts, y_pred_pts)
+    assert loss_seg == loss_pts
+
+    X = pd.DataFrame({"range10": range(10)})
+
+    loss_seg_X = metric(y_true_seg, y_pred_seg, X)
+    loss_pts_X = metric(y_true_pts, y_pred_pts, X)
+    assert loss_seg_X == loss_pts_X
diff --git a/sktime/utils/tests/test_multiindex.py b/sktime/utils/tests/test_multiindex.py
index 2023f4d5d..b2b5bbc56 100644
--- a/sktime/utils/tests/test_multiindex.py
+++ b/sktime/utils/tests/test_multiindex.py
@@ -1,10 +1,11 @@
 """Testing multiindex utilities."""
 
+import numpy as np
 import pandas as pd
 import pytest
 
 from sktime.tests.test_switch import run_test_module_changed
-from sktime.utils.multiindex import flatten_multiindex, rename_multiindex
+from sktime.utils.multiindex import apply_split, flatten_multiindex, rename_multiindex
 
 
 @pytest.mark.skipif(
@@ -79,3 +80,21 @@ def test_rename_multiindex():
 
     assert isinstance(orig, pd.Index)
     assert (expected_orig == orig).all()
+
+
+@pytest.mark.skipif(
+    not run_test_module_changed(["sktime.utils.multiindex"]),
+    reason="Run if multiindex module has changed.",
+)
+def test_apply_split():
+    """Test apply_split() and check shapes"""
+    mi = pd.MultiIndex.from_product([["a", "b"], [0, 42]])
+
+    iloc_ix = [1]
+
+    result = apply_split(mi, iloc_ix)
+
+    expected = np.array([2, 3])
+
+    assert isinstance(result, np.ndarray)
+    assert np.array_equal(result, expected)