Repository: sktime/sktime

Issue:
[BUG] `unequal_length` clusterer bugs out in unequal data set
**Describe the bug**
`TimeSeriesKMeansTslearn` is a clusterer which is able to be fitted with unequal length of time series within a data set (according to tags). But it does not seems to be doing what is it suppose to do. Possible bug/wrong usage/missing preprocessing step?

**To Reproduce**
```python
from sktime.datasets import load_acsf1
from sktime.registry import all_estimators
from sktime.clustering.k_means._k_means_tslearn import TimeSeriesKMeansTslearn
RANDOM_STATE= 2
no_of_unknown_clusters = 5

# import data
X, _ = load_acsf1(return_type='pd-multiindex')

# remove last 10 rows from the last appliance to simulate unequal time series 
X_mod = X.iloc[:-10]

# instantiate clusterer which can handle unequal time series
# all_estimators('clusterer', as_dataframe=True, filter_tags={'capability:unequal_length': True})
unequal_clst = TimeSeriesKMeansTslearn(n_clusters=no_of_unknown_clusters, n_jobs=-1, random_state=RANDOM_STATE)

# fit the clusterer
unequal_clst.fit(X_mod) # error
```

```
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[119], line 18
     15 unequal_clst = TimeSeriesKMeansTslearn(n_clusters=no_of_unknown_clusters, n_jobs=-1, random_state=RANDOM_STATE)
     17 # fit the clusterer
---> 18 unequal_clst.fit(X_mod)

File c:\Users\agpgago\data-science\forecast-wizard-test\venv\lib\site-packages\sktime\clustering\base.py:110, in BaseClusterer.fit(self, X, y)
    107 # reset estimator at the start of fit
    108 self.reset()
--> 110 X = self._check_clusterer_input(X)
    112 multithread = self.get_tag("capability:multithreading")
    113 if multithread:

File c:\Users\agpgago\data-science\forecast-wizard-test\venv\lib\site-packages\sktime\clustering\base.py:422, in BaseClusterer._check_clusterer_input(self, X, enforce_min_instances)
    420 unequal = not X_metadata["is_equal_length"]
    421 self._check_capabilities(missing, multivariate, unequal)
--> 422 return convert_to(
    423     X,
    424     to_type=self.get_tag("X_inner_mtype"),
    425     as_scitype="Panel",
    426 )

File c:\Users\agpgago\data-science\forecast-wizard-test\venv\lib\site-packages\sktime\datatypes\_convert.py:263, in convert_to(obj, to_type, as_scitype, store, store_behaviour, return_to_mtype)
    260 from_type = infer_mtype(obj=obj, as_scitype=as_scitype)
...
    610 X_values = X_coerced.values
--> 611 X_3d = X_values.reshape(n_instances, n_timepoints, n_columns).swapaxes(1, 2)
    613 return X_3d

ValueError: cannot reshape array of size 291990 into shape (200,1460,1)
```

For the above to work, I actually have to align the time period of all time series into the same range in order for the clusterer to work. Doesn't that defeat the purpose of having a clusterer that works for unequal length time series.
```python
from sktime.transformations.panel.padder import PaddingTransformer

clst_pipe = PaddingTransformer() * unequal_clst
clst_pipe.fit(X_mod) # no errors
```

**Expected behavior**
Fitted clusterer on multiple time series which are unequal in length

**Versions**
<details>

```bash
System:
    python: 3.10.9 | packaged by Anaconda, Inc. | (main, Mar  1 2023, 18:18:15) [MSC v.1916 64 bit (AMD64)]
executable: [c:\Users\agpgago\data-science\forecast-wizard-test\venv\Scripts\python.exe](file:///C:/Users/agpgago/data-science/forecast-wizard-test/venv/Scripts/python.exe)
   machine: Windows-10-10.0.19042-SP0

Python dependencies:
          pip: 23.3.2
       sktime: 0.28.0
      sklearn: 1.4.1.post1
       skbase: 0.7.2
        numpy: 1.26.4
        scipy: 1.12.0
       pandas: 2.1.4
   matplotlib: 3.8.3
       joblib: 1.3.2
        numba: 0.58.1
  statsmodels: 0.14.1
     pmdarima: 2.0.4
statsforecast: 1.7.3
      tsfresh: 0.20.2
      tslearn: 0.6.3
        torch: None
   tensorflow: 2.15.0
tensorflow_probability: None
```
</details>




Discussion:
Yes, I think the tag is simply wrong, thanks for reporting.

I think the tag is wrong because this interfaces `tslearn`, and `tslearn` uses vanilla 3D `numpy` internally, which implies unequal length.

Resolving patch:
diff --git a/sktime/clustering/k_means/_k_means_tslearn.py b/sktime/clustering/k_means/_k_means_tslearn.py
--- a/sktime/clustering/k_means/_k_means_tslearn.py
+++ b/sktime/clustering/k_means/_k_means_tslearn.py
@@ -107,7 +107,7 @@ class TimeSeriesKMeansTslearn(_TslearnAdapter, BaseClusterer):
         # estimator type
         # --------------
         "capability:multivariate": True,
-        "capability:unequal_length": True,
+        "capability:unequal_length": False,
     }
 
     # defines the name of the attribute containing the tslearn estimator
diff --git a/sktime/clustering/k_shapes.py b/sktime/clustering/k_shapes.py
--- a/sktime/clustering/k_shapes.py
+++ b/sktime/clustering/k_shapes.py
@@ -57,7 +57,7 @@ class TimeSeriesKShapes(_TslearnAdapter, BaseClusterer):
         # estimator type
         # --------------
         "capability:multivariate": True,
-        "capability:unequal_length": True,
+        "capability:unequal_length": False,
     }
 
     # defines the name of the attribute containing the tslearn estimator


Test patch:
diff --git a/sktime/utils/_testing/scenarios_clustering.py b/sktime/utils/_testing/scenarios_clustering.py
--- a/sktime/utils/_testing/scenarios_clustering.py
+++ b/sktime/utils/_testing/scenarios_clustering.py
@@ -7,7 +7,11 @@
 
 __all__ = ["scenarios_clustering"]
 
+from inspect import isclass
+
 from sktime.base import BaseObject
+from sktime.registry import scitype
+from sktime.utils._testing.hierarchical import _make_hierarchical
 from sktime.utils._testing.panel import _make_panel_X, make_clustering_problem
 from sktime.utils._testing.scenarios import TestScenario
 
@@ -43,11 +47,52 @@ def get_args(self, key, obj=None, deepcopy_args=True):
 
         return super().get_args(key=key, obj=obj, deepcopy_args=deepcopy_args)
 
+    def is_applicable(self, obj):
+        """Check whether scenario is applicable to obj.
+
+        Parameters
+        ----------
+        obj : class or object to check against scenario
+
+        Returns
+        -------
+        applicable: bool
+            True if self is applicable to obj, False if not
+        """
+
+        def get_tag(obj, tag_name):
+            if isclass(obj):
+                return obj.get_class_tag(tag_name)
+            else:
+                return obj.get_tag(tag_name)
+
+        # applicable only if obj inherits from BaseClassifier, BaseEarlyClassifier or
+        #   BaseRegressor. currently we test both classifiers and regressors using these
+        #   scenarios
+        if scitype(obj) != "clusterer":
+            return False
+
+        # if X is multivariate, applicable only if can handle multivariate
+        is_multivariate = not self.get_tag("X_univariate")
+        if is_multivariate and not get_tag(obj, "capability:multivariate"):
+            return False
+
+        # if X is unequal length, applicable only if can handle unequal length
+        is_unequal_length = self.get_tag("X_unequal_length")
+        if is_unequal_length and not get_tag(obj, "capability:unequal_length"):
+            return False
+
+        return True
+
 
 class ClustererFitPredict(ClustererTestScenario):
     """Fit/predict with panel Xmake_clustering_problem."""
 
-    _tags = {"X_univariate": True, "is_enabled": True}
+    _tags = {
+        "X_univariate": True,
+        "X_unequal_length": False,
+        "is_enabled": True,
+    }
 
     args = {
         "fit": {"X": make_clustering_problem(random_state=RAND_SEED)},
@@ -56,6 +101,39 @@ class ClustererFitPredict(ClustererTestScenario):
     default_method_sequence = ["fit", "predict"]
 
 
+class ClustererFitPredictUnequalLength(ClustererTestScenario):
+    """Fit/predict with univariate panel X, unequal length series."""
+
+    _tags = {
+        "X_univariate": True,
+        "X_unequal_length": True,
+        "is_enabled": True,
+    }
+
+    @property
+    def args(self):
+        X_unequal_length = _make_hierarchical(
+            hierarchy_levels=(10,),
+            min_timepoints=10,
+            max_timepoints=15,
+            random_state=RAND_SEED,
+        )
+        X_unequal_length_test = _make_hierarchical(
+            hierarchy_levels=(5,),
+            min_timepoints=10,
+            max_timepoints=15,
+            random_state=RAND_SEED,
+        )
+        return {
+            "fit": {"X": X_unequal_length},
+            "predict": {"X": X_unequal_length_test},
+        }
+
+    default_method_sequence = ["fit", "predict", "predict_proba", "decision_function"]
+    default_arg_sequence = ["fit", "predict", "predict", "predict"]
+
+
 scenarios_clustering = [
     ClustererFitPredict,
+    ClustererFitPredictUnequalLength,
 ]