Repository: alan-turing-institute/sktime
Pull request #3303 (2022-08-20 15:44:34)

[BUG] Fix `write_ndarray_to_tsfile` for `classLabel = False`
#### Reference Issues/PRs
Fixes the following issue when reading a tsfile written with `write_ndarray_to_tsfile`

```
---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
/tmp/ipykernel_1092/2889448795.py in <module>
----> 1 df = load_from_tsfile_to_dataframe("./amex_train_1.ts/AMEX-Data/AMEX-Data.ts", replace_missing_vals_with='NaN', return_separate_X_and_y=False)

/tmp/ipykernel_1092/3989994789.py in load_from_tsfile_to_dataframe(full_file_path_and_name, return_separate_X_and_y, replace_missing_vals_with)
    159                         print(has_data_tag)
    160                         raise IOError(
--> 161                             "a full set of metadata has not been provided "
    162                             "before the data"
    163                         )

OSError: a full set of metadata has not been provided before the data
```


#### What does this implement/fix? Explain your changes.
The issue in `load_from_tsfile_to_dataframe` happend because `has_class_labels_tag = False`. This is because the line 

```python
elif line.startswith("@classlabel"):
     ....
     has_class_labels_tag = True
```
is never  executed since the `line.startswith("@classlabel")` is `False`

#### Does your contribution introduce a new dependency? If yes, which one?

No

#### What should a reviewer concentrate their feedback on?

How should the tag "@classlabel" be named,  `write_ndarray_to_tsfile` and `load_from_tsfile_to_dataframe` depend on each other here. Another solution would be to allow multiple way of writing this tag in `load_from_tsfile_to_dataframe`


##### For all contributions
- [ ] I've added myself to the [list of contributors](https://github.com/alan-turing-institute/sktime/blob/main/.all-contributorsrc).
- [ ] Optionally, I've updated sktime's [CODEOWNERS](https://github.com/alan-turing-institute/sktime/blob/main/CODEOWNERS) to receive notifications about future changes to these files.
- [x] I've added unit tests and made sure they pass locally.
- [x] The PR title starts with either [ENH], [MNT], [DOC], or [BUG] indicating whether the PR topic is related to enhancement, maintenance, documentation, or bug.

fkiraly: @achieveordie, I think you recently looked at these I/O utilities - do you have any comments?
fkiraly: @paulbauriegel, thanks for the fix!

Just started the CI to check.

Btw, do you know whether there is a test that checks for the `classLabel=False` case? If not, would appreciate a test, or advice what the test should check.
paulbauriegel: @fkiraly As far as I understand you have two test functions  in `sktime/datasets/tests/test_data_io.py`

- `test_load_from_tsfile_to_dataframe` to test the load function
-  `test_write_dataframe_to_ts_success`/`test_write_dataframe_to_ts_fail` to test if writing works in general

So far there is no function that actually checks what is written. So I think it would make sense to implement a test that verifies the output of `write_ndarray_to_tsfile`.
I can write such a test if you would like me to.
fkiraly: >I can write such a test if you would like me to.

that would be neat, much obliged!

Patch:
diff --git a/sktime/datasets/_data_io.py b/sktime/datasets/_data_io.py
index 4106cf731..dbe764114 100644
--- a/sktime/datasets/_data_io.py
+++ b/sktime/datasets/_data_io.py
@@ -1573,9 +1573,9 @@ def write_ndarray_to_tsfile(
     # write class label line
     if class_label is not None:
         space_separated_class_label = " ".join(str(label) for label in class_label)
-        file.write(f"@classLabel true {space_separated_class_label}\n")
+        file.write(f"@classlabel true {space_separated_class_label}\n")
     else:
-        file.write("@class_label false\n")
+        file.write("@classlabel false\n")
     # begin writing the core data for each case
     # which are the series and the class value list if there is any
     file.write("@data\n")
diff --git a/sktime/dists_kernels/__init__.py b/sktime/dists_kernels/__init__.py
index 18ff4b12c..292301f36 100644
--- a/sktime/dists_kernels/__init__.py
+++ b/sktime/dists_kernels/__init__.py
@@ -6,6 +6,7 @@ from sktime.dists_kernels._base import (
     BasePairwiseTransformerPanel,
 )
 from sktime.dists_kernels.compose_tab_to_panel import AggrDist, FlatDist
+from sktime.dists_kernels.dtw import DtwDist
 from sktime.dists_kernels.edit_dist import EditDist
 from sktime.dists_kernels.scipy_dist import ScipyDist
 
@@ -13,6 +14,7 @@ __all__ = [
     "BasePairwiseTransformer",
     "BasePairwiseTransformerPanel",
     "AggrDist",
+    "DtwDist",
     "EditDist",
     "FlatDist",
     "ScipyDist",
diff --git a/sktime/dists_kernels/algebra.py b/sktime/dists_kernels/algebra.py
new file mode 100644
index 000000000..394aee6ef
--- /dev/null
+++ b/sktime/dists_kernels/algebra.py
@@ -0,0 +1,189 @@
+# -*- coding: utf-8 -*-
+"""Arithmetics with distances/kernels, e.g., addition, multiplication."""
+
+__author__ = ["fkiraly"]
+
+import numpy as np
+
+from sktime.base import _HeterogenousMetaEstimator
+from sktime.dists_kernels._base import BasePairwiseTransformerPanel
+
+SUPPORTED_MTYPES = ["pd-multiindex", "nested_univ", "df-list", "numpy3D"]
+
+
+class CombinedDistance(BasePairwiseTransformerPanel, _HeterogenousMetaEstimator):
+    """Distances combined via arithmetic operation, e.g., addition, multiplication.
+
+    `CombinedDistance` creates a pairwise trafo from multiple other pairwise trafos,
+    by performing an arithmetic operation (np.ufunc) on the multiple distance matrices.
+
+    For a list of transformers `trafo1`, `trafo2`, ..., `trafoN`, ufunc `operation`,
+    this compositor behaves as follows:
+    `transform(X, X2)` - computes `dist1 = trafo1.transform(X, X2)`,
+        `dist2 = trafo2.transform(X, X2)`, ..., distN =  trafoN.transform(X, X2)`,
+        all of shape `(len(X), len(X2)`, then applies `operation` entry-wise,
+        to obtain a single matrix `dist` of shape `(len(X), len(X2)`
+        Example: if `operation = np.sum`, then `dist` is
+        the entry-wise sum of `dist1`, `dist2`, ..., `distN`
+
+    Parameters
+    ----------
+    pw_trafos : list of sktime pairwise panel distances, or
+        list of tuples (str, transformer) of sktime pairwise panel distances
+        distances combined to a single distance using the operation
+    operation : None, str, function, or numpy ufunc, optional, default = None = mean
+        if str, must be one of "mean", "+" (add), "*" (multiply), "max", "min"
+        if func, must be of signature (1D iterable) -> float
+        operation carried out on the distance matrices distances
+
+    Examples
+    --------
+    >>> from sktime.dists_kernels.algebra import CombinedDistance
+    >>> from sktime.dists_kernels.dtw import DtwDist
+    >>> from sktime.datasets import load_unit_test
+    >>>
+    >>> X, _ = load_unit_test()
+    >>> X = X[0:3]
+    >>> sum_dist = CombinedDistance([DtwDist(), DtwDist(weighted=True)], "+")
+    >>> dist_mat = sum_dist.transform(X)
+    """
+
+    _tags = {
+        "X_inner_mtype": SUPPORTED_MTYPES,
+        "capability:missing_values": True,  # can estimator handle missing data?
+        "capability:multivariate": True,  # can estimator handle multivariate data?
+        "capability:unequal_length": True,  # can dist handle unequal length panels?
+    }
+
+    def __init__(self, pw_trafos, operation=None):
+
+        self.pw_trafos = pw_trafos
+        self.pw_trafos_ = self._check_estimators(
+            self.pw_trafos, cls_type=BasePairwiseTransformerPanel
+        )
+        self.operation = operation
+        self._operation = self._resolve_operation(operation)
+
+        super(CombinedDistance, self).__init__()
+
+        # abbreviate for readability
+        ests = self.pw_trafos_
+
+        # set property tags based on tags of components
+        self._anytagis_then_set("fit_is_empty", False, True, ests)
+        self._anytagis_then_set("capability:missing_values", False, True, ests)
+        self._anytagis_then_set("capability:multivariate", False, True, ests)
+        self._anytagis_then_set("capability:unequal_length", False, True, ests)
+
+    def _resolve_operation(self, operation):
+        """Coerce operation to a numpy.ufunc."""
+        alias_dict = {
+            None: np.mean,
+            "mean": np.mean,
+            "+": np.add,
+            "add": np.add,
+            "*": np.multiply,
+            "mult": np.multiply,
+            "multiply": np.multiply,
+            "min": np.min,
+            "max": np.max,
+        }
+
+        if operation in alias_dict.keys():
+            return alias_dict[operation]
+        else:
+            return operation
+
+    @property
+    def _pw_trafos(self):
+        return self._get_estimator_tuples(self.pw_trafos, clone_ests=False)
+
+    @_pw_trafos.setter
+    def _pw_trafos(self, value):
+        self.pw_trafos = value
+
+    def _transform(self, X, X2=None):
+        """Compute distance/kernel matrix.
+
+        private _transform containing core logic, called from public transform
+
+        Parameters
+        ----------
+        X: sktime Panel data container
+        X2: sktime Panel data container
+
+        Returns
+        -------
+        distmat: np.array of shape [n, m]
+            (i,j)-th entry contains distance/kernel between X.iloc[i] and X2.iloc[j]
+        """
+        ests = self._get_estimator_list(self.pw_trafos_)
+
+        distmats = [est.transform(X=X, X2=X2) for est in ests]
+        distmat_stack = np.stack(distmats)
+
+        operation = self._operation
+        if isinstance(operation, np.ufunc):
+            distmat = operation.reduce(distmat_stack)
+        else:
+            distmat = np.apply_over_axes(operation, distmat_stack, 0)
+            distmat = distmat.squeeze(axis=0)
+
+        return distmat
+
+    def get_params(self, deep=True):
+        """Get parameters of estimator in `steps`.
+
+        Parameters
+        ----------
+        deep : boolean, optional, default=True
+            If True, will return the parameters for this estimator and
+            contained sub-objects that are estimators.
+
+        Returns
+        -------
+        params : mapping of string to any
+            Parameter names mapped to their values.
+        """
+        return self._get_params("_pw_trafos", deep=deep)
+
+    def set_params(self, **kwargs):
+        """Set the parameters of estimator in `steps`.
+
+        Valid parameter keys can be listed with ``get_params()``.
+
+        Returns
+        -------
+        self : returns an instance of self.
+        """
+        self._set_params("_pw_trafos", **kwargs)
+        return self
+
+    @classmethod
+    def get_test_params(cls, parameter_set="default"):
+        """Return testing parameter settings for the estimator.
+
+        Parameters
+        ----------
+        parameter_set : str, default="default"
+            Name of the set of test parameters to return, for use in tests. If no
+            special parameters are defined for a value, will return `"default"` set.
+
+        Returns
+        -------
+        params : dict or list of dict, default={}
+            Parameters to create testing instances of the class.
+            Each dict are parameters to construct an "interesting" test instance, i.e.,
+            `MyClass(**params)` or `MyClass(**params[i])` creates a valid test instance.
+            `create_test_instance` uses the first (or only) dictionary in `params`.
+        """
+        from sktime.dists_kernels.compose_tab_to_panel import AggrDist
+        from sktime.dists_kernels.dtw import DtwDist
+
+        params1 = {"pw_trafos": [AggrDist.create_test_instance()]}
+        params2 = {
+            "pw_trafos": [AggrDist.create_test_instance(), DtwDist()],
+            "operation": "+",
+        }
+
+        return [params1, params2]
diff --git a/sktime/transformations/base.py b/sktime/transformations/base.py
index eb1bef898..93f7ebd05 100644
--- a/sktime/transformations/base.py
+++ b/sktime/transformations/base.py
@@ -278,6 +278,20 @@ class BaseTransformer(BaseEstimator):
         else:
             return NotImplemented
 
+    def __neg__(self):
+        """Magic unary - (negation) method, return OptionalPassthrough of self.
+
+        Intuition: `OptionalPassthrough` is "not having transformer", as an option.
+
+        Returns
+        -------
+        `OptionalPassthrough` object, containing `self`, with `passthrough=False`.
+            The `passthrough` parameter can be set via `set_params`.
+        """
+        from sktime.transformations.series.compose import OptionalPassthrough
+
+        return OptionalPassthrough(self, passthrough=False)
+
     def __getitem__(self, key):
         """Magic [...] method, return column subsetted transformer.
 
diff --git a/sktime/transformations/compose.py b/sktime/transformations/compose.py
index 93c412dab..a655521ad 100644
--- a/sktime/transformations/compose.py
+++ b/sktime/transformations/compose.py
@@ -8,6 +8,11 @@ from sklearn import clone
 from sktime.base import _HeterogenousMetaEstimator
 from sktime.transformations._delegate import _DelegatedTransformer
 from sktime.transformations.base import BaseTransformer
+from sktime.transformations.series.compose import (
+    ColumnwiseTransformer,
+    OptionalPassthrough,
+    YtoX,
+)
 from sktime.utils.multiindex import flatten_multiindex
 from sktime.utils.sklearn import (
     is_sklearn_classifier,
@@ -15,12 +20,29 @@ from sktime.utils.sklearn import (
     is_sklearn_transformer,
 )
 
-__author__ = ["fkiraly", "mloning", "miraep8"]
+__author__ = ["fkiraly", "mloning", "miraep8", "aiwalter", "SveaMeyer13"]
 __all__ = [
-    "TransformerPipeline",
+    "ColumnwiseTransformer",
     "FeatureUnion",
     "FitInTransform",
     "MultiplexTransformer",
+    "OptionalPassthrough",
+    "TransformerPipeline",
+    "YtoX",
+]
+
+
+# mtypes for Series, Panel, Hierarchical,
+# with exception of some ambiguous and discouraged mtypes
+CORE_MTYPES = [
+    "pd.DataFrame",
+    "np.ndarray",
+    "pd.Series",
+    "pd-multiindex",
+    "df-list",
+    "nested_univ",
+    "numpy3D",
+    "pd_multiindex_hier",
 ]
 
 
@@ -128,16 +150,7 @@ class TransformerPipeline(BaseTransformer, _HeterogenousMetaEstimator):
 
     _tags = {
         # we let all X inputs through to be handled by first transformer
-        "X_inner_mtype": [
-            "pd.DataFrame",
-            "np.ndarray",
-            "pd.Series",
-            "pd-multiindex",
-            "df-list",
-            "nested_univ",
-            "numpy3D",
-            "pd_multiindex_hier",
-        ],
+        "X_inner_mtype": CORE_MTYPES,
         "univariate-only": False,
     }
 
diff --git a/sktime/transformations/series/compose.py b/sktime/transformations/series/compose.py
index 58fc0fe71..47399917e 100644
--- a/sktime/transformations/series/compose.py
+++ b/sktime/transformations/series/compose.py
@@ -6,12 +6,35 @@
 __author__ = ["aiwalter", "SveaMeyer13", "fkiraly"]
 __all__ = ["OptionalPassthrough", "ColumnwiseTransformer", "YtoX"]
 
+from warnings import warn
+
 import pandas as pd
 from sklearn.utils.metaestimators import if_delegate_has_method
 
 from sktime.transformations.base import BaseTransformer
 from sktime.utils.validation.series import check_series
 
+warn(
+    "transformations.series.compose is deprecated and will be removed in"
+    " version 0.15.0. All estimators in it will be moved to transformations.compose. "
+    "Please change imports to transformations.compose to avoid breakage.",
+    DeprecationWarning,
+)
+
+
+# mtypes for Series, Panel, Hierarchical,
+# with exception of some ambiguous and discouraged mtypes
+CORE_MTYPES = [
+    "pd.DataFrame",
+    "np.ndarray",
+    "pd.Series",
+    "pd-multiindex",
+    "df-list",
+    "nested_univ",
+    "numpy3D",
+    "pd_multiindex_hier",
+]
+
 
 class OptionalPassthrough(BaseTransformer):
     """Wrap an existing transformer to tune whether to include it in a pipeline.
@@ -73,7 +96,7 @@ class OptionalPassthrough(BaseTransformer):
         "scitype:transform-output": "Series",
         # what scitype is returned: Primitives, Series, Panel
         "scitype:instancewise": True,  # is this an instance-wise transform?
-        "X_inner_mtype": ["pd.DataFrame", "pd.Series"],
+        "X_inner_mtype": CORE_MTYPES,
         # which mtypes do _fit/_predict support for X?
         "y_inner_mtype": "None",  # which mtypes do _fit/_predict support for y?
         "univariate-only": False,


Test patch:
diff --git a/sktime/transformations/tests/test_compose.py b/sktime/transformations/tests/test_compose.py
index 24be02488..7acd7e8c0 100644
--- a/sktime/transformations/tests/test_compose.py
+++ b/sktime/transformations/tests/test_compose.py
@@ -13,6 +13,7 @@ from sktime.datatypes import get_examples
 from sktime.transformations.compose import FeatureUnion, TransformerPipeline
 from sktime.transformations.panel.padder import PaddingTransformer
 from sktime.transformations.series.boxcox import LogTransformer
+from sktime.transformations.series.compose import OptionalPassthrough
 from sktime.transformations.series.exponent import ExponentTransformer
 from sktime.transformations.series.impute import Imputer
 from sktime.transformations.series.subset import ColumnSelect
@@ -224,3 +225,18 @@ def test_subset_getitem():
     )
     _assert_array_almost_equal(t_both.fit_transform(X), X_theta[["b__0", "b__2"]])
     _assert_array_almost_equal(t_none.fit_transform(X), X_theta)
+
+
+def test_dunder_neg():
+    """Test the neg dunder method, for wrapping in OptionalPassthrough."""
+    X = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
+
+    t = ExponentTransformer(power=2)
+
+    tp = -t
+
+    assert isinstance(tp, OptionalPassthrough)
+    assert not tp.get_params()["passthrough"]
+    assert isinstance(tp.get_params()["transformer"], ExponentTransformer)
+
+    _assert_array_almost_equal(tp.fit_transform(X), X)