Repository: sktime/sktime

Issue:
[ENH] Adding MLP DL model for classification
**Is your feature request related to a problem? Please describe.**
On the topic of migrating sktime-dl models to sktime, adding MLP model for classification from sktime-dl


Discussion:

Resolving patch:
diff --git a/sktime/classification/deep_learning/mlp.py b/sktime/classification/deep_learning/mlp.py
new file mode 100644
--- /dev/null
+++ b/sktime/classification/deep_learning/mlp.py
@@ -0,0 +1,176 @@
+# -*- coding: utf-8 -*-
+"""Multi Layer Perceptron Network (MLP) for classification."""
+
+__author__ = ["James-Large", "AurumnPegasus"]
+__all__ = ["MLPClassifier"]
+
+from sklearn.utils import check_random_state
+
+from sktime.classification.deep_learning.base import BaseDeepClassifier
+from sktime.networks.mlp import MLPNetwork
+from sktime.utils.validation._dependencies import _check_dl_dependencies
+
+_check_dl_dependencies(severity="warning")
+
+
+class MLPClassifier(BaseDeepClassifier):
+    """Multi Layer Perceptron Network (MLP), as described in [1].
+
+    Parameters
+    ----------
+    should inherited fields be listed here?
+    n_epochs       : int, default = 2000
+        the number of epochs to train the model
+    batch_size      : int, default = 16
+        the number of samples per gradient update.
+    random_state    : int or None, default=None
+        Seed for random number generation.
+    verbose         : boolean, default = False
+        whether to output extra information
+    loss            : string, default="mean_squared_error"
+        fit parameter for the keras model
+    optimizer       : keras.optimizer, default=keras.optimizers.Adam(),
+    metrics         : list of strings, default=["accuracy"],
+    activation      : string or a tf callable, default="sigmoid"
+        Activation function used in the output linear layer.
+        List of available activation functions:
+        https://keras.io/api/layers/activations/
+    use_bias        : boolean, default = True
+        whether the layer uses a bias vector.
+    optimizer       : keras.optimizers object, default = Adam(lr=0.01)
+        specify the optimizer and the learning rate to be used.
+
+    Notes
+    -----
+    .. .. [1]  Network originally defined in:
+    @inproceedings{wang2017time, title={Time series classification from
+    scratch with deep neural networks: A strong baseline}, author={Wang,
+    Zhiguang and Yan, Weizhong and Oates, Tim}, booktitle={2017
+    International joint conference on neural networks (IJCNN)}, pages={
+    1578--1585}, year={2017}, organization={IEEE} }
+
+    Adapted from the implementation from source code
+    https://github.com/hfawaz/dl-4-tsc/blob/master/classifiers/mlp.py
+
+    Examples
+    --------
+    >>> from sktime.classification.deep_learning.mlp import MLPClassifier
+    >>> from sktime.datasets import load_unit_test
+    >>> X_train, y_train = load_unit_test(split="train", return_X_y=True)
+    >>> X_test, y_test = load_unit_test(split="test", return_X_y=True)
+    >>> mlp = MLPClassifier()
+    >>> mlp.fit(X_train, y_train)
+    MLPClassifier(...)
+    """
+
+    def __init__(
+        self,
+        n_epochs=2000,
+        batch_size=16,
+        callbacks=None,
+        verbose=False,
+        loss="categorical_crossentropy",
+        metrics=None,
+        random_state=None,
+        activation="sigmoid",
+        use_bias=True,
+        optimizer=None,
+    ):
+        _check_dl_dependencies(severity="error")
+        super(MLPClassifier, self).__init__()
+        self.callbacks = callbacks
+        self.n_epochs = n_epochs
+        self.batch_size = batch_size
+        self.verbose = verbose
+        self.loss = loss
+        self.metrics = metrics
+        self.random_state = random_state
+        self.activation = activation
+        self.use_bias = use_bias
+        self.optimizer = optimizer
+        self.history = None
+        self._network = MLPNetwork()
+
+    def build_model(self, input_shape, n_classes, **kwargs):
+        """Construct a compiled, un-trained, keras model that is ready for training.
+
+        In sktime, time series are stored in numpy arrays of shape (d,m), where d
+        is the number of dimensions, m is the series length. Keras/tensorflow assume
+        data is in shape (m,d). This method also assumes (m,d). Transpose should
+        happen in fit.
+
+        Parameters
+        ----------
+        input_shape : tuple
+            The shape of the data fed into the input layer, should be (m,d)
+        n_classes: int
+            The number of classes, which becomes the size of the output layer
+
+        Returns
+        -------
+        output : a compiled Keras Model
+        """
+        import tensorflow as tf
+        from tensorflow import keras
+
+        tf.random.set_seed(self.random_state)
+
+        if self.metrics is None:
+            metrics = ["accuracy"]
+        else:
+            metrics = self.metrics
+        input_layer, output_layer = self._network.build_network(input_shape, **kwargs)
+
+        output_layer = keras.layers.Dense(
+            units=n_classes, activation=self.activation, use_bias=self.use_bias
+        )(output_layer)
+
+        self.optimizer_ = (
+            keras.optimizers.Adam(learning_rate=0.01)
+            if self.optimizer is None
+            else self.optimizer
+        )
+
+        model = keras.models.Model(inputs=input_layer, outputs=output_layer)
+        model.compile(
+            loss=self.loss,
+            optimizer=self.optimizer_,
+            metrics=metrics,
+        )
+        return model
+
+    def _fit(self, X, y):
+        """Fit the classifier on the training set (X, y).
+
+        Parameters
+        ----------
+        X : np.ndarray of shape = (n_instances (n), n_dimensions (d), series_length (m))
+            The training input samples.
+        y : np.ndarray of shape n
+            The training data class labels.
+
+        Returns
+        -------
+        self : object
+        """
+        if self.callbacks is None:
+            self._callbacks = []
+
+        y_onehot = self.convert_y_to_keras(y)
+        # Transpose to conform to Keras input style.
+        X = X.transpose(0, 2, 1)
+
+        check_random_state(self.random_state)
+        self.input_shape = X.shape[1:]
+        self.model_ = self.build_model(self.input_shape, self.n_classes_)
+        if self.verbose:
+            self.model_.summary()
+        self.history = self.model_.fit(
+            X,
+            y_onehot,
+            batch_size=self.batch_size,
+            epochs=self.n_epochs,
+            verbose=self.verbose,
+            callbacks=self._callbacks,
+        )
+        return self
diff --git a/sktime/networks/mlp.py b/sktime/networks/mlp.py
new file mode 100644
--- /dev/null
+++ b/sktime/networks/mlp.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+"""Multi Layer Perceptron (MLP) (minus the final output layer)."""
+
+__author__ = ["James-Large", "Withington", "AurumnPegasus"]
+
+from sktime.networks.base import BaseDeepNetwork
+from sktime.utils.validation._dependencies import _check_dl_dependencies
+
+_check_dl_dependencies(severity="warning")
+
+
+class MLPNetwork(BaseDeepNetwork):
+    """Establish the network structure for a MLP.
+
+    Adapted from the implementation used in [1]
+
+    Parameters
+    ----------
+    random_state    : int, default = 0
+        seed to any needed random actions
+
+    Notes
+    -----
+    Adapted from the implementation from source code
+    https://github.com/hfawaz/dl-4-tsc/blob/master/classifiers/mlp.py
+
+    References
+    ----------
+    .. [1]  Network originally defined in:
+    @inproceedings{wang2017time, title={Time series classification from
+    scratch with deep neural networks: A strong baseline}, author={Wang,
+    Zhiguang and Yan, Weizhong and Oates, Tim}, booktitle={2017
+    International joint conference on neural networks (IJCNN)}, pages={
+    1578--1585}, year={2017}, organization={IEEE} }
+    """
+
+    _tags = {"python_dependencies": "tensorflow"}
+
+    def __init__(
+        self,
+        random_state=0,
+    ):
+        _check_dl_dependencies(severity="error")
+        self.random_state = random_state
+
+    def build_network(self, input_shape, **kwargs):
+        """Construct a network and return its input and output layers.
+
+        Arguments
+        ---------
+        input_shape : tuple of shape = (series_length (m), n_dimensions (d))
+            The shape of the data fed into the input layer
+
+        Returns
+        -------
+        input_layer : a keras layer
+        output_layer : a keras layer
+        """
+        from tensorflow import keras
+
+        # flattened because multivariate should be on same axis
+        input_layer = keras.layers.Input(input_shape)
+        input_layer_flattened = keras.layers.Flatten()(input_layer)
+
+        layer_1 = keras.layers.Dropout(0.1)(input_layer_flattened)
+        layer_1 = keras.layers.Dense(500, activation="relu")(layer_1)
+
+        layer_2 = keras.layers.Dropout(0.2)(layer_1)
+        layer_2 = keras.layers.Dense(500, activation="relu")(layer_2)
+
+        layer_3 = keras.layers.Dropout(0.2)(layer_2)
+        layer_3 = keras.layers.Dense(500, activation="relu")(layer_3)
+
+        output_layer = keras.layers.Dropout(0.3)(layer_3)
+
+        return input_layer, output_layer
diff --git a/sktime/registry/_base_classes.py b/sktime/registry/_base_classes.py
--- a/sktime/registry/_base_classes.py
+++ b/sktime/registry/_base_classes.py
@@ -70,6 +70,7 @@
 )
 from sktime.forecasting.base import BaseForecaster
 from sktime.forecasting.model_selection._split import BaseSplitter
+from sktime.networks.base import BaseDeepNetwork
 from sktime.param_est.base import BaseParamFitter
 from sktime.performance_metrics.base import BaseMetric
 from sktime.regression.base import BaseRegressor
@@ -90,6 +91,7 @@
     ("early_classifier", BaseEarlyClassifier, "early time series classifier"),
     ("forecaster", BaseForecaster, "forecaster"),
     ("metric", BaseMetric, "performance metric"),
+    ("network", BaseDeepNetwork, "deep learning network"),
     ("param_est", BaseParamFitter, "parameter fitting estimator"),
     ("regressor", BaseRegressor, "time series regressor"),
     ("series-annotator", BaseSeriesAnnotator, "time series annotator"),


Test patch:
diff --git a/sktime/registry/tests/test_lookup.py b/sktime/registry/tests/test_lookup.py
--- a/sktime/registry/tests/test_lookup.py
+++ b/sktime/registry/tests/test_lookup.py
@@ -20,7 +20,13 @@
 )
 
 # some scitypes have no associated tags yet
-SCITYPES_WITHOUT_TAGS = ["series-annotator", "clusterer", "object", "splitter"]
+SCITYPES_WITHOUT_TAGS = [
+    "series-annotator",
+    "clusterer",
+    "object",
+    "splitter",
+    "network",
+]
 
 # shorthands for easy reading
 b = BASE_CLASS_SCITYPE_LIST
diff --git a/sktime/tests/_config.py b/sktime/tests/_config.py
--- a/sktime/tests/_config.py
+++ b/sktime/tests/_config.py
@@ -51,6 +51,10 @@
         "test_fit_idempotent",
         "test_persistence_via_pickle",
     ],
+    "MLPClassifier": [
+        "test_fit_idempotent",
+        "test_persistence_via_pickle",
+    ],
     "CNNRegressor": [
         "test_fit_idempotent",
         "test_persistence_via_pickle",