Repository: sktime/sktime

Issue:
[BUG] testing deep learning base class fails due to optimizer type
**Describe the bug**

 test_deep_estimator_full[keras-adamax] is failing due to a type check in keras. See, for example

https://github.com/sktime/sktime/actions/runs/3495913658/jobs/5855642783

"ValueError: `optimizer` of type <class 'keras.optimizers.optimizer_v2.adamax.Adamax'> cannot be serialized, it should either be absent/None/str/tf.keras.optimizers.Optimizer object"

it could be that the wrong type of Optimizer is being used, output states

Optimizer  = <class 'keras.optimizers.optimizer_experimental.optimizer.Optimizer'>
whereas it is expecting
tf.keras.optimizers.Optimizer

![image](https://user-images.githubusercontent.com/9594042/202723857-05d98e12-f03b-40e5-a4ac-97405b330638.png)

I don't want to hack around in the testing suite, I'll let someone who understands it better do that,




Discussion:
yes, this is new - must be coming from an update in `keras`?
I assume so, since nothing has changed our side. 
temporarily skipped here:
https://github.com/sktime/sktime/pull/3817

not resolved - still needs a bugfix.

@achieveordie, I think you wrote the test so might have a good idea on what´s going on?
@fkiraly, the bug pointed out by @TonyBagnall  is on sktime its side.
The problem resides in your build system. I would use https://python-poetry.org/. This would raise an error if you have an incompatible version of keras installed.
You should fix your external packages as you wrap them and they change all the time.
I also received warnings that "things" have changed.
Things seem to be breaking after their update. To ensure smooth sailing, we could replace `keras.optimizers` with `tf.keras.optimizers` in our import statement.

Of course, one can argue that our build should hold more specific versions as said by @RikStarmans, but I'm not sure what would be the "right" approach. On one hand, the current code enables us to catch and correct such changes while on the other we'd ensure the test suite works without fail.

We should choose the latter for releases but the case here is for dev.
> You should fix your external packages as you wrap them and they change all the time.
I also received warnings that "things" have changed.

@RikStarmans, there is a longer discussion on fixing bounds for package versions
https://github.com/sktime/sktime/issues/1480
this has only posts by me, but I remember there being long discussions, possibly in issues referencing this or on slack.

Resolving patch:
diff --git a/sktime/classification/deep_learning/base.py b/sktime/classification/deep_learning/base.py
--- a/sktime/classification/deep_learning/base.py
+++ b/sktime/classification/deep_learning/base.py
@@ -15,7 +15,6 @@
 from sklearn.utils import check_random_state
 
 from sktime.classification.base import BaseClassifier
-from sktime.exceptions import NotFittedError
 from sktime.utils.validation._dependencies import _check_soft_dependencies
 
 
@@ -245,12 +244,14 @@ def save(self, path=None):
             _check_soft_dependencies("h5py")
             import h5py
 
-            with h5py.File(
-                "disk_less", "w", driver="core", backing_store=False
-            ) as h5file:
-                self.model_.save(h5file)
-                h5file.flush()
-                in_memory_model = h5file.id.get_file_image()
+            in_memory_model = None
+            if self.model_ is not None:
+                with h5py.File(
+                    "disk_less", "w", driver="core", backing_store=False
+                ) as h5file:
+                    self.model_.save(h5file)
+                    h5file.flush()
+                    in_memory_model = h5file.id.get_file_image()
 
             in_memory_history = pickle.dumps(self.history.history)
 
@@ -269,13 +270,12 @@ def save(self, path=None):
                 f"but found of type:{type(path)}."
             )
 
-        if self.model_ is None:
-            raise NotFittedError("Model not built yet, call it via `.fit()`")
-
         path = Path(path) if isinstance(path, str) else path
         path.mkdir()
 
-        self.model_.save(path / "keras/")
+        if self.model_ is not None:
+            self.model_.save(path / "keras/")
+
         with open(path / "history", "wb") as history_writer:
             pickle.dump(self.history.history, history_writer)
 
@@ -323,13 +323,16 @@ def load_from_serial(cls, serial):
             )
 
         serial, in_memory_model, in_memory_history = serial
-        with TemporaryFile() as store_:
-            store_.write(in_memory_model)
-            h5file = h5py.File(store_, "r")
-            cls.model_ = load_model(h5file)
-            cls.history = pickle.loads(in_memory_history)
-            h5file.close()
+        if in_memory_model is None:
+            cls.model_ = None
+        else:
+            with TemporaryFile() as store_:
+                store_.write(in_memory_model)
+                h5file = h5py.File(store_, "r")
+                cls.model_ = load_model(h5file)
+                h5file.close()
 
+        cls.history = pickle.loads(in_memory_history)
         return pickle.loads(serial)
 
     @classmethod
@@ -359,9 +362,13 @@ def load_from_path(cls, serial):
                     continue
                 zip_file.extract(file, temp_unzip_loc)
 
-        cls.model_ = keras.models.load_model(temp_unzip_loc / "keras/")
-        rmtree(temp_unzip_loc)
+        keras_location = temp_unzip_loc / "keras"
+        if keras_location.exists():
+            cls.model_ = keras.models.load_model(keras_location)
+        else:
+            cls.model_ = None
 
+        rmtree(temp_unzip_loc)
         cls.history = keras.callbacks.History()
         with ZipFile(serial, mode="r") as file:
             cls.history.set_params(pickle.loads(file.open("history").read()))
diff --git a/sktime/regression/deep_learning/base.py b/sktime/regression/deep_learning/base.py
--- a/sktime/regression/deep_learning/base.py
+++ b/sktime/regression/deep_learning/base.py
@@ -13,7 +13,6 @@
 
 import numpy as np
 
-from sktime.exceptions import NotFittedError
 from sktime.regression.base import BaseRegressor
 from sktime.utils.validation._dependencies import _check_soft_dependencies
 
@@ -202,12 +201,14 @@ def save(self, path=None):
             _check_soft_dependencies("h5py")
             import h5py
 
-            with h5py.File(
-                "disk_less", "w", driver="core", backing_store=False
-            ) as h5file:
-                self.model_.save(h5file)
-                h5file.flush()
-                in_memory_model = h5file.id.get_file_image()
+            in_memory_model = None
+            if self.model_ is not None:
+                with h5py.File(
+                    "disk_less", "w", driver="core", backing_store=False
+                ) as h5file:
+                    self.model_.save(h5file)
+                    h5file.flush()
+                    in_memory_model = h5file.id.get_file_image()
 
             in_memory_history = pickle.dumps(self.history.history)
 
@@ -226,13 +227,12 @@ def save(self, path=None):
                 f"but found of type:{type(path)}."
             )
 
-        if self.model_ is None:
-            raise NotFittedError("Model not built yet, call it via `.fit()`")
-
         path = Path(path) if isinstance(path, str) else path
         path.mkdir()
 
-        self.model_.save(path / "keras/")
+        if self.model_ is not None:
+            self.model_.save(path / "keras/")
+
         with open(path / "history", "wb") as history_writer:
             pickle.dump(self.history.history, history_writer)
 
@@ -280,13 +280,16 @@ def load_from_serial(cls, serial):
             )
 
         serial, in_memory_model, in_memory_history = serial
-        with TemporaryFile() as store_:
-            store_.write(in_memory_model)
-            h5file = h5py.File(store_, "r")
-            cls.model_ = load_model(h5file)
-            cls.history = pickle.loads(in_memory_history)
-            h5file.close()
+        if in_memory_model is None:
+            cls.model_ = None
+        else:
+            with TemporaryFile() as store_:
+                store_.write(in_memory_model)
+                h5file = h5py.File(store_, "r")
+                cls.model_ = load_model(h5file)
+                h5file.close()
 
+        cls.history = pickle.loads(in_memory_history)
         return pickle.loads(serial)
 
     @classmethod
@@ -316,9 +319,13 @@ def load_from_path(cls, serial):
                     continue
                 zip_file.extract(file, temp_unzip_loc)
 
-        cls.model_ = keras.models.load_model(temp_unzip_loc / "keras/")
-        rmtree(temp_unzip_loc)
+        keras_location = temp_unzip_loc / "keras"
+        if keras_location.exists():
+            cls.model_ = keras.models.load_model(keras_location)
+        else:
+            cls.model_ = None
 
+        rmtree(temp_unzip_loc)
         cls.history = keras.callbacks.History()
         with ZipFile(serial, mode="r") as file:
             cls.history.set_params(pickle.loads(file.open("history").read()))


Test patch:
diff --git a/sktime/classification/tests/test_base.py b/sktime/classification/tests/test_base.py
--- a/sktime/classification/tests/test_base.py
+++ b/sktime/classification/tests/test_base.py
@@ -475,17 +475,15 @@ def test_deep_estimator_empty():
 
 
 @pytest.mark.skipif(
-    not _check_soft_dependencies("keras", severity="none"),
+    not _check_soft_dependencies("tensorflow", severity="none"),
     reason="skip test if required soft dependency not available",
 )
-@pytest.mark.xfail(reason="known failure of unknown cause, see #3816")
-@pytest.mark.parametrize("optimizer", [None, "adam", "keras-adamax"])
+@pytest.mark.parametrize("optimizer", [None, "adam", "object-adamax"])
 def test_deep_estimator_full(optimizer):
     """Check if serialization works for full dummy."""
-    from keras.optimizers import Adamax
-    from tensorflow.keras.optimizers import Optimizer, serialize
+    from tensorflow.keras.optimizers import Adamax, Optimizer, serialize
 
-    if optimizer == "keras-adamax":
+    if optimizer == "object-adamax":
         optimizer = Adamax()
 
     full_dummy = _DummyDeepClassifierFull(optimizer)
@@ -501,8 +499,7 @@ def test_deep_estimator_full(optimizer):
 
         # assert weights of optimizers are same
         assert (
-            full_dummy.optimizer.get_weights()
-            == deserialized_full.optimizer.get_weights()
+            full_dummy.optimizer.variables() == deserialized_full.optimizer.variables()
         )
 
         # remove optimizers from both to do full dict check,