Repository: sktime/sktime

Issue:
[BUG] sporadic bug with contracted TDE not meeting accuracy
#1301 had this sporadic bug with a failure of `test_contracted_tde_on_unit_test_data`, when TDE is tested.

This seems to be an instance where it does not reach a certain accuracy. Is it really a good idea to test for performance, in a case where it is inherently stochastic and no random seed is set?

@TonyBagnall 

```
=================================== FAILURES ===================================
____________________ test_contracted_tde_on_unit_test_data _____________________

    def test_contracted_tde_on_unit_test_data():
        """Test of contracted TDE on unit test data."""
        # load unit test data
        X_train, y_train = load_unit_test(split="train", return_X_y=True)
        X_test, y_test = load_unit_test(split="test", return_X_y=True)
    
        # train contracted TDE
        tde = TemporalDictionaryEnsemble(
            time_limit_in_minutes=0.025,
            max_ensemble_size=5,
            randomly_selected_params=5,
            random_state=0,
        )
        tde.fit(X_train, y_train)
    
        assert len(tde.estimators_) > 1
>       assert accuracy_score(y_test, tde.predict(X_test)) >= 0.8
E       AssertionError: assert 0.7727272727272727 >= 0.8
E        +  where 0.7727272727272727 = accuracy_score(array(['1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2',\n       '2', '2', '2', '2', '2', '2', '2', '2', '2'], dtype='<U1'), array(['2', '2', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2',\n       '2', '1', '2', '2', '1', '2', '2', '2', '1'], dtype='<U1'))
E        +    where array(['2', '2', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2',\n       '2', '1', '2', '2', '1', '2', '2', '2', '1'], dtype='<U1') = <bound method BaseClassifier.predict of TemporalDictionaryEnsemble(bigrams=None, dim_threshold=0.85, max_dims=20,\n    ...ms=5,\n                           save_train_predictions=False,\n                           time_limit_in_minutes=0.025)>(                                                dim_0\n0   0      501.0\n1      328.0\n2      195.0\n3      ...\n1   0     ...\n3      ...\n20  0       95.0\n1       63.0\n2       45.0\n3      ...\n21  0      193.0\n1      195.0\n2       90.0\n3      ...)
E        +      where <bound method BaseClassifier.predict of TemporalDictionaryEnsemble(bigrams=None, dim_threshold=0.85, max_dims=20,\n    ...ms=5,\n                           save_train_predictions=False,\n                           time_limit_in_minutes=0.025)> = TemporalDictionaryEnsemble(bigrams=None, dim_threshold=0.85, max_dims=20,\n                           max_ensemble_size...ams=5,\n                           save_train_predictions=False,\n                           time_limit_in_minutes=0.025).predict

X_test     =                                                 dim_0
0   0      501.0
1      328.0
2      195.0
3      ...
1   0     ...
3      ...
20  0       95.0
1       63.0
2       45.0
3      ...
21  0      193.0
1      195.0
2       90.0
3      ...
X_train    =                                                 dim_0
0   0      573.0
1      375.0
2      301.0
3      ...
1   0     ...
3      ...
18  0       95.0
1       63.0
2       45.0
3      ...
19  0       82.0
1       93.0
2       51.0
3      ...
tde        = TemporalDictionaryEnsemble(bigrams=None, dim_threshold=0.85, max_dims=20,
                           max_ensemble_size...ams=5,
                           save_train_predictions=False,
                           time_limit_in_minutes=0.025)
y_test     = array(['1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2',
       '2', '2', '2', '2', '2', '2', '2', '2', '2'], dtype='<U1')
y_train    = array(['1', '1', '1', '1', '1', '1', '1', '1', '1', '1', '2', '2', '2',
       '2', '2', '2', '2', '2', '2', '2'], dtype='<U1')

../../../../hostedtoolcache/Python/3.9.6/x64/lib/python3.9/site-packages/sktime/classification/dictionary_based/tests/test_tde.py:58: AssertionError
```


Discussion:

Resolving patch:
diff --git a/sktime/classification/dictionary_based/_tde.py b/sktime/classification/dictionary_based/_tde.py
--- a/sktime/classification/dictionary_based/_tde.py
+++ b/sktime/classification/dictionary_based/_tde.py
@@ -80,6 +80,9 @@ class TemporalDictionaryEnsemble(BaseClassifier):
     time_limit_in_minutes : int, default=0
         Time contract to limit build time in minutes, overriding n_estimators.
         Default of 0 means n_estimators is used.
+    contract_max_n_parameter_samples : int, default=np.inf
+        Max number of parameter combinations to consider when time_limit_in_minutes is
+        set.
     save_train_predictions : bool, default=False
         Save the ensemble member train predictions in fit for use in _get_train_probs
         leave-one-out cross-validation.
@@ -156,6 +159,7 @@ def __init__(
         dim_threshold=0.85,
         max_dims=20,
         time_limit_in_minutes=0.0,
+        contract_max_n_parameter_samples=np.inf,
         save_train_predictions=False,
         n_jobs=1,
         random_state=None,
@@ -172,6 +176,7 @@ def __init__(
         self.max_dims = max_dims
 
         self.time_limit_in_minutes = time_limit_in_minutes
+        self.contract_max_n_parameter_samples = contract_max_n_parameter_samples
         self.save_train_predictions = save_train_predictions
 
         self.random_state = random_state
@@ -208,7 +213,6 @@ def _fit(self, X, y):
                 "ensemble member parameters will be fully randomly selected.",
             )
 
-        time_limit = self.time_limit_in_minutes * 60
         self.n_instances, self.n_dims, self.series_length = X.shape
         self.n_classes = np.unique(y).shape[0]
         self.classes_ = class_distribution(np.asarray(y).reshape(-1, 1))[0][0]
@@ -248,8 +252,10 @@ def _fit(self, X, y):
         train_time = 0
         if time_limit > 0:
             n_parameter_samples = 0
+            contract_max_n_parameter_samples = self.contract_max_n_parameter_samples
         else:
             n_parameter_samples = self.n_parameter_samples
+            contract_max_n_parameter_samples = np.inf
 
         rng = check_random_state(self.random_state)
 
@@ -263,7 +269,11 @@ def _fit(self, X, y):
 
         # use time limit or n_parameter_samples if limit is 0
         while (
-            train_time < time_limit or num_classifiers < n_parameter_samples
+            (
+                train_time < time_limit
+                and num_classifiers < contract_max_n_parameter_samples
+            )
+            or num_classifiers < n_parameter_samples
         ) and len(possible_parameters) > 0:
             if num_classifiers < self.randomly_selected_params:
                 parameters = possible_parameters.pop(


Test patch:
diff --git a/sktime/classification/dictionary_based/tests/test_tde.py b/sktime/classification/dictionary_based/tests/test_tde.py
--- a/sktime/classification/dictionary_based/tests/test_tde.py
+++ b/sktime/classification/dictionary_based/tests/test_tde.py
@@ -31,31 +31,32 @@ def test_tde_on_unit_test_data():
     # test loocv train estimate
     train_probas = tde._get_train_probs(X_train, y_train)
     train_preds = tde.classes_[np.argmax(train_probas, axis=1)]
-    assert accuracy_score(y_train, train_preds) >= 0.85
+    assert accuracy_score(y_train, train_preds) >= 0.75
 
     # test oob estimate
     train_probas = tde._get_train_probs(X_train, y_train, train_estimate_method="oob")
     train_preds = tde.classes_[np.argmax(train_probas, axis=1)]
-    assert accuracy_score(y_train, train_preds) >= 0.8
+    assert accuracy_score(y_train, train_preds) >= 0.75
 
 
-def test_contracted_tde_on_unit_test_data():
-    """Test of contracted TDE on unit test data."""
-    # load unit test data
-    X_train, y_train = load_unit_test(split="train", return_X_y=True)
-    X_test, y_test = load_unit_test(split="test", return_X_y=True)
-
-    # train contracted TDE
-    tde = TemporalDictionaryEnsemble(
-        time_limit_in_minutes=0.025,
-        max_ensemble_size=5,
-        randomly_selected_params=5,
-        random_state=0,
-    )
-    tde.fit(X_train, y_train)
-
-    assert len(tde.estimators_) > 1
-    assert accuracy_score(y_test, tde.predict(X_test)) >= 0.8
+# def test_contracted_tde_on_unit_test_data():
+#     """Test of contracted TDE on unit test data."""
+#     # load unit test data
+#     X_train, y_train = load_unit_test(split="train", return_X_y=True)
+#     X_test, y_test = load_unit_test(split="test", return_X_y=True)
+#
+#     # train contracted TDE
+#     tde = TemporalDictionaryEnsemble(
+#         time_limit_in_minutes=0.25,
+#         contract_max_n_parameter_samples=10,
+#         max_ensemble_size=5,
+#         randomly_selected_params=5,
+#         random_state=0,
+#     )
+#     tde.fit(X_train, y_train)
+#
+#     assert len(tde.estimators_) > 1
+#     assert accuracy_score(y_test, tde.predict(X_test)) >= 0.75
 
 
 def test_tde_on_basic_motions():
diff --git a/sktime/classification/interval_based/tests/test_drcif.py b/sktime/classification/interval_based/tests/test_drcif.py
--- a/sktime/classification/interval_based/tests/test_drcif.py
+++ b/sktime/classification/interval_based/tests/test_drcif.py
@@ -26,21 +26,25 @@ def test_drcif_on_unit_test_data():
     # test train estimate
     train_probas = drcif._get_train_probs(X_train, y_train)
     train_preds = drcif.classes_[np.argmax(train_probas, axis=1)]
-    assert accuracy_score(y_train, train_preds) >= 0.85
+    assert accuracy_score(y_train, train_preds) >= 0.75
 
 
-def test_contracted_drcif_on_unit_test_data():
-    """Test of contracted DrCIF on unit test data."""
-    # load unit test data
-    X_train, y_train = load_unit_test(split="train", return_X_y=True)
-    X_test, y_test = load_unit_test(split="test", return_X_y=True)
-
-    # train contracted DrCIF
-    drcif = DrCIF(time_limit_in_minutes=0.025, random_state=0)
-    drcif.fit(X_train, y_train)
-
-    assert len(drcif.estimators_) > 1
-    assert accuracy_score(y_test, drcif.predict(X_test)) >= 0.8
+# def test_contracted_drcif_on_unit_test_data():
+#     """Test of contracted DrCIF on unit test data."""
+#     # load unit test data
+#     X_train, y_train = load_unit_test(split="train", return_X_y=True)
+#     X_test, y_test = load_unit_test(split="test", return_X_y=True)
+#
+#     # train contracted DrCIF
+#     drcif = DrCIF(
+#         time_limit_in_minutes=0.25,
+#         contract_max_n_estimators=10,
+#         random_state=0,
+#     )
+#     drcif.fit(X_train, y_train)
+#
+#     assert len(drcif.estimators_) > 1
+#     assert accuracy_score(y_test, drcif.predict(X_test)) >= 0.75
 
 
 def test_drcif_on_basic_motions():
diff --git a/sktime/classification/kernel_based/tests/test_arsenal.py b/sktime/classification/kernel_based/tests/test_arsenal.py
--- a/sktime/classification/kernel_based/tests/test_arsenal.py
+++ b/sktime/classification/kernel_based/tests/test_arsenal.py
@@ -28,21 +28,26 @@ def test_arsenal_on_unit_test_data():
     # test train estimate
     train_probas = arsenal._get_train_probs(X_train, y_train)
     train_preds = arsenal.classes_[np.argmax(train_probas, axis=1)]
-    assert accuracy_score(y_train, train_preds) >= 0.85
+    assert accuracy_score(y_train, train_preds) >= 0.75
 
 
-def test_contracted_arsenal_on_unit_test_data():
-    """Test of contracted Arsenal on unit test data."""
-    # load unit test data
-    X_train, y_train = load_unit_test(split="train", return_X_y=True)
-    X_test, y_test = load_unit_test(split="test", return_X_y=True)
-
-    # train contracted DrCIF
-    arsenal = Arsenal(time_limit_in_minutes=0.025, random_state=0)
-    arsenal.fit(X_train, y_train)
-
-    assert len(arsenal.estimators_) > 1
-    assert accuracy_score(y_test, arsenal.predict(X_test)) >= 0.8
+# def test_contracted_arsenal_on_unit_test_data():
+#     """Test of contracted Arsenal on unit test data."""
+#     # load unit test data
+#     X_train, y_train = load_unit_test(split="train", return_X_y=True)
+#     X_test, y_test = load_unit_test(split="test", return_X_y=True)
+#
+#     # train contracted Arsenal
+#     arsenal = Arsenal(
+#         time_limit_in_minutes=0.25,
+#         contract_max_n_estimators=5,
+#         num_kernels=500,
+#         random_state=0,
+#     )
+#     arsenal.fit(X_train, y_train)
+#
+#     assert len(arsenal.estimators_) > 1
+#     assert accuracy_score(y_test, arsenal.predict(X_test)) >= 0.75
 
 
 def test_arsenal_on_basic_motions():