Repository: sktime/sktime
Pull request: [MNT] Test case to ensure an AssertionError is thrown if EXCLUDED_TESTS_BY_TEST is not updated


#### Reference Issues/PRs
fixes #7618 

#### What does this implement/fix? Explain your changes.
Wrote a test case to ensure an AssertionError is thrown if EXCLUDED_TESTS_BY_TEST is not updated.

#### PR-checklist
- [x] The PR title starts with either [ENH], [MNT], [DOC], or [BUG]. [BUG] - bugfix, [MNT] - CI, test framework, [ENH] - adding or improving code, [DOC] - writing or improving documentation or docstrings.

Problem statement:
[ENH] `EXCLUDED_TESTS_BY_TEST` should be automatically in sync when test_params are added to estimators
**Describe the bug**
`EXCLUDED_TESTS_BY_TEST` does not update automatically when test_params are added to estimators and there are no tests to ensure the same.

**Expected behavior**
A test should ensure that `EXCLUDED_TESTS_BY_TEST` is up to date with estimators that do not have at least 2 test parameters.

Hints from issue discussion:
I will be looking into this.
Thanks! I think it should not update automatically, I would simply add another test that checks whether the list is up to date, i.e., estimators that pass that specific test should not be on the list.

Commit message: Wrote a test case to ensure an AssertionError is thrown if EXCLUDED_TESTS_BY_TEST is not updated

Patch to review:
diff --git a/sktime/tests/_config.py b/sktime/tests/_config.py
index 98b3067f446..10c2d80d198 100644
--- a/sktime/tests/_config.py
+++ b/sktime/tests/_config.py
@@ -265,37 +265,33 @@
 # exclude tests but keyed by test name
 EXCLUDED_TESTS_BY_TEST = {
     "test_get_test_params_coverage": [
-        "Arsenal",
-        "BaggingForecaster",
         "BOSSEnsemble",
+        "BaggingForecaster",
+        "CAPA",
         "CNTCClassifier",
         "CNTCNetwork",
         "CNTCRegressor",
         "CanonicalIntervalForest",
+        "CircularBinarySegmentation",
         "ClaSPTransformer",
         "ClearSky",
-        "ClustererPipeline",
-        "ColumnConcatenator",
         "ColumnEnsembleClassifier",
         "ColumnwiseTransformer",
         "ContractableBOSS",
         "DOBIN",
         "DWTTransformer",
-        "DerivativeSlopeTransformer",
         "DilationMappingTransformer",
-        "DirectTabularRegressionForecaster",
         "DirRecTabularRegressionForecaster",
         "DirRecTimeSeriesRegressionForecaster",
+        "DirectTabularRegressionForecaster",
         "DirectTimeSeriesRegressionForecaster",
         "DistFromAligner",
         "DistanceFeatures",
         "DontUpdate",
         "DummyRegressor",
         "ElasticEnsemble",
-        "ElbowClassPairwise",
         "EnbPIForecaster",
         "FeatureSelection",
-        "Filter",
         "FittedParamExtractor",
         "ForecastingOptunaSearchCV",
         "FreshPRINCE",
@@ -303,8 +299,8 @@
         "HCrystalBallAdapter",
         "HIVECOTEV1",
         "HIVECOTEV2",
-        "HOG1DTransformer",
         "Hidalgo",
+        "HierarchicalProphet",
         "HolidayFeatures",
         "InceptionTimeNetwork",
         "IndividualBOSS",
@@ -320,11 +316,13 @@
         "MCDCNNRegressor",
         "MLPNetwork",
         "MUSE",
+        "MVCAPA",
         "MatrixProfile",
         "MatrixProfileTransformer",
         "MiniRocket",
         "MiniRocketMultivariate",
         "MiniRocketMultivariateVariable",
+        "MovingWindow",
         "MultiRocket",
         "MultioutputTabularRegressionForecaster",
         "MultioutputTimeSeriesRegressionForecaster",
@@ -333,6 +331,7 @@
         "PAA",
         "PAAlegacy",
         "PCATransformer",
+        "PELT",
         "PaddingTransformer",
         "ParamFitterPipeline",
         "PlateauFinder",
@@ -341,7 +340,6 @@
         "Prophet",
         "ProphetPiecewiseLinearTrendForecaster",
         "Prophetverse",
-        "HierarchicalProphet",
         "RandomIntervalClassifier",
         "RandomIntervalFeatureExtractor",
         "RandomIntervalSegmenter",
@@ -352,12 +350,12 @@
         "RecursiveTabularRegressionForecaster",
         "RecursiveTimeSeriesRegressionForecaster",
         "ReducerTransform",
-        "RegressorPipeline",
         "Rocket",
         "SAX",
         "SAXlegacy",
         "SFA",
         "SFAFast",
+        "SeededBinarySegmentation",
         "ShapeletTransform",
         "ShapeletTransformClassifier",
         "SignatureClassifier",
@@ -365,26 +363,23 @@
         "SlidingWindowSegmenter",
         "SlopeTransformer",
         "StackingForecaster",
+        "StatThresholdAnomaliser",
         "SummaryClassifier",
-        "SupervisedIntervals",
         "SupervisedTimeSeriesForest",
         "TEASER",
         "TSBootstrapAdapter",
         "TSFreshClassifier",
-        "Tabularizer",
         "TapNetNetwork",
         "TemporalDictionaryEnsemble",
-        "ThetaLinesTransformer",
+        "ThetaModularForecaster",
         "TimeBinner",
         "TimeSeriesForestClassifier",
         "TimeSeriesForestRegressor",
         "TimeSeriesKMedoids",
         "TimeSeriesKernelKMeans",
-        "ThetaModularForecaster",
         "TruncationTransformer",
         "UnobservedComponents",
         "WEASEL",
-        "WeightedEnsembleClassifier",
         "WhiteNoiseAugmenter",
     ]
 }
diff --git a/sktime/tests/tests/test_test_utils.py b/sktime/tests/tests/test_test_utils.py
index b783671a56a..65d272023ba 100644
--- a/sktime/tests/tests/test_test_utils.py
+++ b/sktime/tests/tests/test_test_utils.py
@@ -1,10 +1,33 @@
 """Tests for the test utilities."""
 
-from sktime.tests._config import EXCLUDE_ESTIMATORS
+from sktime.registry import all_estimators
+from sktime.tests._config import EXCLUDE_ESTIMATORS, EXCLUDED_TESTS_BY_TEST
 from sktime.tests.test_switch import run_test_for_class
 from sktime.utils.dependencies import _check_estimator_deps
 
 
+def test_excluded_tests_by_test():
+    """Test that EXCLUDED_TESTS_BY_TEST contains estimators with <2 test params."""
+    all_ests = all_estimators()
+    filtered_estimators = [
+        x[0]
+        for x in all_ests
+        if (
+            (
+                len(x[1].get_test_params()) < 2
+                or isinstance(x[1].get_test_params(), dict)
+            )
+            and (
+                len(x[1].get_param_names())
+                - len(x[1].get_class_tags().get("reserved_params", []))
+                > 0
+            )
+        )
+    ]
+    for _, excluded_estimators in EXCLUDED_TESTS_BY_TEST.items():
+        assert excluded_estimators == filtered_estimators
+
+
 def test_exclude_estimators():
     """Test that EXCLUDE_ESTIMATORS is a list of strings."""
     assert isinstance(EXCLUDE_ESTIMATORS, list)


Code review comments:
diff_hunk: @@ -1,10 +1,33 @@
 """Tests for the test utilities."""
 
-from sktime.tests._config import EXCLUDE_ESTIMATORS
+from sktime.registry import all_estimators
+from sktime.tests._config import EXCLUDE_ESTIMATORS, EXCLUDED_TESTS_BY_TEST
 from sktime.tests.test_switch import run_test_for_class
 from sktime.utils.dependencies import _check_estimator_deps
 
 
+def test_excluded_tests_by_test():
+    """Test that EXCLUDED_TESTS_BY_TEST contains estimators with <2 test params."""
+    all_ests = all_estimators()
+    filtered_estimators = [
+        x[0]
+        for x in all_ests
+        if (
+            (
+                len(x[1].get_test_params()) < 2
+                or isinstance(x[1].get_test_params(), dict)
+            )
+            and (
+                len(x[1].get_param_names())
+                - len(x[1].get_class_tags().get("reserved_params", []))
+                > 0
+            )
+        )
+    ]
+    for _, excluded_estimators in EXCLUDED_TESTS_BY_TEST.items():
+        assert excluded_estimators == filtered_estimators
path: sktime/tests/tests/test_test_utils.py
text: @user1:
since working on this list is a "good first issue", it would be nice to have an error message which explains what to do in order to fis this failure, for the assertion.

@author:
will do it once I figure out why these errors are being thrown during checks.

diff_hunk: @@ -1,10 +1,33 @@
 """Tests for the test utilities."""
 
-from sktime.tests._config import EXCLUDE_ESTIMATORS
+from sktime.registry import all_estimators
+from sktime.tests._config import EXCLUDE_ESTIMATORS, EXCLUDED_TESTS_BY_TEST
 from sktime.tests.test_switch import run_test_for_class
 from sktime.utils.dependencies import _check_estimator_deps
 
 
+def test_excluded_tests_by_test():
+    """Test that EXCLUDED_TESTS_BY_TEST contains estimators with <2 test params."""
+    all_ests = all_estimators()
+    filtered_estimators = [
+        x[0]
+        for x in all_ests
+        if (
+            (
+                len(x[1].get_test_params()) < 2
+                or isinstance(x[1].get_test_params(), dict)
+            )
+            and (
+                len(x[1].get_param_names())
+                - len(x[1].get_class_tags().get("reserved_params", []))
+                > 0
+            )
+        )
+    ]
+    for _, excluded_estimators in EXCLUDED_TESTS_BY_TEST.items():
+        assert excluded_estimators == filtered_estimators
path: sktime/tests/tests/test_test_utils.py
text: @user1:
```suggestion
        assert set(excluded_estimators) == set(filtered_estimators)
```