Repository: openml/openml-python

Issue:
Checking active datasets
https://github.com/openml/openml-python/blob/3dc6dee62c1835e9b4f2bc9b6f9208d902263606/openml/datasets/functions.py#L295-L296

Maybe we should `list_datasets(status='active')` and if a `dataset_id` from the iterable of `dataset_ids` that we want to check is in the results, we can mark it as True, otherwise False. 

Discussion:

Patch:
diff --git a/doc/progress.rst b/doc/progress.rst
index c3aaf8d1..7dc63334 100644
--- a/doc/progress.rst
+++ b/doc/progress.rst
@@ -8,6 +8,7 @@ Changelog
 
 0.11.1
 ~~~~~~
+* MAINT #671: Improved the performance of ``check_datasets_active`` by only querying the given list of datasets in contrast to querying all datasets. Modified the corresponding unit test.
 
 0.11.0
 ~~~~~~
diff --git a/examples/30_extended/run_setup_tutorial.py b/examples/30_extended/run_setup_tutorial.py
index a46bf969..cea38e06 100644
--- a/examples/30_extended/run_setup_tutorial.py
+++ b/examples/30_extended/run_setup_tutorial.py
@@ -34,14 +34,12 @@ In this tutorial we will
 
 import numpy as np
 import openml
-import sklearn.ensemble
-import sklearn.impute
-import sklearn.preprocessing
 from sklearn.pipeline import make_pipeline, Pipeline
 from sklearn.compose import ColumnTransformer
 from sklearn.impute import SimpleImputer
 from sklearn.preprocessing import OneHotEncoder, FunctionTransformer
-from sklearn.experimental import enable_hist_gradient_boosting
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.decomposition import TruncatedSVD
 
 
 openml.config.start_using_configuration_for_example()
@@ -58,9 +56,6 @@ task = openml.tasks.get_task(6)
 # many potential hyperparameters. Of course, the model can be as complex and as
 # easy as you want it to be
 
-from sklearn.ensemble import HistGradientBoostingClassifier
-from sklearn.decomposition import TruncatedSVD
-
 
 # Helper functions to return required columns for ColumnTransformer
 def cont(X):
@@ -77,18 +72,16 @@ cat_imp = make_pipeline(
     TruncatedSVD(),
 )
 ct = ColumnTransformer([("cat", cat_imp, cat), ("cont", "passthrough", cont)])
-model_original = sklearn.pipeline.Pipeline(
-    steps=[("transform", ct), ("estimator", HistGradientBoostingClassifier()),]
-)
+model_original = Pipeline(steps=[("transform", ct), ("estimator", RandomForestClassifier()),])
 
 # Let's change some hyperparameters. Of course, in any good application we
 # would tune them using, e.g., Random Search or Bayesian Optimization, but for
 # the purpose of this tutorial we set them to some specific values that might
 # or might not be optimal
 hyperparameters_original = {
-    "estimator__loss": "auto",
-    "estimator__learning_rate": 0.15,
-    "estimator__max_iter": 50,
+    "estimator__criterion": "gini",
+    "estimator__n_estimators": 50,
+    "estimator__max_depth": 10,
     "estimator__min_samples_leaf": 1,
 }
 model_original.set_params(**hyperparameters_original)
diff --git a/examples/30_extended/study_tutorial.py b/examples/30_extended/study_tutorial.py
index c02a5c03..3c93a7e8 100644
--- a/examples/30_extended/study_tutorial.py
+++ b/examples/30_extended/study_tutorial.py
@@ -15,13 +15,7 @@ tasks, all required information about a study can be retrieved.
 
 import uuid
 
-import numpy as np
-import sklearn.tree
-from sklearn.pipeline import make_pipeline, Pipeline
-from sklearn.compose import ColumnTransformer
-from sklearn.impute import SimpleImputer
-from sklearn.decomposition import TruncatedSVD
-from sklearn.preprocessing import OneHotEncoder, FunctionTransformer
+from sklearn.ensemble import RandomForestClassifier
 
 import openml
 
@@ -71,45 +65,25 @@ evaluations = openml.evaluations.list_evaluations(
 )
 print(evaluations.head())
 
-###########################################################from openml.testing import cat, cont#################
+############################################################################
 # Uploading studies
 # =================
 #
 # Creating a study is as simple as creating any kind of other OpenML entity.
 # In this examples we'll create a few runs for the OpenML-100 benchmark
 # suite which is available on the OpenML test server.
-
 openml.config.start_using_configuration_for_example()
 
-# Model that can handle missing values
-from sklearn.experimental import enable_hist_gradient_boosting
-from sklearn.ensemble import HistGradientBoostingClassifier
-
-
-# Helper functions to return required columns for ColumnTransformer
-def cont(X):
-    return X.dtypes != "category"
-
-
-def cat(X):
-    return X.dtypes == "category"
+# Model to be used
+clf = RandomForestClassifier()
 
+# We'll create a study with one run on 3 datasets present in the suite
+tasks = [115, 259, 307]
 
-cat_imp = make_pipeline(
-    SimpleImputer(strategy="most_frequent"),
-    OneHotEncoder(handle_unknown="ignore", sparse=False),
-    TruncatedSVD(),
-)
-ct = ColumnTransformer(
-    [("cat", cat_imp, cat), ("cont", FunctionTransformer(lambda x: x, validate=False), cont)]
-)
-clf = sklearn.pipeline.Pipeline(
-    steps=[("transform", ct), ("estimator", HistGradientBoostingClassifier()),]
-)
-
+# To verify
 suite = openml.study.get_suite(1)
-# We'll create a study with one run on three random datasets each
-tasks = np.random.choice(suite.tasks, size=3, replace=False)
+print(all([t_id in suite.tasks for t_id in tasks]))
+
 run_ids = []
 for task_id in tasks:
     task = openml.tasks.get_task(task_id)
diff --git a/openml/datasets/functions.py b/openml/datasets/functions.py
index 28bde17f..c2eb8ee7 100644
--- a/openml/datasets/functions.py
+++ b/openml/datasets/functions.py
@@ -333,14 +333,23 @@ def _load_features_from_file(features_file: str) -> Dict:
         return xml_dict["oml:data_features"]
 
 
-def check_datasets_active(dataset_ids: List[int]) -> Dict[int, bool]:
+def check_datasets_active(
+    dataset_ids: List[int],
+    raise_error_if_not_exist: bool = True,
+) -> Dict[int, bool]:
     """
     Check if the dataset ids provided are active.
 
+    Raises an error if a dataset_id in the given list
+    of dataset_ids does not exist on the server.
+
     Parameters
     ----------
     dataset_ids : List[int]
         A list of integers representing dataset ids.
+    raise_error_if_not_exist : bool (default=True)
+        Flag that if activated can raise an error, if one or more of the
+        given dataset ids do not exist on the server.
 
     Returns
     -------
@@ -353,7 +362,8 @@ def check_datasets_active(dataset_ids: List[int]) -> Dict[int, bool]:
     for did in dataset_ids:
         dataset = dataset_list.get(did, None)
         if dataset is None:
-            raise ValueError("Could not find dataset {} in OpenML dataset list.".format(did))
+            if raise_error_if_not_exist:
+                raise ValueError(f'Could not find dataset {did} in OpenML dataset list.')
         else:
             active[did] = dataset["status"] == "active"
 
diff --git a/openml/extensions/sklearn/extension.py b/openml/extensions/sklearn/extension.py
index edb14487..0339667b 100644
--- a/openml/extensions/sklearn/extension.py
+++ b/openml/extensions/sklearn/extension.py
@@ -1546,7 +1546,9 @@ class SklearnExtension(Extension):
         fold_no: int,
         y_train: Optional[np.ndarray] = None,
         X_test: Optional[Union[np.ndarray, scipy.sparse.spmatrix, pd.DataFrame]] = None,
-    ) -> Tuple[np.ndarray, pd.DataFrame, "OrderedDict[str, float]", Optional[OpenMLRunTrace]]:
+    ) -> Tuple[
+        np.ndarray, Optional[pd.DataFrame], "OrderedDict[str, float]", Optional[OpenMLRunTrace]
+    ]:
         """Run a model on a repeat,fold,subsample triplet of the task and return prediction
         information.
 
@@ -1581,19 +1583,21 @@ class SklearnExtension(Extension):
         -------
         pred_y : np.ndarray
             Predictions on the training/test set, depending on the task type.
-            For supervised tasks, predicitons are on the test set.
-            For unsupervised tasks, predicitons are on the training set.
-        proba_y : pd.DataFrame
+            For supervised tasks, predictions are on the test set.
+            For unsupervised tasks, predictions are on the training set.
+        proba_y : pd.DataFrame, optional
             Predicted probabilities for the test set.
             None, if task is not Classification or Learning Curve prediction.
         user_defined_measures : OrderedDict[str, float]
             User defined measures that were generated on this fold
-        trace : Optional[OpenMLRunTrace]]
+        trace : OpenMLRunTrace, optional
             arff trace object from a fitted model and the trace content obtained by
             repeatedly calling ``run_model_on_task``
         """
 
-        def _prediction_to_probabilities(y: np.ndarray, model_classes: List[Any]) -> pd.DataFrame:
+        def _prediction_to_probabilities(
+            y: np.ndarray, model_classes: List[Any], class_labels: Optional[List[str]]
+        ) -> pd.DataFrame:
             """Transforms predicted probabilities to match with OpenML class indices.
 
             Parameters
@@ -1603,28 +1607,26 @@ class SklearnExtension(Extension):
                 training data).
             model_classes : list
                 List of classes known_predicted by the model, ordered by their index.
+            class_labels : list
+                List of classes as stored in the task object fetched from server.
 
             Returns
             -------
             pd.DataFrame
             """
+            if class_labels is None:
+                raise ValueError("The task has no class labels")
 
-            if isinstance(task, (OpenMLClassificationTask, OpenMLLearningCurveTask)):
-                if task.class_labels is not None:
-                    if isinstance(y_train, np.ndarray) and isinstance(task.class_labels[0], str):
-                        # mapping (decoding) the predictions to the categories
-                        # creating a separate copy to not change the expected pred_y type
-                        y = [task.class_labels[pred] for pred in y]
-                else:
-                    raise ValueError("The task has no class labels")
-            else:
-                return None
+            if isinstance(y_train, np.ndarray) and isinstance(class_labels[0], str):
+                # mapping (decoding) the predictions to the categories
+                # creating a separate copy to not change the expected pred_y type
+                y = [class_labels[pred] for pred in y]  # list or numpy array of predictions
 
-            # y: list or numpy array of predictions
             # model_classes: sklearn classifier mapping from original array id to
             # prediction index id
             if not isinstance(model_classes, list):
                 raise ValueError("please convert model classes to list prior to calling this fn")
+
             # DataFrame allows more accurate mapping of classes as column names
             result = pd.DataFrame(
                 0, index=np.arange(len(y)), columns=model_classes, dtype=np.float32
@@ -1639,10 +1641,6 @@ class SklearnExtension(Extension):
             if X_test is None:
                 raise TypeError("argument X_test must not be of type None")
 
-        # TODO: if possible, give a warning if model is already fitted (acceptable
-        # in case of custom experimentation,
-        # but not desirable if we want to upload to OpenML).
-
         model_copy = sklearn.base.clone(model, safe=True)
         # sanity check: prohibit users from optimizing n_jobs
         self._prevent_optimize_n_jobs(model_copy)
@@ -1732,10 +1730,7 @@ class SklearnExtension(Extension):
                 proba_y = model_copy.predict_proba(X_test)
                 proba_y = pd.DataFrame(proba_y, columns=model_classes)  # handles X_test as numpy
             except AttributeError:  # predict_proba is not available when probability=False
-                if task.class_labels is not None:
-                    proba_y = _prediction_to_probabilities(pred_y, model_classes)
-                else:
-                    raise ValueError("The task has no class labels")
+                proba_y = _prediction_to_probabilities(pred_y, model_classes, task.class_labels)
 
             if task.class_labels is not None:
                 if proba_y.shape[1] != len(task.class_labels):
@@ -1759,6 +1754,7 @@ class SklearnExtension(Extension):
                         # adding missing columns with 0 probability
                         if col not in model_classes:
                             proba_y[col] = 0
+                    # We re-order the columns to move possibly added missing columns into place.
                     proba_y = proba_y[task.class_labels]
             else:
                 raise ValueError("The task has no class labels")


Test patch:
diff --git a/tests/test_datasets/test_dataset_functions.py b/tests/test_datasets/test_dataset_functions.py
index c6e6f78f..707b6f9c 100644
--- a/tests/test_datasets/test_dataset_functions.py
+++ b/tests/test_datasets/test_dataset_functions.py
@@ -227,9 +227,13 @@ class TestOpenMLDataset(TestBase):
     def test_check_datasets_active(self):
         # Have to test on live because there is no deactivated dataset on the test server.
         openml.config.server = self.production_server
-        active = openml.datasets.check_datasets_active([2, 17])
+        active = openml.datasets.check_datasets_active(
+            [2, 17, 79],
+            raise_error_if_not_exist=False,
+        )
         self.assertTrue(active[2])
         self.assertFalse(active[17])
+        self.assertIsNone(active.get(79))
         self.assertRaisesRegex(
             ValueError,
             "Could not find dataset 79 in OpenML dataset list.",