Repository: openml/openml-python

Issue:
Improve error message when API key is not provided
The error message for a missing API key looks very generic and is not very user friendly:
```
Traceback (most recent call last):
  File "/home/feurerm/sync_dir/projects/openml/python/local/example.py", line 13, in <module>
    run = openml.runs.run_model_on_task(clf, task)  # run classifier on splits
  File "/home/feurerm/sync_dir/projects/openml/python/openml/runs/functions.py", line 140, in run_model_on_task
    n_jobs=n_jobs,
  File "/home/feurerm/sync_dir/projects/openml/python/openml/runs/functions.py", line 228, in run_flow_on_task
    flow_id = flow_exists(flow.name, flow.external_version)
  File "/home/feurerm/sync_dir/projects/openml/python/openml/flows/functions.py", line 256, in flow_exists
    "flow/exists", "post", data={"name": name, "external_version": external_version},
  File "/home/feurerm/sync_dir/projects/openml/python/openml/_api_calls.py", line 67, in _perform_api_call
    response = __read_url(url, request_method, data)
  File "/home/feurerm/sync_dir/projects/openml/python/openml/_api_calls.py", line 206, in __read_url
    request_method=request_method, url=url, data=data, md5_checksum=md5_checksum
  File "/home/feurerm/sync_dir/projects/openml/python/openml/_api_calls.py", line 237, in _send_request
    __check_response(response=response, url=url, file_elements=files)
  File "/home/feurerm/sync_dir/projects/openml/python/openml/_api_calls.py", line 284, in __check_response
    raise __parse_server_exception(response, url, file_elements=file_elements)
openml.exceptions.OpenMLServerException: https://test.openml.org/api/v1/xml/flow/exists returned code 102: No authentication (Please provide API key for all requests other than HTTP GET) - None

Process finished with exit code 1
```
Something like
```
The API call https://test.openml.org/api/v1/xml/flow/exists requires authentication via an API key. Please configure OpenML-Python to use your API as described in this example: https://openml.github.io/openml-python/main/examples/20_basic/introduction_tutorial.html#authentication
``` 

Discussion:
eddiebergman: Was debugging with it xx, it's not ready yet, still some errors it seems

Patch:
diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index bc4a04bc..773dda6f 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -29,10 +29,7 @@ jobs:
         python-version: 3.8
     - name: Install dependencies
       run: |
-        pip install -e .[docs,examples,examples_unix]
-        # dependency "fanova" does not work with numpy 1.24 or later
-        # https://github.com/automl/fanova/issues/108
-        pip install numpy==1.23.5
+          pip install -e .[docs,examples]
     - name: Make docs
       run: |
         cd doc
diff --git a/examples/40_paper/2018_kdd_rijn_example.py b/examples/40_paper/2018_kdd_rijn_example.py
index 7ec60fe5..6522013e 100644
--- a/examples/40_paper/2018_kdd_rijn_example.py
+++ b/examples/40_paper/2018_kdd_rijn_example.py
@@ -39,151 +39,150 @@ if sys.platform == "win32":  # noqa
     exit()
 
 # DEPRECATED EXAMPLE -- Avoid running this code in our CI/CD pipeline
-print("This example is deprecated, remove this code to use it manually.")
-exit()
-
-import json
-import fanova
-import matplotlib.pyplot as plt
-import pandas as pd
-import seaborn as sns
-
-import openml
-
-
-##############################################################################
-# With the advent of automated machine learning, automated hyperparameter
-# optimization methods are by now routinely used in data mining. However, this
-# progress is not yet matched by equal progress on automatic analyses that
-# yield information beyond performance-optimizing hyperparameter settings.
-# In this example, we aim to answer the following two questions: Given an
-# algorithm, what are generally its most important hyperparameters?
-#
-# This work is carried out on the OpenML-100 benchmark suite, which can be
-# obtained by ``openml.study.get_suite('OpenML100')``. In this example, we
-# conduct the experiment on the Support Vector Machine (``flow_id=7707``)
-# with specific kernel (we will perform a post-process filter operation for
-# this). We should set some other experimental parameters (number of results
-# per task, evaluation measure and the number of trees of the internal
-# functional Anova) before the fun can begin.
-#
-# Note that we simplify the example in several ways:
-#
-# 1) We only consider numerical hyperparameters
-# 2) We consider all hyperparameters that are numerical (in reality, some
-#    hyperparameters might be inactive (e.g., ``degree``) or irrelevant
-#    (e.g., ``random_state``)
-# 3) We assume all hyperparameters to be on uniform scale
-#
-# Any difference in conclusion between the actual paper and the presented
-# results is most likely due to one of these simplifications. For example,
-# the hyperparameter C looks rather insignificant, whereas it is quite
-# important when it is put on a log-scale. All these simplifications can be
-# addressed by defining a ConfigSpace. For a more elaborated example that uses
-# this, please see:
-# https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py # noqa F401
-
-suite = openml.study.get_suite("OpenML100")
-flow_id = 7707
-parameter_filters = {"sklearn.svm.classes.SVC(17)_kernel": "sigmoid"}
-evaluation_measure = "predictive_accuracy"
-limit_per_task = 500
-limit_nr_tasks = 15
-n_trees = 16
-
-fanova_results = []
-# we will obtain all results from OpenML per task. Practice has shown that this places the bottleneck on the
-# communication with OpenML, and for iterated experimenting it is better to cache the results in a local file.
-for idx, task_id in enumerate(suite.tasks):
-    if limit_nr_tasks is not None and idx >= limit_nr_tasks:
-        continue
-    print(
-        "Starting with task %d (%d/%d)"
-        % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks)
-    )
-    # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop)
-    evals = openml.evaluations.list_evaluations_setups(
-        evaluation_measure,
-        flows=[flow_id],
-        tasks=[task_id],
-        size=limit_per_task,
-        output_format="dataframe",
-    )
-
-    performance_column = "value"
-    # make a DataFrame consisting of all hyperparameters (which is a dict in setup['parameters']) and the performance
-    # value (in setup['value']). The following line looks a bit complicated, but combines 2 tasks: a) combine
-    # hyperparameters and performance data in a single dict, b) cast hyperparameter values to the appropriate format
-    # Note that the ``json.loads(...)`` requires the content to be in JSON format, which is only the case for
-    # scikit-learn setups (and even there some legacy setups might violate this requirement). It will work for the
-    # setups that belong to the flows embedded in this example though.
-    try:
-        setups_evals = pd.DataFrame(
-            [
-                dict(
-                    **{name: json.loads(value) for name, value in setup["parameters"].items()},
-                    **{performance_column: setup[performance_column]}
-                )
-                for _, setup in evals.iterrows()
-            ]
+print("This example is deprecated, remove the `if False` in this code to use it manually.")
+if False:
+    import json
+    import fanova
+    import matplotlib.pyplot as plt
+    import pandas as pd
+    import seaborn as sns
+
+    import openml
+
+
+    ##############################################################################
+    # With the advent of automated machine learning, automated hyperparameter
+    # optimization methods are by now routinely used in data mining. However, this
+    # progress is not yet matched by equal progress on automatic analyses that
+    # yield information beyond performance-optimizing hyperparameter settings.
+    # In this example, we aim to answer the following two questions: Given an
+    # algorithm, what are generally its most important hyperparameters?
+    #
+    # This work is carried out on the OpenML-100 benchmark suite, which can be
+    # obtained by ``openml.study.get_suite('OpenML100')``. In this example, we
+    # conduct the experiment on the Support Vector Machine (``flow_id=7707``)
+    # with specific kernel (we will perform a post-process filter operation for
+    # this). We should set some other experimental parameters (number of results
+    # per task, evaluation measure and the number of trees of the internal
+    # functional Anova) before the fun can begin.
+    #
+    # Note that we simplify the example in several ways:
+    #
+    # 1) We only consider numerical hyperparameters
+    # 2) We consider all hyperparameters that are numerical (in reality, some
+    #    hyperparameters might be inactive (e.g., ``degree``) or irrelevant
+    #    (e.g., ``random_state``)
+    # 3) We assume all hyperparameters to be on uniform scale
+    #
+    # Any difference in conclusion between the actual paper and the presented
+    # results is most likely due to one of these simplifications. For example,
+    # the hyperparameter C looks rather insignificant, whereas it is quite
+    # important when it is put on a log-scale. All these simplifications can be
+    # addressed by defining a ConfigSpace. For a more elaborated example that uses
+    # this, please see:
+    # https://github.com/janvanrijn/openml-pimp/blob/d0a14f3eb480f2a90008889f00041bdccc7b9265/examples/plot/plot_fanova_aggregates.py # noqa F401
+
+    suite = openml.study.get_suite("OpenML100")
+    flow_id = 7707
+    parameter_filters = {"sklearn.svm.classes.SVC(17)_kernel": "sigmoid"}
+    evaluation_measure = "predictive_accuracy"
+    limit_per_task = 500
+    limit_nr_tasks = 15
+    n_trees = 16
+
+    fanova_results = []
+    # we will obtain all results from OpenML per task. Practice has shown that this places the bottleneck on the
+    # communication with OpenML, and for iterated experimenting it is better to cache the results in a local file.
+    for idx, task_id in enumerate(suite.tasks):
+        if limit_nr_tasks is not None and idx >= limit_nr_tasks:
+            continue
+        print(
+            "Starting with task %d (%d/%d)"
+            % (task_id, idx + 1, len(suite.tasks) if limit_nr_tasks is None else limit_nr_tasks)
         )
-    except json.decoder.JSONDecodeError as e:
-        print("Task %d error: %s" % (task_id, e))
-        continue
-    # apply our filters, to have only the setups that comply to the hyperparameters we want
-    for filter_key, filter_value in parameter_filters.items():
-        setups_evals = setups_evals[setups_evals[filter_key] == filter_value]
-    # in this simplified example, we only display numerical and float hyperparameters. For categorical hyperparameters,
-    # the fanova library needs to be informed by using a configspace object.
-    setups_evals = setups_evals.select_dtypes(include=["int64", "float64"])
-    # drop rows with unique values. These are by definition not an interesting hyperparameter, e.g., ``axis``,
-    # ``verbose``.
-    setups_evals = setups_evals[
-        [
-            c
-            for c in list(setups_evals)
-            if len(setups_evals[c].unique()) > 1 or c == performance_column
-        ]
-    ]
-    # We are done with processing ``setups_evals``. Note that we still might have some irrelevant hyperparameters, e.g.,
-    # ``random_state``. We have dropped some relevant hyperparameters, i.e., several categoricals. Let's check it out:
-
-    # determine x values to pass to fanova library
-    parameter_names = [
-        pname for pname in setups_evals.columns.to_numpy() if pname != performance_column
-    ]
-    evaluator = fanova.fanova.fANOVA(
-        X=setups_evals[parameter_names].to_numpy(),
-        Y=setups_evals[performance_column].to_numpy(),
-        n_trees=n_trees,
-    )
-    for idx, pname in enumerate(parameter_names):
+        # note that we explicitly only include tasks from the benchmark suite that was specified (as per the for-loop)
+        evals = openml.evaluations.list_evaluations_setups(
+            evaluation_measure,
+            flows=[flow_id],
+            tasks=[task_id],
+            size=limit_per_task,
+            output_format="dataframe",
+        )
+
+        performance_column = "value"
+        # make a DataFrame consisting of all hyperparameters (which is a dict in setup['parameters']) and the performance
+        # value (in setup['value']). The following line looks a bit complicated, but combines 2 tasks: a) combine
+        # hyperparameters and performance data in a single dict, b) cast hyperparameter values to the appropriate format
+        # Note that the ``json.loads(...)`` requires the content to be in JSON format, which is only the case for
+        # scikit-learn setups (and even there some legacy setups might violate this requirement). It will work for the
+        # setups that belong to the flows embedded in this example though.
         try:
-            fanova_results.append(
-                {
-                    "hyperparameter": pname.split(".")[-1],
-                    "fanova": evaluator.quantify_importance([idx])[(idx,)]["individual importance"],
-                }
+            setups_evals = pd.DataFrame(
+                [
+                    dict(
+                        **{name: json.loads(value) for name, value in setup["parameters"].items()},
+                        **{performance_column: setup[performance_column]}
+                    )
+                    for _, setup in evals.iterrows()
+                ]
             )
-        except RuntimeError as e:
-            # functional ANOVA sometimes crashes with a RuntimeError, e.g., on tasks where the performance is constant
-            # for all configurations (there is no variance). We will skip these tasks (like the authors did in the
-            # paper).
+        except json.decoder.JSONDecodeError as e:
             print("Task %d error: %s" % (task_id, e))
             continue
+        # apply our filters, to have only the setups that comply to the hyperparameters we want
+        for filter_key, filter_value in parameter_filters.items():
+            setups_evals = setups_evals[setups_evals[filter_key] == filter_value]
+        # in this simplified example, we only display numerical and float hyperparameters. For categorical hyperparameters,
+        # the fanova library needs to be informed by using a configspace object.
+        setups_evals = setups_evals.select_dtypes(include=["int64", "float64"])
+        # drop rows with unique values. These are by definition not an interesting hyperparameter, e.g., ``axis``,
+        # ``verbose``.
+        setups_evals = setups_evals[
+            [
+                c
+                for c in list(setups_evals)
+                if len(setups_evals[c].unique()) > 1 or c == performance_column
+            ]
+        ]
+        # We are done with processing ``setups_evals``. Note that we still might have some irrelevant hyperparameters, e.g.,
+        # ``random_state``. We have dropped some relevant hyperparameters, i.e., several categoricals. Let's check it out:
 
-# transform ``fanova_results`` from a list of dicts into a DataFrame
-fanova_results = pd.DataFrame(fanova_results)
-
-##############################################################################
-# make the boxplot of the variance contribution. Obviously, we can also use
-# this data to make the Nemenyi plot, but this relies on the rather complex
-# ``Orange`` dependency (``pip install Orange3``). For the complete example,
-# the reader is referred to the more elaborate script (referred to earlier)
-fig, ax = plt.subplots()
-sns.boxplot(x="hyperparameter", y="fanova", data=fanova_results, ax=ax)
-ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha="right")
-ax.set_ylabel("Variance Contribution")
-ax.set_xlabel(None)
-plt.tight_layout()
-plt.show()
+        # determine x values to pass to fanova library
+        parameter_names = [
+            pname for pname in setups_evals.columns.to_numpy() if pname != performance_column
+        ]
+        evaluator = fanova.fanova.fANOVA(
+            X=setups_evals[parameter_names].to_numpy(),
+            Y=setups_evals[performance_column].to_numpy(),
+            n_trees=n_trees,
+        )
+        for idx, pname in enumerate(parameter_names):
+            try:
+                fanova_results.append(
+                    {
+                        "hyperparameter": pname.split(".")[-1],
+                        "fanova": evaluator.quantify_importance([idx])[(idx,)]["individual importance"],
+                    }
+                )
+            except RuntimeError as e:
+                # functional ANOVA sometimes crashes with a RuntimeError, e.g., on tasks where the performance is constant
+                # for all configurations (there is no variance). We will skip these tasks (like the authors did in the
+                # paper).
+                print("Task %d error: %s" % (task_id, e))
+                continue
+
+    # transform ``fanova_results`` from a list of dicts into a DataFrame
+    fanova_results = pd.DataFrame(fanova_results)
+
+    ##############################################################################
+    # make the boxplot of the variance contribution. Obviously, we can also use
+    # this data to make the Nemenyi plot, but this relies on the rather complex
+    # ``Orange`` dependency (``pip install Orange3``). For the complete example,
+    # the reader is referred to the more elaborate script (referred to earlier)
+    fig, ax = plt.subplots()
+    sns.boxplot(x="hyperparameter", y="fanova", data=fanova_results, ax=ax)
+    ax.set_xticklabels(ax.get_xticklabels(), rotation=45, ha="right")
+    ax.set_ylabel("Variance Contribution")
+    ax.set_xlabel(None)
+    plt.tight_layout()
+    plt.show()
diff --git a/openml/_api_calls.py b/openml/_api_calls.py
index 4d1d1767..3509f18e 100644
--- a/openml/_api_calls.py
+++ b/openml/_api_calls.py
@@ -24,6 +24,7 @@ from . import config
 from .__version__ import __version__
 from .exceptions import (
     OpenMLHashException,
+    OpenMLNotAuthorizedError,
     OpenMLServerError,
     OpenMLServerException,
     OpenMLServerNoResult,
@@ -36,6 +37,8 @@ DATA_TYPE = Dict[str, Union[str, int]]
 FILE_ELEMENTS_TYPE = Dict[str, Union[str, Tuple[str, str]]]
 DATABASE_CONNECTION_ERRCODE = 107
 
+API_TOKEN_HELP_LINK = "https://openml.github.io/openml-python/main/examples/20_basic/introduction_tutorial.html#authentication"  # noqa: S105
+
 
 def _robot_delay(n: int) -> float:
     wait = (1 / (1 + math.exp(-(n * 0.5 - 4)))) * 60
@@ -456,21 +459,28 @@ def __parse_server_exception(
     url: str,
     file_elements: FILE_ELEMENTS_TYPE | None,
 ) -> OpenMLServerError:
-    if response.status_code == 414:
+    if response.status_code == requests.codes.URI_TOO_LONG:
         raise OpenMLServerError(f"URI too long! ({url})")
 
+    # OpenML has a sophisticated error system where information about failures is provided,
+    # in the response body itself.
+    # First, we need to parse it out.
     try:
         server_exception = xmltodict.parse(response.text)
     except xml.parsers.expat.ExpatError as e:
         raise e
     except Exception as e:
-        # OpenML has a sophisticated error system
-        # where information about failures is provided. try to parse this
+        # If we failed to parse it out, then something has gone wrong in the body we have sent back
+        # from the server and there is little extra information we can capture.
         raise OpenMLServerError(
             f"Unexpected server error when calling {url}. Please contact the developers!\n"
             f"Status code: {response.status_code}\n{response.text}",
         ) from e
 
+    # Now we can parse out the specific error codes that we return. These
+    # are in addition to the typical HTTP error codes, but encode more
+    # specific informtion. You can find these codes here:
+    # https://github.com/openml/OpenML/blob/develop/openml_OS/views/pages/api_new/v1/xml/pre.php
     server_error = server_exception["oml:error"]
     code = int(server_error["oml:code"])
     message = server_error["oml:message"]
@@ -496,4 +506,21 @@ def __parse_server_exception(
         )
     else:
         full_message = f"{message} - {additional_information}"
+
+    if code in [
+        102,  # flow/exists post
+        137,  # dataset post
+        350,  # dataset/42 delete
+        310,  # flow/<something> post
+        320,  # flow/42 delete
+        400,  # run/42 delete
+        460,  # task/42 delete
+    ]:
+        msg = (
+            f"The API call {url} requires authentication via an API key.\nPlease configure "
+            "OpenML-Python to use your API as described in this example:"
+            "\nhttps://openml.github.io/openml-python/main/examples/20_basic/introduction_tutorial.html#authentication"
+        )
+        return OpenMLNotAuthorizedError(message=msg)
+
     return OpenMLServerException(code=code, message=full_message, url=url)
diff --git a/openml/config.py b/openml/config.py
index a412c0cc..a244a317 100644
--- a/openml/config.py
+++ b/openml/config.py
@@ -10,9 +10,10 @@ import os
 import platform
 import shutil
 import warnings
+from contextlib import contextmanager
 from io import StringIO
 from pathlib import Path
-from typing import Any, cast
+from typing import Any, Iterator, cast
 from typing_extensions import Literal, TypedDict
 from urllib.parse import urlparse
 
@@ -174,11 +175,11 @@ def get_server_base_url() -> str:
 apikey: str = _defaults["apikey"]
 show_progress: bool = _defaults["show_progress"]
 # The current cache directory (without the server name)
-_root_cache_directory = Path(_defaults["cachedir"])
+_root_cache_directory: Path = Path(_defaults["cachedir"])
 avoid_duplicate_runs = _defaults["avoid_duplicate_runs"]
 
-retry_policy = _defaults["retry_policy"]
-connection_n_retries = _defaults["connection_n_retries"]
+retry_policy: Literal["human", "robot"] = _defaults["retry_policy"]
+connection_n_retries: int = _defaults["connection_n_retries"]
 
 
 def set_retry_policy(value: Literal["human", "robot"], n_retries: int | None = None) -> None:
@@ -497,6 +498,18 @@ start_using_configuration_for_example = (
 stop_using_configuration_for_example = ConfigurationForExamples.stop_using_configuration_for_example
 
 
+@contextmanager
+def overwrite_config_context(config: dict[str, Any]) -> Iterator[_Config]:
+    """A context manager to temporarily override variables in the configuration."""
+    existing_config = get_config_as_dict()
+    merged_config = {**existing_config, **config}
+
+    _setup(merged_config)  # type: ignore
+    yield merged_config  # type: ignore
+
+    _setup(existing_config)
+
+
 __all__ = [
     "get_cache_directory",
     "set_root_cache_directory",
diff --git a/openml/runs/functions.py b/openml/runs/functions.py
index c6af4a48..b16af0b8 100644
--- a/openml/runs/functions.py
+++ b/openml/runs/functions.py
@@ -364,7 +364,7 @@ def get_run_trace(run_id: int) -> OpenMLRunTrace:
     return OpenMLRunTrace.trace_from_xml(trace_xml)
 
 
-def initialize_model_from_run(run_id: int) -> Any:
+def initialize_model_from_run(run_id: int, *, strict_version: bool = True) -> Any:
     """
     Initialized a model based on a run_id (i.e., using the exact
     same parameter settings)
@@ -373,6 +373,8 @@ def initialize_model_from_run(run_id: int) -> Any:
     ----------
     run_id : int
         The Openml run_id
+    strict_version: bool (default=True)
+        See `flow_to_model` strict_version.
 
     Returns
     -------
@@ -382,7 +384,7 @@ def initialize_model_from_run(run_id: int) -> Any:
     # TODO(eddiebergman): I imagine this is None if it's not published,
     # might need to raise an explicit error for that
     assert run.setup_id is not None
-    return initialize_model(run.setup_id)
+    return initialize_model(setup_id=run.setup_id, strict_version=strict_version)
 
 
 def initialize_model_from_trace(
diff --git a/openml/setups/functions.py b/openml/setups/functions.py
index 0bcd2b4e..87738463 100644
--- a/openml/setups/functions.py
+++ b/openml/setups/functions.py
@@ -265,7 +265,7 @@ def __list_setups(
     return setups
 
 
-def initialize_model(setup_id: int) -> Any:
+def initialize_model(setup_id: int, *, strict_version: bool = True) -> Any:
     """
     Initialized a model based on a setup_id (i.e., using the exact
     same parameter settings)
@@ -274,6 +274,8 @@ def initialize_model(setup_id: int) -> Any:
     ----------
     setup_id : int
         The Openml setup_id
+    strict_version: bool (default=True)
+        See `flow_to_model` strict_version.
 
     Returns
     -------
@@ -294,7 +296,7 @@ def initialize_model(setup_id: int) -> Any:
                 subflow = flow
             subflow.parameters[hyperparameter.parameter_name] = hyperparameter.value
 
-    return flow.extension.flow_to_model(flow)
+    return flow.extension.flow_to_model(flow, strict_version=strict_version)
 
 
 def _to_dict(
diff --git a/openml/utils.py b/openml/utils.py
index 66c4df80..82859fd4 100644
--- a/openml/utils.py
+++ b/openml/utils.py
@@ -234,7 +234,7 @@ def _delete_entity(entity_type: str, entity_id: int) -> bool:
                     " please open an issue at: https://github.com/openml/openml/issues/new"
                 ),
             ) from e
-        raise
+        raise e
 
 
 @overload


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 81c7c0d5..79ee2bbd 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -23,6 +23,7 @@ Possible Future: class TestBase from openml/testing.py can be included
 # License: BSD 3-Clause
 from __future__ import annotations
 
+from collections.abc import Iterator
 import logging
 import os
 import shutil
@@ -195,55 +196,90 @@ def pytest_addoption(parser):
 def _expected_static_cache_state(root_dir: Path) -> list[Path]:
     _c_root_dir = root_dir / "org" / "openml" / "test"
     res_paths = [root_dir, _c_root_dir]
-    
+
     for _d in ["datasets", "tasks", "runs", "setups"]:
         res_paths.append(_c_root_dir / _d)
 
-    for _id in ["-1","2"]:
+    for _id in ["-1", "2"]:
         tmp_p = _c_root_dir / "datasets" / _id
-        res_paths.extend([
-            tmp_p / "dataset.arff",
-            tmp_p / "features.xml",
-            tmp_p / "qualities.xml",
-            tmp_p / "description.xml",
-        ])
+        res_paths.extend(
+            [
+                tmp_p / "dataset.arff",
+                tmp_p / "features.xml",
+                tmp_p / "qualities.xml",
+                tmp_p / "description.xml",
+            ]
+        )
 
     res_paths.append(_c_root_dir / "datasets" / "30" / "dataset_30.pq")
     res_paths.append(_c_root_dir / "runs" / "1" / "description.xml")
     res_paths.append(_c_root_dir / "setups" / "1" / "description.xml")
-    
+
     for _id in ["1", "3", "1882"]:
         tmp_p = _c_root_dir / "tasks" / _id
-        res_paths.extend([
-            tmp_p / "datasplits.arff",
-            tmp_p / "task.xml",
-        ])
-    
+        res_paths.extend(
+            [
+                tmp_p / "datasplits.arff",
+                tmp_p / "task.xml",
+            ]
+        )
+
     return res_paths
 
 
 def assert_static_test_cache_correct(root_dir: Path) -> None:
     for p in _expected_static_cache_state(root_dir):
-        assert p.exists(), f"Expected path {p} does not exist"
-    
+        assert p.exists(), f"Expected path {p} exists"
+
 
 @pytest.fixture(scope="class")
 def long_version(request):
     request.cls.long_version = request.config.getoption("--long")
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def test_files_directory() -> Path:
     return Path(__file__).parent / "files"
 
 
-@pytest.fixture()
+@pytest.fixture(scope="session")
 def test_api_key() -> str:
     return "c0c42819af31e706efe1f4b88c23c6c1"
 
 
-@pytest.fixture(autouse=True)
-def verify_cache_state(test_files_directory) -> None:
+@pytest.fixture(autouse=True, scope="function")
+def verify_cache_state(test_files_directory) -> Iterator[None]:
     assert_static_test_cache_correct(test_files_directory)
     yield
     assert_static_test_cache_correct(test_files_directory)
+
+
+@pytest.fixture(autouse=True, scope="session")
+def as_robot() -> Iterator[None]:
+    policy = openml.config.retry_policy
+    n_retries = openml.config.connection_n_retries
+    openml.config.set_retry_policy("robot", n_retries=20)
+    yield
+    openml.config.set_retry_policy(policy, n_retries)
+
+
+@pytest.fixture(autouse=True, scope="session")
+def with_test_server():
+    openml.config.start_using_configuration_for_example()
+    yield
+    openml.config.stop_using_configuration_for_example()
+
+
+@pytest.fixture(autouse=True)
+def with_test_cache(test_files_directory, request):
+    if not test_files_directory.exists():
+        raise ValueError(
+            f"Cannot find test cache dir, expected it to be {test_files_directory!s}!",
+        )
+    _root_cache_directory = openml.config._root_cache_directory
+    tmp_cache = test_files_directory / request.node.name
+    openml.config.set_root_cache_directory(tmp_cache)
+    yield
+    openml.config.set_root_cache_directory(_root_cache_directory)
+    if tmp_cache.exists():
+        shutil.rmtree(tmp_cache)
diff --git a/tests/test_evaluations/test_evaluations_example.py b/tests/test_evaluations/test_evaluations_example.py
index bf5b03f3..a0980f5f 100644
--- a/tests/test_evaluations/test_evaluations_example.py
+++ b/tests/test_evaluations/test_evaluations_example.py
@@ -3,35 +3,47 @@ from __future__ import annotations
 
 import unittest
 
+from openml.config import overwrite_config_context
+
 
 class TestEvaluationsExample(unittest.TestCase):
     def test_example_python_paper(self):
         # Example script which will appear in the upcoming OpenML-Python paper
         # This test ensures that the example will keep running!
-
-        import matplotlib.pyplot as plt
-        import numpy as np
-
-        import openml
-
-        df = openml.evaluations.list_evaluations_setups(
-            "predictive_accuracy",
-            flows=[8353],
-            tasks=[6],
-            output_format="dataframe",
-            parameters_in_separate_columns=True,
-        )  # Choose an SVM flow, for example 8353, and a task.
-
-        hp_names = ["sklearn.svm.classes.SVC(16)_C", "sklearn.svm.classes.SVC(16)_gamma"]
-        df[hp_names] = df[hp_names].astype(float).apply(np.log)
-        C, gamma, score = df[hp_names[0]], df[hp_names[1]], df["value"]
-
-        cntr = plt.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r")
-        plt.colorbar(cntr, label="accuracy")
-        plt.xlim((min(C), max(C)))
-        plt.ylim((min(gamma), max(gamma)))
-        plt.xlabel("C (log10)", size=16)
-        plt.ylabel("gamma (log10)", size=16)
-        plt.title("SVM performance landscape", size=20)
-
-        plt.tight_layout()
+        with overwrite_config_context(
+            {
+                "server": "https://www.openml.org/api/v1/xml",
+                "apikey": None,
+            }
+        ):
+            import matplotlib.pyplot as plt
+            import numpy as np
+
+            import openml
+
+            df = openml.evaluations.list_evaluations_setups(
+                "predictive_accuracy",
+                flows=[8353],
+                tasks=[6],
+                output_format="dataframe",
+                parameters_in_separate_columns=True,
+            )  # Choose an SVM flow, for example 8353, and a task.
+
+            assert len(df) > 0, (
+                "No evaluation found for flow 8353 on task 6, could "
+                "be that this task is not available on the test server."
+            )
+
+            hp_names = ["sklearn.svm.classes.SVC(16)_C", "sklearn.svm.classes.SVC(16)_gamma"]
+            df[hp_names] = df[hp_names].astype(float).apply(np.log)
+            C, gamma, score = df[hp_names[0]], df[hp_names[1]], df["value"]
+
+            cntr = plt.tricontourf(C, gamma, score, levels=12, cmap="RdBu_r")
+            plt.colorbar(cntr, label="accuracy")
+            plt.xlim((min(C), max(C)))
+            plt.ylim((min(gamma), max(gamma)))
+            plt.xlabel("C (log10)", size=16)
+            plt.ylabel("gamma (log10)", size=16)
+            plt.title("SVM performance landscape", size=20)
+
+            plt.tight_layout()
diff --git a/tests/test_openml/test_api_calls.py b/tests/test_openml/test_api_calls.py
index 37cf6591..51123b0d 100644
--- a/tests/test_openml/test_api_calls.py
+++ b/tests/test_openml/test_api_calls.py
@@ -9,8 +9,9 @@ import minio
 import pytest
 
 import openml
+from openml.config import ConfigurationForExamples
 import openml.testing
-from openml._api_calls import _download_minio_bucket
+from openml._api_calls import _download_minio_bucket, API_TOKEN_HELP_LINK
 
 
 class TestConfig(openml.testing.TestBase):
@@ -99,3 +100,26 @@ def test_download_minio_failure(mock_minio, tmp_path: Path) -> None:
 
     with pytest.raises(ValueError):
         _download_minio_bucket(source=some_url, destination=tmp_path)
+
+
+@pytest.mark.parametrize(
+    "endpoint, method",
+    [
+        # https://github.com/openml/OpenML/blob/develop/openml_OS/views/pages/api_new/v1/xml/pre.php
+        ("flow/exists", "post"),  # 102
+        ("dataset", "post"),  # 137
+        ("dataset/42", "delete"),  # 350
+        # ("flow/owned", "post"),  # 310 - Couldn't find what would trigger this
+        ("flow/42", "delete"),  # 320
+        ("run/42", "delete"),  # 400
+        ("task/42", "delete"),  # 460
+    ],
+)
+def test_authentication_endpoints_requiring_api_key_show_relevant_help_link(
+    endpoint: str,
+    method: str,
+) -> None:
+    # We need to temporarily disable the API key to test the error message
+    with openml.config.overwrite_config_context({"apikey": None}):
+        with pytest.raises(openml.exceptions.OpenMLNotAuthorizedError, match=API_TOKEN_HELP_LINK):
+            openml._api_calls._perform_api_call(call=endpoint, request_method=method, data=None)
diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py
index d43a8bab..2bd9ee0e 100644
--- a/tests/test_runs/test_run_functions.py
+++ b/tests/test_runs/test_run_functions.py
@@ -1905,6 +1905,16 @@ class TestRun(TestBase):
         _run_id = run.run_id
         assert delete_run(_run_id)
 
+    @unittest.skipIf(
+        Version(sklearn.__version__) < Version("0.20"),
+        reason="SimpleImputer doesn't handle mixed type DataFrame as input",
+    )
+    def test_initialize_model_from_run_nonstrict(self):
+        # We cannot guarantee that a run with an older version exists on the server.
+        # Thus, we test it simply with a run that we know exists that might not be loose.
+        # This tests all lines of code for OpenML but not the initialization, which we do not want to guarantee anyhow.
+        _ = openml.runs.initialize_model_from_run(run_id=1, strict_version=False)
+
 
 @mock.patch.object(requests.Session, "delete")
 def test_delete_run_not_owned(mock_delete, test_files_directory, test_api_key):
diff --git a/tests/test_utils/test_utils.py b/tests/test_utils/test_utils.py
index cae94791..d900671b 100644
--- a/tests/test_utils/test_utils.py
+++ b/tests/test_utils/test_utils.py
@@ -8,37 +8,6 @@ import openml
 from openml.testing import _check_dataset
 
 
-@pytest.fixture(autouse=True)
-def as_robot():
-    policy = openml.config.retry_policy
-    n_retries = openml.config.connection_n_retries
-    openml.config.set_retry_policy("robot", n_retries=20)
-    yield
-    openml.config.set_retry_policy(policy, n_retries)
-
-
-@pytest.fixture(autouse=True)
-def with_test_server():
-    openml.config.start_using_configuration_for_example()
-    yield
-    openml.config.stop_using_configuration_for_example()
-
-
-@pytest.fixture(autouse=True)
-def with_test_cache(test_files_directory, request):
-    if not test_files_directory.exists():
-        raise ValueError(
-            f"Cannot find test cache dir, expected it to be {test_files_directory!s}!",
-        )
-    _root_cache_directory = openml.config._root_cache_directory
-    tmp_cache = test_files_directory / request.node.name
-    openml.config.set_root_cache_directory(tmp_cache)
-    yield
-    openml.config.set_root_cache_directory(_root_cache_directory)
-    if tmp_cache.exists():
-        shutil.rmtree(tmp_cache)
-
-
 @pytest.fixture()
 def min_number_tasks_on_test_server() -> int:
     """After a reset at least 1068 tasks are on the test server"""