Repository: automl/auto-sklearn

Issue:
[Dev only] MOO ensemble can fail to return pareto front if automl instance consutrcted with single metric 
If constructing the automl instance with only a single metric but then using multiple metrics during ensemble, querying for the pareto front will fail with:
```python
File "/home/skantify/code/automlbenchmark-fairness-private/frameworks/autosklearn_moo/lib/auto-sklearn/autosklearn/automl.py", line 1656, in _load_pareto_set
    raise ValueError("Pareto set is only available for two or more metrics.")
```


Discussion:
I guess this is an odd case and doesn't seem easy to fix. `fit_ensemble()` doesn't take different metrics and the only source of metrics is the `init`. I'll close but if this is an issue in the future, I guess we can re-open

Resolving patch:
diff --git a/autosklearn/automl.py b/autosklearn/automl.py
--- a/autosklearn/automl.py
+++ b/autosklearn/automl.py
@@ -1502,6 +1502,7 @@ def fit_ensemble(
         ensemble_nbest: Optional[int] = None,
         ensemble_class: Optional[AbstractEnsemble] = EnsembleSelection,
         ensemble_kwargs: Optional[Dict[str, Any]] = None,
+        metrics: Scorer | Sequence[Scorer] | None = None,
     ):
         check_is_fitted(self)
 
@@ -1532,6 +1533,10 @@ def fit_ensemble(
         else:
             self._is_dask_client_internally_created = False
 
+        metrics = metrics if metrics is not None else self._metrics
+        if not isinstance(metrics, Sequence):
+            metrics = [metrics]
+
         # Use the current thread to start the ensemble builder process
         # The function ensemble_builder_process will internally create a ensemble
         # builder in the provide dask client
@@ -1541,7 +1546,7 @@ def fit_ensemble(
             backend=copy.deepcopy(self._backend),
             dataset_name=dataset_name if dataset_name else self._dataset_name,
             task=task if task else self._task,
-            metrics=self._metrics,
+            metrics=metrics if metrics is not None else self._metrics,
             ensemble_class=(
                 ensemble_class if ensemble_class is not None else self._ensemble_class
             ),
@@ -1652,9 +1657,6 @@ def _load_best_individual_model(self):
         return ensemble
 
     def _load_pareto_set(self) -> Sequence[VotingClassifier | VotingRegressor]:
-        if len(self._metrics) <= 1:
-            raise ValueError("Pareto set is only available for two or more metrics.")
-
         if self._ensemble_class is not None:
             self.ensemble_ = self._backend.load_ensemble(self._seed)
         else:
@@ -1662,10 +1664,7 @@ def _load_pareto_set(self) -> Sequence[VotingClassifier | VotingRegressor]:
 
         # If no ensemble is loaded we cannot do anything
         if not self.ensemble_:
-
-            raise ValueError(
-                "Pareto set can only be accessed if an ensemble is available."
-            )
+            raise ValueError("Pareto set only available if ensemble can be loaded.")
 
         if isinstance(self.ensemble_, AbstractMultiObjectiveEnsemble):
             pareto_set = self.ensemble_.get_pareto_set()
diff --git a/autosklearn/ensemble_building/builder.py b/autosklearn/ensemble_building/builder.py
--- a/autosklearn/ensemble_building/builder.py
+++ b/autosklearn/ensemble_building/builder.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Dict, Iterable, Sequence, Type, cast
+from typing import Any, Iterable, Mapping, Sequence, Type, cast
 
 import logging.handlers
 import multiprocessing
@@ -46,7 +46,7 @@ def __init__(
         task_type: int,
         metrics: Sequence[Scorer],
         ensemble_class: Type[AbstractEnsemble] = EnsembleSelection,
-        ensemble_kwargs: Dict[str, Any] | None = None,
+        ensemble_kwargs: Mapping[str, Any] | None = None,
         ensemble_nbest: int | float = 50,
         max_models_on_disc: int | float | None = 100,
         seed: int = 1,
@@ -71,9 +71,11 @@ def __init__(
         metrics: Sequence[Scorer]
             Metrics to optimize the ensemble for. These must be non-duplicated.
 
-        ensemble_class
+        ensemble_class: Type[AbstractEnsemble]
+            Implementation of the ensemble algorithm.
 
-        ensemble_kwargs
+        ensemble_kwargs: Mapping[str, Any] | None
+            Arguments passed to the constructor of the ensemble algorithm.
 
         ensemble_nbest: int | float = 50
 
@@ -169,6 +171,8 @@ def __init__(
         self.validation_performance_ = np.inf
 
         # Data we may need
+        # TODO: The test data is needlessly loaded but automl_common has no concept of
+        # these and is perhaps too rigid
         datamanager: XYDataManager = self.backend.load_datamanager()
         self._X_test: SUPPORTED_FEAT_TYPES | None = datamanager.data.get("X_test", None)
         self._y_test: np.ndarray | None = datamanager.data.get("Y_test", None)
@@ -442,6 +446,17 @@ def main(
             self.logger.debug("Found no runs")
             raise RuntimeError("Found no runs")
 
+        # We load in `X_data` if we need it
+        if any(m._needs_X for m in self.metrics):
+            ensemble_X_data = self.X_data("ensemble")
+
+            if ensemble_X_data is None:
+                msg = "No `X_data` for 'ensemble' which was required by metrics"
+                self.logger.debug(msg)
+                raise RuntimeError(msg)
+        else:
+            ensemble_X_data = None
+
         # Calculate the loss for those that require it
         requires_update = self.requires_loss_update(runs)
         if self.read_at_most is not None:
@@ -450,9 +465,7 @@ def main(
         for run in requires_update:
             run.record_modified_times()  # So we don't count as modified next time
             run.losses = {
-                metric.name: self.loss(
-                    run, metric=metric, X_data=self.X_data("ensemble")
-                )
+                metric.name: self.loss(run, metric=metric, X_data=ensemble_X_data)
                 for metric in self.metrics
             }
 
@@ -549,15 +562,14 @@ def main(
             return self.ensemble_history, self.ensemble_nbest
 
         targets = cast(np.ndarray, self.targets("ensemble"))  # Sure they exist
-        X_data = self.X_data("ensemble")
 
         ensemble = self.fit_ensemble(
             candidates=candidates,
-            X_data=X_data,
             targets=targets,
             runs=runs,
             ensemble_class=self.ensemble_class,
             ensemble_kwargs=self.ensemble_kwargs,
+            X_data=ensemble_X_data,
             task=self.task_type,
             metrics=self.metrics,
             precision=self.precision,
@@ -587,7 +599,15 @@ def main(
 
             run_preds = [r.predictions(kind, precision=self.precision) for r in models]
             pred = ensemble.predict(run_preds)
-            X_data = self.X_data(kind)
+
+            if any(m._needs_X for m in self.metrics):
+                X_data = self.X_data(kind)
+                if X_data is None:
+                    msg = f"No `X` data for '{kind}' which was required by metrics"
+                    self.logger.debug(msg)
+                    raise RuntimeError(msg)
+            else:
+                X_data = None
 
             scores = calculate_scores(
                 solution=pred_targets,
@@ -597,10 +617,19 @@ def main(
                 X_data=X_data,
                 scoring_functions=None,
             )
+
+            # TODO only one metric in history
+            #
+            #   We should probably return for all metrics but this makes
+            #   automl::performance_history a lot more complicated, will
+            #   tackle in a future PR
+            first_metric = self.metrics[0]
             performance_stamp[f"ensemble_{score_name}_score"] = scores[
-                self.metrics[0].name
+                first_metric.name
             ]
-            self.ensemble_history.append(performance_stamp)
+
+        # Add the performance stamp to the history
+        self.ensemble_history.append(performance_stamp)
 
         # Lastly, delete any runs that need to be deleted. We save this as the last step
         # so that we have an ensemble saved that is up to date. If we do not do so,
@@ -805,13 +834,13 @@ def candidate_selection(
 
     def fit_ensemble(
         self,
-        candidates: list[Run],
-        X_data: SUPPORTED_FEAT_TYPES,
-        targets: np.ndarray,
+        candidates: Sequence[Run],
+        runs: Sequence[Run],
         *,
-        runs: list[Run],
+        targets: np.ndarray | None = None,
         ensemble_class: Type[AbstractEnsemble] = EnsembleSelection,
-        ensemble_kwargs: Dict[str, Any] | None = None,
+        ensemble_kwargs: Mapping[str, Any] | None = None,
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
         task: int | None = None,
         metrics: Sequence[Scorer] | None = None,
         precision: int | None = None,
@@ -825,24 +854,24 @@ def fit_ensemble(
 
         Parameters
         ----------
-        candidates: list[Run]
+        candidates: Sequence[Run]
             List of runs to build an ensemble from
 
-        X_data: SUPPORTED_FEAT_TYPES
-            The base level data.
+        runs: Sequence[Run]
+            List of all runs (also pruned ones and dummy runs)
 
-        targets: np.ndarray
+        targets: np.ndarray | None = None
             The targets to build the ensemble with
 
-        runs: list[Run]
-            List of all runs (also pruned ones and dummy runs)
-
-        ensemble_class: AbstractEnsemble
+        ensemble_class: Type[AbstractEnsemble]
             Implementation of the ensemble algorithm.
 
-        ensemble_kwargs: Dict[str, Any]
+        ensemble_kwargs: Mapping[str, Any] | None
             Arguments passed to the constructor of the ensemble algorithm.
 
+        X_data: SUPPORTED_FEAT_TYPES | None = None
+            The base level data.
+
         task: int | None = None
             The kind of task performed
 
@@ -859,24 +888,42 @@ def fit_ensemble(
         -------
         AbstractEnsemble
         """
-        task = task if task is not None else self.task_type
+        # Validate we have targets if None specified
+        if targets is None:
+            targets = self.targets("ensemble")
+            if targets is None:
+                path = self.backend._get_targets_ensemble_filename()
+                raise ValueError(f"`fit_ensemble` could not find any targets at {path}")
+
         ensemble_class = (
             ensemble_class if ensemble_class is not None else self.ensemble_class
         )
-        ensemble_kwargs = (
-            ensemble_kwargs if ensemble_kwargs is not None else self.ensemble_kwargs
-        )
-        ensemble_kwargs = ensemble_kwargs if ensemble_kwargs is not None else {}
-        metrics = metrics if metrics is not None else self.metrics
-        rs = random_state if random_state is not None else self.random_state
 
-        ensemble = ensemble_class(
-            task_type=task,
-            metrics=metrics,
-            random_state=rs,
-            backend=self.backend,
-            **ensemble_kwargs,
-        )  # type: AbstractEnsemble
+        # Create the ensemble_kwargs, favouring in order:
+        # 1) function kwargs, 2) function params 3) init_kwargs 4) init_params
+
+        # Collect func params in dict if they're not None
+        params = {
+            k: v
+            for k, v in [
+                ("task_type", task),
+                ("metrics", metrics),
+                ("random_state", random_state),
+            ]
+            if v is not None
+        }
+
+        kwargs = {
+            "backend": self.backend,
+            "task_type": self.task_type,
+            "metrics": self.metrics,
+            "random_state": self.random_state,
+            **(self.ensemble_kwargs or {}),
+            **params,
+            **(ensemble_kwargs or {}),
+        }
+
+        ensemble = ensemble_class(**kwargs)  # type: AbstractEnsemble
 
         self.logger.debug(f"Fitting ensemble on {len(candidates)} models")
         start_time = time.time()
@@ -995,7 +1042,8 @@ def loss(
         self,
         run: Run,
         metric: Scorer,
-        X_data: SUPPORTED_FEAT_TYPES,
+        *,
+        X_data: SUPPORTED_FEAT_TYPES | None = None,
         kind: str = "ensemble",
     ) -> float:
         """Calculate the loss for a run
@@ -1008,6 +1056,9 @@ def loss(
         metric: Scorer
             The metric to calculate the loss of
 
+        X_data: SUPPORTED_FEAT_TYPES | None = None
+            Any X_data required to be passed to the metric
+
         kind: str = "ensemble"
             The kind of targets to use for the run
 
diff --git a/autosklearn/estimators.py b/autosklearn/estimators.py
--- a/autosklearn/estimators.py
+++ b/autosklearn/estimators.py
@@ -601,6 +601,7 @@ def fit_ensemble(
         ensemble_kwargs: Optional[Dict[str, Any]] = None,
         ensemble_nbest: Optional[int] = None,
         ensemble_class: Optional[AbstractEnsemble] = EnsembleSelection,
+        metrics: Scorer | Sequence[Scorer] | None = None,
     ):
         """Fit an ensemble to models trained during an optimization process.
 
@@ -650,12 +651,13 @@ def fit_ensemble(
             to obtain only use the single best model instead of an
             ensemble.
 
+        metrics: Scorer | Sequence[Scorer] | None = None
+            A metric or list of metrics to score the ensemble with
+
         Returns
         -------
         self
-
         """
-
         # User specified `ensemble_size` explicitly, warn them about deprecation
         if ensemble_size is not None:
             # Keep consistent behaviour
@@ -708,6 +710,7 @@ def fit_ensemble(
             ensemble_nbest=ensemble_nbest,
             ensemble_class=ensemble_class,
             ensemble_kwargs=ensemble_kwargs,
+            metrics=metrics,
         )
         return self
 


Test patch:
diff --git a/test/fixtures/ensemble_building.py b/test/fixtures/ensemble_building.py
--- a/test/fixtures/ensemble_building.py
+++ b/test/fixtures/ensemble_building.py
@@ -164,6 +164,10 @@ def _make(
                 backend.save_additional_data(
                     datamanager.data["Y_train"], what="targets_ensemble"
                 )
+            if "X_train" in datamanager.data:
+                backend.save_additional_data(
+                    datamanager.data["X_train"], what="input_ensemble"
+                )
 
         builder = EnsembleBuilder(
             backend=backend,
diff --git a/test/fixtures/metrics.py b/test/fixtures/metrics.py
new file mode 100644
--- /dev/null
+++ b/test/fixtures/metrics.py
@@ -0,0 +1,26 @@
+from typing import Any
+
+import numpy as np
+
+from autosklearn.metrics import accuracy, make_scorer
+
+
+def _accuracy_requiring_X_data(
+    y_true: np.ndarray,
+    y_pred: np.ndarray,
+    X_data: Any,
+) -> float:
+    """Dummy metric that needs X Data"""
+    if X_data is None:
+        raise ValueError()
+    return accuracy(y_true, y_pred)
+
+
+acc_with_X_data = make_scorer(
+    name="acc_with_X_data",
+    score_func=_accuracy_requiring_X_data,
+    needs_X=True,
+    optimum=1,
+    worst_possible_result=0,
+    greater_is_better=True,
+)
diff --git a/test/test_automl/__init__.py b/test/test_automl/__init__.py
--- a/test/test_automl/__init__.py
+++ b/test/test_automl/__init__.py
@@ -1 +0,0 @@
-# -*- encoding: utf-8 -*-
diff --git a/test/test_automl/test_construction.py b/test/test_automl/test_construction.py
--- a/test/test_automl/test_construction.py
+++ b/test/test_automl/test_construction.py
@@ -1,9 +1,4 @@
-"""Property based Tests
-
-These test are for checking properties of already fitted models. Any test that does
-tests using cases should not modify the state as these models are cached between tests
-to reduce training time.
-"""
+"""Test things related to only constructing an AutoML instance"""
 from typing import Any, Dict, Optional, Union
 
 from autosklearn.automl import AutoML
diff --git a/test/test_automl/test_dataset_compression.py b/test/test_automl/test_dataset_compression.py
--- a/test/test_automl/test_dataset_compression.py
+++ b/test/test_automl/test_dataset_compression.py
@@ -1,3 +1,4 @@
+"""Test things related to how AutoML compresses the dataset size"""
 from typing import Any, Callable, Dict
 
 import numpy as np
diff --git a/test/test_automl/test_dummy_predictions.py b/test/test_automl/test_dummy_predictions.py
--- a/test/test_automl/test_dummy_predictions.py
+++ b/test/test_automl/test_dummy_predictions.py
@@ -1,3 +1,7 @@
+"""Test the dummy predictor of AutoML
+
+Dummy models can serve as an early warning of issues with parameters during fit
+"""
 from __future__ import annotations
 
 from typing import Callable, Sequence, Tuple
@@ -183,6 +187,11 @@ def test_crash_due_to_memory_exception(
 
 
 def test_raises_if_no_metric_set(make_automl: Callable[..., AutoML]) -> None:
+    """
+    Expects
+    -------
+    * raise if there was no metric set when calling `_do_dummy_prediction()`
+    """
     automl = make_automl()
     with pytest.raises(ValueError, match="Metric/Metrics was/were not set"):
         automl._do_dummy_prediction()
@@ -193,10 +202,17 @@ def test_raises_invalid_metric(
     make_automl: Callable[..., AutoML],
     make_sklearn_dataset: Callable[..., XYDataManager],
 ) -> None:
+    """
+    Expects
+    -------
+    * Should raise an error if the given metric is not applicable to a given task type
+    """
+    # `precision` is not applicable to MULTICLASS_CLASSIFICATION
     dataset = "iris"
     task = MULTICLASS_CLASSIFICATION
+    metrics = [accuracy, precision]
 
-    automl = make_automl(metrics=[accuracy, precision])
+    automl = make_automl(metrics=metrics)
     automl._logger = mock_logger
 
     datamanager = make_sklearn_dataset(
diff --git a/test/test_automl/test_fit.py b/test/test_automl/test_fit.py
--- a/test/test_automl/test_fit.py
+++ b/test/test_automl/test_fit.py
@@ -1,3 +1,4 @@
+"""Test specific ways of calling `fit` of AutoML"""
 from typing import Any, Callable, Dict, Optional, Tuple, Union
 
 import numpy as np
diff --git a/test/test_automl/test_fit_pipeline.py b/test/test_automl/test_fit_pipeline.py
--- a/test/test_automl/test_fit_pipeline.py
+++ b/test/test_automl/test_fit_pipeline.py
@@ -1 +1 @@
-"""TODO"""
+"""Test specific ways of calling `fit_pipeline`"""
diff --git a/test/test_automl/test_model_predict.py b/test/test_automl/test_model_predict.py
--- a/test/test_automl/test_model_predict.py
+++ b/test/test_automl/test_model_predict.py
@@ -1,3 +1,4 @@
+"""Test the _model_predict helper function such that it shapes output correctly"""
 from typing import Callable, Dict, Tuple
 
 import warnings
@@ -19,7 +20,10 @@
 
 
 class WarningModel:
+    """Simple model that returns incorrect shape and issues warning"""
+
     def predict(self, X: np.ndarray) -> np.ndarray:
+        """Shout a warning during prediction"""
         warnings.warn("shout")
         return X
 
diff --git a/test/test_automl/test_outputs.py b/test/test_automl/test_outputs.py
deleted file mode 100644
--- a/test/test_automl/test_outputs.py
+++ /dev/null
@@ -1,118 +0,0 @@
-from pathlib import Path
-
-from autosklearn.automl import AutoML
-from autosklearn.ensemble_building.builder import CANDIDATES_FILENAME
-
-from pytest import mark
-from pytest_cases import parametrize_with_cases
-from pytest_cases.filters import has_tag
-
-import test.test_automl.cases as cases
-from test.conftest import DEFAULT_SEED
-
-# Some filters
-has_ensemble = has_tag("fitted") & ~has_tag("no_ensemble")
-no_ensemble = has_tag("fitted") & has_tag("no_ensemble")
-
-
-@mark.todo
-def test_datamanager_stored_contents() -> None:
-    ...
-
-
-@parametrize_with_cases("automl", cases=cases, filter=has_ensemble)
-def test_paths_created(automl: AutoML) -> None:
-    """
-    Parameters
-    ----------
-    automl : AutoML
-        A previously fitted automl
-
-    Expects
-    -------
-    * The given paths should exist after the automl has been run and fitted
-    """
-    assert automl._backend is not None
-
-    partial = Path(automl._backend.internals_directory)
-    expected = [
-        partial / fixture
-        for fixture in (
-            "true_targets_ensemble.npy",
-            f"start_time_{DEFAULT_SEED}",
-            "datamanager.pkl",
-            "runs",
-        )
-    ]
-
-    for path in expected:
-        assert path.exists()
-
-
-@parametrize_with_cases("automl", cases=cases, filter=has_ensemble)
-def test_paths_created_with_ensemble(automl: AutoML) -> None:
-    """
-    Parameters
-    ----------
-    automl : AutoML
-        A previously fitted automl
-
-    Expects
-    -------
-    * The given paths for an automl with an ensemble should include paths
-    specific to ensemble building
-    """
-    assert automl._backend is not None
-
-    partial = Path(automl._backend.internals_directory)
-    expected = [
-        partial / fixture
-        for fixture in (
-            "ensembles",
-            "ensemble_history.json",
-            CANDIDATES_FILENAME,
-        )
-    ]
-
-    for path in expected:
-        assert path.exists()
-
-
-@parametrize_with_cases("automl", cases=cases, filter=has_ensemble)
-def test_at_least_one_model_and_predictions(automl: AutoML) -> None:
-    """
-    Expects
-    -------
-    * There should be at least one models saved
-    * Each model saved should have predictions for the ensemble
-    """
-    assert automl._backend is not None
-    runs_dir = Path(automl._backend.get_runs_directory())
-
-    runs = list(runs_dir.iterdir())
-    assert len(runs) > 0
-
-    at_least_one = False
-    for run in runs:
-        prediction_files = run.glob("predictions_ensemble*.npy")
-        model_files = run.glob("*.*.model")
-
-        if any(prediction_files):
-            at_least_one = True
-            assert any(model_files), "Run produced prediction but no model"
-
-    assert at_least_one, "No runs produced predictions"
-
-
-@parametrize_with_cases("automl", cases=cases, filter=has_ensemble)
-def test_at_least_one_ensemble(automl: AutoML) -> None:
-    """
-    Expects
-    -------
-    * There should be at least one ensemble generated
-    """
-    assert automl._backend is not None
-    ens_dir = Path(automl._backend.get_ensemble_dir())
-
-    # TODO make more generic
-    assert len(list(ens_dir.glob("*.ensemble"))) > 0
diff --git a/test/test_automl/test_pareto_front.py b/test/test_automl/test_pareto_front.py
new file mode 100644
--- /dev/null
+++ b/test/test_automl/test_pareto_front.py
@@ -0,0 +1,36 @@
+"""Test the output of loading the pareto set from an automl instance"""
+from autosklearn.automl import AutoML
+
+from pytest_cases import parametrize_with_cases
+from pytest_cases.filters import has_tag
+
+import test.test_automl.cases as cases
+
+has_ensemble = has_tag("fitted") & ~has_tag("no_ensemble")
+
+single_objective = has_ensemble & ~has_tag("multiobjective")
+multi_objective = has_ensemble & has_tag("multiobjective")
+
+
+@parametrize_with_cases("automl", cases=cases, filter=single_objective)
+def test_can_output_pareto_front_singleobjective(automl: AutoML) -> None:
+    """
+    Expects
+    -------
+    * Non-multiobjective instances should have a pareto set of size 1
+    """
+    pareto_set = automl._load_pareto_set()
+
+    assert len(pareto_set) == 1
+
+
+@parametrize_with_cases("automl", cases=cases, filter=multi_objective)
+def test_can_output_pareto_front_multiobjective(automl: AutoML) -> None:
+    """
+    Expects
+    -------
+    * Multiobjective ensembles should return >= 1, #TODO should test it's pareto optimal
+    """
+    pareto_set = automl._load_pareto_set()
+
+    assert len(pareto_set) >= 1
diff --git a/test/test_automl/test_performance.py b/test/test_automl/test_performance.py
--- a/test/test_automl/test_performance.py
+++ b/test/test_automl/test_performance.py
@@ -0,0 +1,47 @@
+"""Test the performance of automl instances after fitting"""
+
+import numpy as np
+from sklearn.ensemble import VotingClassifier, VotingRegressor
+
+from autosklearn.automl import AutoML
+
+from pytest_cases import parametrize_with_cases
+
+import test.test_automl.cases as cases
+
+
+@parametrize_with_cases("automl", cases.case_classifier_fitted_holdout_multiobjective)
+def test_performance_with_multiobjective(automl: AutoML) -> None:
+    """
+    Expects
+    -------
+    * Auto-sklearn can predict/predict_proba and has a model
+    * Each ensemble in the pareto_set can predict/predict_proba
+    """
+    # TODO: This test is hyperspecific to this one case
+    #
+    #   Long term we probably want to return additional info about the case so we can
+    #   test things for other than this case
+
+    # Check that the predict function works
+    X = np.array([[1.0, 1.0, 1.0, 1.0]])
+
+    assert automl.predict_proba(X).shape == (1, 3)
+    assert automl.predict(X).shape == (1,)
+
+    pareto_front = automl._load_pareto_set()
+    for ensemble in pareto_front:
+
+        assert isinstance(ensemble, (VotingClassifier, VotingRegressor))
+
+        y_pred = ensemble.predict_proba(X)
+        assert y_pred.shape == (1, 3)
+
+        y_pred = ensemble.predict(X)
+        assert y_pred in ["setosa", "versicolor", "virginica"]
+
+    statistics = automl.sprint_statistics()
+    assert "Metrics" in statistics
+    assert ("Best validation score: 0.9" in statistics) or (
+        "Best validation score: 1.0" in statistics
+    ), statistics
diff --git a/test/test_automl/test_performance_over_time.py b/test/test_automl/test_performance_over_time.py
--- a/test/test_automl/test_performance_over_time.py
+++ b/test/test_automl/test_performance_over_time.py
@@ -1,3 +1,4 @@
+"""Test the performance over time functionality of automl instances"""
 from autosklearn.automl import AutoML
 
 from pytest_cases import parametrize_with_cases
diff --git a/test/test_automl/test_post_fit.py b/test/test_automl/test_post_fit.py
--- a/test/test_automl/test_post_fit.py
+++ b/test/test_automl/test_post_fit.py
@@ -1,11 +1,20 @@
-import numpy as np
-from sklearn.ensemble import VotingClassifier, VotingRegressor
+"""Check the internal state of the automl instances after it has been fitted"""
+
+from pathlib import Path
 
 from autosklearn.automl import AutoML
+from autosklearn.ensemble_building.builder import CANDIDATES_FILENAME
 
+from pytest import mark
 from pytest_cases import parametrize_with_cases
+from pytest_cases.filters import has_tag
 
 import test.test_automl.cases as cases
+from test.conftest import DEFAULT_SEED
+
+# Some filters
+has_ensemble = has_tag("fitted") & ~has_tag("no_ensemble")
+no_ensemble = has_tag("fitted") & has_tag("no_ensemble")
 
 
 @parametrize_with_cases("automl", cases=cases, has_tag=["fitted", "holdout"])
@@ -52,7 +61,7 @@ def test_cv_loaded_models(automl: AutoML) -> None:
     assert set(automl.cv_models_.keys()) == set(ensemble_identifiers)
 
 
-@parametrize_with_cases("automl", cases=cases, has_tag=["fitted", "no_ensemble"])
+@parametrize_with_cases("automl", cases=cases, has_tag=no_ensemble)
 def test_no_ensemble(automl: AutoML) -> None:
     """
     Parameters
@@ -71,36 +80,99 @@ def test_no_ensemble(automl: AutoML) -> None:
     assert len(automl.cv_models_) == 0
 
 
-@parametrize_with_cases("automl", cases, has_tag=["multiobjective"])
-def test__load_pareto_front(automl: AutoML) -> None:
+@mark.todo
+def test_datamanager_stored_contents() -> None:
     """
-    Parameters
-    ----------
-    automl : AutoML
-        An AutoML object fitted with multiple objective metrics
+    Expects
+    -------
+    * TODO
+    """
+    ...
+
+
+@parametrize_with_cases("automl", cases=cases, filter=has_ensemble)
+def test_paths_created(automl: AutoML) -> None:
+    """
+    Expects
+    -------
+    * The given paths should exist after the automl has been run and fitted
+    """
+    assert automl._backend is not None
+
+    partial = Path(automl._backend.internals_directory)
+    expected = [
+        partial / fixture
+        for fixture in (
+            "true_targets_ensemble.npy",
+            f"start_time_{DEFAULT_SEED}",
+            "datamanager.pkl",
+            "runs",
+        )
+    ]
+
+    for path in expected:
+        assert path.exists()
+
+
+@parametrize_with_cases("automl", cases=cases, filter=has_ensemble)
+def test_paths_created_with_ensemble(automl: AutoML) -> None:
+    """
+    Expects
+    -------
+    * The given paths for an automl with an ensemble should include paths
+    specific to ensemble building
+    """
+    assert automl._backend is not None
+
+    partial = Path(automl._backend.internals_directory)
+    expected = [
+        partial / fixture
+        for fixture in (
+            "ensembles",
+            "ensemble_history.json",
+            CANDIDATES_FILENAME,
+        )
+    ]
+
+    for path in expected:
+        assert path.exists()
 
+
+@parametrize_with_cases("automl", cases=cases, filter=has_ensemble)
+def test_at_least_one_model_and_predictions(automl: AutoML) -> None:
+    """
     Expects
     -------
-    * Auto-sklearn can predict and has a model
-    * _load_pareto_front returns one scikit-learn ensemble
-    """
-    # Check that the predict function works
-    X = np.array([[1.0, 1.0, 1.0, 1.0]])
-
-    assert automl.predict_proba(X).shape == (1, 3)
-    assert automl.predict(X).shape == (1,)
-
-    pareto_front = automl._load_pareto_set()
-    assert len(pareto_front) == 1
-    for ensemble in pareto_front:
-        assert isinstance(ensemble, (VotingClassifier, VotingRegressor))
-        y_pred = ensemble.predict_proba(X)
-        assert y_pred.shape == (1, 3)
-        y_pred = ensemble.predict(X)
-        assert y_pred in ["setosa", "versicolor", "virginica"]
-
-    statistics = automl.sprint_statistics()
-    assert "Metrics" in statistics
-    assert ("Best validation score: 0.9" in statistics) or (
-        "Best validation score: 1.0" in statistics
-    ), statistics
+    * There should be at least one models saved
+    * Each model saved should have predictions for the ensemble
+    """
+    assert automl._backend is not None
+    runs_dir = Path(automl._backend.get_runs_directory())
+
+    runs = list(runs_dir.iterdir())
+    assert len(runs) > 0
+
+    at_least_one = False
+    for run in runs:
+        prediction_files = run.glob("predictions_ensemble*.npy")
+        model_files = run.glob("*.*.model")
+
+        if any(prediction_files):
+            at_least_one = True
+            assert any(model_files), "Run produced prediction but no model"
+
+    assert at_least_one, "No runs produced predictions"
+
+
+@parametrize_with_cases("automl", cases=cases, filter=has_ensemble)
+def test_at_least_one_ensemble(automl: AutoML) -> None:
+    """
+    Expects
+    -------
+    * There should be at least one ensemble generated
+    """
+    assert automl._backend is not None
+    ens_dir = Path(automl._backend.get_ensemble_dir())
+
+    # TODO make more generic
+    assert len(list(ens_dir.glob("*.ensemble"))) > 0
diff --git a/test/test_automl/test_predict.py b/test/test_automl/test_predict.py
--- a/test/test_automl/test_predict.py
+++ b/test/test_automl/test_predict.py
@@ -1 +1 @@
-"""TODO"""
+"""Test predictions of an automl instance"""
diff --git a/test/test_automl/test_refit.py b/test/test_automl/test_refit.py
--- a/test/test_automl/test_refit.py
+++ b/test/test_automl/test_refit.py
@@ -1,3 +1,4 @@
+"""Test the refitting functionality of an automl instance"""
 from typing import Callable, Union
 
 from itertools import repeat
diff --git a/test/test_automl/test_show_models.py b/test/test_automl/test_show_models.py
--- a/test/test_automl/test_show_models.py
+++ b/test/test_automl/test_show_models.py
@@ -1,3 +1,4 @@
+"""Test the show models functinality of an automl instance"""
 from autosklearn.automl import AutoML
 
 from pytest_cases import parametrize_with_cases
diff --git a/test/test_automl/test_sklearn_compliance.py b/test/test_automl/test_sklearn_compliance.py
--- a/test/test_automl/test_sklearn_compliance.py
+++ b/test/test_automl/test_sklearn_compliance.py
@@ -1,7 +1,9 @@
-"""
+"""Test that autosklearn is sklearn compliant
+
 Note
 ----
-This is far from complete at the moment
+* This is far from complete at the moment
+* This should probably be tested on AutoSklearnEstimators not AutoML
 """
 from typing import List, Union
 
diff --git a/test/test_ensemble_builder/test_ensemble_builder.py b/test/test_ensemble_builder/test_ensemble_builder.py
--- a/test/test_ensemble_builder/test_ensemble_builder.py
+++ b/test/test_ensemble_builder/test_ensemble_builder.py
@@ -10,14 +10,15 @@
 
 from autosklearn.automl_common.common.utils.backend import Backend
 from autosklearn.ensemble_building import EnsembleBuilder, Run
-from autosklearn.metrics import make_scorer
+from autosklearn.metrics import Scorer, accuracy, make_scorer
 from autosklearn.util.functional import bound, pairs
 
 import pytest
 from pytest_cases import fixture, parametrize
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 from test.conftest import DEFAULT_SEED
+from test.fixtures.metrics import acc_with_X_data
 
 
 @fixture
@@ -673,6 +674,27 @@ def test_delete_runs_does_not_delete_dummy(
     assert set(loaded.values()) == set(dummy_runs)
 
 
+def test_fit_ensemble_with_no_targets_raises(
+    builder: EnsembleBuilder,
+    make_run: Callable[..., Run],
+) -> None:
+    """
+    Expects
+    -------
+    * If no ensemble targets can be found then `fit_ensemble` should fail
+    """
+    # Delete the targets and then try fit ensemble
+    targets_path = Path(builder.backend._get_targets_ensemble_filename())
+    targets_path.unlink()
+
+    candidates = [make_run(backend=builder.backend) for _ in range(5)]
+    with pytest.raises(ValueError, match="`fit_ensemble` could not find any .*"):
+        builder.fit_ensemble(
+            candidates=candidates,
+            runs=candidates,
+        )
+
+
 def test_fit_ensemble_produces_ensemble(
     builder: EnsembleBuilder,
     make_run: Callable[..., Run],
@@ -682,16 +704,13 @@ def test_fit_ensemble_produces_ensemble(
     -------
     * Should produce an ensemble if all runs have predictions
     """
-    X_data = builder.X_data("ensemble")
     targets = builder.targets("ensemble")
     assert targets is not None
 
     predictions = targets
     runs = [make_run(predictions={"ensemble": predictions}) for _ in range(10)]
 
-    ensemble = builder.fit_ensemble(
-        candidates=runs, X_data=X_data, targets=targets, runs=runs
-    )
+    ensemble = builder.fit_ensemble(candidates=runs, runs=runs)
 
     assert ensemble is not None
 
@@ -823,3 +842,160 @@ def test_deletion_will_not_break_current_ensemble(
 
     for run in new_runs:
         assert run in available_runs
+
+
+@parametrize("metrics", [accuracy, acc_with_X_data, [accuracy, acc_with_X_data]])
+def test_will_build_ensemble_with_different_metrics(
+    make_ensemble_builder: Callable[..., EnsembleBuilder],
+    make_run: Callable[..., Run],
+    metrics: Scorer | list[Scorer],
+) -> None:
+    """
+    Expects
+    -------
+    * Should be able to build a valid ensemble with different combinations of metrics
+    * Should produce a validation score for both "ensemble" and "test" scores
+    """
+    if not isinstance(metrics, list):
+        metrics = [metrics]
+
+    builder = make_ensemble_builder(metrics=metrics)
+
+    # Make some runs and stick them in the same backend as the builder
+    # Dummy just has a terrible loss for all metrics
+    make_run(
+        dummy=True,
+        losses={m.name: 1000 for m in metrics},
+        backend=builder.backend,
+    )
+
+    # "Proper" runs will have the correct targets and so be better than dummy
+    run_predictions = {
+        "ensemble": builder.targets("ensemble"),
+        "test": builder.targets("test"),
+    }
+    for _ in range(5):
+        make_run(predictions=run_predictions, backend=builder.backend)
+
+    history, nbest = builder.main()
+
+    # Should only produce one step
+    assert len(history) == 1
+    hist = history[0]
+
+    # Each of these two keys should be present
+    for key in ["ensemble_optimization_score", "ensemble_test_score"]:
+        assert key in hist
+
+        # TODO should be updated in next PR
+        #   Each of these scores should contain all the metrics
+        # for metric in metrics:
+        #   assert metric.name in hist[key]
+
+
+@parametrize("n_least_prioritized", [1, 2, 3, 4])
+@parametrize("metrics", [accuracy, acc_with_X_data, [accuracy, acc_with_X_data]])
+def test_fit_ensemble_kwargs_priorities(
+    make_ensemble_builder: Callable[..., EnsembleBuilder],
+    make_run: Callable[..., Run],
+    metrics: Scorer | list[Scorer],
+    n_least_prioritized: int,
+) -> None:
+    """
+    Expects
+    -------
+    * Should favour 1) function kwargs, 2) function params 3) init_kwargs 4) init_params
+    """
+    if not isinstance(metrics, list):
+        metrics = [metrics]
+
+    class FakeEnsembleClass:
+        def __init__(self, *args, **kwargs):
+            self.args = args
+            self.kwargs = kwargs
+
+        def fit(*args, **kwargs) -> None:
+            pass
+
+    # We establish the priorty order and give each one of them a custom metric
+    priority = ["function_kwargs", "function_params", "init_kwargs", "init_params"]
+
+    # We reverse the priority and use the `n_least_prioritized` ones
+    # with `n_least_prioritized = 3`
+    #   reversed =  ["init_params", "init_kwargs", "function_params", "function_kwargs"]
+    #   used =      ["init_params", "init_kwargs", "function_params"]
+    #   highest =   "function_params"
+    reversed_priority = list(reversed(priority))
+    used = reversed_priority[:n_least_prioritized]
+    highest_priority = used[-1]
+
+    def S(name: str) -> Scorer:
+        return make_scorer(name, lambda: None)
+
+    # We now pass in all the places this arguments could be specified
+    # Naming them specifically to make it more clear in setup below
+    builder_metric = [S("init_params")] if "init_params" in used else None
+    fit_ensemble_metric = [S("function_params")] if "function_params" in used else None
+
+    builder_ensemble_kwargs = (
+        {"metrics": [S("init_kwargs")]} if "init_kwargs" in used else None
+    )
+    fit_ensemble_kwargs = (
+        {"metrics": [S("function_kwargs")]} if "function_kwargs" in used else None
+    )
+
+    builder = make_ensemble_builder(
+        metrics=builder_metric,
+        ensemble_kwargs=builder_ensemble_kwargs,
+    )
+
+    candidates = [make_run() for _ in range(5)]  # Just so something can be run
+
+    ensemble = builder.fit_ensemble(
+        metrics=fit_ensemble_metric,
+        ensemble_class=FakeEnsembleClass,
+        ensemble_kwargs=fit_ensemble_kwargs,
+        candidates=candidates,
+        runs=candidates,
+    )
+
+    # These are the final metrics passed to the ensemble builder when constructed
+    passed_metrics = ensemble.kwargs["metrics"]
+    metric = passed_metrics[0]
+
+    assert metric.name == highest_priority
+
+
+@parametrize("metric, should_be_loaded", [(accuracy, False), (acc_with_X_data, True)])
+def test_X_data_only_loaded_when_required(
+    make_ensemble_builder: Callable[..., EnsembleBuilder],
+    make_run: Callable[..., Run],
+    metric: Scorer,
+    should_be_loaded: bool,
+) -> None:
+    """
+    Expects
+    -------
+    * Should only load X_train if it's required
+    * TODO should only load X_test if it's required
+    """
+    metrics = [metric]
+    builder = make_ensemble_builder(metrics=metrics)
+
+    # Make a dummy which is required for the whole pipeline to run
+    make_run(dummy=True, losses={metric.name: 1000}, backend=builder.backend)
+
+    # Make a run that has no losses recorded, forcing us to use the metric
+    make_run(
+        dummy=False,
+        predictions={"ensemble": builder.targets("ensemble")},
+        losses=None,
+        backend=builder.backend,
+    )
+
+    ret_value = builder.X_data()
+    builder.X_data = Mock(return_value=ret_value)
+
+    builder.main()
+
+    assert builder.X_data.called == should_be_loaded