Repository: mad-lab-fau/tpcp

Issue:
Allow scoring functions to return arbitary additional information

Comments:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9a20a93..c1dfe08 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Pipelines, Optimize objects, and `Scorer` are now `Generic`. This improves typing (in particular with VsCode), but 
   means a little bit more typing (pun intended), when creating new Pipelines and Optimizers
   (https://github.com/mad-lab-fau/tpcp/pull/29)
+- Added option for scoring function to return arbitrary additional information using the `NoAgg` wrapper
+  (https://github.com/mad-lab-fau/tpcp/pull/31)
 
 ### Changed
 
diff --git a/docs/modules/validate.rst b/docs/modules/validate.rst
index 3dd0a40..449f4d3 100644
--- a/docs/modules/validate.rst
+++ b/docs/modules/validate.rst
@@ -14,7 +14,8 @@ Classes
    :toctree: generated/validate
    :template: class_with_private.rst
 
-   Scorer
+    Scorer
+    NoAgg
 
 Functions
 ---------
diff --git a/tpcp/validate/__init__.py b/tpcp/validate/__init__.py
index c375dae..003b915 100644
--- a/tpcp/validate/__init__.py
+++ b/tpcp/validate/__init__.py
@@ -1,5 +1,5 @@
 """Module for all helper methods to evaluate algorithms."""
-from tpcp.validate._scorer import Scorer, aggregate_scores
+from tpcp.validate._scorer import NoAgg, Scorer, aggregate_scores
 from tpcp.validate._validate import cross_validate
 
-__all__ = ["Scorer", "cross_validate", "aggregate_scores"]
+__all__ = ["Scorer", "NoAgg", "cross_validate", "aggregate_scores"]
diff --git a/tpcp/validate/_scorer.py b/tpcp/validate/_scorer.py
index d85f540..0c859b7 100644
--- a/tpcp/validate/_scorer.py
+++ b/tpcp/validate/_scorer.py
@@ -26,10 +26,13 @@ from typing_extensions import Protocol
 from tpcp._dataset import Dataset, Dataset_
 from tpcp._pipeline import Pipeline, Pipeline_
 from tpcp._utils._score import _ERROR_SCORE_TYPE
-from tpcp.exceptions import ScorerFailed
+from tpcp.exceptions import ScorerFailed, ValidationError
+
+T = TypeVar("T")
 
 SingleScoreType = float
-MultiScoreType = Dict[str, float]
+MultiScoreType = Dict[str, Union[float, "NoAgg"]]
+ScoreType = Union[SingleScoreType, MultiScoreType]
 ScoreType_ = TypeVar("ScoreType_", SingleScoreType, MultiScoreType)
 
 IndividualScoreType = Union[Dict[str, List[float]], List[float]]
@@ -59,6 +62,38 @@ class ScoreCallback(Protocol[Pipeline_, Dataset_, ScoreType_]):
         ...
 
 
+class NoAgg(Generic[T]):
+    """Wrapper to wrap one or multiple output values of a scorer to prevent aggregation of these values.
+
+    If one of the values in the return dictionary of a multi-value score function is wrapped with this class,
+    the scorer will not aggregate the value.
+    This allows to pass arbitary data from the score function through the scorer.
+    As example, this could be some general metadata, some non-numeric scores, or an array of values (e.g. when the
+    actual score is the mean of such values).
+
+    Examples
+    --------
+    >>> def score_func(pipe, dataset):
+    ...     ...
+    ...     return {"score_val_1": score, "some_metadata": NoAgg(metadata)}
+    >>> my_scorer = Scorer(score_func)
+
+    """
+
+    _value: T
+
+    def __init__(self, _value: T):
+        self._value = _value
+
+    def __repr__(self):
+        """Show the represnentation of the object."""
+        return f"{self.__class__.__name__}({repr(self._value)})"
+
+    def get_value(self) -> T:
+        """Return the value wrapped by NoAgg."""
+        return self._value
+
+
 class Scorer(Generic[Pipeline_, Dataset_, ScoreType_]):
     """A scorer to score multiple data points of a dataset and average the results.
 
@@ -173,18 +208,22 @@ class Scorer(Generic[Pipeline_, Dataset_, ScoreType_]):
 ScorerTypes = Union[ScoreFunc[Pipeline_, Dataset_, ScoreType_], Scorer[Pipeline_, Dataset_, ScoreType_], None]
 
 
-def _validate_score_return_val(value: Union[SingleScoreType, MultiScoreType, float]):
+def _validate_score_return_val(value: ScoreType):
     """We expect a scorer to return either a numeric value or a dictionary of such values."""
     if isinstance(value, (int, float)):
         return
     if isinstance(value, dict):
         for v in value.values():
-            if not isinstance(v, (int, float)):
+            if not isinstance(v, (int, float, NoAgg)):
                 break
         else:
             return
-    raise ValueError(
-        "The scoring function must return either a dictionary of numeric values or a single numeric value."
+
+    raise ValidationError(
+        "The scoring function must have one of the following return types:\n"
+        "1. dictionary of numeric values or values wrapped by `NoAgg`.\n"
+        "2. single numeric value.\n\n"
+        f"You return value was {value}"
     )
 
 
@@ -283,9 +322,23 @@ def aggregate_scores(
     agg_scores: Dict[str, float] = {}
     # Invert the dict and calculate the mean per score:
     for key in score_names:
-        # If the scorer raised an error, there will only be a single value. This value will be used for all
-        # scores then
-        score_array = [score[key] if isinstance(score, dict) else score for score in scores]
+        key_is_no_agg = False
+        score_array = []
+        for score in scores:
+            if isinstance(score, dict):
+                score_val = score[key]
+                if isinstance(score_val, NoAgg):
+                    # If one of the values are wrapped in NoAgg, we will not aggregate the values and only remove the
+                    # NoAgg warpper.
+                    key_is_no_agg = True
+                    score_array.append(score_val.get_value())
+                else:
+                    score_array.append(score_val)
+            else:
+                # If the scorer raised an error, there will only be a single value. This value will be used for all
+                # scores then
+                score_array.append(score)
         inv_scores[key] = score_array
-        agg_scores[key] = agg_method(score_array)
+        if not key_is_no_agg:
+            agg_scores[key] = agg_method(score_array)
     return agg_scores, inv_scores


Test patch:
diff --git a/tests/test_pipelines/test_scorer.py b/tests/test_pipelines/test_scorer.py
index 3aae28b..2cf23ef 100644
--- a/tests/test_pipelines/test_scorer.py
+++ b/tests/test_pipelines/test_scorer.py
@@ -12,8 +12,9 @@ from tests.test_pipelines.conftest import (
     dummy_multi_score_func,
     dummy_single_score_func,
 )
+from tpcp.exceptions import ValidationError
 from tpcp.validate import Scorer
-from tpcp.validate._scorer import _passthrough_scoring, _validate_scorer
+from tpcp.validate._scorer import NoAgg, _passthrough_scoring, _validate_scorer
 
 
 class TestScorerCalls:
@@ -69,6 +70,27 @@ class TestScorer:
             else:
                 assert v == np.mean(data.groups)
 
+    def test_score_return_val_multi_score_no_agg(self):
+        def multi_score_func(pipeline, data_point):
+            return {"score_1": data_point.groups[0], "no_agg_score": NoAgg(str(data_point.groups))}
+
+        scorer = Scorer(multi_score_func)
+        pipe = DummyOptimizablePipeline()
+        data = DummyDataset()
+        agg, single = scorer(pipe, data, np.nan)
+        assert isinstance(single, dict)
+        for k, v in single.items():
+            assert len(v) == len(data)
+            # Our Dummy scorer, returns the groupname of the dataset as string in the no-agg case
+            if k == "no_agg_score":
+                assert all(np.array(v) == [str(d.groups) for d in data])
+            else:
+                assert all(np.array(v) == data.groups)
+        assert isinstance(agg, dict)
+        assert "score_1" in agg
+        assert "no_agg_score" not in agg
+        assert agg["score_1"] == np.mean(data.groups)
+
     @pytest.mark.parametrize("err_val", (np.nan, 1))
     def test_scoring_return_err_val(self, err_val):
         scorer = Scorer(dummy_error_score_func)
@@ -128,15 +150,17 @@ class TestScorer:
         assert str(e.value) == "Dummy Error for 0"
 
     @pytest.mark.parametrize("error_score", ("raise", 0))
-    @pytest.mark.parametrize("bad_scorer", (lambda x, y: "test", lambda x, y: {"val": "test"}))
+    @pytest.mark.parametrize(
+        "bad_scorer", (lambda x, y: "test", lambda x, y: {"val": "test"}, lambda x, y: NoAgg(None))
+    )
     def test_bad_scorer(self, error_score, bad_scorer):
         """Check that we catch cases where the scoring func returns invalid values independent of the error_score val"""
         scorer = Scorer(bad_scorer)
         pipe = DummyOptimizablePipeline()
         data = DummyDataset()
-        with pytest.raises(ValueError) as e:
+        with pytest.raises(ValidationError) as e:
             scorer(pipe, data, error_score)
-        assert "The scoring function must return" in str(e.value)
+        assert "The scoring function must have one" in str(e.value)
 
     def test_kwargs_passed(self):
         kwargs = {"a": 3, "b": "test"}
@@ -174,6 +198,9 @@ class TestScorer:
         pipe = DummyOptimizablePipeline()
         scorer(pipeline=pipe, dataset=DummyDataset(), error_score=np.nan)
 
+    def test_no_agg_scoring(self):
+        pass
+
 
 def _dummy_func(x):
     return x