Repository: lastmile-ai/aiconfig (language: python)

Issue:
[AIC-py][eval] make e2e test a little more robust
[AIC-py][eval] make e2e test a little more robust

---
Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/lastmile-ai/aiconfig/pull/435).
* __->__ #435
* #432


Discussion / hints:

Fix patch:
diff --git a/python/requirements.txt b/python/requirements.txt
index 80b58143d..80f2e93e6 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -14,5 +14,5 @@ nest_asyncio
 prompt_toolkit
 mock
 pytest-asyncio
-lastmile-utils==0.0.8
+lastmile-utils==0.0.9
 hypothesis==6.91.0
\ No newline at end of file
diff --git a/python/src/aiconfig/eval/api/__init__.py b/python/src/aiconfig/eval/api/__init__.py
index 156f13b02..d046f0ff1 100644
--- a/python/src/aiconfig/eval/api/__init__.py
+++ b/python/src/aiconfig/eval/api/__init__.py
@@ -14,19 +14,17 @@
     TestSuiteWithInputsSettings,
     run_test_suite_with_inputs,
     run_test_suite_outputs_only,
-    SampleEvaluationFunction,
 )
 
-from ..metrics import (
-    brevity,
-    substring_match,
-)
+from ..common import Metric
+
+from ..metrics import brevity, substring_match
 
 __all__ = [
+    "Metric",
     "brevity",
     "substring_match",
     "run_test_suite_with_inputs",
     "run_test_suite_outputs_only",
     "TestSuiteWithInputsSettings",
-    "SampleEvaluationFunction",
 ]
diff --git a/python/src/aiconfig/eval/common.py b/python/src/aiconfig/eval/common.py
index 7d6f270e3..815893c9d 100644
--- a/python/src/aiconfig/eval/common.py
+++ b/python/src/aiconfig/eval/common.py
@@ -1,3 +1,5 @@
+from dataclasses import dataclass
+import json
 from typing import Any, Generic
 from abc import abstractmethod
 from typing import Any, Generic, Protocol, TypeVar
@@ -10,10 +12,58 @@
 T_OutputDatum = TypeVar("T_OutputDatum", contravariant=True)
 
 
+class SampleEvaluationFunction(Protocol, Generic[T_OutputDatum]):
+    @abstractmethod
+    def __call__(self, output_datum: T_OutputDatum) -> float:
+        pass
+
+
 class EvaluationMetricMetadata(cu.Record, Generic[T_OutputDatum]):
     """A record to tie together metadata about an evaluation metric
     to ensure that numbers are interpreted as intended.
 
+    IMPORTANT NOTE:
+        **This property is part of the contract of implementing a metric**:
+
+        `id` must uniquely identify the metric to disambiguate different
+          conceptual quantities in case they happen to share a name.
+
+          If you write a metric, to write an automated test for this property,
+          see `test_metric_library_id_properties()`.
+
+          Illustration of the property:
+            ```
+                # Helper function
+                def extract_id(matcher, matcher_input):
+                    return matcher(matcher_input).interpretation.id
+
+                # These two metrics share everything but the substring.
+                matcher1 = substring_match("hello")
+                matcher2 = substring_match("world")
+
+                # Run both on the same input.
+                the_input = "hello world"
+
+                # They have distinct IDs because of the two different substrings.
+                assert extract_id(matcher1, the_input) != extract_id(matcher2, the_input)
+             ```
+
+
+        `id` must however be a _constant_ with respect to the metric input.
+            Illustration of the property:
+
+            ```
+                the_matcher = substring_match("hello")
+                input1 = "hello world"
+                input2 = "the quick brown fox"
+
+                assert extract_id(the_matcher, input1) == extract_id(the_matcher, input2)
+            ```
+
+          See `metrics.substring_match()` for an example of how to set an id.
+
+
+
     Assumptions:
     * Metric type is float
         (bools and ints have to be represented as floats; tensors are not supported)
@@ -30,10 +80,37 @@ class EvaluationMetricMetadata(cu.Record, Generic[T_OutputDatum]):
           Error count must fall between 0 and inf.
     """
 
+    @property
+    def id(self) -> str:
+        return cu.hash_id(
+            f"{self.name}{self.description}{self.best_value}{self.worst_value}params={self._serialize_extra_metadata()}".encode(
+                "utf-8"
+            )
+        )
+
+    def _serialize_extra_metadata(self) -> str:
+        return json.dumps(self.extra_metadata, sort_keys=True)
+
     name: str
     description: str
     best_value: float
     worst_value: float
+    # e.g. {"substring": "hello", "case_sensitive": False}
+    extra_metadata: dict[str, Any] = {}
+
+
+@dataclass(frozen=True)
+class Metric(Generic[T_OutputDatum]):
+    calculate: SampleEvaluationFunction[T_OutputDatum]
+    interpretation: EvaluationMetricMetadata[T_OutputDatum]
+
+
+    def __call__(self, output_datum: T_OutputDatum) -> Any:
+        """
+        For convenience, make a Metric callable.
+        Similar to torch Module `forward()`.
+        """
+        return self.calculate(output_datum)
 
 
 class SampleMetricValue(cu.Record, Generic[T_OutputDatum]):
@@ -73,9 +150,3 @@ def check_value_range(cls, values: dict[str, Any]) -> dict[str, Any]:
             )
 
         return values
-
-
-class SampleEvaluationFunction(Protocol, Generic[T_OutputDatum]):
-    @abstractmethod
-    def __call__(self, output_datum: T_OutputDatum) -> SampleMetricValue[T_OutputDatum]:
-        pass
diff --git a/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
index b83491a83..e599203dc 100644
--- a/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
+++ b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
@@ -162,22 +162,52 @@
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " <function brevity at 0x15d4195a0>\n",
+      " Metric(calculate=<function _calculate_brevity at 0x142b18ca0>, interpretation={\n",
+      "  \"name\": \"brevity\",\n",
+      "  \"description\": \"Absolute text length\",\n",
+      "  \"best_value\": 1.0,\n",
+      "  \"worst_value\": Infinity,\n",
+      "  \"extra_metadata\": {}\n",
+      "})\n",
       "\n",
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " functools.partial(<function contains_substring at 0x15d419480>, substring='Magnolia Bakery', case_sensitive=False)\n",
+      " Metric(calculate=<function substring_match.<locals>._fn at 0x142aeb880>, interpretation={\n",
+      "  \"name\": \"substring_match\",\n",
+      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
+      "  \"best_value\": 1.0,\n",
+      "  \"worst_value\": 0.0,\n",
+      "  \"extra_metadata\": {\n",
+      "    \"substring\": \"Magnolia Bakery\",\n",
+      "    \"case_sensitive\": false\n",
+      "  }\n",
+      "})\n",
       "\n",
       "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " <function brevity at 0x15d4195a0>\n",
+      " Metric(calculate=<function _calculate_brevity at 0x142b18ca0>, interpretation={\n",
+      "  \"name\": \"brevity\",\n",
+      "  \"description\": \"Absolute text length\",\n",
+      "  \"best_value\": 1.0,\n",
+      "  \"worst_value\": Infinity,\n",
+      "  \"extra_metadata\": {}\n",
+      "})\n",
       "\n",
       "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " functools.partial(<function contains_substring at 0x15d419480>, substring='Empire State Building', case_sensitive=False)\n"
+      " Metric(calculate=<function substring_match.<locals>._fn at 0x142aeb910>, interpretation={\n",
+      "  \"name\": \"substring_match\",\n",
+      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
+      "  \"best_value\": 1.0,\n",
+      "  \"worst_value\": 0.0,\n",
+      "  \"extra_metadata\": {\n",
+      "    \"substring\": \"Empire State Building\",\n",
+      "    \"case_sensitive\": false\n",
+      "  }\n",
+      "})\n"
      ]
     }
    ],
@@ -225,6 +255,7 @@
        "      <th>input</th>\n",
        "      <th>aiconfig_output</th>\n",
        "      <th>value</th>\n",
+       "      <th>metric_id</th>\n",
        "      <th>metric_name</th>\n",
        "      <th>metric_description</th>\n",
        "      <th>best_possible_value</th>\n",
@@ -235,10 +266,9 @@
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>1. Start at Top of the Rock for skyline views.\n",
-       "2. Visit Empire State Building, admire city panoramas. \n",
-       "3. Explore Grand Central Terminal, appreciate architecture, dine, shop.</td>\n",
-       "      <td>174.0</td>\n",
+       "      <td>Visit Chelsea Market for diverse cuisine. Join Chinatown Food Tour for Chinese dishes. Explore Little Italy on a culinary walking tour.</td>\n",
+       "      <td>135.0</td>\n",
+       "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
        "      <td>1.0</td>\n",
@@ -247,11 +277,10 @@
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>1. Start at Top of the Rock for skyline views.\n",
-       "2. Visit Empire State Building, admire city panoramas. \n",
-       "3. Explore Grand Central Terminal, appreciate architecture, dine, shop.</td>\n",
+       "      <td>Visit Chelsea Market for diverse cuisine. Join Chinatown Food Tour for Chinese dishes. Explore Little Italy on a culinary walking tour.</td>\n",
        "      <td>0.0</td>\n",
-       "      <td>contains_substring</td>\n",
+       "      <td>2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054</td>\n",
+       "      <td>substring_match</td>\n",
        "      <td>1.0 (pass) if contains given substring</td>\n",
        "      <td>1.0</td>\n",
        "      <td>0.0</td>\n",
@@ -259,10 +288,11 @@
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Start at Top of the Rock for skyline views.\n",
-       "2. Visit Empire State Building, admire city panoramas. \n",
-       "3. Explore Grand Central Terminal, appreciate architecture, dine, shop.</td>\n",
-       "      <td>174.0</td>\n",
+       "      <td>1. Top of the Rock\n",
+       "2. Empire State Building\n",
+       "3. NBC Studio Tour</td>\n",
+       "      <td>62.0</td>\n",
+       "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
        "      <td>1.0</td>\n",
@@ -271,11 +301,12 @@
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Start at Top of the Rock for skyline views.\n",
-       "2. Visit Empire State Building, admire city panoramas. \n",
-       "3. Explore Grand Central Terminal, appreciate architecture, dine, shop.</td>\n",
+       "      <td>1. Top of the Rock\n",
+       "2. Empire State Building\n",
+       "3. NBC Studio Tour</td>\n",
        "      <td>1.0</td>\n",
-       "      <td>contains_substring</td>\n",
+       "      <td>e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519</td>\n",
+       "      <td>substring_match</td>\n",
        "      <td>1.0 (pass) if contains given substring</td>\n",
        "      <td>1.0</td>\n",
        "      <td>0.0</td>\n",
@@ -291,25 +322,27 @@
        "2   iconic midtown skyscrapers   \n",
        "3   iconic midtown skyscrapers   \n",
        "\n",
-       "                                                                                                                                                                  aiconfig_output  \\\n",
-       "0  1. Start at Top of the Rock for skyline views.\n",
-       "2. Visit Empire State Building, admire city panoramas. \n",
-       "3. Explore Grand Central Terminal, appreciate architecture, dine, shop.   \n",
-       "1  1. Start at Top of the Rock for skyline views.\n",
-       "2. Visit Empire State Building, admire city panoramas. \n",
-       "3. Explore Grand Central Terminal, appreciate architecture, dine, shop.   \n",
-       "2  1. Start at Top of the Rock for skyline views.\n",
-       "2. Visit Empire State Building, admire city panoramas. \n",
-       "3. Explore Grand Central Terminal, appreciate architecture, dine, shop.   \n",
-       "3  1. Start at Top of the Rock for skyline views.\n",
-       "2. Visit Empire State Building, admire city panoramas. \n",
-       "3. Explore Grand Central Terminal, appreciate architecture, dine, shop.   \n",
+       "                                                                                                                           aiconfig_output  \\\n",
+       "0  Visit Chelsea Market for diverse cuisine. Join Chinatown Food Tour for Chinese dishes. Explore Little Italy on a culinary walking tour.   \n",
+       "1  Visit Chelsea Market for diverse cuisine. Join Chinatown Food Tour for Chinese dishes. Explore Little Italy on a culinary walking tour.   \n",
+       "2                                                                           1. Top of the Rock\n",
+       "2. Empire State Building\n",
+       "3. NBC Studio Tour   \n",
+       "3                                                                           1. Top of the Rock\n",
+       "2. Empire State Building\n",
+       "3. NBC Studio Tour   \n",
        "\n",
-       "   value         metric_name                      metric_description  \\\n",
-       "0  174.0             brevity                    Absolute text length   \n",
-       "1    0.0  contains_substring  1.0 (pass) if contains given substring   \n",
-       "2  174.0             brevity                    Absolute text length   \n",
-       "3    1.0  contains_substring  1.0 (pass) if contains given substring   \n",
+       "   value                                                         metric_id  \\\n",
+       "0  135.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "1    0.0  2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054   \n",
+       "2   62.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "3    1.0  e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519   \n",
+       "\n",
+       "       metric_name                      metric_description  \\\n",
+       "0          brevity                    Absolute text length   \n",
+       "1  substring_match  1.0 (pass) if contains given substring   \n",
+       "2          brevity                    Absolute text length   \n",
+       "3  substring_match  1.0 (pass) if contains given substring   \n",
        "\n",
        "   best_possible_value  worst_possible_value  \n",
        "0                  1.0                   inf  \n",
@@ -370,7 +403,7 @@
        "      <th></th>\n",
        "      <th>metric_name</th>\n",
        "      <th>brevity</th>\n",
-       "      <th>contains_substring</th>\n",
+       "      <th>substring_match</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>input</th>\n",
@@ -382,14 +415,14 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>different kinds of cuisines</th>\n",
-       "      <th>1. Start at Top of the Rock for skyline views.\\n2. Visit Empire State Building, admire city panoramas. \\n3. Explore Grand Central Terminal, appreciate architecture, dine, shop.</th>\n",
-       "      <td>174.0</td>\n",
+       "      <th>Visit Chelsea Market for diverse cuisine. Join Chinatown Food Tour for Chinese dishes. Explore Little Italy on a culinary walking tour.</th>\n",
+       "      <td>135.0</td>\n",
        "      <td>0.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>iconic midtown skyscrapers</th>\n",
-       "      <th>1. Start at Top of the Rock for skyline views.\\n2. Visit Empire State Building, admire city panoramas. \\n3. Explore Grand Central Terminal, appreciate architecture, dine, shop.</th>\n",
-       "      <td>174.0</td>\n",
+       "      <th>1. Top of the Rock\\n2. Empire State Building\\n3. NBC Studio Tour</th>\n",
+       "      <td>62.0</td>\n",
        "      <td>1.0</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
@@ -397,15 +430,15 @@
        "</div>"
       ],
       "text/plain": [
-       "metric_name                                                                                                                                                                                                   brevity  \\\n",
-       "input                       aiconfig_output                                                                                                                                                                             \n",
-       "different kinds of cuisines 1. Start at Top of the Rock for skyline views.\\n2. Visit Empire State Building, admire city panoramas. \\n3. Explore Grand Central Terminal, appreciate architecture, dine, shop.    174.0   \n",
-       "iconic midtown skyscrapers  1. Start at Top of the Rock for skyline views.\\n2. Visit Empire State Building, admire city panoramas. \\n3. Explore Grand Central Terminal, appreciate architecture, dine, shop.    174.0   \n",
+       "metric_name                                                                                                                                                          brevity  \\\n",
+       "input                       aiconfig_output                                                                                                                                    \n",
+       "different kinds of cuisines Visit Chelsea Market for diverse cuisine. Join Chinatown Food Tour for Chinese dishes. Explore Little Italy on a culinary walking tour.    135.0   \n",
+       "iconic midtown skyscrapers  1. Top of the Rock\\n2. Empire State Building\\n3. NBC Studio Tour                                                                            62.0   \n",
        "\n",
-       "metric_name                                                                                                                                                                                                   contains_substring  \n",
-       "input                       aiconfig_output                                                                                                                                                                                       \n",
-       "different kinds of cuisines 1. Start at Top of the Rock for skyline views.\\n2. Visit Empire State Building, admire city panoramas. \\n3. Explore Grand Central Terminal, appreciate architecture, dine, shop.                 0.0  \n",
-       "iconic midtown skyscrapers  1. Start at Top of the Rock for skyline views.\\n2. Visit Empire State Building, admire city panoramas. \\n3. Explore Grand Central Terminal, appreciate architecture, dine, shop.                 1.0  "
+       "metric_name                                                                                                                                                          substring_match  \n",
+       "input                       aiconfig_output                                                                                                                                           \n",
+       "different kinds of cuisines Visit Chelsea Market for diverse cuisine. Join Chinatown Food Tour for Chinese dishes. Explore Little Italy on a culinary walking tour.              0.0  \n",
+       "iconic midtown skyscrapers  1. Top of the Rock\\n2. Empire State Building\\n3. NBC Studio Tour                                                                                     1.0  "
       ]
      },
      "execution_count": 5,
@@ -492,22 +525,52 @@
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " <function brevity at 0x15d4195a0>\n",
+      " Metric(calculate=<function _calculate_brevity at 0x142b18ca0>, interpretation={\n",
+      "  \"name\": \"brevity\",\n",
+      "  \"description\": \"Absolute text length\",\n",
+      "  \"best_value\": 1.0,\n",
+      "  \"worst_value\": Infinity,\n",
+      "  \"extra_metadata\": {}\n",
+      "})\n",
       "\n",
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " functools.partial(<function contains_substring at 0x15d419480>, substring='Magnolia Bakery', case_sensitive=False)\n",
+      " Metric(calculate=<function substring_match.<locals>._fn at 0x142ba5ab0>, interpretation={\n",
+      "  \"name\": \"substring_match\",\n",
+      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
+      "  \"best_value\": 1.0,\n",
+      "  \"worst_value\": 0.0,\n",
+      "  \"extra_metadata\": {\n",
+      "    \"substring\": \"Magnolia Bakery\",\n",
+      "    \"case_sensitive\": false\n",
+      "  }\n",
+      "})\n",
       "\n",
       "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " <function brevity at 0x15d4195a0>\n",
+      " Metric(calculate=<function _calculate_brevity at 0x142b18ca0>, interpretation={\n",
+      "  \"name\": \"brevity\",\n",
+      "  \"description\": \"Absolute text length\",\n",
+      "  \"best_value\": 1.0,\n",
+      "  \"worst_value\": Infinity,\n",
+      "  \"extra_metadata\": {}\n",
+      "})\n",
       "\n",
       "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " functools.partial(<function contains_substring at 0x15d419480>, substring='Empire State Building', case_sensitive=False)\n"
+      " Metric(calculate=<function substring_match.<locals>._fn at 0x142ba5b40>, interpretation={\n",
+      "  \"name\": \"substring_match\",\n",
+      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
+      "  \"best_value\": 1.0,\n",
+      "  \"worst_value\": 0.0,\n",
+      "  \"extra_metadata\": {\n",
+      "    \"substring\": \"Empire State Building\",\n",
+      "    \"case_sensitive\": false\n",
+      "  }\n",
+      "})\n"
      ]
     }
    ],
@@ -555,6 +618,7 @@
        "      <th>input</th>\n",
        "      <th>aiconfig_output</th>\n",
        "      <th>value</th>\n",
+       "      <th>metric_id</th>\n",
        "      <th>metric_name</th>\n",
        "      <th>metric_description</th>\n",
        "      <th>best_possible_value</th>\n",
@@ -567,6 +631,7 @@
        "      <td>Missing</td>\n",
        "      <td>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</td>\n",
        "      <td>160.0</td>\n",
+       "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
        "      <td>1.0</td>\n",
@@ -577,7 +642,8 @@
        "      <td>Missing</td>\n",
        "      <td>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</td>\n",
        "      <td>0.0</td>\n",
-       "      <td>contains_substring</td>\n",
+       "      <td>2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054</td>\n",
+       "      <td>substring_match</td>\n",
        "      <td>1.0 (pass) if contains given substring</td>\n",
        "      <td>1.0</td>\n",
        "      <td>0.0</td>\n",
@@ -587,6 +653,7 @@
        "      <td>Missing</td>\n",
        "      <td>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</td>\n",
        "      <td>267.0</td>\n",
+       "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
        "      <td>1.0</td>\n",
@@ -597,7 +664,8 @@
        "      <td>Missing</td>\n",
        "      <td>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</td>\n",
        "      <td>1.0</td>\n",
-       "      <td>contains_substring</td>\n",
+       "      <td>e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519</td>\n",
+       "      <td>substring_match</td>\n",
        "      <td>1.0 (pass) if contains given substring</td>\n",
        "      <td>1.0</td>\n",
        "      <td>0.0</td>\n",
@@ -619,11 +687,17 @@
        "2  1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.   \n",
        "3  1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.   \n",
        "\n",
-       "   value         metric_name                      metric_description  \\\n",
-       "0  160.0             brevity                    Absolute text length   \n",
-       "1    0.0  contains_substring  1.0 (pass) if contains given substring   \n",
-       "2  267.0             brevity                    Absolute text length   \n",
-       "3    1.0  contains_substring  1.0 (pass) if contains given substring   \n",
+       "   value                                                         metric_id  \\\n",
+       "0  160.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "1    0.0  2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054   \n",
+       "2  267.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "3    1.0  e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519   \n",
+       "\n",
+       "       metric_name                      metric_description  \\\n",
+       "0          brevity                    Absolute text length   \n",
+       "1  substring_match  1.0 (pass) if contains given substring   \n",
+       "2          brevity                    Absolute text length   \n",
+       "3  substring_match  1.0 (pass) if contains given substring   \n",
        "\n",
        "   best_possible_value  worst_possible_value  \n",
        "0                  1.0                   inf  \n",
@@ -680,7 +754,7 @@
        "    <tr style=\"text-align: right;\">\n",
        "      <th>metric_name</th>\n",
        "      <th>brevity</th>\n",
-       "      <th>contains_substring</th>\n",
+       "      <th>substring_match</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>aiconfig_output</th>\n",
@@ -709,10 +783,10 @@
        "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.    267.0   \n",
        "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                               160.0   \n",
        "\n",
-       "metric_name                                                                                                                                                                                                                                                                  contains_substring  \n",
-       "aiconfig_output                                                                                                                                                                                                                                                                                  \n",
-       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.                 1.0  \n",
-       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                            0.0  "
+       "metric_name                                                                                                                                                                                                                                                                  substring_match  \n",
+       "aiconfig_output                                                                                                                                                                                                                                                                               \n",
+       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.              1.0  \n",
+       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                         0.0  "
       ]
      },
      "execution_count": 9,
diff --git a/python/src/aiconfig/eval/examples/travel/travel_promptfoo_config.yaml b/python/src/aiconfig/eval/examples/travel/travel_promptfoo_config.yaml
index 49e3847f2..aad3a1d4e 100644
--- a/python/src/aiconfig/eval/examples/travel/travel_promptfoo_config.yaml
+++ b/python/src/aiconfig/eval/examples/travel/travel_promptfoo_config.yaml
@@ -15,6 +15,6 @@ tests:
       question: "iconic midtown skyscrapers"
     assert:
       - type: python
-        value: output.lower().find('empire state building') != -1
+        value: output.lower().find('empire state') != -1
       - type: python
         value: len(output) < 5000
diff --git a/python/src/aiconfig/eval/lib.py b/python/src/aiconfig/eval/lib.py
index 4e089162c..a5d74cffd 100644
--- a/python/src/aiconfig/eval/lib.py
+++ b/python/src/aiconfig/eval/lib.py
@@ -8,21 +8,21 @@
 from result import Ok, Result
 
 from aiconfig.eval.common import (
-    SampleEvaluationFunction,
+    Metric,
     SampleMetricValue,
     T_InputDatum,
     T_OutputDatum,
 )
 
-logger = logging.getLogger(__name__)
 logging.basicConfig(format=cu.LOGGER_FMT)
+logger = logging.getLogger(__name__)
 
 
 # Each test is a (input_datum, evaluation_fn) pair
-UserTestSuiteWithInputs = Sequence[Tuple[str, SampleEvaluationFunction[str]]]
+UserTestSuiteWithInputs = Sequence[Tuple[str, Metric[str]]]
 
 # Each test is a (output_datum, evaluation_fn) pair
-UserTestSuiteOutputsOnly = Sequence[Tuple[str, SampleEvaluationFunction[str]]]
+UserTestSuiteOutputsOnly = Sequence[Tuple[str, Metric[str]]]
 
 
 class TestSuiteWithInputsSettings(cu.Record):
@@ -82,10 +82,10 @@ class SampleEvaluationResult(Generic[T_InputDatum, T_OutputDatum], cu.Record):
 class SampleEvaluationParams(Generic[T_InputDatum, T_OutputDatum]):
     input_sample: T_InputDatum | None
     output_sample: T_OutputDatum
-    evaluation_fn: SampleEvaluationFunction[T_OutputDatum]
+    metric: Metric[T_OutputDatum]
 
     def __str__(self) -> str:
-        return f"\nSampleEvaluationParams:\n\t{self.output_sample=}\n\t{self.evaluation_fn=}"
+        return f"\nSampleEvaluationParams:\n\t{self.output_sample=}\n\t{self.metric=}"
 
 
 def evaluate(
@@ -98,14 +98,19 @@ def evaluate(
     results: Sequence[SampleEvaluationResult[T_InputDatum, T_OutputDatum]] = []
 
     for eval_params in evaluation_params_list:
-        sample, evaluation_fn = (
+        sample, metric = (
             eval_params.output_sample,
-            eval_params.evaluation_fn,
+            eval_params.metric,
         )
+        evaluation_fn = metric.calculate
         res_ = evaluation_fn(sample)
         logger.debug(f"{res_=}")
         result = SampleEvaluationResult(
-            input_datum=eval_params.input_sample, output_datum=sample, metric_value=res_
+            input_datum=eval_params.input_sample,
+            output_datum=sample,
+            metric_value=SampleMetricValue(
+                value=res_, interpretation=metric.interpretation
+            ),
         )
         results.append(result)
 
@@ -122,6 +127,7 @@ def eval_res_to_df(
                 input=sample_res.input_datum,
                 aiconfig_output=sample_res.output_datum,
                 value=sample_res.metric_value.value,
+                metric_id=sample_res.metric_value.interpretation.id,
                 metric_name=sample_res.metric_value.interpretation.name,
                 metric_description=sample_res.metric_value.interpretation.description,
                 best_possible_value=sample_res.metric_value.interpretation.best_value,
@@ -141,11 +147,11 @@ async def user_test_suite_with_inputs_to_eval_params_list(
     test_suite: UserTestSuiteWithInputs, prompt_name: str, aiconfig: AIConfigRuntime
 ) -> Result[Sequence[SampleEvaluationParams[TextInput, TextOutput]], str]:
     out: list[SampleEvaluationParams[TextInput, TextOutput]] = []
-    grouped: dict[str, list[SampleEvaluationFunction[str]]] = {}
-    for input_datum, eval_fn in test_suite:
+    grouped: dict[str, list[Metric[str]]] = {}
+    for input_datum, metric in test_suite:
         if input_datum not in grouped:
             grouped[input_datum] = []
-        grouped[input_datum].append(eval_fn)
+        grouped[input_datum].append(metric)
 
     all_inputs = list(grouped.keys())
 
@@ -154,7 +160,13 @@ async def _run(input_datum: str) -> Result[TextOutput, str]:
             TextOutput
         )
 
-    res_outputs = await cu.result_reduce_list_all_ok_async(list(map(_run, all_inputs)))
+    # TODO: fix the race condition and then use gather
+    # https://github.com/lastmile-ai/aiconfig/issues/434
+    res_outputs_: list[Result[TextOutput, str]] = []
+    for input_datum in all_inputs:
+        res_outputs_.append(await _run(input_datum))
+    res_outputs = cu.result_reduce_list_all_ok(res_outputs_)
+    # res_outputs = await cu.result_reduce_list_all_ok_async(list(map(_run, all_inputs)))
 
     def _zip_inputs_outputs(outputs: list[TextOutput]):
         # This zip is safe because we have defined an order for the keys in `all_inputs`
@@ -165,13 +177,13 @@ def _zip_inputs_outputs(outputs: list[TextOutput]):
         # of awaitables in aws.
         outputs_by_input = dict(zip(all_inputs, outputs))
 
-        for input_datum, eval_fns in grouped.items():
-            for eval_fn in eval_fns:
+        for input_datum, metrics in grouped.items():
+            for metric in metrics:
                 out.append(
                     SampleEvaluationParams(
                         input_sample=TextInput(input_datum),
                         output_sample=outputs_by_input[input_datum],
-                        evaluation_fn=eval_fn,
+                        metric=metric,
                     )
                 )
         return out
@@ -184,11 +196,9 @@ def user_test_suite_outputs_only_to_eval_params_list(
 ) -> Sequence[SampleEvaluationParams[TextInput, TextOutput]]:
     return [
         SampleEvaluationParams(
-            input_sample=None,
-            output_sample=TextOutput(output_datum),
-            evaluation_fn=eval_fn,
+            input_sample=None, output_sample=TextOutput(output_datum), metric=metric
         )
-        for output_datum, eval_fn in test_suite
+        for output_datum, metric in test_suite
     ]
 
 
@@ -200,7 +210,8 @@ async def run_aiconfig_helper(
     }
 
     try:
-        return Ok(await runtime.run_and_get_output_text(prompt_name, params, run_with_dependencies=True))  # type: ignore[fixme, no-untyped-call]
+        out = Ok(await runtime.run_and_get_output_text(prompt_name, params, run_with_dependencies=True))  # type: ignore[fixme, no-untyped-call]
+        return out
     except Exception as e:
         return cu.ErrWithTraceback(e)
 
diff --git a/python/src/aiconfig/eval/metrics.py b/python/src/aiconfig/eval/metrics.py
index f61ed8158..eb0176021 100644
--- a/python/src/aiconfig/eval/metrics.py
+++ b/python/src/aiconfig/eval/metrics.py
@@ -1,57 +1,50 @@
-from functools import partial
 
 from aiconfig.eval.common import (
     EvaluationMetricMetadata,
-    SampleEvaluationFunction,
-    SampleMetricValue,
+    Metric,
 )
 
 
-def check_substring(output_datum: str, substring: str, case_sensitive: bool) -> bool:
+def _check_substring(output_datum: str, substring: str, case_sensitive: bool) -> bool:
     if case_sensitive:
         return substring in output_datum
     else:
         return substring.lower() in output_datum.lower()
 
 
-def contains_substring(
-    output_datum: str, substring: str, case_sensitive: bool
-) -> SampleMetricValue[str]:
-    return SampleMetricValue(
-        value=float(
-            check_substring(output_datum, substring, case_sensitive=case_sensitive)
-        ),
+def substring_match(substring: str, case_sensitive: bool = True) -> Metric[str]:
+    def _fn(output_datum: str) -> float:
+        return float(
+            _check_substring(
+                output_datum=output_datum,
+                substring=substring,
+                case_sensitive=case_sensitive,
+            )
+        )
+    return Metric(
+        calculate=_fn,
         interpretation=EvaluationMetricMetadata(
-            name="contains_substring",
+            name="substring_match",
             description="1.0 (pass) if contains given substring",
             best_value=1.0,
             worst_value=0.0,
+            extra_metadata=dict(substring=substring, case_sensitive=case_sensitive),
         ),
     )
 
 
-def substring_match(
-    substring: str, case_sensitive: bool = True
-) -> SampleEvaluationFunction[str]:
-    """Convenience function for running `contains_substring()` on a fixed substring.
-    Can be used directly to construct a user test suite."""
-    return partial(
-        contains_substring, substring=substring, case_sensitive=case_sensitive
-    )
+def _calculate_brevity(output_datum: str) -> float:
+    if len(output_datum) == 0:
+        raise ValueError("Brevity is meaningless for empty string.")
+    return float(len(output_datum))
 
 
-def brevity(output_datum: str) -> SampleMetricValue[str]:
-    def _fn(output_datum: str) -> float:
-        if len(output_datum) == 0:
-            raise ValueError("Brevity is meaningless for empty string.")
-        return float(len(output_datum))
-
-    return SampleMetricValue(
-        value=_fn(output_datum),
-        interpretation=EvaluationMetricMetadata(
-            name="brevity",
-            description="Absolute text length",
-            best_value=1.0,
-            worst_value=float("inf"),
-        ),
-    )
+brevity: Metric[str] = Metric(
+    calculate=_calculate_brevity,
+    interpretation=EvaluationMetricMetadata(
+        name="brevity",
+        description="Absolute text length",
+        best_value=1.0,
+        worst_value=float("inf"),
+    ),
+)


Test patch:
diff --git a/python/tests/test_eval.py b/python/tests/test_eval.py
index f8f486903..13caba6c9 100644
--- a/python/tests/test_eval.py
+++ b/python/tests/test_eval.py
@@ -1,5 +1,5 @@
 import os
-from typing import Any
+from typing import Any, TypeVar
 import hypothesis.strategies as st
 import lastmile_utils.lib.core.api as cu
 import pandas as pd
@@ -15,10 +15,12 @@
 )
 from aiconfig.eval.lib import TestSuiteWithInputsSpec, run_test_suite_helper
 
-from hypothesis import given
+import hypothesis
 
 from aiconfig.model_parser import InferenceOptions
 
+T_MetricParams = TypeVar("T_MetricParams")
+
 
 def current_dir():
     return os.path.dirname(os.path.realpath(__file__))
@@ -44,11 +46,11 @@ async def run_and_get_output_text(
 
 
 def test_metrics():
-    assert brevity("hello").value == 5.0
+    assert brevity("hello") == 5.0
 
-    assert substring_match("lo w")("hello world").value == 1.0
-    assert substring_match("hello", case_sensitive=False)("HELLO world").value == 1.0
-    assert substring_match("hello", case_sensitive=True)("HELLO world").value == 0.0
+    assert substring_match("lo w")("hello world") == 1.0
+    assert substring_match("hello", case_sensitive=False)("HELLO world") == 1.0
+    assert substring_match("hello", case_sensitive=True)("HELLO world") == 0.0
 
 
 @pytest.mark.asyncio
@@ -84,7 +86,7 @@ async def test_run_with_outputs_only_basic():
     assert out["value"].equals(exp["value"])  # type: ignore
 
 
-@given(st.data())
+@hypothesis.given(st.data())
 @pytest.mark.asyncio
 async def test_run_test_suite_outputs_only(data: st.DataObject):
     metrics = [brevity, substring_match("hello")]
@@ -102,6 +104,7 @@ async def test_run_test_suite_outputs_only(data: st.DataObject):
         "input",
         "aiconfig_output",
         "value",
+        "metric_id",
         "metric_name",
         "metric_description",
         "best_possible_value",
@@ -117,7 +120,7 @@ async def test_run_test_suite_outputs_only(data: st.DataObject):
     ).all()
 
 
-@given(st.data())
+@hypothesis.given(st.data())
 @pytest.mark.asyncio
 async def test_run_test_suite_with_inputs(data: st.DataObject):
     """In test_run_test_suite_outputs_only, we test the user-facing function (e2e)
@@ -137,8 +140,6 @@ async def test_run_test_suite_with_inputs(data: st.DataObject):
         )
     )
 
-    input_data, _ = cu.unzip(user_test_suite_with_inputs)
-
     mock_aiconfig = MockAIConfigRuntime()
 
     out = await run_test_suite_helper(
@@ -157,18 +158,31 @@ async def test_run_test_suite_with_inputs(data: st.DataObject):
                 "input",
                 "aiconfig_output",
                 "value",
+                "metric_id",
                 "metric_name",
                 "metric_description",
                 "best_possible_value",
                 "worst_possible_value",
             ]
-            inputs = df["input"].astype(str).tolist()  # type: ignore[no-untyped-call]
-            assert set(df["input"]) == set(input_data)  # type: ignore[no-untyped-call]
+
+            input_pairs = {
+                (input_datum, metric.interpretation.id)
+                for input_datum, metric in user_test_suite_with_inputs
+            }
+            result_pairs = set(  # type: ignore[no-untyped-call]
+                df[["input", "metric_id"]].itertuples(index=False, name=None)  # type: ignore[no-untyped-call]
+            )
+
+            assert input_pairs == result_pairs
 
             df_brevity = df[df["metric_name"] == "brevity"]
             assert (
                 df_brevity["aiconfig_output"].apply(len)  # type: ignore[no-untyped-call]
                 == df_brevity["value"]  # type: ignore[no-untyped-call]
             ).all()
+
+            df_substring = df[df["metric_name"] == "substring_match"]
+            assert (df_substring["value"].apply(lambda x: x in {0.0, 0.1})).all()  # type: ignore[no-untyped-call]
+
         case Err(e):
             assert False, f"expected Ok, got Err({e})"