Repository: lastmile-ai/aiconfig (language: python)

Issue:
[AIC-py][eval] fix the types pt 1
[AIC-py][eval] fix the types pt 1

---
Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/lastmile-ai/aiconfig/pull/523).
* #539
* __->__ #523
* #522
* #513


Discussion / hints:

Fix patch:
diff --git a/python/requirements.txt b/python/requirements.txt
index ce2c38030..456c48461 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -14,6 +14,6 @@ nest_asyncio
 prompt_toolkit
 mock
 pytest-asyncio
-lastmile-utils==0.0.10
+lastmile-utils==0.0.11
 hypothesis==6.91.0
 nltk
\ No newline at end of file
diff --git a/python/src/aiconfig/eval/api/__init__.py b/python/src/aiconfig/eval/api/__init__.py
index 4869a18f6..f04ad9683 100644
--- a/python/src/aiconfig/eval/api/__init__.py
+++ b/python/src/aiconfig/eval/api/__init__.py
@@ -9,7 +9,7 @@
     TestSuiteWithInputsSettings,
 )
 """
-from .. import metrics
+from .. import common, metrics
 
 # pyright: reportWildcardImportFromLibrary=false
 from ..lib import TestSuiteWithInputsSettings, run_test_suite_outputs_only, run_test_suite_with_inputs
@@ -17,6 +17,7 @@
 
 __all__ = [
     "Metric",
+    "common",
     "metrics",
     "brevity",
     "substring_match",
diff --git a/python/src/aiconfig/eval/common.py b/python/src/aiconfig/eval/common.py
index 7d2ebd468..36b77e15d 100644
--- a/python/src/aiconfig/eval/common.py
+++ b/python/src/aiconfig/eval/common.py
@@ -1,16 +1,25 @@
 import json
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import Any, Generic, Protocol, TypeVar
+from typing import Any, Generic, NewType, Protocol, Type, TypeVar
 
 import lastmile_utils.lib.core.api as cu
-from pydantic import root_validator
+import result
+from aiconfig.eval import common
+from pydantic import BaseModel
+from result import Result
 
 T_InputDatum = TypeVar("T_InputDatum", contravariant=True)
 T_OutputDatum = TypeVar("T_OutputDatum", contravariant=True)
 
+T_Evaluable = TypeVar("T_Evaluable", contravariant=True)
 
-@dataclass
+T_BaseModel = TypeVar("T_BaseModel", bound=BaseModel)
+
+SerializedJSON = NewType("SerializedJSON", str)
+
+
+@dataclass(eq=False)
 class CustomMetricValue(ABC):
     """
     Subclass this if you want your metric to return a type not included in MetricValue.
@@ -21,16 +30,28 @@ class CustomMetricValue(ABC):
     """
 
 
-MetricValue = int | float | str | bool | CustomMetricValue
+T_MetricValue = TypeVar("T_MetricValue", int, float, str, bool, CustomMetricValue, covariant=True)
 
 
-class EvaluationFunction(Protocol, Generic[T_OutputDatum]):
+class CompletionTextToSerializedJSON(Protocol):
     @abstractmethod
-    async def __call__(self, output_datum: T_OutputDatum) -> MetricValue:
+    def __call__(self, output_datum: str) -> Result[common.SerializedJSON, str]:
         pass
 
 
-class EvaluationMetricMetadata(cu.Record, Generic[T_OutputDatum]):
+@dataclass
+class CustomMetricPydanticObject(CustomMetricValue, Generic[T_BaseModel]):
+    data: T_BaseModel
+
+
+class EvaluationFunction(Protocol, Generic[T_Evaluable, T_MetricValue]):
+    @abstractmethod
+    async def __call__(self, datum: T_Evaluable) -> T_MetricValue:
+        pass
+
+
+class EvaluationMetricMetadata(cu.Record, Generic[T_Evaluable, T_MetricValue]):
+
     """A record to tie together metadata about an evaluation metric
     to ensure that numbers are interpreted as intended.
 
@@ -65,8 +86,8 @@ def _serialize_extra_metadata(self) -> str:
 
     name: str
     description: str
-    best_value: MetricValue | None = None
-    worst_value: MetricValue | None = None
+    best_value: T_MetricValue | None = None
+    worst_value: T_MetricValue | None = None
     # e.g. {"substring": "hello", "case_sensitive": False}
     extra_metadata: dict[str, Any] = {}
 
@@ -77,25 +98,26 @@ def __repr__(self) -> str:
         return f"EvaluationMetricMetadata({s_json})"
 
 
-class SampleMetricValue(cu.Record, Generic[T_OutputDatum]):
-    value: MetricValue | None
-    metric_metadata: EvaluationMetricMetadata[T_OutputDatum]
+@dataclass
+class SampleMetricValue(Generic[T_Evaluable, T_MetricValue]):
+    value: T_MetricValue | None
+    metric_metadata: EvaluationMetricMetadata[T_Evaluable, T_MetricValue]
 
-    @root_validator(pre=True)
-    def check_value_range(cls, values: dict[str, Any]) -> dict[str, Any]:
+    def __post_init__(self) -> None:
+        metric_metadata = self.metric_metadata
         worst_value, best_value = (
-            values["metric_metadata"].worst_value,
-            values["metric_metadata"].best_value,
+            metric_metadata.worst_value,
+            metric_metadata.best_value,
         )
-        value = values["value"]
+        value = self.value
         if worst_value is None and best_value is None:
             # fine
-            return values
+            return
         elif worst_value is None or best_value is None:
             raise ValueError(
                 f"""
-                    [{values["metric_metadata"].name}]
-                    {values["metric_metadata"].description}
+                    [{metric_metadata.name}]
+                    {metric_metadata.description}
 
                     You must define both worst_value and best_value, or neither.
                     You defined worst_value = {worst_value} and best_value = {best_value}.
@@ -103,27 +125,43 @@ def check_value_range(cls, values: dict[str, Any]) -> dict[str, Any]:
             )
         elif worst_value == best_value:
             raise ValueError("best_value and worst_value cannot be equal")
-        elif value is not None and worst_value < best_value and not worst_value <= value <= best_value:
+        elif value is not None and worst_value < best_value and not worst_value <= value <= best_value:  # type: ignore[fixme]
             raise ValueError(
                 f"""
-                    [{values["metric_metadata"].name}]
-                    {values["metric_metadata"].description}
+                    [{metric_metadata.name}]
+                    {metric_metadata.description}
 
                     Value {value} is not in range [{worst_value}, {best_value}]. 
                     You defined worst_value = {worst_value} and best_value = {best_value},
                     but got value outside that range.
                 """
             )
-        elif value is not None and worst_value > best_value and not worst_value >= value >= best_value:
+        elif value is not None and worst_value > best_value and not worst_value >= value >= best_value:  # type: ignore[fixme]
             raise ValueError(
                 f"""
-                    [{values["metric_metadata"].name}]
-                    {values["metric_metadata"].description}
+                    [{metric_metadata.name}]
+                    {metric_metadata.description}
 
                     Value {value} is not in range [{worst_value}, {best_value}]. 
                     You defined worst_value = {worst_value} and best_value = {best_value},
                     but got value outside that range.
                 """
             )
-        else:
-            return values
+
+
+class TextRatingsData(cu.Record):
+    conciseness_rating: int
+    conciseness_confidence: float
+    conciseness_reasoning: str
+
+
+def get_llm_structured_response(
+    input_text: str,
+    chat_completion_create: CompletionTextToSerializedJSON,
+    basemodel_type: Type[common.T_BaseModel],
+) -> Result[common.T_BaseModel, str]:
+    return result.do(
+        cu.safe_model_validate_json(response_ok, basemodel_type)
+        # get the serialized JSON response
+        for response_ok in chat_completion_create(input_text)
+    )
diff --git a/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
index 9bd7ba9ac..e8f97daec 100644
--- a/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
+++ b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
@@ -61,7 +61,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -70,6 +70,18 @@
      "text": [
       "Imports and set log level\n"
      ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/pydantic/_internal/_fields.py:128: UserWarning: Field \"model_parsers\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
     }
    ],
    "source": [
@@ -83,12 +95,12 @@
     "pd.set_option(\"display.max_colwidth\", None)\n",
     "\n",
     "from aiconfig.eval.api import (\n",
-    "    brevity,\n",
-    "    substring_match,\n",
     "    run_test_suite_with_inputs,\n",
     "    TestSuiteWithInputsSettings,\n",
     ")\n",
     "\n",
+    "from aiconfig.eval.api import metrics\n",
+    "\n",
     "jupyter_utils.set_log_level(logging.WARNING)\n",
     "\n"
    ]
@@ -102,7 +114,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -160,16 +172,16 @@
     "test_suite_with_inputs = []\n",
     "for test_input, substring in test_inputs_with_substrings:\n",
     "    # Add the brevity metric\n",
-    "    test_fn1 = brevity\n",
+    "    test_fn1 = metrics.brevity\n",
     "    test_suite_with_inputs.append((test_input, test_fn1))\n",
     "    # Add substring check function\n",
-    "    test_fn2 = substring_match(substring, case_sensitive=False)\n",
+    "    test_fn2 = metrics.substring_match(substring, case_sensitive=False)\n",
     "    test_suite_with_inputs.append((test_input, test_fn2))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -181,52 +193,56 @@
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " Metric(calculate=<function _calculate_brevity at 0x11f79ade0>, interpretation={\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
       "  \"best_value\": 1.0,\n",
       "  \"worst_value\": Infinity,\n",
-      "  \"extra_metadata\": {}\n",
-      "})\n",
+      "  \"extra_metadata\": {},\n",
+      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "}))\n",
       "\n",
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " Metric(calculate=<function substring_match.<locals>._fn at 0x11f799b20>, interpretation={\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x1036cfac0>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
-      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": 0.0,\n",
+      "  \"description\": \"True (pass) if contains given substring\",\n",
+      "  \"best_value\": true,\n",
+      "  \"worst_value\": false,\n",
       "  \"extra_metadata\": {\n",
       "    \"substring\": \"Magnolia Bakery\",\n",
       "    \"case_sensitive\": false\n",
-      "  }\n",
-      "})\n",
+      "  },\n",
+      "  \"id\": \"0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6\"\n",
+      "}))\n",
       "\n",
       "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " Metric(calculate=<function _calculate_brevity at 0x11f79ade0>, interpretation={\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
       "  \"best_value\": 1.0,\n",
       "  \"worst_value\": Infinity,\n",
-      "  \"extra_metadata\": {}\n",
-      "})\n",
+      "  \"extra_metadata\": {},\n",
+      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "}))\n",
       "\n",
       "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " Metric(calculate=<function substring_match.<locals>._fn at 0x11f799ee0>, interpretation={\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x144090e50>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
-      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": 0.0,\n",
+      "  \"description\": \"True (pass) if contains given substring\",\n",
+      "  \"best_value\": true,\n",
+      "  \"worst_value\": false,\n",
       "  \"extra_metadata\": {\n",
       "    \"substring\": \"Empire State Building\",\n",
       "    \"case_sensitive\": false\n",
-      "  }\n",
-      "})\n"
+      "  },\n",
+      "  \"id\": \"53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f\"\n",
+      "}))\n"
      ]
     }
    ],
@@ -239,7 +255,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -285,10 +301,10 @@
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>1. Experience culinary delights at Chelsea Market.\n",
-       "2. Immerse in NYC's Chinatown food tour. \n",
-       "3. Indulge in Italian tradition at Little Italy.</td>\n",
-       "      <td>141.0</td>\n",
+       "      <td>1. Explore Chelsea Market's cuisine.\n",
+       "2. Visit Little Italy for authentic fare.\n",
+       "3. Savor Asian delicacies in Flushing, Queens.</td>\n",
+       "      <td>125</td>\n",
        "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
@@ -298,23 +314,23 @@
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>1. Experience culinary delights at Chelsea Market.\n",
-       "2. Immerse in NYC's Chinatown food tour. \n",
-       "3. Indulge in Italian tradition at Little Italy.</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054</td>\n",
+       "      <td>1. Explore Chelsea Market's cuisine.\n",
+       "2. Visit Little Italy for authentic fare.\n",
+       "3. Savor Asian delicacies in Flushing, Queens.</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6</td>\n",
        "      <td>substring_match</td>\n",
-       "      <td>1.0 (pass) if contains given substring</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>True (pass) if contains given substring</td>\n",
+       "      <td>True</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Visit \"Top of the Rock\" at Rockefeller Center.\n",
-       "2. Tour Empire State Building.\n",
-       "3. Explore Grand Central Terminal.</td>\n",
-       "      <td>115.0</td>\n",
+       "      <td>1. Empire State Building Observatory - 86th floor views.\n",
+       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
+       "3. One World Trade Center - One World Observatory visit.</td>\n",
+       "      <td>170</td>\n",
        "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
@@ -324,15 +340,15 @@
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Visit \"Top of the Rock\" at Rockefeller Center.\n",
-       "2. Tour Empire State Building.\n",
-       "3. Explore Grand Central Terminal.</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519</td>\n",
+       "      <td>1. Empire State Building Observatory - 86th floor views.\n",
+       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
+       "3. One World Trade Center - One World Observatory visit.</td>\n",
+       "      <td>True</td>\n",
+       "      <td>53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f</td>\n",
        "      <td>substring_match</td>\n",
-       "      <td>1.0 (pass) if contains given substring</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>True (pass) if contains given substring</td>\n",
+       "      <td>True</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -345,40 +361,40 @@
        "2   iconic midtown skyscrapers   \n",
        "3   iconic midtown skyscrapers   \n",
        "\n",
-       "                                                                                                                                 aiconfig_output  \\\n",
-       "0  1. Experience culinary delights at Chelsea Market.\n",
-       "2. Immerse in NYC's Chinatown food tour. \n",
-       "3. Indulge in Italian tradition at Little Italy.   \n",
-       "1  1. Experience culinary delights at Chelsea Market.\n",
-       "2. Immerse in NYC's Chinatown food tour. \n",
-       "3. Indulge in Italian tradition at Little Italy.   \n",
-       "2                            1. Visit \"Top of the Rock\" at Rockefeller Center.\n",
-       "2. Tour Empire State Building.\n",
-       "3. Explore Grand Central Terminal.   \n",
-       "3                            1. Visit \"Top of the Rock\" at Rockefeller Center.\n",
-       "2. Tour Empire State Building.\n",
-       "3. Explore Grand Central Terminal.   \n",
+       "                                                                                                                                                              aiconfig_output  \\\n",
+       "0                                               1. Explore Chelsea Market's cuisine.\n",
+       "2. Visit Little Italy for authentic fare.\n",
+       "3. Savor Asian delicacies in Flushing, Queens.   \n",
+       "1                                               1. Explore Chelsea Market's cuisine.\n",
+       "2. Visit Little Italy for authentic fare.\n",
+       "3. Savor Asian delicacies in Flushing, Queens.   \n",
+       "2  1. Empire State Building Observatory - 86th floor views.\n",
+       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
+       "3. One World Trade Center - One World Observatory visit.   \n",
+       "3  1. Empire State Building Observatory - 86th floor views.\n",
+       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
+       "3. One World Trade Center - One World Observatory visit.   \n",
        "\n",
        "   value                                                         metric_id  \\\n",
-       "0  141.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "1    0.0  2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054   \n",
-       "2  115.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "3    1.0  e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519   \n",
+       "0    125  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "1  False  0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6   \n",
+       "2    170  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "3   True  53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f   \n",
        "\n",
-       "       metric_name                      metric_description  \\\n",
-       "0          brevity                    Absolute text length   \n",
-       "1  substring_match  1.0 (pass) if contains given substring   \n",
-       "2          brevity                    Absolute text length   \n",
-       "3  substring_match  1.0 (pass) if contains given substring   \n",
+       "       metric_name                       metric_description  \\\n",
+       "0          brevity                     Absolute text length   \n",
+       "1  substring_match  True (pass) if contains given substring   \n",
+       "2          brevity                     Absolute text length   \n",
+       "3  substring_match  True (pass) if contains given substring   \n",
        "\n",
-       "   best_possible_value  worst_possible_value  \n",
-       "0                  1.0                   inf  \n",
-       "1                  1.0                   0.0  \n",
-       "2                  1.0                   inf  \n",
-       "3                  1.0                   0.0  "
+       "  best_possible_value worst_possible_value  \n",
+       "0                 1.0                  inf  \n",
+       "1                True                False  \n",
+       "2                 1.0                  inf  \n",
+       "3                True                False  "
       ]
      },
-     "execution_count": 9,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -397,7 +413,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -442,33 +458,33 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>different kinds of cuisines</th>\n",
-       "      <th>1. Experience culinary delights at Chelsea Market.\\n2. Immerse in NYC's Chinatown food tour. \\n3. Indulge in Italian tradition at Little Italy.</th>\n",
-       "      <td>141.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <th>1. Explore Chelsea Market's cuisine.\\n2. Visit Little Italy for authentic fare.\\n3. Savor Asian delicacies in Flushing, Queens.</th>\n",
+       "      <td>125</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>iconic midtown skyscrapers</th>\n",
-       "      <th>1. Visit \"Top of the Rock\" at Rockefeller Center.\\n2. Tour Empire State Building.\\n3. Explore Grand Central Terminal.</th>\n",
-       "      <td>115.0</td>\n",
-       "      <td>1.0</td>\n",
+       "      <th>1. Empire State Building Observatory - 86th floor views.\\n2. Rockefeller Center - Top of the Rock and ice skating.\\n3. One World Trade Center - One World Observatory visit.</th>\n",
+       "      <td>170</td>\n",
+       "      <td>True</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "metric_name                                                                                                                                                                  brevity  \\\n",
-       "input                       aiconfig_output                                                                                                                                            \n",
-       "different kinds of cuisines 1. Experience culinary delights at Chelsea Market.\\n2. Immerse in NYC's Chinatown food tour. \\n3. Indulge in Italian tradition at Little Italy.    141.0   \n",
-       "iconic midtown skyscrapers  1. Visit \"Top of the Rock\" at Rockefeller Center.\\n2. Tour Empire State Building.\\n3. Explore Grand Central Terminal.                              115.0   \n",
+       "metric_name                                                                                                                                                                                              brevity  \\\n",
+       "input                       aiconfig_output                                                                                                                                                                        \n",
+       "different kinds of cuisines 1. Explore Chelsea Market's cuisine.\\n2. Visit Little Italy for authentic fare.\\n3. Savor Asian delicacies in Flushing, Queens.                                                  125   \n",
+       "iconic midtown skyscrapers  1. Empire State Building Observatory - 86th floor views.\\n2. Rockefeller Center - Top of the Rock and ice skating.\\n3. One World Trade Center - One World Observatory visit.     170   \n",
        "\n",
-       "metric_name                                                                                                                                                                  substring_match  \n",
-       "input                       aiconfig_output                                                                                                                                                   \n",
-       "different kinds of cuisines 1. Experience culinary delights at Chelsea Market.\\n2. Immerse in NYC's Chinatown food tour. \\n3. Indulge in Italian tradition at Little Italy.              0.0  \n",
-       "iconic midtown skyscrapers  1. Visit \"Top of the Rock\" at Rockefeller Center.\\n2. Tour Empire State Building.\\n3. Explore Grand Central Terminal.                                        1.0  "
+       "metric_name                                                                                                                                                                                              substring_match  \n",
+       "input                       aiconfig_output                                                                                                                                                                               \n",
+       "different kinds of cuisines 1. Explore Chelsea Market's cuisine.\\n2. Visit Little Italy for authentic fare.\\n3. Savor Asian delicacies in Flushing, Queens.                                                        False  \n",
+       "iconic midtown skyscrapers  1. Empire State Building Observatory - 86th floor views.\\n2. Rockefeller Center - Top of the Rock and ice skating.\\n3. One World Trade Center - One World Observatory visit.            True  "
       ]
      },
-     "execution_count": 10,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -488,7 +504,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -504,11 +520,11 @@
     "\n",
     "\n",
     "from aiconfig.eval.api import (\n",
-    "    brevity,\n",
-    "    substring_match,\n",
     "    run_test_suite_outputs_only,\n",
     ")\n",
     "\n",
+    "from aiconfig.eval.api import metrics\n",
+    "\n",
     "\n",
     "# This is similar to \"test_inputs_with_substrings\" above, but we have the AIConfig *outputs*\n",
     "# in the test cases, rather than the inputs. The library will evaluate these strings directly\n",
@@ -529,10 +545,10 @@
     "test_suite_outputs_only = []\n",
     "for test_output, substring in test_outputs_with_substrings:\n",
     "    # Add the brevity metric\n",
-    "    test_fn1 = brevity\n",
+    "    test_fn1 = metrics.brevity\n",
     "    test_suite_outputs_only.append((test_output, test_fn1))\n",
     "    # Add substring check function\n",
-    "    test_fn2 = substring_match(substring, case_sensitive=False)\n",
+    "    test_fn2 = metrics.substring_match(substring, case_sensitive=False)\n",
     "    test_suite_outputs_only.append(\n",
     "        (test_output, test_fn2)\n",
     "    )"
@@ -540,7 +556,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -552,52 +568,56 @@
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " Metric(calculate=<function _calculate_brevity at 0x11f79ade0>, interpretation={\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
       "  \"best_value\": 1.0,\n",
       "  \"worst_value\": Infinity,\n",
-      "  \"extra_metadata\": {}\n",
-      "})\n",
+      "  \"extra_metadata\": {},\n",
+      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "}))\n",
       "\n",
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " Metric(calculate=<function substring_match.<locals>._fn at 0x13b20cae0>, interpretation={\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x14412c1f0>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
-      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": 0.0,\n",
+      "  \"description\": \"True (pass) if contains given substring\",\n",
+      "  \"best_value\": true,\n",
+      "  \"worst_value\": false,\n",
       "  \"extra_metadata\": {\n",
       "    \"substring\": \"Magnolia Bakery\",\n",
       "    \"case_sensitive\": false\n",
-      "  }\n",
-      "})\n",
+      "  },\n",
+      "  \"id\": \"0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6\"\n",
+      "}))\n",
       "\n",
       "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " Metric(calculate=<function _calculate_brevity at 0x11f79ade0>, interpretation={\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
       "  \"best_value\": 1.0,\n",
       "  \"worst_value\": Infinity,\n",
-      "  \"extra_metadata\": {}\n",
-      "})\n",
+      "  \"extra_metadata\": {},\n",
+      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "}))\n",
       "\n",
       "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " Metric(calculate=<function substring_match.<locals>._fn at 0x13b20c720>, interpretation={\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x14412dcf0>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
-      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": 0.0,\n",
+      "  \"description\": \"True (pass) if contains given substring\",\n",
+      "  \"best_value\": true,\n",
+      "  \"worst_value\": false,\n",
       "  \"extra_metadata\": {\n",
       "    \"substring\": \"Empire State Building\",\n",
       "    \"case_sensitive\": false\n",
-      "  }\n",
-      "})\n"
+      "  },\n",
+      "  \"id\": \"53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f\"\n",
+      "}))\n"
      ]
     }
    ],
@@ -610,7 +630,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -657,7 +677,7 @@
        "      <th>0</th>\n",
        "      <td>Missing</td>\n",
        "      <td>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</td>\n",
-       "      <td>160.0</td>\n",
+       "      <td>160</td>\n",
        "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
@@ -668,18 +688,18 @@
        "      <th>1</th>\n",
        "      <td>Missing</td>\n",
        "      <td>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6</td>\n",
        "      <td>substring_match</td>\n",
-       "      <td>1.0 (pass) if contains given substring</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>True (pass) if contains given substring</td>\n",
+       "      <td>True</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>Missing</td>\n",
        "      <td>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</td>\n",
-       "      <td>267.0</td>\n",
+       "      <td>267</td>\n",
        "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
@@ -690,12 +710,12 @@
        "      <th>3</th>\n",
        "      <td>Missing</td>\n",
        "      <td>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519</td>\n",
+       "      <td>True</td>\n",
+       "      <td>53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f</td>\n",
        "      <td>substring_match</td>\n",
-       "      <td>1.0 (pass) if contains given substring</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>True (pass) if contains given substring</td>\n",
+       "      <td>True</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -715,25 +735,25 @@
        "3  1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.   \n",
        "\n",
        "   value                                                         metric_id  \\\n",
-       "0  160.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "1    0.0  2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054   \n",
-       "2  267.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "3    1.0  e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519   \n",
+       "0    160  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "1  False  0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6   \n",
+       "2    267  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "3   True  53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f   \n",
        "\n",
-       "       metric_name                      metric_description  \\\n",
-       "0          brevity                    Absolute text length   \n",
-       "1  substring_match  1.0 (pass) if contains given substring   \n",
-       "2          brevity                    Absolute text length   \n",
-       "3  substring_match  1.0 (pass) if contains given substring   \n",
+       "       metric_name                       metric_description  \\\n",
+       "0          brevity                     Absolute text length   \n",
+       "1  substring_match  True (pass) if contains given substring   \n",
+       "2          brevity                     Absolute text length   \n",
+       "3  substring_match  True (pass) if contains given substring   \n",
        "\n",
-       "   best_possible_value  worst_possible_value  \n",
-       "0                  1.0                   inf  \n",
-       "1                  1.0                   0.0  \n",
-       "2                  1.0                   inf  \n",
-       "3                  1.0                   0.0  "
+       "  best_possible_value worst_possible_value  \n",
+       "0                 1.0                  inf  \n",
+       "1                True                False  \n",
+       "2                 1.0                  inf  \n",
+       "3                True                False  "
       ]
      },
-     "execution_count": 13,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -749,7 +769,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -792,31 +812,31 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</th>\n",
-       "      <td>267.0</td>\n",
-       "      <td>1.0</td>\n",
+       "      <td>267</td>\n",
+       "      <td>True</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</th>\n",
-       "      <td>160.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>160</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "metric_name                                                                                                                                                                                                                                                                  brevity  \\\n",
-       "aiconfig_output                                                                                                                                                                                                                                                                        \n",
-       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.    267.0   \n",
-       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                               160.0   \n",
+       "metric_name                                                                                                                                                                                                                                                                 brevity  \\\n",
+       "aiconfig_output                                                                                                                                                                                                                                                                       \n",
+       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.     267   \n",
+       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                160   \n",
        "\n",
-       "metric_name                                                                                                                                                                                                                                                                  substring_match  \n",
-       "aiconfig_output                                                                                                                                                                                                                                                                               \n",
-       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.              1.0  \n",
-       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                         0.0  "
+       "metric_name                                                                                                                                                                                                                                                                 substring_match  \n",
+       "aiconfig_output                                                                                                                                                                                                                                                                              \n",
+       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.            True  \n",
+       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                      False  "
       ]
      },
-     "execution_count": 14,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -826,6 +846,49 @@
     "df_result.set_index([ \"aiconfig_output\", \"metric_name\"])\\\n",
     "        .value.unstack(\"metric_name\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/pydantic/_internal/_fields.py:128: UserWarning: Field \"model_parsers\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CustomMetricRecord(data={\n",
+       "  \"conciseness_rating\": 5,\n",
+       "  \"conciseness_confidence\": 8.0,\n",
+       "  \"conciseness_reasoning\": \"The text is short and to the point.\"\n",
+       "})"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import openai\n",
+    "\n",
+    "from aiconfig.eval.api import metrics\n",
+    "\n",
+    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
+    "\n",
+    "await metrics.gpt3_5_text_ratings(\"one two three\")"
+   ]
   }
  ],
  "metadata": {
diff --git a/python/src/aiconfig/eval/lib.py b/python/src/aiconfig/eval/lib.py
index 819546d0f..03040f8f4 100644
--- a/python/src/aiconfig/eval/lib.py
+++ b/python/src/aiconfig/eval/lib.py
@@ -1,11 +1,12 @@
+import asyncio
 import logging
 from dataclasses import dataclass
-from typing import Generic, NewType, Sequence, Tuple, TypeVar
+from typing import Any, Generic, NewType, Sequence, Tuple, TypeVar
 
 import lastmile_utils.lib.core.api as cu
 import pandas as pd
 from aiconfig.Config import AIConfigRuntime
-from aiconfig.eval.common import MetricValue, SampleMetricValue, T_InputDatum, T_OutputDatum
+from aiconfig.eval.common import SampleMetricValue, T_InputDatum, T_MetricValue, T_OutputDatum
 from aiconfig.eval.metrics import Metric
 from result import Err, Ok, Result
 
@@ -13,11 +14,12 @@
 LOGGER = logging.getLogger(__name__)
 
 
+# TODO: figure out a way to do heterogenous list without Any
 # Each test is a (input_datum, Metric) pair
-UserTestSuiteWithInputs = Sequence[Tuple[str, Metric[str]]]
+UserTestSuiteWithInputs = Sequence[Tuple[str, Metric[str, Any]]]
 
 # Each test is a (output_datum, Metric) pair
-UserTestSuiteOutputsOnly = Sequence[Tuple[str, Metric[str]]]
+UserTestSuiteOutputsOnly = Sequence[Tuple[str, Metric[str, Any]]]
 
 
 class TestSuiteWithInputsSettings(cu.Record):
@@ -62,68 +64,75 @@ class NumericalEvalDataset(cu.Record):
 # TODO:
 # GenericBeforeBaseModelWarning: Classes should inherit from `BaseModel` before generic classes (e.g. `typing.Generic[T]`) for pydantic generics to work properly.
 # But swapping the order breaks
-class SampleEvaluationResult(Generic[T_InputDatum, T_OutputDatum], cu.Record):
+class SampleEvaluationResult(Generic[T_InputDatum, T_OutputDatum, T_MetricValue], cu.Record):
     input_datum: T_InputDatum | None
     output_datum: T_OutputDatum
-    metric_value: SampleMetricValue[T_OutputDatum]
-
-    Tuple[T_OutputDatum, SampleMetricValue[T_OutputDatum]]
-
-
-DatasetEvaluationResult = Sequence[SampleEvaluationResult[T_InputDatum, T_OutputDatum]]
+    metric_value: SampleMetricValue[T_OutputDatum, T_MetricValue]
 
 
 @dataclass(frozen=True)
-class SampleEvaluationParams(Generic[T_InputDatum, T_OutputDatum]):
+class SampleEvaluationParams(Generic[T_InputDatum, T_OutputDatum, T_MetricValue]):
     # input_sample doesn't _need_ to be here, because we already have
     # output_sample ready to input to eval.
     # input_sample is here for documentation/debugging.
     input_sample: T_InputDatum | None
     output_sample: T_OutputDatum
-    metric: Metric[T_OutputDatum]
+    metric: Metric[T_OutputDatum, T_MetricValue]
 
     def __str__(self) -> str:
         return f"\nSampleEvaluationParams:\n\t{self.output_sample=}\n\t{self.metric=}"
 
 
-async def evaluate(
-    evaluation_params_list: Sequence[SampleEvaluationParams[T_InputDatum, T_OutputDatum]],
-) -> Result[DatasetEvaluationResult[T_InputDatum, T_OutputDatum], str]:
-    results: Sequence[SampleEvaluationResult[T_InputDatum, T_OutputDatum]] = []
+# TODO: don't use Any.
+DatasetEvaluationResult = Sequence[SampleEvaluationResult[T_InputDatum, T_OutputDatum, Any]]
+DatasetEvaluationParams = Sequence[SampleEvaluationParams[T_InputDatum, T_OutputDatum, Any]]
+MetricList = list[Metric[T_OutputDatum, Any]]
 
-    for eval_params in evaluation_params_list:
-        sample, metric = (
-            eval_params.output_sample,
-            eval_params.metric,
-        )
 
-        async def _calculate() -> MetricValue:
-            return await metric.evaluation_fn(sample)
-
-        def _ok_with_log(res_: Result[MetricValue, str]) -> MetricValue | None:
-            match res_:
-                case Ok(res):
-                    return res
-                case Err(e):
-                    LOGGER.error(f"Error evaluating {eval_params=}: {e}")
-                    return None
-
-        # TODO: figure out the right timeout
-        res_ = await cu.run_thunk_safe(_calculate(), timeout=1)
-        result = SampleEvaluationResult(
-            input_datum=eval_params.input_sample,
-            output_datum=sample,
-            metric_value=SampleMetricValue(value=_ok_with_log(res_), metric_metadata=metric.metric_metadata),
-        )
-        results.append(result)
+async def _evaluate_for_sample(
+    eval_params: SampleEvaluationParams[T_InputDatum, T_OutputDatum, T_MetricValue]
+) -> SampleEvaluationResult[T_InputDatum, T_OutputDatum, T_MetricValue]:
+    sample, metric = (
+        eval_params.output_sample,
+        eval_params.metric,
+    )
+
+    async def _calculate() -> T_MetricValue:
+        return await metric.evaluation_fn(sample)
+
+    def _ok_with_log(res_: Result[T_MetricValue, str]) -> T_MetricValue | None:
+        match res_:
+            case Ok(res):
+                return res
+            case Err(e):
+                LOGGER.error(f"Error evaluating {eval_params=}: {e}")
+                return None
+
+    # TODO: figure out the right timeout
+    res_ = await cu.run_thunk_safe(_calculate(), timeout=1)
+    result = SampleEvaluationResult(
+        input_datum=eval_params.input_sample,
+        output_datum=sample,
+        metric_value=SampleMetricValue(
+            #
+            value=_ok_with_log(res_),
+            metric_metadata=metric.metric_metadata,
+        ),
+    )
+    return result
 
-    return Ok(results)
+
+async def evaluate(
+    evaluation_params_list: DatasetEvaluationParams[T_InputDatum, T_OutputDatum],
+) -> Result[DatasetEvaluationResult[T_InputDatum, T_OutputDatum], str]:
+    return Ok(await asyncio.gather(*map(_evaluate_for_sample, evaluation_params_list)))
 
 
 def eval_res_to_df(
     eval_res: DatasetEvaluationResult[T_InputDatum, T_OutputDatum],
 ) -> pd.DataFrame:
-    records: list[dict[str, None | MetricValue | T_InputDatum | T_OutputDatum]] = []
+    # TODO: dont use Any
+    records: list[dict[str, Any]] = []
     for sample_res in eval_res:
         records.append(
             dict(
@@ -148,17 +157,17 @@ def eval_res_to_df(
 
 async def user_test_suite_with_inputs_to_eval_params_list(
     test_suite: UserTestSuiteWithInputs, prompt_name: str, aiconfig: AIConfigRuntime
-) -> Result[Sequence[SampleEvaluationParams[TextInput, TextOutput]], str]:
+) -> Result[DatasetEvaluationParams[TextInput, TextOutput], str]:
     """
     Example in/out:
         [("hello", brevity)] -> [SampleEvaluationParams("hello", "output_is_world", brevity)]
     """
-    out: list[SampleEvaluationParams[TextInput, TextOutput]] = []
+    out: DatasetEvaluationParams[TextInput, TextOutput] = []
 
     # Group by input so that we only run each input through the AIConfig once.
     # This is sort of an optimization because the user can give the same input
     # multiple times (with different metrics).
-    input_to_metrics_mapping: dict[str, list[Metric[str]]] = {}
+    input_to_metrics_mapping: dict[str, MetricList[TextOutput]] = {}
     for input_datum, metric in test_suite:
         if input_datum not in input_to_metrics_mapping:
             input_to_metrics_mapping[input_datum] = []
@@ -202,7 +211,7 @@ def _zip_inputs_outputs(outputs: list[TextOutput]):
 
 def user_test_suite_outputs_only_to_eval_params_list(
     test_suite: UserTestSuiteOutputsOnly,
-) -> Sequence[SampleEvaluationParams[TextInput, TextOutput]]:
+) -> DatasetEvaluationParams[TextInput, TextOutput]:
     """
     Example: [("the_output_is_world", brevity)] -> [SampleEvaluationParams(None, "the_output_is_world", brevity)
     """
@@ -241,7 +250,7 @@ async def run_test_suite_helper(
 ) -> Result[pd.DataFrame, str]:
     async def _get_eval_params_list(
         test_suite_spec: TestSuiteSpec,
-    ) -> Result[Sequence[SampleEvaluationParams[TextInput, TextOutput]], str]:
+    ) -> Result[DatasetEvaluationParams[TextInput, TextOutput], str]:
         match test_suite_spec:
             case TestSuiteWithInputsSpec(test_suite=test_suite, prompt_name=prompt_name, aiconfig=aiconfig):
                 return await user_test_suite_with_inputs_to_eval_params_list(test_suite, prompt_name, aiconfig)
diff --git a/python/src/aiconfig/eval/metrics.py b/python/src/aiconfig/eval/metrics.py
index ca0c7366d..09af6ca25 100644
--- a/python/src/aiconfig/eval/metrics.py
+++ b/python/src/aiconfig/eval/metrics.py
@@ -1,70 +1,59 @@
-from functools import total_ordering
-from typing import Any, Generic
+import json
+import sys
+from abc import abstractmethod
+from functools import partial, total_ordering
+from typing import Any, Callable, Generic, Protocol, Type
 
+import lastmile_utils.lib.core.api as cu
 import nltk
 import pandas as pd
-from aiconfig.eval.common import CustomMetricValue, EvaluationFunction, EvaluationMetricMetadata, T_OutputDatum
+from aiconfig.eval import common
+from aiconfig.eval.common import (
+    CustomMetricValue,
+    EvaluationFunction,
+    EvaluationMetricMetadata,
+    T_BaseModel,
+    T_Evaluable,
+    T_MetricValue,
+    TextRatingsData,
+)
+from aiconfig.eval.openai import OpenAIChatCompletionCreate, default_openai_chat_completion_create, make_fn_completion_text_to_serialized_json
 from attr import dataclass
-from nltk.sentiment.vader import SentimentIntensityAnalyzer
+from nltk.sentiment.vader import SentimentIntensityAnalyzer as NLTKSentimentIntensityAnalyzer
+from result import Err, Ok, Result
 
 
 @dataclass(frozen=True)
-class Metric(Generic[T_OutputDatum]):
+class Metric(Generic[T_Evaluable, T_MetricValue]):
     """See metrics.py for examples."""
 
-    evaluation_fn: EvaluationFunction[T_OutputDatum]
-    metric_metadata: EvaluationMetricMetadata[T_OutputDatum]
+    evaluation_fn: EvaluationFunction[T_Evaluable, T_MetricValue]
+    metric_metadata: EvaluationMetricMetadata[T_Evaluable, T_MetricValue]
 
-    async def __call__(self, output_datum: T_OutputDatum) -> Any:
+    async def __call__(self, datum: T_Evaluable) -> T_MetricValue:
         """
         For convenience, make a Metric callable.
         Similar to torch Module `forward()`.
         """
-        return await self.evaluation_fn(output_datum)
+        return await self.evaluation_fn(datum)
 
 
-def _check_substring(output_datum: str, substring: str, case_sensitive: bool) -> bool:
+def _check_substring(
+    output_datum: str,
+    substring: str,
+    #
+    case_sensitive: bool,
+) -> bool:
     if case_sensitive:
         return substring in output_datum
     else:
         return substring.lower() in output_datum.lower()
 
 
-def substring_match(substring: str, case_sensitive: bool = True) -> Metric[str]:
-    async def _fn(output_datum: str) -> bool:
-        return _check_substring(
-            output_datum=output_datum,
-            substring=substring,
-            case_sensitive=case_sensitive,
-        )
-
-    return Metric(
-        evaluation_fn=_fn,
-        metric_metadata=EvaluationMetricMetadata(
-            name="substring_match",
-            description="True (pass) if contains given substring",
-            best_value=True,
-            worst_value=False,
-            extra_metadata=dict(substring=substring, case_sensitive=case_sensitive),
-        ),
-    )
-
-
-async def _calculate_brevity(output_datum: str) -> int:
-    if len(output_datum) == 0:
+async def _calculate_brevity(datum: str) -> int:
+    if len(datum) == 0:
         raise ValueError("Brevity is meaningless for empty string.")
-    return len(output_datum)
-
-
-brevity: Metric[str] = Metric(
-    evaluation_fn=_calculate_brevity,
-    metric_metadata=EvaluationMetricMetadata(
-        name="brevity",
-        description="Absolute text length",
-        best_value=1.0,
-        worst_value=float("inf"),
-    ),
-)
+    return len(datum)
 
 
 @dataclass
@@ -98,52 +87,251 @@ def __lt__(self, other: Any) -> bool:
         return self.pos - self.neg < other.pos - other.neg
 
 
-async def _get_sentiment_scores(output_datum: str) -> TextSentimentScores:
-    nltk.download("vader_lexicon", quiet=True)  # type: ignore
-    sid = SentimentIntensityAnalyzer()
-    mapping: dict[str, float] = sid.polarity_scores(output_datum)  # type: ignore
+class GetPolarityScores(Protocol):
+    @abstractmethod
+    def __call__(self, text: str) -> dict[str, float]:
+        pass
+
+
+def _get_nltk_polarity_scores(text: str, model: str) -> dict[str, float]:
+    nltk.download(model, quiet=True)  # type: ignore
+    return NLTKSentimentIntensityAnalyzer().polarity_scores(text)  # type: ignore
+
+
+def _get_sentiment_scores(output_datum: str, get_polarity_scores: GetPolarityScores) -> TextSentimentScores:
+    mapping: dict[str, float] = get_polarity_scores(output_datum)
     highest: str = pd.Series(mapping).idxmax()  # type: ignore
     return TextSentimentScores(mapping=mapping, **mapping, highest=highest)
 
 
-async def _get_sentiment(output_datum: str) -> str:
-    scores = await _get_sentiment_scores(output_datum)
-    return scores.highest
+def make_get_sentiment_scores(get_polarity_scores: GetPolarityScores) -> EvaluationFunction[str, TextSentimentScores]:
+    async def _f(datum: str) -> TextSentimentScores:
+        return _get_sentiment_scores(datum, get_polarity_scores)
+
+    return _f
+
+
+def make_get_sentiment_class(get_polarity_scores: GetPolarityScores) -> EvaluationFunction[str, str]:
+    async def _f(datum: str) -> str:
+        scores = _get_sentiment_scores(datum, get_polarity_scores)
+        return scores.highest
+
+    return _f
+
 
+def make_get_overall_positive_sentiment(get_polarity_scores: GetPolarityScores) -> EvaluationFunction[str, TextOverallPositiveSentiment]:
+    async def _f(datum: str) -> TextOverallPositiveSentiment:
+        scores = _get_sentiment_scores(datum, get_polarity_scores)
+        return TextOverallPositiveSentiment(pos=scores.pos, neg=scores.neg)
 
-async def _get_overall_positive_sentiment(output_datum: str) -> TextOverallPositiveSentiment:
-    scores = await _get_sentiment_scores(output_datum)
-    return TextOverallPositiveSentiment(pos=scores.pos, neg=scores.neg)
+    return _f
 
 
-sentiment_scores: Metric[str] = Metric(
-    evaluation_fn=_get_sentiment_scores,
+def make_sentiment_scores_metric(
+    get_polarity_scores: GetPolarityScores,
+    make_evaluation_fn: Callable[[GetPolarityScores], EvaluationFunction[str, T_MetricValue]],
+    name: str,
+    description: str,
+    best_value: T_MetricValue | None = None,
+    worst_value: T_MetricValue | None = None,
+) -> Metric[str, T_MetricValue]:
+    evaluation_fn: EvaluationFunction[str, T_MetricValue] = make_evaluation_fn(get_polarity_scores)
+    out: Metric[str, T_MetricValue] = Metric(
+        evaluation_fn=evaluation_fn,
+        metric_metadata=EvaluationMetricMetadata(
+            #
+            name=name,
+            description=description,
+            #
+            best_value=best_value,
+            worst_value=worst_value,
+        ),
+    )
+    return out
+
+
+def make_structured_llm_metric(
+    chat_completion_create: common.CompletionTextToSerializedJSON,
+    eval_llm_name: str,
+    pydantic_basemodel_type: Type[T_BaseModel],
+    metric_name: str,
+    metric_description: str,
+    field_descriptions: dict[str, str] = {},
+) -> Metric[str, common.CustomMetricPydanticObject[T_BaseModel]]:
+    def _make_evaluation_fn(basemodel_type: Type[T_BaseModel]) -> EvaluationFunction[str, common.CustomMetricPydanticObject[T_BaseModel]]:
+        async def _evaluation_fn(datum: str) -> common.CustomMetricPydanticObject[T_BaseModel]:
+            resp = common.get_llm_structured_response(
+                input_text=datum,
+                chat_completion_create=chat_completion_create,
+                basemodel_type=basemodel_type,
+            )
+
+            # Intentional: unwrap and raise here to conform to the Metric interface.
+            match resp:
+                case Err(e):
+                    raise ValueError(f"Error getting structured response: {e}")
+                case Ok(data):
+                    return common.CustomMetricPydanticObject(data=data)
+
+        return _evaluation_fn
+
+    return Metric(
+        evaluation_fn=_make_evaluation_fn(pydantic_basemodel_type),
+        metric_metadata=EvaluationMetricMetadata(
+            name=metric_name,
+            description=metric_description,
+            extra_metadata=dict(
+                basemodel_type_name=pydantic_basemodel_type.__name__,
+                eval_llm_name=eval_llm_name,
+                field_descriptions_json=json.dumps(field_descriptions, sort_keys=True),
+            ),
+        ),
+    )
+
+
+def _make_openai_structured_llm_metric_helper(
+    eval_llm_name: str,
+    pydantic_basemodel_type: Type[T_BaseModel],
+    metric_name: str,
+    metric_description: str,
+    field_descriptions: dict[str, str],
+    openai_chat_completion_create: OpenAIChatCompletionCreate | None = None,
+) -> Result[Metric[str, common.CustomMetricPydanticObject[T_BaseModel]], str]:
+    schema = pydantic_basemodel_type.model_json_schema()
+    properties = schema["properties"]
+    required = schema["required"]
+
+    if not field_descriptions.keys() <= properties.keys():
+        return Err(
+            f"""
+                The following field_descriptions keys are not in the schema:
+                {set(field_descriptions.keys()) - set(properties.keys())}
+            """
+        )
+
+    def _with_description(key: str, value: dict[str, str]) -> dict[str, str]:
+        if key in field_descriptions:
+            return cu.dict_union_allow_replace(value, {"description": field_descriptions[key]})
+        return value
+
+    properties = {k: _with_description(k, v) for k, v in properties.items()}
+
+    required = required or list(properties.keys())
+
+    openai_eval_llm_chat_completion_create: common.CompletionTextToSerializedJSON = make_fn_completion_text_to_serialized_json(
+        eval_llm_name=eval_llm_name,
+        properties=properties,
+        required=required,
+        openai_chat_completion_create=(openai_chat_completion_create or default_openai_chat_completion_create),
+    )
+
+    return Ok(
+        make_structured_llm_metric(
+            openai_eval_llm_chat_completion_create,
+            eval_llm_name=eval_llm_name,
+            pydantic_basemodel_type=pydantic_basemodel_type,
+            metric_name=metric_name,
+            metric_description=metric_description,
+            field_descriptions=field_descriptions,
+        )
+    )
+
+
+## User interface
+
+
+# 1. functions that return metrics intended to be called directly
+
+
+def make_openai_structured_llm_metric(
+    eval_llm_name: str,
+    pydantic_basemodel_type: Type[T_BaseModel],
+    metric_name: str,
+    metric_description: str,
+    field_descriptions: dict[str, str] = {},
+    openai_chat_completion_create: OpenAIChatCompletionCreate | None = None,
+) -> Metric[str, common.CustomMetricPydanticObject[T_BaseModel]]:
+    res_metric = _make_openai_structured_llm_metric_helper(
+        eval_llm_name=eval_llm_name,
+        pydantic_basemodel_type=pydantic_basemodel_type,
+        metric_name=metric_name,
+        metric_description=metric_description,
+        field_descriptions=field_descriptions,
+        openai_chat_completion_create=openai_chat_completion_create,
+    )
+
+    # User interface: unwrap and raise
+    match res_metric:
+        case Ok(metric):
+            return metric
+        case Err(e):
+            raise ValueError(f"Error making metric: {e}")
+
+
+def substring_match(substring: str, case_sensitive: bool = True) -> Metric[str, bool]:
+    async def _fn(datum: str) -> bool:
+        return _check_substring(
+            output_datum=datum,
+            substring=substring,
+            case_sensitive=case_sensitive,
+        )
+
+    return Metric(
+        evaluation_fn=_fn,
+        metric_metadata=EvaluationMetricMetadata(
+            name="substring_match",
+            description="True (pass) if contains given substring",
+            best_value=True,
+            worst_value=False,
+            extra_metadata=dict(substring=substring, case_sensitive=case_sensitive),
+        ),
+    )
+
+
+# 2. literal metrics
+
+brevity: Metric[str, int] = Metric(
+    evaluation_fn=_calculate_brevity,
     metric_metadata=EvaluationMetricMetadata(
-        name="sentiment_scores",
-        description="Sentiment scores container object",
-        best_value=None,
-        worst_value=None,
+        name="brevity",
+        description="Absolute text length",
+        best_value=1,
+        worst_value=sys.maxsize,
     ),
 )
 
 
-sentiment_class: Metric[str] = Metric(
-    evaluation_fn=_get_sentiment,
-    metric_metadata=EvaluationMetricMetadata(
-        name="sentiment_class",
-        description="top sentiment class",
-        best_value=None,
-        worst_value=None,
+gpt3_5_text_ratings = make_openai_structured_llm_metric(
+    eval_llm_name="gpt-3.5-turbo-0613",
+    pydantic_basemodel_type=TextRatingsData,
+    metric_name="text_ratings",
+    metric_description="Text ratings",
+    field_descriptions=dict(
+        conciseness_rating="1 to 5 rating of conciseness",
+        conciseness_confidence="0 to 1.0 rating of confidence in conciseness rating",
+        conciseness_reasoning="reasoning behind the conciseness rating",
     ),
 )
 
+nltk_sentiment_scores_vader = make_sentiment_scores_metric(
+    get_polarity_scores=partial(_get_nltk_polarity_scores, model="vader_lexicon"),
+    make_evaluation_fn=make_get_sentiment_scores,
+    name="nltk_sentiment_scores_vader",
+    description="NLTK sentiment scores using Vader",
+)
 
-sentiment_score_overall_positive: Metric[str] = Metric(
-    evaluation_fn=_get_overall_positive_sentiment,
-    metric_metadata=EvaluationMetricMetadata(
-        name="sentiment_score_overall_positive",
-        description="Positive minus negative",
-        best_value=TextOverallPositiveSentiment(pos=1.0, neg=0.0),
-        worst_value=TextOverallPositiveSentiment(pos=0.0, neg=1.0),
-    ),
+nltk_sentiment_class_vader = make_sentiment_scores_metric(
+    get_polarity_scores=partial(_get_nltk_polarity_scores, model="vader_lexicon"),
+    make_evaluation_fn=make_get_sentiment_class,
+    name="nltk_sentiment_class_vader",
+    description="Highest-probability NLTK sentiment class using Vader",
+)
+
+nltk_sentiment_score_overall_positive = make_sentiment_scores_metric(
+    get_polarity_scores=partial(_get_nltk_polarity_scores, model="vader_lexicon"),
+    make_evaluation_fn=make_get_overall_positive_sentiment,
+    name="nltk_sentiment_score_overall_positive",
+    description="Positive minus negative",
+    best_value=TextOverallPositiveSentiment(pos=1.0, neg=0.0),
+    worst_value=TextOverallPositiveSentiment(pos=0.0, neg=1.0),
 )
diff --git a/python/src/aiconfig/eval/openai.py b/python/src/aiconfig/eval/openai.py
new file mode 100644
index 000000000..10c96ddda
--- /dev/null
+++ b/python/src/aiconfig/eval/openai.py
@@ -0,0 +1,88 @@
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Protocol
+
+import lastmile_utils.lib.core.api as cu
+import openai
+import openai.types.chat as openai_types
+from aiconfig.eval import common
+from result import Err, Ok, Result
+
+
+@dataclass
+class OpenAIChatCompletionParams:
+    # TODO: type better
+    messages: list[openai_types.ChatCompletionMessageParam]
+    model: str
+    temperature: float
+    tools: list[openai_types.ChatCompletionToolParam]
+
+
+class OpenAIChatCompletionCreate(Protocol):
+    @abstractmethod
+    def __call__(self, completion_params: OpenAIChatCompletionParams) -> Result[openai_types.ChatCompletion, str]:
+        pass
+
+
+def default_openai_chat_completion_create(completion_params: OpenAIChatCompletionParams) -> Result[openai_types.ChatCompletion, str]:
+    try:
+        result = openai.chat.completions.create(
+            messages=completion_params.messages,
+            model=completion_params.model,
+            temperature=completion_params.temperature,
+            tools=completion_params.tools,
+            stream=False,
+        )
+        return Ok(result)
+    except Exception as e:
+        return cu.ErrWithTraceback(e)
+
+
+def extract_json_from_chat_completion(chat_completion: openai_types.ChatCompletion) -> Result[common.SerializedJSON, str]:
+    choice = chat_completion.choices[0]
+    message = choice.message
+    if message.tool_calls is None:
+        return Err("No tool calls found")
+
+    return Ok(common.SerializedJSON(message.tool_calls[0].function.arguments))
+
+
+def make_fn_completion_text_to_serialized_json(
+    eval_llm_name: str,
+    properties: dict[str, dict[str, str]],
+    required: list[str],
+    openai_chat_completion_create: OpenAIChatCompletionCreate,
+) -> common.CompletionTextToSerializedJSON:
+    def _chat_completion_create(output_datum: str) -> Result[common.SerializedJSON, str]:
+        openai_chat_completion_params = _make_openai_completion_params(output_datum, eval_llm_name, properties, required)
+        return openai_chat_completion_create(openai_chat_completion_params).and_then(extract_json_from_chat_completion)
+
+    out: common.CompletionTextToSerializedJSON = _chat_completion_create
+    return out
+
+
+def _make_openai_completion_params(
+    input_text: str, eval_llm_name: str, properties: dict[str, dict[str, str]], required: list[str]
+) -> OpenAIChatCompletionParams:
+    return OpenAIChatCompletionParams(
+        messages=[
+            {"role": "system", "content": "Call the function with arguments based on the provided text."},
+            {"role": "user", "content": input_text},
+        ],
+        model=eval_llm_name,
+        temperature=0,
+        tools=[
+            {
+                "type": "function",
+                "function": {
+                    "name": "dummy",
+                    "description": "Submit ratings that apply to a piece of text",
+                    "parameters": {
+                        "type": "object",
+                        "properties": properties,
+                        "required": required,
+                    },
+                },
+            },
+        ],
+    )


Test patch:
diff --git a/python/tests/test_eval.py b/python/tests/test_eval.py
index 1c78799d5..33c13e248 100644
--- a/python/tests/test_eval.py
+++ b/python/tests/test_eval.py
@@ -1,23 +1,49 @@
-# type: ignore[no-untyped-call]
 import itertools
 import logging
 import os
-from typing import Any
+from typing import Any, cast
 
+import aiconfig.eval.openai as lib_openai
 import hypothesis
 import hypothesis.strategies as st
 import lastmile_utils.lib.core.api as cu
+import openai.types.chat as openai_chat_types
+import openai.types.chat.chat_completion as openai_chat_completion_types
+import openai.types.chat.chat_completion_message_tool_call as openai_tool_call_types
 import pandas as pd
 import pytest
 from aiconfig.Config import AIConfigRuntime
+from aiconfig.eval import common
 from aiconfig.eval.api import TestSuiteWithInputsSettings, metrics, run_test_suite_outputs_only, run_test_suite_with_inputs
-from aiconfig.eval.lib import TestSuiteWithInputsSpec, run_test_suite_helper
+from aiconfig.eval.lib import MetricList, TestSuiteWithInputsSpec, run_test_suite_helper
 from aiconfig.model_parser import InferenceOptions
-from result import Err, Ok
+from result import Err, Ok, Result
 
 brevity = metrics.brevity
 substring_match = metrics.substring_match
 
+MOCK_NLTK_SENTIMENT_SCORE_MAPPING = {
+    "nltk is amazing": {"pos": 0.9, "neu": 0.1, "neg": 0.0, "compound": 0.9},
+    "whats for dinner?": {"pos": 0.0, "neu": 0.9, "neg": 0.1, "compound": -0.9},
+    "oh, bother": {"pos": 0.0, "neu": 0.1, "neg": 0.9, "compound": -0.9},
+}
+
+
+def _compute_mock_sentiment_class_mapping(score_mapping: dict[str, dict[str, float]]) -> dict[str, str]:
+    out: dict[str, str] = {}
+    for k, scores in score_mapping.items():
+        max_class, max_score = "", float("-inf")
+        for class_, score in scores.items():
+            if score > max_score:
+                max_score = score
+                max_class = class_
+        out[k] = max_class
+
+    return out
+
+
+MOCK_NLTK_SENTIMENT_CLASS_MAPPING = _compute_mock_sentiment_class_mapping(MOCK_NLTK_SENTIMENT_SCORE_MAPPING)
+
 
 def set_pd():
     pd.set_option("display.max_rows", 50)
@@ -41,7 +67,7 @@ async def run_and_get_output_text(
         prompt_name: str,
         params: dict[Any, Any] | None = None,
         options: InferenceOptions | None = None,
-        **kwargs,
+        **kwargs,  # type: ignore
     ) -> str:
         """
         This overrides the real method for mocking, but the output doesn't matter very much.
@@ -92,7 +118,7 @@ async def test_run_with_outputs_only_basic():
     out = await run_test_suite_outputs_only(test_suite)
     exp = pd.DataFrame(
         data={
-            "value": [11.0, True],
+            "value": [11, True],
         }
     )
     assert out["value"].equals(exp["value"])  # type: ignore
@@ -122,11 +148,11 @@ async def test_run_test_suite_outputs_only(data: st.DataObject):
         "best_possible_value",
         "worst_possible_value",
     ]
-    inputs = df["input"].astype(str).tolist()
-    assert cu.only(inputs) == Ok("Missing")
+    inputs = df["input"].astype(str).tolist()  # type: ignore
+    assert cu.only(inputs) == Ok("Missing")  # type: ignore
 
-    df_brevity = df[df["metric_name"] == "brevity"]
-    assert (df_brevity["aiconfig_output"].apply(len) == df_brevity["value"]).all()
+    df_brevity = df[df["metric_name"] == "brevity"]  # type: ignore
+    assert (df_brevity["aiconfig_output"].apply(len) == df_brevity["value"]).all()  # type: ignore
 
 
 @hypothesis.given(st.data())
@@ -181,48 +207,71 @@ async def test_run_test_suite_with_inputs(data: st.DataObject):
 
             assert input_pairs == result_pairs
 
-            df_brevity = df[df["metric_name"] == "brevity"]
-            assert (df_brevity["aiconfig_output"].apply(len) == df_brevity["value"]).all()
+            df_brevity = df[df["metric_name"] == "brevity"]  # type: ignore
+            assert (df_brevity["aiconfig_output"].apply(len) == df_brevity["value"]).all()  # type: ignore
 
-            df_substring = df[df["metric_name"] == "substring_match"]
-            assert (df_substring["value"].apply(lambda x: x in {False, True})).all()
+            df_substring = df[df["metric_name"] == "substring_match"]  # type: ignore
+            assert (df_substring["value"].apply(lambda x: x in {False, True})).all()  # type: ignore
 
         case Err(e):
             assert False, f"expected Ok, got Err({e})"
 
 
+def _make_mock_nltk_metrics() -> MetricList[str]:
+    def _mock_get_nltk_polarity_scores(text: str) -> dict[str, float]:
+        return MOCK_NLTK_SENTIMENT_SCORE_MAPPING[text]
+
+    mock_nltk_sentiment_scores_vader = metrics.make_sentiment_scores_metric(
+        get_polarity_scores=_mock_get_nltk_polarity_scores,
+        make_evaluation_fn=metrics.make_get_sentiment_scores,
+        name="nltk_sentiment_scores_vader",
+        description="NLTK sentiment scores using Vader",
+    )
+
+    mock_nltk_sentiment_class_vader = metrics.make_sentiment_scores_metric(
+        get_polarity_scores=_mock_get_nltk_polarity_scores,
+        make_evaluation_fn=metrics.make_get_sentiment_class,
+        name="nltk_sentiment_class_vader",
+        description="Highest-probability NLTK sentiment class using Vader",
+    )
+
+    mock_nltk_sentiment_score_overall_positive = metrics.make_sentiment_scores_metric(
+        get_polarity_scores=_mock_get_nltk_polarity_scores,
+        make_evaluation_fn=metrics.make_get_overall_positive_sentiment,
+        name="nltk_sentiment_score_overall_positive",
+        description="Positive minus negative",
+        best_value=metrics.TextOverallPositiveSentiment(pos=1.0, neg=0.0),
+        worst_value=metrics.TextOverallPositiveSentiment(pos=0.0, neg=1.0),
+    )
+
+    return [mock_nltk_sentiment_scores_vader, mock_nltk_sentiment_class_vader, mock_nltk_sentiment_score_overall_positive]
+
+
 @pytest.mark.asyncio
 async def test_custom_metric_type():
+    mock_nltk_metrics = _make_mock_nltk_metrics()
     user_test_suite_outputs_only = list(
         itertools.product(
             ["nltk is amazing", "whats for dinner?", "oh, bother"],
-            [
-                metrics.sentiment_scores,
-                metrics.sentiment_class,
-                metrics.sentiment_score_overall_positive,
-            ],
+            mock_nltk_metrics,
         )
     )
     df = await run_test_suite_outputs_only(user_test_suite_outputs_only)
-    result = df.set_index(["metric_name", "aiconfig_output"]).value.unstack(0).to_dict()
-    assert result["sentiment_class"] == {
-        "nltk is amazing": "pos",
-        "whats for dinner?": "neu",
-        "oh, bother": "neg",
-    }
+    result = df.set_index(["metric_name", "aiconfig_output"]).value.unstack(0).to_dict()  # type: ignore
+    assert result["nltk_sentiment_class_vader"] == MOCK_NLTK_SENTIMENT_CLASS_MAPPING
 
-    assert all(isinstance(v, metrics.TextSentimentScores) for v in result["sentiment_scores"].values())
+    assert all(isinstance(v, metrics.TextSentimentScores) for v in result["nltk_sentiment_scores_vader"].values())  # type: ignore
 
-    assert all(isinstance(v, metrics.TextOverallPositiveSentiment) for v in result["sentiment_score_overall_positive"].values())
+    assert all(isinstance(v, metrics.TextOverallPositiveSentiment) for v in result["nltk_sentiment_score_overall_positive"].values())  # type: ignore
 
     neutral = metrics.TextOverallPositiveSentiment(pos=0.0, neg=0.0)
 
-    assert result["sentiment_score_overall_positive"]["nltk is amazing"] > neutral
-    assert result["sentiment_score_overall_positive"]["oh, bother"] < neutral
+    assert result["nltk_sentiment_score_overall_positive"]["nltk is amazing"] > neutral
+    assert result["nltk_sentiment_score_overall_positive"]["oh, bother"] < neutral
 
 
 @pytest.mark.asyncio
-async def test_exception_metric(caplog):
+async def test_exception_metric(caplog: pytest.LogCaptureFixture):
     user_test_suite_outputs_only = list(
         itertools.product(
             ["Hundred Acre Wood", ""],
@@ -231,8 +280,102 @@ async def test_exception_metric(caplog):
     )
     with caplog.at_level(logging.ERROR):
         df = await run_test_suite_outputs_only(user_test_suite_outputs_only)
-    mapping = df.query("metric_name=='brevity'").set_index("aiconfig_output").value.to_dict()
+    mapping: dict[str, Any] = df.query("metric_name=='brevity'").set_index("aiconfig_output").value.to_dict()  # type: ignore
     assert mapping["Hundred Acre Wood"] == 17.0
-    assert pd.isnull(mapping[""])
+    assert pd.isnull(mapping[""])  # type: ignore
 
     assert any("Brevity is meaningless for empty string." in record.msg for record in caplog.records)
+
+
+def _mock_response(function_args: common.SerializedJSON) -> openai_chat_types.ChatCompletion:
+    return openai_chat_types.ChatCompletion(
+        id="123",
+        choices=[
+            openai_chat_completion_types.Choice(
+                index=0,
+                message=openai_chat_types.ChatCompletionMessage(
+                    content=None,
+                    role="assistant",
+                    tool_calls=[
+                        openai_chat_types.ChatCompletionMessageToolCall(
+                            id="cm-tk-1",
+                            type="function",
+                            function=openai_tool_call_types.Function(
+                                name="dummy",
+                                arguments=function_args,
+                            ),
+                        )
+                    ],
+                ),
+                finish_reason="stop",
+            )
+        ],
+        created=0,
+        model="",
+        object="chat.completion",
+    )
+
+
+def _make_mock_openai_chat_completion_create(function_arguments_return: common.SerializedJSON) -> lib_openai.OpenAIChatCompletionCreate:
+    def _mock_openai_chat_completion_create(
+        completion_params: lib_openai.OpenAIChatCompletionParams,
+    ) -> Result[openai_chat_types.ChatCompletion, str]:
+        return Ok(
+            _mock_response(
+                function_arguments_return,
+            )
+        )
+
+    return _mock_openai_chat_completion_create
+
+
+@pytest.mark.asyncio
+async def test_openai_structured_eval():
+    _mock_create = _make_mock_openai_chat_completion_create(
+        common.SerializedJSON('{"conciseness_rating": 5, "conciseness_confidence": 0.9, "conciseness_reasoning": "I think it\'s pretty concise."}')
+    )
+    mock_metric = metrics.make_openai_structured_llm_metric(
+        eval_llm_name="gpt-3.5-turbo-0613",
+        pydantic_basemodel_type=common.TextRatingsData,
+        metric_name="text_ratings",
+        metric_description="Text ratings",
+        field_descriptions=dict(
+            conciseness_rating="1 to 5 rating of conciseness",
+            conciseness_confidence="0 to 1.0 rating of confidence in conciseness rating",
+            conciseness_reasoning="reasoning behind the conciseness rating",
+        ),
+        openai_chat_completion_create=_mock_create,
+    )
+
+    user_test_suite_outputs_only = [
+        ("one two three", mock_metric),
+    ]
+    df = await run_test_suite_outputs_only(user_test_suite_outputs_only)
+    metric_data = cast(common.CustomMetricPydanticObject[metrics.TextRatingsData], df.loc[0, "value"]).data
+    assert isinstance(metric_data, common.TextRatingsData)
+    metric_json = metric_data.to_dict()
+    assert metric_json == {"conciseness_rating": 5, "conciseness_confidence": 0.9, "conciseness_reasoning": "I think it's pretty concise."}
+
+
+@pytest.mark.asyncio
+async def test_bad_structured_eval_metric():
+    _mock_create = _make_mock_openai_chat_completion_create(
+        common.SerializedJSON('{"conciseness_rating": 5, "conciseness_confidence": 0.9, "conciseness_reasoning": "I think it\'s pretty concise."}')
+    )
+
+    with pytest.raises(ValueError) as exc:
+        _ = metrics.make_openai_structured_llm_metric(
+            eval_llm_name="gpt-3.5-turbo-0613",
+            pydantic_basemodel_type=common.TextRatingsData,
+            metric_name="text_ratings",
+            metric_description="Text ratings",
+            field_descriptions=dict(
+                fake_field="123",
+                conciseness_rating="1 to 5 rating of conciseness",
+                conciseness_confidence="0 to 1.0 rating of confidence in conciseness rating",
+                conciseness_reasoning="reasoning behind the conciseness rating",
+            ),
+            openai_chat_completion_create=_mock_create,
+        )
+
+    assert "The following field_descriptions keys are not in the schema" in str(exc)