Repository: lastmile-ai/aiconfig (language: python)

Issue:
[AIC-py][eval] add structured model-based evals
[AIC-py][eval] add structured model-based evals

- add `make_structured_llm_metric()` which takes some configs and creates a metric
that queries an LLM with the text to evaluate and some evaluation criteria,
and returns a structured eval result
- wrap with `make_openai_structured_llm_metric()`
- make example `gpt3_5_text_ratings()`
- add tests that mock the openai call

---
Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/lastmile-ai/aiconfig/pull/541).
* #542
* __->__ #541


Discussion / hints:

Fix patch:
diff --git a/python/requirements.txt b/python/requirements.txt
index ce2c38030..456c48461 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -14,6 +14,6 @@ nest_asyncio
 prompt_toolkit
 mock
 pytest-asyncio
-lastmile-utils==0.0.10
+lastmile-utils==0.0.11
 hypothesis==6.91.0
 nltk
\ No newline at end of file
diff --git a/python/src/aiconfig/eval/api/__init__.py b/python/src/aiconfig/eval/api/__init__.py
index 4869a18f6..f04ad9683 100644
--- a/python/src/aiconfig/eval/api/__init__.py
+++ b/python/src/aiconfig/eval/api/__init__.py
@@ -9,7 +9,7 @@
     TestSuiteWithInputsSettings,
 )
 """
-from .. import metrics
+from .. import common, metrics
 
 # pyright: reportWildcardImportFromLibrary=false
 from ..lib import TestSuiteWithInputsSettings, run_test_suite_outputs_only, run_test_suite_with_inputs
@@ -17,6 +17,7 @@
 
 __all__ = [
     "Metric",
+    "common",
     "metrics",
     "brevity",
     "substring_match",
diff --git a/python/src/aiconfig/eval/common.py b/python/src/aiconfig/eval/common.py
index 7d2ebd468..c617d5d3a 100644
--- a/python/src/aiconfig/eval/common.py
+++ b/python/src/aiconfig/eval/common.py
@@ -1,14 +1,21 @@
 import json
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import Any, Generic, Protocol, TypeVar
+from typing import Any, Generic, NewType, Protocol, Type, TypeVar
 
 import lastmile_utils.lib.core.api as cu
-from pydantic import root_validator
+import result
+from aiconfig.eval import common
+from pydantic import BaseModel, root_validator
+from result import Result
 
 T_InputDatum = TypeVar("T_InputDatum", contravariant=True)
 T_OutputDatum = TypeVar("T_OutputDatum", contravariant=True)
 
+T_BaseModel = TypeVar("T_BaseModel", bound=BaseModel)
+
+SerializedJSON = NewType("SerializedJSON", str)
+
 
 @dataclass
 class CustomMetricValue(ABC):
@@ -21,9 +28,20 @@ class CustomMetricValue(ABC):
     """
 
 
+class CompletionTextToSerializedJSON(Protocol):
+    @abstractmethod
+    def __call__(self, output_datum: str) -> Result[common.SerializedJSON, str]:
+        pass
+
+
 MetricValue = int | float | str | bool | CustomMetricValue
 
 
+@dataclass
+class CustomMetricPydanticObject(CustomMetricValue, Generic[T_BaseModel]):
+    data: T_BaseModel
+
+
 class EvaluationFunction(Protocol, Generic[T_OutputDatum]):
     @abstractmethod
     async def __call__(self, output_datum: T_OutputDatum) -> MetricValue:
@@ -127,3 +145,21 @@ def check_value_range(cls, values: dict[str, Any]) -> dict[str, Any]:
             )
         else:
             return values
+
+
+class TextRatingsData(cu.Record):
+    conciseness_rating: int
+    conciseness_confidence: float
+    conciseness_reasoning: str
+
+
+def get_llm_structured_response(
+    input_text: str,
+    chat_completion_create: CompletionTextToSerializedJSON,
+    basemodel_type: Type[common.T_BaseModel],
+) -> Result[common.T_BaseModel, str]:
+    return result.do(
+        cu.safe_model_validate_json(response_ok, basemodel_type)
+        # get the serialized JSON response
+        for response_ok in chat_completion_create(input_text)
+    )
diff --git a/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
index 9bd7ba9ac..e8f97daec 100644
--- a/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
+++ b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
@@ -61,7 +61,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
@@ -70,6 +70,18 @@
      "text": [
       "Imports and set log level\n"
      ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/pydantic/_internal/_fields.py:128: UserWarning: Field \"model_parsers\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
     }
    ],
    "source": [
@@ -83,12 +95,12 @@
     "pd.set_option(\"display.max_colwidth\", None)\n",
     "\n",
     "from aiconfig.eval.api import (\n",
-    "    brevity,\n",
-    "    substring_match,\n",
     "    run_test_suite_with_inputs,\n",
     "    TestSuiteWithInputsSettings,\n",
     ")\n",
     "\n",
+    "from aiconfig.eval.api import metrics\n",
+    "\n",
     "jupyter_utils.set_log_level(logging.WARNING)\n",
     "\n"
    ]
@@ -102,7 +114,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -160,16 +172,16 @@
     "test_suite_with_inputs = []\n",
     "for test_input, substring in test_inputs_with_substrings:\n",
     "    # Add the brevity metric\n",
-    "    test_fn1 = brevity\n",
+    "    test_fn1 = metrics.brevity\n",
     "    test_suite_with_inputs.append((test_input, test_fn1))\n",
     "    # Add substring check function\n",
-    "    test_fn2 = substring_match(substring, case_sensitive=False)\n",
+    "    test_fn2 = metrics.substring_match(substring, case_sensitive=False)\n",
     "    test_suite_with_inputs.append((test_input, test_fn2))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -181,52 +193,56 @@
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " Metric(calculate=<function _calculate_brevity at 0x11f79ade0>, interpretation={\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
       "  \"best_value\": 1.0,\n",
       "  \"worst_value\": Infinity,\n",
-      "  \"extra_metadata\": {}\n",
-      "})\n",
+      "  \"extra_metadata\": {},\n",
+      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "}))\n",
       "\n",
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " Metric(calculate=<function substring_match.<locals>._fn at 0x11f799b20>, interpretation={\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x1036cfac0>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
-      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": 0.0,\n",
+      "  \"description\": \"True (pass) if contains given substring\",\n",
+      "  \"best_value\": true,\n",
+      "  \"worst_value\": false,\n",
       "  \"extra_metadata\": {\n",
       "    \"substring\": \"Magnolia Bakery\",\n",
       "    \"case_sensitive\": false\n",
-      "  }\n",
-      "})\n",
+      "  },\n",
+      "  \"id\": \"0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6\"\n",
+      "}))\n",
       "\n",
       "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " Metric(calculate=<function _calculate_brevity at 0x11f79ade0>, interpretation={\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
       "  \"best_value\": 1.0,\n",
       "  \"worst_value\": Infinity,\n",
-      "  \"extra_metadata\": {}\n",
-      "})\n",
+      "  \"extra_metadata\": {},\n",
+      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "}))\n",
       "\n",
       "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " Metric(calculate=<function substring_match.<locals>._fn at 0x11f799ee0>, interpretation={\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x144090e50>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
-      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": 0.0,\n",
+      "  \"description\": \"True (pass) if contains given substring\",\n",
+      "  \"best_value\": true,\n",
+      "  \"worst_value\": false,\n",
       "  \"extra_metadata\": {\n",
       "    \"substring\": \"Empire State Building\",\n",
       "    \"case_sensitive\": false\n",
-      "  }\n",
-      "})\n"
+      "  },\n",
+      "  \"id\": \"53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f\"\n",
+      "}))\n"
      ]
     }
    ],
@@ -239,7 +255,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -285,10 +301,10 @@
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>1. Experience culinary delights at Chelsea Market.\n",
-       "2. Immerse in NYC's Chinatown food tour. \n",
-       "3. Indulge in Italian tradition at Little Italy.</td>\n",
-       "      <td>141.0</td>\n",
+       "      <td>1. Explore Chelsea Market's cuisine.\n",
+       "2. Visit Little Italy for authentic fare.\n",
+       "3. Savor Asian delicacies in Flushing, Queens.</td>\n",
+       "      <td>125</td>\n",
        "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
@@ -298,23 +314,23 @@
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>1. Experience culinary delights at Chelsea Market.\n",
-       "2. Immerse in NYC's Chinatown food tour. \n",
-       "3. Indulge in Italian tradition at Little Italy.</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054</td>\n",
+       "      <td>1. Explore Chelsea Market's cuisine.\n",
+       "2. Visit Little Italy for authentic fare.\n",
+       "3. Savor Asian delicacies in Flushing, Queens.</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6</td>\n",
        "      <td>substring_match</td>\n",
-       "      <td>1.0 (pass) if contains given substring</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>True (pass) if contains given substring</td>\n",
+       "      <td>True</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Visit \"Top of the Rock\" at Rockefeller Center.\n",
-       "2. Tour Empire State Building.\n",
-       "3. Explore Grand Central Terminal.</td>\n",
-       "      <td>115.0</td>\n",
+       "      <td>1. Empire State Building Observatory - 86th floor views.\n",
+       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
+       "3. One World Trade Center - One World Observatory visit.</td>\n",
+       "      <td>170</td>\n",
        "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
@@ -324,15 +340,15 @@
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Visit \"Top of the Rock\" at Rockefeller Center.\n",
-       "2. Tour Empire State Building.\n",
-       "3. Explore Grand Central Terminal.</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519</td>\n",
+       "      <td>1. Empire State Building Observatory - 86th floor views.\n",
+       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
+       "3. One World Trade Center - One World Observatory visit.</td>\n",
+       "      <td>True</td>\n",
+       "      <td>53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f</td>\n",
        "      <td>substring_match</td>\n",
-       "      <td>1.0 (pass) if contains given substring</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>True (pass) if contains given substring</td>\n",
+       "      <td>True</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -345,40 +361,40 @@
        "2   iconic midtown skyscrapers   \n",
        "3   iconic midtown skyscrapers   \n",
        "\n",
-       "                                                                                                                                 aiconfig_output  \\\n",
-       "0  1. Experience culinary delights at Chelsea Market.\n",
-       "2. Immerse in NYC's Chinatown food tour. \n",
-       "3. Indulge in Italian tradition at Little Italy.   \n",
-       "1  1. Experience culinary delights at Chelsea Market.\n",
-       "2. Immerse in NYC's Chinatown food tour. \n",
-       "3. Indulge in Italian tradition at Little Italy.   \n",
-       "2                            1. Visit \"Top of the Rock\" at Rockefeller Center.\n",
-       "2. Tour Empire State Building.\n",
-       "3. Explore Grand Central Terminal.   \n",
-       "3                            1. Visit \"Top of the Rock\" at Rockefeller Center.\n",
-       "2. Tour Empire State Building.\n",
-       "3. Explore Grand Central Terminal.   \n",
+       "                                                                                                                                                              aiconfig_output  \\\n",
+       "0                                               1. Explore Chelsea Market's cuisine.\n",
+       "2. Visit Little Italy for authentic fare.\n",
+       "3. Savor Asian delicacies in Flushing, Queens.   \n",
+       "1                                               1. Explore Chelsea Market's cuisine.\n",
+       "2. Visit Little Italy for authentic fare.\n",
+       "3. Savor Asian delicacies in Flushing, Queens.   \n",
+       "2  1. Empire State Building Observatory - 86th floor views.\n",
+       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
+       "3. One World Trade Center - One World Observatory visit.   \n",
+       "3  1. Empire State Building Observatory - 86th floor views.\n",
+       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
+       "3. One World Trade Center - One World Observatory visit.   \n",
        "\n",
        "   value                                                         metric_id  \\\n",
-       "0  141.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "1    0.0  2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054   \n",
-       "2  115.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "3    1.0  e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519   \n",
+       "0    125  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "1  False  0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6   \n",
+       "2    170  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "3   True  53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f   \n",
        "\n",
-       "       metric_name                      metric_description  \\\n",
-       "0          brevity                    Absolute text length   \n",
-       "1  substring_match  1.0 (pass) if contains given substring   \n",
-       "2          brevity                    Absolute text length   \n",
-       "3  substring_match  1.0 (pass) if contains given substring   \n",
+       "       metric_name                       metric_description  \\\n",
+       "0          brevity                     Absolute text length   \n",
+       "1  substring_match  True (pass) if contains given substring   \n",
+       "2          brevity                     Absolute text length   \n",
+       "3  substring_match  True (pass) if contains given substring   \n",
        "\n",
-       "   best_possible_value  worst_possible_value  \n",
-       "0                  1.0                   inf  \n",
-       "1                  1.0                   0.0  \n",
-       "2                  1.0                   inf  \n",
-       "3                  1.0                   0.0  "
+       "  best_possible_value worst_possible_value  \n",
+       "0                 1.0                  inf  \n",
+       "1                True                False  \n",
+       "2                 1.0                  inf  \n",
+       "3                True                False  "
       ]
      },
-     "execution_count": 9,
+     "execution_count": 4,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -397,7 +413,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -442,33 +458,33 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>different kinds of cuisines</th>\n",
-       "      <th>1. Experience culinary delights at Chelsea Market.\\n2. Immerse in NYC's Chinatown food tour. \\n3. Indulge in Italian tradition at Little Italy.</th>\n",
-       "      <td>141.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <th>1. Explore Chelsea Market's cuisine.\\n2. Visit Little Italy for authentic fare.\\n3. Savor Asian delicacies in Flushing, Queens.</th>\n",
+       "      <td>125</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>iconic midtown skyscrapers</th>\n",
-       "      <th>1. Visit \"Top of the Rock\" at Rockefeller Center.\\n2. Tour Empire State Building.\\n3. Explore Grand Central Terminal.</th>\n",
-       "      <td>115.0</td>\n",
-       "      <td>1.0</td>\n",
+       "      <th>1. Empire State Building Observatory - 86th floor views.\\n2. Rockefeller Center - Top of the Rock and ice skating.\\n3. One World Trade Center - One World Observatory visit.</th>\n",
+       "      <td>170</td>\n",
+       "      <td>True</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "metric_name                                                                                                                                                                  brevity  \\\n",
-       "input                       aiconfig_output                                                                                                                                            \n",
-       "different kinds of cuisines 1. Experience culinary delights at Chelsea Market.\\n2. Immerse in NYC's Chinatown food tour. \\n3. Indulge in Italian tradition at Little Italy.    141.0   \n",
-       "iconic midtown skyscrapers  1. Visit \"Top of the Rock\" at Rockefeller Center.\\n2. Tour Empire State Building.\\n3. Explore Grand Central Terminal.                              115.0   \n",
+       "metric_name                                                                                                                                                                                              brevity  \\\n",
+       "input                       aiconfig_output                                                                                                                                                                        \n",
+       "different kinds of cuisines 1. Explore Chelsea Market's cuisine.\\n2. Visit Little Italy for authentic fare.\\n3. Savor Asian delicacies in Flushing, Queens.                                                  125   \n",
+       "iconic midtown skyscrapers  1. Empire State Building Observatory - 86th floor views.\\n2. Rockefeller Center - Top of the Rock and ice skating.\\n3. One World Trade Center - One World Observatory visit.     170   \n",
        "\n",
-       "metric_name                                                                                                                                                                  substring_match  \n",
-       "input                       aiconfig_output                                                                                                                                                   \n",
-       "different kinds of cuisines 1. Experience culinary delights at Chelsea Market.\\n2. Immerse in NYC's Chinatown food tour. \\n3. Indulge in Italian tradition at Little Italy.              0.0  \n",
-       "iconic midtown skyscrapers  1. Visit \"Top of the Rock\" at Rockefeller Center.\\n2. Tour Empire State Building.\\n3. Explore Grand Central Terminal.                                        1.0  "
+       "metric_name                                                                                                                                                                                              substring_match  \n",
+       "input                       aiconfig_output                                                                                                                                                                               \n",
+       "different kinds of cuisines 1. Explore Chelsea Market's cuisine.\\n2. Visit Little Italy for authentic fare.\\n3. Savor Asian delicacies in Flushing, Queens.                                                        False  \n",
+       "iconic midtown skyscrapers  1. Empire State Building Observatory - 86th floor views.\\n2. Rockefeller Center - Top of the Rock and ice skating.\\n3. One World Trade Center - One World Observatory visit.            True  "
       ]
      },
-     "execution_count": 10,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -488,7 +504,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -504,11 +520,11 @@
     "\n",
     "\n",
     "from aiconfig.eval.api import (\n",
-    "    brevity,\n",
-    "    substring_match,\n",
     "    run_test_suite_outputs_only,\n",
     ")\n",
     "\n",
+    "from aiconfig.eval.api import metrics\n",
+    "\n",
     "\n",
     "# This is similar to \"test_inputs_with_substrings\" above, but we have the AIConfig *outputs*\n",
     "# in the test cases, rather than the inputs. The library will evaluate these strings directly\n",
@@ -529,10 +545,10 @@
     "test_suite_outputs_only = []\n",
     "for test_output, substring in test_outputs_with_substrings:\n",
     "    # Add the brevity metric\n",
-    "    test_fn1 = brevity\n",
+    "    test_fn1 = metrics.brevity\n",
     "    test_suite_outputs_only.append((test_output, test_fn1))\n",
     "    # Add substring check function\n",
-    "    test_fn2 = substring_match(substring, case_sensitive=False)\n",
+    "    test_fn2 = metrics.substring_match(substring, case_sensitive=False)\n",
     "    test_suite_outputs_only.append(\n",
     "        (test_output, test_fn2)\n",
     "    )"
@@ -540,7 +556,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
@@ -552,52 +568,56 @@
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " Metric(calculate=<function _calculate_brevity at 0x11f79ade0>, interpretation={\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
       "  \"best_value\": 1.0,\n",
       "  \"worst_value\": Infinity,\n",
-      "  \"extra_metadata\": {}\n",
-      "})\n",
+      "  \"extra_metadata\": {},\n",
+      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "}))\n",
       "\n",
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " Metric(calculate=<function substring_match.<locals>._fn at 0x13b20cae0>, interpretation={\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x14412c1f0>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
-      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": 0.0,\n",
+      "  \"description\": \"True (pass) if contains given substring\",\n",
+      "  \"best_value\": true,\n",
+      "  \"worst_value\": false,\n",
       "  \"extra_metadata\": {\n",
       "    \"substring\": \"Magnolia Bakery\",\n",
       "    \"case_sensitive\": false\n",
-      "  }\n",
-      "})\n",
+      "  },\n",
+      "  \"id\": \"0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6\"\n",
+      "}))\n",
       "\n",
       "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " Metric(calculate=<function _calculate_brevity at 0x11f79ade0>, interpretation={\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
       "  \"best_value\": 1.0,\n",
       "  \"worst_value\": Infinity,\n",
-      "  \"extra_metadata\": {}\n",
-      "})\n",
+      "  \"extra_metadata\": {},\n",
+      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "}))\n",
       "\n",
       "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " Metric(calculate=<function substring_match.<locals>._fn at 0x13b20c720>, interpretation={\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x14412dcf0>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
-      "  \"description\": \"1.0 (pass) if contains given substring\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": 0.0,\n",
+      "  \"description\": \"True (pass) if contains given substring\",\n",
+      "  \"best_value\": true,\n",
+      "  \"worst_value\": false,\n",
       "  \"extra_metadata\": {\n",
       "    \"substring\": \"Empire State Building\",\n",
       "    \"case_sensitive\": false\n",
-      "  }\n",
-      "})\n"
+      "  },\n",
+      "  \"id\": \"53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f\"\n",
+      "}))\n"
      ]
     }
    ],
@@ -610,7 +630,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -657,7 +677,7 @@
        "      <th>0</th>\n",
        "      <td>Missing</td>\n",
        "      <td>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</td>\n",
-       "      <td>160.0</td>\n",
+       "      <td>160</td>\n",
        "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
@@ -668,18 +688,18 @@
        "      <th>1</th>\n",
        "      <td>Missing</td>\n",
        "      <td>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</td>\n",
-       "      <td>0.0</td>\n",
-       "      <td>2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054</td>\n",
+       "      <td>False</td>\n",
+       "      <td>0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6</td>\n",
        "      <td>substring_match</td>\n",
-       "      <td>1.0 (pass) if contains given substring</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>True (pass) if contains given substring</td>\n",
+       "      <td>True</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>Missing</td>\n",
        "      <td>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</td>\n",
-       "      <td>267.0</td>\n",
+       "      <td>267</td>\n",
        "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
@@ -690,12 +710,12 @@
        "      <th>3</th>\n",
        "      <td>Missing</td>\n",
        "      <td>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519</td>\n",
+       "      <td>True</td>\n",
+       "      <td>53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f</td>\n",
        "      <td>substring_match</td>\n",
-       "      <td>1.0 (pass) if contains given substring</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>True (pass) if contains given substring</td>\n",
+       "      <td>True</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -715,25 +735,25 @@
        "3  1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.   \n",
        "\n",
        "   value                                                         metric_id  \\\n",
-       "0  160.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "1    0.0  2b849774ca14c9e75f3492f100b190cd0ff0a5e4a5a7f21f318b1ba0ea239054   \n",
-       "2  267.0  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "3    1.0  e09e60224d3b9ca180ca934cbabf904a4e63843ee5f8bf060c062504dceab519   \n",
+       "0    160  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "1  False  0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6   \n",
+       "2    267  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
+       "3   True  53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f   \n",
        "\n",
-       "       metric_name                      metric_description  \\\n",
-       "0          brevity                    Absolute text length   \n",
-       "1  substring_match  1.0 (pass) if contains given substring   \n",
-       "2          brevity                    Absolute text length   \n",
-       "3  substring_match  1.0 (pass) if contains given substring   \n",
+       "       metric_name                       metric_description  \\\n",
+       "0          brevity                     Absolute text length   \n",
+       "1  substring_match  True (pass) if contains given substring   \n",
+       "2          brevity                     Absolute text length   \n",
+       "3  substring_match  True (pass) if contains given substring   \n",
        "\n",
-       "   best_possible_value  worst_possible_value  \n",
-       "0                  1.0                   inf  \n",
-       "1                  1.0                   0.0  \n",
-       "2                  1.0                   inf  \n",
-       "3                  1.0                   0.0  "
+       "  best_possible_value worst_possible_value  \n",
+       "0                 1.0                  inf  \n",
+       "1                True                False  \n",
+       "2                 1.0                  inf  \n",
+       "3                True                False  "
       ]
      },
-     "execution_count": 13,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -749,7 +769,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
@@ -792,31 +812,31 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</th>\n",
-       "      <td>267.0</td>\n",
-       "      <td>1.0</td>\n",
+       "      <td>267</td>\n",
+       "      <td>True</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</th>\n",
-       "      <td>160.0</td>\n",
-       "      <td>0.0</td>\n",
+       "      <td>160</td>\n",
+       "      <td>False</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "metric_name                                                                                                                                                                                                                                                                  brevity  \\\n",
-       "aiconfig_output                                                                                                                                                                                                                                                                        \n",
-       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.    267.0   \n",
-       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                               160.0   \n",
+       "metric_name                                                                                                                                                                                                                                                                 brevity  \\\n",
+       "aiconfig_output                                                                                                                                                                                                                                                                       \n",
+       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.     267   \n",
+       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                160   \n",
        "\n",
-       "metric_name                                                                                                                                                                                                                                                                  substring_match  \n",
-       "aiconfig_output                                                                                                                                                                                                                                                                               \n",
-       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.              1.0  \n",
-       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                         0.0  "
+       "metric_name                                                                                                                                                                                                                                                                 substring_match  \n",
+       "aiconfig_output                                                                                                                                                                                                                                                                              \n",
+       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.            True  \n",
+       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                      False  "
       ]
      },
-     "execution_count": 14,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -826,6 +846,49 @@
     "df_result.set_index([ \"aiconfig_output\", \"metric_name\"])\\\n",
     "        .value.unstack(\"metric_name\")"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/pydantic/_internal/_fields.py:128: UserWarning: Field \"model_parsers\" has conflict with protected namespace \"model_\".\n",
+      "\n",
+      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
+      "  warnings.warn(\n",
+      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "CustomMetricRecord(data={\n",
+       "  \"conciseness_rating\": 5,\n",
+       "  \"conciseness_confidence\": 8.0,\n",
+       "  \"conciseness_reasoning\": \"The text is short and to the point.\"\n",
+       "})"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import openai\n",
+    "\n",
+    "from aiconfig.eval.api import metrics\n",
+    "\n",
+    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
+    "\n",
+    "await metrics.gpt3_5_text_ratings(\"one two three\")"
+   ]
   }
  ],
  "metadata": {
diff --git a/python/src/aiconfig/eval/metrics.py b/python/src/aiconfig/eval/metrics.py
index ca0c7366d..d51821d8a 100644
--- a/python/src/aiconfig/eval/metrics.py
+++ b/python/src/aiconfig/eval/metrics.py
@@ -1,11 +1,16 @@
+import json
 from functools import total_ordering
-from typing import Any, Generic
+from typing import Any, Generic, Type
 
+import lastmile_utils.lib.core.api as cu
 import nltk
 import pandas as pd
-from aiconfig.eval.common import CustomMetricValue, EvaluationFunction, EvaluationMetricMetadata, T_OutputDatum
+from aiconfig.eval import common
+from aiconfig.eval.common import CustomMetricValue, EvaluationFunction, EvaluationMetricMetadata, T_BaseModel, T_OutputDatum, TextRatingsData
+from aiconfig.eval.openai import OpenAIChatCompletionCreate, default_openai_chat_completion_create, make_fn_completion_text_to_serialized_json
 from attr import dataclass
 from nltk.sentiment.vader import SentimentIntensityAnalyzer
+from result import Err, Ok, Result
 
 
 @dataclass(frozen=True)
@@ -147,3 +152,128 @@ async def _get_overall_positive_sentiment(output_datum: str) -> TextOverallPosit
         worst_value=TextOverallPositiveSentiment(pos=0.0, neg=1.0),
     ),
 )
+
+
+def make_structured_llm_metric(
+    chat_completion_create: common.CompletionTextToSerializedJSON,
+    eval_llm_name: str,
+    pydantic_basemodel_type: Type[T_BaseModel],
+    metric_name: str,
+    metric_description: str,
+    field_descriptions: dict[str, str] = {},
+) -> Metric[str]:
+    def _make_evaluation_fn(basemodel_type: Type[T_BaseModel]) -> EvaluationFunction[str]:
+        async def _evaluation_fn(output_datum: str) -> common.CustomMetricPydanticObject[T_BaseModel]:
+            resp = common.get_llm_structured_response(
+                input_text=output_datum,
+                chat_completion_create=chat_completion_create,
+                basemodel_type=basemodel_type,
+            )
+
+            # Intentional: unwrap and raise here to conform to the Metric interface.
+            match resp:
+                case Err(e):
+                    raise ValueError(f"Error getting structured response: {e}")
+                case Ok(data):
+                    return common.CustomMetricPydanticObject(data=data)
+
+        return _evaluation_fn
+
+    return Metric(
+        evaluation_fn=_make_evaluation_fn(pydantic_basemodel_type),
+        metric_metadata=EvaluationMetricMetadata(
+            name=metric_name,
+            description=metric_description,
+            extra_metadata=dict(
+                basemodel_type_name=pydantic_basemodel_type.__name__,
+                eval_llm_name=eval_llm_name,
+                field_descriptions_json=json.dumps(field_descriptions, sort_keys=True),
+            ),
+        ),
+    )
+
+
+def _make_openai_structured_llm_metric_helper(
+    eval_llm_name: str,
+    pydantic_basemodel_type: Type[T_BaseModel],
+    metric_name: str,
+    metric_description: str,
+    field_descriptions: dict[str, str],
+    openai_chat_completion_create: OpenAIChatCompletionCreate | None = None,
+) -> Result[Metric[str], str]:
+    schema = pydantic_basemodel_type.model_json_schema()
+    properties = schema["properties"]
+    required = schema["required"]
+
+    if not field_descriptions.keys() <= properties.keys():
+        return Err(
+            f"""
+                The following field_descriptions keys are not in the schema:
+                {set(field_descriptions.keys()) - set(properties.keys())}
+            """
+        )
+
+    def _with_description(key: str, value: dict[str, str]) -> dict[str, str]:
+        if key in field_descriptions:
+            return cu.dict_union_allow_replace(value, {"description": field_descriptions[key]})
+        return value
+
+    properties = {k: _with_description(k, v) for k, v in properties.items()}
+
+    required = required or list(properties.keys())
+
+    openai_eval_llm_chat_completion_create: common.CompletionTextToSerializedJSON = make_fn_completion_text_to_serialized_json(
+        eval_llm_name=eval_llm_name,
+        properties=properties,
+        required=required,
+        openai_chat_completion_create=(openai_chat_completion_create or default_openai_chat_completion_create),
+    )
+
+    return Ok(
+        make_structured_llm_metric(
+            openai_eval_llm_chat_completion_create,
+            eval_llm_name=eval_llm_name,
+            pydantic_basemodel_type=pydantic_basemodel_type,
+            metric_name=metric_name,
+            metric_description=metric_description,
+            field_descriptions=field_descriptions,
+        )
+    )
+
+
+def make_openai_structured_llm_metric(
+    eval_llm_name: str,
+    pydantic_basemodel_type: Type[T_BaseModel],
+    metric_name: str,
+    metric_description: str,
+    field_descriptions: dict[str, str] = {},
+    openai_chat_completion_create: OpenAIChatCompletionCreate | None = None,
+) -> Metric[str]:
+    res_metric = _make_openai_structured_llm_metric_helper(
+        eval_llm_name=eval_llm_name,
+        pydantic_basemodel_type=pydantic_basemodel_type,
+        metric_name=metric_name,
+        metric_description=metric_description,
+        field_descriptions=field_descriptions,
+        openai_chat_completion_create=openai_chat_completion_create,
+    )
+
+    # User interface: unwrap and raise
+    match res_metric:
+        case Ok(metric):
+            return metric
+        case Err(e):
+            raise ValueError(f"Error making metric: {e}")
+
+
+gpt3_5_text_ratings = make_openai_structured_llm_metric(
+    eval_llm_name="gpt-3.5-turbo-0613",
+    pydantic_basemodel_type=TextRatingsData,
+    metric_name="text_ratings",
+    metric_description="Text ratings",
+    field_descriptions=dict(
+        conciseness_rating="1 to 5 rating of conciseness",
+        conciseness_confidence="0 to 1.0 rating of confidence in conciseness rating",
+        conciseness_reasoning="reasoning behind the conciseness rating",
+    ),
+)
diff --git a/python/src/aiconfig/eval/openai.py b/python/src/aiconfig/eval/openai.py
new file mode 100644
index 000000000..b7a27982f
--- /dev/null
+++ b/python/src/aiconfig/eval/openai.py
@@ -0,0 +1,87 @@
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Protocol
+
+import lastmile_utils.lib.core.api as cu
+import openai
+import openai.types.chat as openai_types
+from aiconfig.eval import common
+from result import Err, Ok, Result
+
+
+@dataclass
+class OpenAIChatCompletionParams:
+    # TODO: type better
+    messages: list[openai_types.ChatCompletionMessageParam]
+    model: str
+    temperature: float
+    tools: list[openai_types.ChatCompletionToolParam]
+
+
+class OpenAIChatCompletionCreate(Protocol):
+    @abstractmethod
+    def __call__(self, completion_params: OpenAIChatCompletionParams) -> Result[openai_types.ChatCompletion, str]:
+        pass
+
+
+def default_openai_chat_completion_create(completion_params: OpenAIChatCompletionParams) -> Result[openai_types.ChatCompletion, str]:
+    try:
+        result = openai.chat.completions.create(
+            messages=completion_params.messages,
+            model=completion_params.model,
+            temperature=completion_params.temperature,
+            tools=completion_params.tools,
+        )
+        return Ok(result)
+    except Exception as e:
+        return cu.ErrWithTraceback(e)
+
+
+def extract_json_from_chat_completion(chat_completion: openai_types.ChatCompletion) -> Result[common.SerializedJSON, str]:
+    choice = chat_completion.choices[0]
+    message = choice.message
+    if message.tool_calls is None:
+        return Err("No tool calls found")
+
+    return Ok(common.SerializedJSON(message.tool_calls[0].function.arguments))
+
+
+def make_fn_completion_text_to_serialized_json(
+    eval_llm_name: str,
+    properties: dict[str, dict[str, str]],
+    required: list[str],
+    openai_chat_completion_create: OpenAIChatCompletionCreate,
+) -> common.CompletionTextToSerializedJSON:
+    def _chat_completion_create(output_datum: str) -> Result[common.SerializedJSON, str]:
+        openai_chat_completion_params = _make_openai_completion_params(output_datum, eval_llm_name, properties, required)
+        return openai_chat_completion_create(openai_chat_completion_params).and_then(extract_json_from_chat_completion)
+
+    out: common.CompletionTextToSerializedJSON = _chat_completion_create
+    return out
+
+
+def _make_openai_completion_params(
+    input_text: str, eval_llm_name: str, properties: dict[str, dict[str, str]], required: list[str]
+) -> OpenAIChatCompletionParams:
+    return OpenAIChatCompletionParams(
+        messages=[
+            {"role": "system", "content": "Call the function with arguments based on the provided text."},
+            {"role": "user", "content": input_text},
+        ],
+        model=eval_llm_name,
+        temperature=0,
+        tools=[
+            {
+                "type": "function",
+                "function": {
+                    "name": "dummy",
+                    "description": "Submit ratings that apply to a piece of text",
+                    "parameters": {
+                        "type": "object",
+                        "properties": properties,
+                        "required": required,
+                    },
+                },
+            },
+        ],
+    )


Test patch:
diff --git a/python/tests/test_eval.py b/python/tests/test_eval.py
index 1c78799d5..98bf1126d 100644
--- a/python/tests/test_eval.py
+++ b/python/tests/test_eval.py
@@ -1,19 +1,23 @@
-# type: ignore[no-untyped-call]
 import itertools
 import logging
 import os
-from typing import Any
+from typing import Any, cast
 
+import aiconfig.eval.openai as lib_openai
 import hypothesis
 import hypothesis.strategies as st
 import lastmile_utils.lib.core.api as cu
+import openai.types.chat as openai_chat_types
+import openai.types.chat.chat_completion as openai_chat_completion_types
+import openai.types.chat.chat_completion_message_tool_call as openai_tool_call_types
 import pandas as pd
 import pytest
 from aiconfig.Config import AIConfigRuntime
+from aiconfig.eval import common
 from aiconfig.eval.api import TestSuiteWithInputsSettings, metrics, run_test_suite_outputs_only, run_test_suite_with_inputs
 from aiconfig.eval.lib import TestSuiteWithInputsSpec, run_test_suite_helper
 from aiconfig.model_parser import InferenceOptions
-from result import Err, Ok
+from result import Err, Ok, Result
 
 brevity = metrics.brevity
 substring_match = metrics.substring_match
@@ -41,7 +45,7 @@ async def run_and_get_output_text(
         prompt_name: str,
         params: dict[Any, Any] | None = None,
         options: InferenceOptions | None = None,
-        **kwargs,
+        **kwargs,  # type: ignore
     ) -> str:
         """
         This overrides the real method for mocking, but the output doesn't matter very much.
@@ -122,11 +126,11 @@ async def test_run_test_suite_outputs_only(data: st.DataObject):
         "best_possible_value",
         "worst_possible_value",
     ]
-    inputs = df["input"].astype(str).tolist()
-    assert cu.only(inputs) == Ok("Missing")
+    inputs = df["input"].astype(str).tolist()  # type: ignore
+    assert cu.only(inputs) == Ok("Missing")  # type: ignore
 
-    df_brevity = df[df["metric_name"] == "brevity"]
-    assert (df_brevity["aiconfig_output"].apply(len) == df_brevity["value"]).all()
+    df_brevity = df[df["metric_name"] == "brevity"]  # type: ignore
+    assert (df_brevity["aiconfig_output"].apply(len) == df_brevity["value"]).all()  # type: ignore
 
 
 @hypothesis.given(st.data())
@@ -181,11 +185,11 @@ async def test_run_test_suite_with_inputs(data: st.DataObject):
 
             assert input_pairs == result_pairs
 
-            df_brevity = df[df["metric_name"] == "brevity"]
-            assert (df_brevity["aiconfig_output"].apply(len) == df_brevity["value"]).all()
+            df_brevity = df[df["metric_name"] == "brevity"]  # type: ignore
+            assert (df_brevity["aiconfig_output"].apply(len) == df_brevity["value"]).all()  # type: ignore
 
-            df_substring = df[df["metric_name"] == "substring_match"]
-            assert (df_substring["value"].apply(lambda x: x in {False, True})).all()
+            df_substring = df[df["metric_name"] == "substring_match"]  # type: ignore
+            assert (df_substring["value"].apply(lambda x: x in {False, True})).all()  # type: ignore
 
         case Err(e):
             assert False, f"expected Ok, got Err({e})"
@@ -204,16 +208,16 @@ async def test_custom_metric_type():
         )
     )
     df = await run_test_suite_outputs_only(user_test_suite_outputs_only)
-    result = df.set_index(["metric_name", "aiconfig_output"]).value.unstack(0).to_dict()
+    result = df.set_index(["metric_name", "aiconfig_output"]).value.unstack(0).to_dict()  # type: ignore
     assert result["sentiment_class"] == {
         "nltk is amazing": "pos",
         "whats for dinner?": "neu",
         "oh, bother": "neg",
     }
 
-    assert all(isinstance(v, metrics.TextSentimentScores) for v in result["sentiment_scores"].values())
+    assert all(isinstance(v, metrics.TextSentimentScores) for v in result["sentiment_scores"].values())  # type: ignore
 
-    assert all(isinstance(v, metrics.TextOverallPositiveSentiment) for v in result["sentiment_score_overall_positive"].values())
+    assert all(isinstance(v, metrics.TextOverallPositiveSentiment) for v in result["sentiment_score_overall_positive"].values())  # type: ignore
 
     neutral = metrics.TextOverallPositiveSentiment(pos=0.0, neg=0.0)
 
@@ -222,7 +226,7 @@ async def test_custom_metric_type():
 
 
 @pytest.mark.asyncio
-async def test_exception_metric(caplog):
+async def test_exception_metric(caplog: pytest.LogCaptureFixture):
     user_test_suite_outputs_only = list(
         itertools.product(
             ["Hundred Acre Wood", ""],
@@ -231,8 +235,102 @@ async def test_exception_metric(caplog):
     )
     with caplog.at_level(logging.ERROR):
         df = await run_test_suite_outputs_only(user_test_suite_outputs_only)
-    mapping = df.query("metric_name=='brevity'").set_index("aiconfig_output").value.to_dict()
+    mapping: dict[str, Any] = df.query("metric_name=='brevity'").set_index("aiconfig_output").value.to_dict()  # type: ignore
     assert mapping["Hundred Acre Wood"] == 17.0
-    assert pd.isnull(mapping[""])
+    assert pd.isnull(mapping[""])  # type: ignore
 
     assert any("Brevity is meaningless for empty string." in record.msg for record in caplog.records)
+
+
+def _mock_response(function_args: common.SerializedJSON) -> openai_chat_types.ChatCompletion:
+    return openai_chat_types.ChatCompletion(
+        id="123",
+        choices=[
+            openai_chat_completion_types.Choice(
+                index=0,
+                message=openai_chat_types.ChatCompletionMessage(
+                    content=None,
+                    role="assistant",
+                    tool_calls=[
+                        openai_chat_types.ChatCompletionMessageToolCall(
+                            id="cm-tk-1",
+                            type="function",
+                            function=openai_tool_call_types.Function(
+                                name="dummy",
+                                arguments=function_args,
+                            ),
+                        )
+                    ],
+                ),
+                finish_reason="stop",
+            )
+        ],
+        created=0,
+        model="",
+        object="chat.completion",
+    )
+
+
+def _make_mock_openai_chat_completion_create(function_arguments_return: common.SerializedJSON) -> lib_openai.OpenAIChatCompletionCreate:
+    def _mock_openai_chat_completion_create(
+        completion_params: lib_openai.OpenAIChatCompletionParams,
+    ) -> Result[openai_chat_types.ChatCompletion, str]:
+        return Ok(
+            _mock_response(
+                function_arguments_return,
+            )
+        )
+
+    return _mock_openai_chat_completion_create
+
+
+@pytest.mark.asyncio
+async def test_openai_structured_eval():
+    _mock_create = _make_mock_openai_chat_completion_create(
+        common.SerializedJSON('{"conciseness_rating": 5, "conciseness_confidence": 0.9, "conciseness_reasoning": "I think it\'s pretty concise."}')
+    )
+    mock_metric = metrics.make_openai_structured_llm_metric(
+        eval_llm_name="gpt-3.5-turbo-0613",
+        pydantic_basemodel_type=common.TextRatingsData,
+        metric_name="text_ratings",
+        metric_description="Text ratings",
+        field_descriptions=dict(
+            conciseness_rating="1 to 5 rating of conciseness",
+            conciseness_confidence="0 to 1.0 rating of confidence in conciseness rating",
+            conciseness_reasoning="reasoning behind the conciseness rating",
+        ),
+        openai_chat_completion_create=_mock_create,
+    )
+
+    user_test_suite_outputs_only = [
+        ("one two three", mock_metric),
+    ]
+    df = await run_test_suite_outputs_only(user_test_suite_outputs_only)
+    metric_data = cast(common.CustomMetricPydanticObject[metrics.TextRatingsData], df.loc[0, "value"]).data
+    assert isinstance(metric_data, common.TextRatingsData)
+    metric_json = metric_data.to_dict()
+    assert metric_json == {"conciseness_rating": 5, "conciseness_confidence": 0.9, "conciseness_reasoning": "I think it's pretty concise."}
+
+
+@pytest.mark.asyncio
+async def test_bad_structured_eval_metric():
+    _mock_create = _make_mock_openai_chat_completion_create(
+        common.SerializedJSON('{"conciseness_rating": 5, "conciseness_confidence": 0.9, "conciseness_reasoning": "I think it\'s pretty concise."}')
+    )
+
+    with pytest.raises(ValueError) as exc:
+        _ = metrics.make_openai_structured_llm_metric(
+            eval_llm_name="gpt-3.5-turbo-0613",
+            pydantic_basemodel_type=common.TextRatingsData,
+            metric_name="text_ratings",
+            metric_description="Text ratings",
+            field_descriptions=dict(
+                fake_field="123",
+                conciseness_rating="1 to 5 rating of conciseness",
+                conciseness_confidence="0 to 1.0 rating of confidence in conciseness rating",
+                conciseness_reasoning="reasoning behind the conciseness rating",
+            ),
+            openai_chat_completion_create=_mock_create,
+        )
+
+    assert "The following field_descriptions keys are not in the schema" in str(exc)