Repository: lastmile-ai/aiconfig (language: python)

Issue:
[AIC-py][eval] automated tests.
[AIC-py][eval] automated tests.

Subsequent PR: fix links in readme

Test:

- added automated property-based tests with hypothesis lib
- Rerun ipynb
- npx

<img width="1455" alt="image" src="https://github.com/lastmile-ai/aiconfig/assets/148090348/d3b38fb8-a8c9-4be9-aaa6-40fbfbadaaea">

---
Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/lastmile-ai/aiconfig/pull/424).
* #426
* #425
* __->__ #424
* #423
* #421
[eval] Promptfoo-style MVP
Evaluation interfaces for AIConfig

(Request from @2timesjay)
Combine config.run + config.get_output_text for efficient chaining
I almost always need the text output after running inference to use for chaining (supported in AI Workbooks). Would love to have these two functions combined instead of always having to do this: 

```
verification_completion = await config.run("verification", params)
verification_response = config.get_output_text("verification")
```


Discussion / hints:

Code is merged, but I didn't mean to auto-close this. Are we aligned that PRs 317 and 381 solve the core problem? @suyoglastmileai , @saqadri 
Clarifying the scope here. This is about small-scale eval, i.e. where the data set is small enough to manually inspect, write test cases, etc. See [this use case](https://docs.google.com/document/d/17tjQgLDmAqyq26XJx4GqVYS73K73GykvhywEiFCozps/edit#bookmark=id.5uogvi8dxrv4).
Similar to 143, might be part of a bigger redesign.

Fix patch:
diff --git a/.gitignore b/.gitignore
index 5e79bc1c5..461b286f0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -136,4 +136,5 @@ dist
 .ipynb_checkpoints
 __pycache__/
 models
-*.egg-info
\ No newline at end of file
+*.egg-info
+.hypothesis
\ No newline at end of file
diff --git a/python/requirements.txt b/python/requirements.txt
index b367dbaba..1a5f05f62 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -14,4 +14,5 @@ nest_asyncio
 prompt_toolkit
 mock
 pytest-asyncio
-lastmile-utils>=0.0.6
\ No newline at end of file
+lastmile-utils>=0.0.6
+hypothesis==6.91.0
\ No newline at end of file
diff --git a/python/src/aiconfig/Config.py b/python/src/aiconfig/Config.py
index a949f4e42..9d3a5031d 100644
--- a/python/src/aiconfig/Config.py
+++ b/python/src/aiconfig/Config.py
@@ -1,6 +1,6 @@
 import json
 import os
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 import requests
 from aiconfig.callback import CallbackEvent, CallbackManager
@@ -274,6 +274,16 @@ async def run(
         await self.callback_manager.run_callbacks(event)
         return response
 
+    async def run_and_get_output_text(
+        self,
+        prompt_name: str,
+        params: dict[Any, Any] | None = None,
+        options: Optional[InferenceOptions] = None,
+        **kwargs,
+    ) -> str:
+        result: Any = await self.run(prompt_name, params, options=options, **kwargs)
+        return self.get_output_text(prompt_name, result[0])
+
     #
     #     Saves this AIConfig to a file.
     #     @param filePath The path to the file to save to.
diff --git a/python/src/aiconfig/eval/README.md b/python/src/aiconfig/eval/README.md
index a46221453..e1e30ec76 100644
--- a/python/src/aiconfig/eval/README.md
+++ b/python/src/aiconfig/eval/README.md
@@ -87,7 +87,7 @@ If you are already familiar with [Promptfoo](https://www.promptfoo.dev/) and/or
 
 Please run the example as follows:
 
-`npx promptfoo@latest eval -c python/src/aiconfig/eval/promptfoo/examples/travel/travel_promptfooconfig.yaml`
+`npx promptfoo@latest eval -c python/src/aiconfig/eval/examples/travel/travel_promptfoo_config.yaml`
 
 You should get one passing test and one failing test:
 
diff --git a/python/src/aiconfig/eval/api/__init__.py b/python/src/aiconfig/eval/api/__init__.py
new file mode 100644
index 000000000..156f13b02
--- /dev/null
+++ b/python/src/aiconfig/eval/api/__init__.py
@@ -0,0 +1,32 @@
+"""This file is officially the eval API.    
+
+Usage:
+
+from aiconfig.eval.api import (
+    brevity,
+    substring_match,
+    run_test_suite_with_inputs,
+    TestSuiteWithInputsSettings,
+)
+"""
+# pyright: reportWildcardImportFromLibrary=false
+from ..lib import (
+    TestSuiteWithInputsSettings,
+    run_test_suite_with_inputs,
+    run_test_suite_outputs_only,
+    SampleEvaluationFunction,
+)
+
+from ..metrics import (
+    brevity,
+    substring_match,
+)
+
+__all__ = [
+    "brevity",
+    "substring_match",
+    "run_test_suite_with_inputs",
+    "run_test_suite_outputs_only",
+    "TestSuiteWithInputsSettings",
+    "SampleEvaluationFunction",
+]
diff --git a/python/src/aiconfig/eval/common.py b/python/src/aiconfig/eval/common.py
new file mode 100644
index 000000000..9203696be
--- /dev/null
+++ b/python/src/aiconfig/eval/common.py
@@ -0,0 +1,61 @@
+
+from typing import Any, Generic
+from abc import abstractmethod
+from typing import Any, Generic, Protocol, TypeVar
+
+import lastmile_utils.lib.core.api as cu
+from pydantic import root_validator
+
+
+
+T_InputDatum = TypeVar("T_InputDatum", contravariant=True)
+T_OutputDatum = TypeVar("T_OutputDatum", contravariant=True)
+
+
+class EvaluationMetricMetadata(cu.Record, Generic[T_OutputDatum]):
+    """A record to tie together metadata about an evaluation metric
+    to ensure that numbers are interpreted as intended.
+
+    Assumptions:
+    * Metric type is float 
+        (bools and ints have to be represented as floats; tensors are not supported)
+    * Value is monotonic in "goodness". i.e. there are two cases:
+        higher is better
+            e.g. accuracy, best_value=1.0, worst_value=0.0
+            The higher the better over the entire range.
+        lower is better
+            e.g. error count, best_value=0.0, worst_value=inf
+            The lower the better over the entire rane.
+    """
+
+    name: str
+    description: str
+    best_value: float
+    worst_value: float
+
+
+class SampleMetricValue(cu.Record, Generic[T_OutputDatum]):
+    value: float
+    interpretation: EvaluationMetricMetadata[T_OutputDatum]
+
+    @root_validator(pre=True)
+    def check_value_range(cls, values: dict[str, Any]) -> dict[str, Any]:
+        wv, bv = (
+            values["interpretation"].worst_value,
+            values["interpretation"].best_value,
+        )
+        value = values["value"]
+        if wv == bv:
+            raise ValueError("best_value and worst_value cannot be equal")
+        if wv < bv and not wv <= value <= bv:
+            raise ValueError(f"value {value} is not in range [{wv}, {bv}] (inclusive)")
+        if wv > bv and not wv >= value >= bv:
+            raise ValueError(f"value {value} is not in range [{bv}, {wv}] (inclusive)")
+
+        return values
+
+
+class SampleEvaluationFunction(Protocol, Generic[T_OutputDatum]):
+    @abstractmethod
+    def __call__(self, output_datum: T_OutputDatum) -> SampleMetricValue[T_OutputDatum]:
+        pass
\ No newline at end of file
diff --git a/python/src/aiconfig/eval/custom_eval/examples/travel/travel_eval.ipynb b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
similarity index 82%
rename from python/src/aiconfig/eval/custom_eval/examples/travel/travel_eval.ipynb
rename to python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
index 03d5f9c8b..9991ed4e0 100644
--- a/python/src/aiconfig/eval/custom_eval/examples/travel/travel_eval.ipynb
+++ b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
@@ -63,7 +63,7 @@
     "\n",
     "pd.set_option(\"display.max_colwidth\", None)\n",
     "\n",
-    "from aiconfig.eval.lib import (\n",
+    "from aiconfig.eval.api import (\n",
     "    brevity,\n",
     "    substring_match,\n",
     "    run_test_suite_with_inputs,\n",
@@ -162,22 +162,22 @@
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " <function brevity at 0x13c7d25f0>\n",
+      " <function brevity at 0x16ab927a0>\n",
       "\n",
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " functools.partial(<function contains_substring at 0x13c7d2b90>, substring='Magnolia Bakery', case_sensitive=False)\n",
+      " functools.partial(<function contains_substring at 0x16ab92710>, substring='Magnolia Bakery', case_sensitive=False)\n",
       "\n",
       "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " <function brevity at 0x13c7d25f0>\n",
+      " <function brevity at 0x16ab927a0>\n",
       "\n",
       "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " functools.partial(<function contains_substring at 0x13c7d2b90>, substring='Empire State Building', case_sensitive=False)\n"
+      " functools.partial(<function contains_substring at 0x16ab92710>, substring='Empire State Building', case_sensitive=False)\n"
      ]
     }
    ],
@@ -227,16 +227,16 @@
        "      <th>value</th>\n",
        "      <th>metric_name</th>\n",
        "      <th>metric_description</th>\n",
-       "      <th>best_value</th>\n",
-       "      <th>worst_value</th>\n",
+       "      <th>best_possible_value</th>\n",
+       "      <th>worst_possible_value</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>Visit Chelsea Market for diverse cuisines and food tours. Explore Queens International Night Market for global dishes and dynamic entertainment. Savor Italy's cuisine at Eataly in Flatiron District.</td>\n",
-       "      <td>198.0</td>\n",
+       "      <td>Visit Chelsea Market for global cuisine exploration, stop by Smorgasburg for diverse street food, end at Eataly for immersive Italian experience.</td>\n",
+       "      <td>145.0</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
        "      <td>1.0</td>\n",
@@ -245,7 +245,7 @@
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>Visit Chelsea Market for diverse cuisines and food tours. Explore Queens International Night Market for global dishes and dynamic entertainment. Savor Italy's cuisine at Eataly in Flatiron District.</td>\n",
+       "      <td>Visit Chelsea Market for global cuisine exploration, stop by Smorgasburg for diverse street food, end at Eataly for immersive Italian experience.</td>\n",
        "      <td>0.0</td>\n",
        "      <td>contains_substring</td>\n",
        "      <td>1.0 (pass) if contains given substring</td>\n",
@@ -255,10 +255,8 @@
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Morning: Empire State Building Observatory for panoramic city views\n",
-       "2. Afternoon: The High Line, beautiful urban walk with a unique view \n",
-       "3. Evening: Top of the Rock Observation Deck, capturing Manhattan skyline.</td>\n",
-       "      <td>215.0</td>\n",
+       "      <td>Day 1: Visit Empire State Building; revel in architecture, history, views. Day 2: Explore Rockefeller Center; admire city panoramas, enjoy other attractions. Day 3: Tour Chrysler Building, appreciate art, stop at Chase Manhattan Plaza.</td>\n",
+       "      <td>235.0</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
        "      <td>1.0</td>\n",
@@ -267,9 +265,7 @@
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Morning: Empire State Building Observatory for panoramic city views\n",
-       "2. Afternoon: The High Line, beautiful urban walk with a unique view \n",
-       "3. Evening: Top of the Rock Observation Deck, capturing Manhattan skyline.</td>\n",
+       "      <td>Day 1: Visit Empire State Building; revel in architecture, history, views. Day 2: Explore Rockefeller Center; admire city panoramas, enjoy other attractions. Day 3: Tour Chrysler Building, appreciate art, stop at Chase Manhattan Plaza.</td>\n",
        "      <td>1.0</td>\n",
        "      <td>contains_substring</td>\n",
        "      <td>1.0 (pass) if contains given substring</td>\n",
@@ -287,27 +283,23 @@
        "2   iconic midtown skyscrapers   \n",
        "3   iconic midtown skyscrapers   \n",
        "\n",
-       "                                                                                                                                                                                                           aiconfig_output  \\\n",
-       "0                   Visit Chelsea Market for diverse cuisines and food tours. Explore Queens International Night Market for global dishes and dynamic entertainment. Savor Italy's cuisine at Eataly in Flatiron District.   \n",
-       "1                   Visit Chelsea Market for diverse cuisines and food tours. Explore Queens International Night Market for global dishes and dynamic entertainment. Savor Italy's cuisine at Eataly in Flatiron District.   \n",
-       "2  1. Morning: Empire State Building Observatory for panoramic city views\n",
-       "2. Afternoon: The High Line, beautiful urban walk with a unique view \n",
-       "3. Evening: Top of the Rock Observation Deck, capturing Manhattan skyline.   \n",
-       "3  1. Morning: Empire State Building Observatory for panoramic city views\n",
-       "2. Afternoon: The High Line, beautiful urban walk with a unique view \n",
-       "3. Evening: Top of the Rock Observation Deck, capturing Manhattan skyline.   \n",
+       "                                                                                                                                                                                                                               aiconfig_output  \\\n",
+       "0                                                                                            Visit Chelsea Market for global cuisine exploration, stop by Smorgasburg for diverse street food, end at Eataly for immersive Italian experience.   \n",
+       "1                                                                                            Visit Chelsea Market for global cuisine exploration, stop by Smorgasburg for diverse street food, end at Eataly for immersive Italian experience.   \n",
+       "2  Day 1: Visit Empire State Building; revel in architecture, history, views. Day 2: Explore Rockefeller Center; admire city panoramas, enjoy other attractions. Day 3: Tour Chrysler Building, appreciate art, stop at Chase Manhattan Plaza.   \n",
+       "3  Day 1: Visit Empire State Building; revel in architecture, history, views. Day 2: Explore Rockefeller Center; admire city panoramas, enjoy other attractions. Day 3: Tour Chrysler Building, appreciate art, stop at Chase Manhattan Plaza.   \n",
        "\n",
        "   value         metric_name                      metric_description  \\\n",
-       "0  198.0             brevity                    Absolute text length   \n",
+       "0  145.0             brevity                    Absolute text length   \n",
        "1    0.0  contains_substring  1.0 (pass) if contains given substring   \n",
-       "2  215.0             brevity                    Absolute text length   \n",
+       "2  235.0             brevity                    Absolute text length   \n",
        "3    1.0  contains_substring  1.0 (pass) if contains given substring   \n",
        "\n",
-       "   best_value  worst_value  \n",
-       "0         1.0          inf  \n",
-       "1         1.0          0.0  \n",
-       "2         1.0          inf  \n",
-       "3         1.0          0.0  "
+       "   best_possible_value  worst_possible_value  \n",
+       "0                  1.0                   inf  \n",
+       "1                  1.0                   0.0  \n",
+       "2                  1.0                   inf  \n",
+       "3                  1.0                   0.0  "
       ]
      },
      "execution_count": 4,
@@ -374,14 +366,14 @@
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>different kinds of cuisines</th>\n",
-       "      <th>Visit Chelsea Market for diverse cuisines and food tours. Explore Queens International Night Market for global dishes and dynamic entertainment. Savor Italy's cuisine at Eataly in Flatiron District.</th>\n",
-       "      <td>198.0</td>\n",
+       "      <th>Visit Chelsea Market for global cuisine exploration, stop by Smorgasburg for diverse street food, end at Eataly for immersive Italian experience.</th>\n",
+       "      <td>145.0</td>\n",
        "      <td>0.0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>iconic midtown skyscrapers</th>\n",
-       "      <th>1. Morning: Empire State Building Observatory for panoramic city views\\n2. Afternoon: The High Line, beautiful urban walk with a unique view \\n3. Evening: Top of the Rock Observation Deck, capturing Manhattan skyline.</th>\n",
-       "      <td>215.0</td>\n",
+       "      <th>Day 1: Visit Empire State Building; revel in architecture, history, views. Day 2: Explore Rockefeller Center; admire city panoramas, enjoy other attractions. Day 3: Tour Chrysler Building, appreciate art, stop at Chase Manhattan Plaza.</th>\n",
+       "      <td>235.0</td>\n",
        "      <td>1.0</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
@@ -389,15 +381,15 @@
        "</div>"
       ],
       "text/plain": [
-       "metric_name                                                                                                                                                                                                                                            brevity  \\\n",
-       "input                       aiconfig_output                                                                                                                                                                                                                      \n",
-       "different kinds of cuisines Visit Chelsea Market for diverse cuisines and food tours. Explore Queens International Night Market for global dishes and dynamic entertainment. Savor Italy's cuisine at Eataly in Flatiron District.                       198.0   \n",
-       "iconic midtown skyscrapers  1. Morning: Empire State Building Observatory for panoramic city views\\n2. Afternoon: The High Line, beautiful urban walk with a unique view \\n3. Evening: Top of the Rock Observation Deck, capturing Manhattan skyline.    215.0   \n",
+       "metric_name                                                                                                                                                                                                                                                              brevity  \\\n",
+       "input                       aiconfig_output                                                                                                                                                                                                                                        \n",
+       "different kinds of cuisines Visit Chelsea Market for global cuisine exploration, stop by Smorgasburg for diverse street food, end at Eataly for immersive Italian experience.                                                                                              145.0   \n",
+       "iconic midtown skyscrapers  Day 1: Visit Empire State Building; revel in architecture, history, views. Day 2: Explore Rockefeller Center; admire city panoramas, enjoy other attractions. Day 3: Tour Chrysler Building, appreciate art, stop at Chase Manhattan Plaza.    235.0   \n",
        "\n",
-       "metric_name                                                                                                                                                                                                                                            contains_substring  \n",
-       "input                       aiconfig_output                                                                                                                                                                                                                                \n",
-       "different kinds of cuisines Visit Chelsea Market for diverse cuisines and food tours. Explore Queens International Night Market for global dishes and dynamic entertainment. Savor Italy's cuisine at Eataly in Flatiron District.                                    0.0  \n",
-       "iconic midtown skyscrapers  1. Morning: Empire State Building Observatory for panoramic city views\\n2. Afternoon: The High Line, beautiful urban walk with a unique view \\n3. Evening: Top of the Rock Observation Deck, capturing Manhattan skyline.                 1.0  "
+       "metric_name                                                                                                                                                                                                                                                              contains_substring  \n",
+       "input                       aiconfig_output                                                                                                                                                                                                                                                  \n",
+       "different kinds of cuisines Visit Chelsea Market for global cuisine exploration, stop by Smorgasburg for diverse street food, end at Eataly for immersive Italian experience.                                                                                                           0.0  \n",
+       "iconic midtown skyscrapers  Day 1: Visit Empire State Building; revel in architecture, history, views. Day 2: Explore Rockefeller Center; admire city panoramas, enjoy other attractions. Day 3: Tour Chrysler Building, appreciate art, stop at Chase Manhattan Plaza.                 1.0  "
       ]
      },
      "execution_count": 5,
@@ -435,7 +427,7 @@
     "print(\"Define outputs to test and criteria, similar to option 1.\")\n",
     "\n",
     "\n",
-    "from aiconfig.eval.lib import (\n",
+    "from aiconfig.eval.api import (\n",
     "    brevity,\n",
     "    substring_match,\n",
     "    run_test_suite_outputs_only,\n",
@@ -484,22 +476,22 @@
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " <function brevity at 0x13c7d25f0>\n",
+      " <function brevity at 0x16ab927a0>\n",
       "\n",
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " functools.partial(<function contains_substring at 0x13c7d2b90>, substring='Magnolia Bakery', case_sensitive=False)\n",
+      " functools.partial(<function contains_substring at 0x16ab92710>, substring='Magnolia Bakery', case_sensitive=False)\n",
       "\n",
       "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " <function brevity at 0x13c7d25f0>\n",
+      " <function brevity at 0x16ab927a0>\n",
       "\n",
       "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " functools.partial(<function contains_substring at 0x13c7d2b90>, substring='Empire State Building', case_sensitive=False)\n"
+      " functools.partial(<function contains_substring at 0x16ab92710>, substring='Empire State Building', case_sensitive=False)\n"
      ]
     }
    ],
@@ -549,8 +541,8 @@
        "      <th>value</th>\n",
        "      <th>metric_name</th>\n",
        "      <th>metric_description</th>\n",
-       "      <th>best_value</th>\n",
-       "      <th>worst_value</th>\n",
+       "      <th>best_possible_value</th>\n",
+       "      <th>worst_possible_value</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -617,11 +609,11 @@
        "2  267.0             brevity                    Absolute text length   \n",
        "3    1.0  contains_substring  1.0 (pass) if contains given substring   \n",
        "\n",
-       "   best_value  worst_value  \n",
-       "0         1.0          inf  \n",
-       "1         1.0          0.0  \n",
-       "2         1.0          inf  \n",
-       "3         1.0          0.0  "
+       "   best_possible_value  worst_possible_value  \n",
+       "0                  1.0                   inf  \n",
+       "1                  1.0                   0.0  \n",
+       "2                  1.0                   inf  \n",
+       "3                  1.0                   0.0  "
       ]
      },
      "execution_count": 8,
diff --git a/python/src/aiconfig/eval/custom_eval/examples/travel/travel_parametrized.aiconfig.json b/python/src/aiconfig/eval/examples/travel/travel_parametrized.aiconfig.json
similarity index 100%
rename from python/src/aiconfig/eval/custom_eval/examples/travel/travel_parametrized.aiconfig.json
rename to python/src/aiconfig/eval/examples/travel/travel_parametrized.aiconfig.json
diff --git a/python/src/aiconfig/eval/promptfoo/examples/travel/travel_promptfooconfig.yaml b/python/src/aiconfig/eval/examples/travel/travel_promptfoo_config.yaml
similarity index 75%
rename from python/src/aiconfig/eval/promptfoo/examples/travel/travel_promptfooconfig.yaml
rename to python/src/aiconfig/eval/examples/travel/travel_promptfoo_config.yaml
index 96f20a514..49e3847f2 100644
--- a/python/src/aiconfig/eval/promptfoo/examples/travel/travel_promptfooconfig.yaml
+++ b/python/src/aiconfig/eval/examples/travel/travel_promptfoo_config.yaml
@@ -1,10 +1,10 @@
 prompts: ["{{question}}"]
 providers:
-  - exec:python -m 'aiconfig.eval.promptfoo.run_aiconfig' ./travel_aiconfig_test_suite_settings.json
+  - exec:python -m 'aiconfig.scripts.run_aiconfig' ./travel_aiconfig_test_suite_settings.json
 tests:
   - description: "Test if output contains the expected value, and measure the length of the output"
     vars:
-      question: Food
+      question: "different kinds of cuisines"
     assert:
       - type: python
         value: output.lower().find('magnolia bakery') != -1
@@ -12,7 +12,7 @@ tests:
         value: len(output) < 5000
   - description: "Test if output contains the expected value, and measure the length of the output"
     vars:
-      question: "Iconic midtown skyscrapers"
+      question: "iconic midtown skyscrapers"
     assert:
       - type: python
         value: output.lower().find('empire state building') != -1
diff --git a/python/src/aiconfig/eval/image.png b/python/src/aiconfig/eval/image.png
deleted file mode 100644
index 3da8dac65..000000000
Binary files a/python/src/aiconfig/eval/image.png and /dev/null differ
diff --git a/python/src/aiconfig/eval/lib.py b/python/src/aiconfig/eval/lib.py
index 2b4daca64..621fdaa80 100644
--- a/python/src/aiconfig/eval/lib.py
+++ b/python/src/aiconfig/eval/lib.py
@@ -1,72 +1,23 @@
 import asyncio
-from functools import partial
 import logging
-from abc import abstractmethod
 from dataclasses import dataclass
-from typing import Any, Dict, Generic, NewType, Protocol, Sequence, Tuple, TypeVar
+from typing import Generic, NewType, Sequence, Tuple, TypeVar
 
 import lastmile_utils.lib.core.api as cu
 import pandas as pd
 from aiconfig.Config import AIConfigRuntime
-from pydantic import root_validator
 from result import Ok, Result
 
+from aiconfig.eval.common import (
+    SampleEvaluationFunction,
+    SampleMetricValue,
+    T_InputDatum,
+    T_OutputDatum,
+)
+
 logger = logging.getLogger(__name__)
 logging.basicConfig(format=cu.LOGGER_FMT)
 
-# API
-
-T_InputDatum = TypeVar("T_InputDatum", contravariant=True)
-T_OutputDatum = TypeVar("T_OutputDatum", contravariant=True)
-
-
-class EvaluationMetricMetadata(cu.Record, Generic[T_OutputDatum]):
-    """A record to tie together metadata about an evaluation metric
-    to ensure that numbers are interpreted as intended.
-
-    Assumptions:
-    * Metric type is float (bools and ints have to be represented as floats; tensors are not supported)
-    * Value is monotonic in "goodness". i.e. there are two cases:
-        higher is better
-            e.g. accuracy, best_value=1.0, worst_value=0.0
-            The higher the better over the entire range.
-        lower is better
-            e.g. error count, best_value=0.0, worst_value=inf
-            The lower the better over the entire rane.
-    """
-
-    name: str
-    description: str
-    best_value: float
-    worst_value: float
-
-
-class SampleMetricValue(cu.Record, Generic[T_OutputDatum]):
-    value: float
-    interpretation: EvaluationMetricMetadata[T_OutputDatum]
-
-    @root_validator(pre=True)
-    def check_value_range(cls, values: Dict[str, Any]) -> Dict[str, Any]:
-        wv, bv = (
-            values["interpretation"].worst_value,
-            values["interpretation"].best_value,
-        )
-        value = values["value"]
-        if wv == bv:
-            raise ValueError("best_value and worst_value cannot be equal")
-        if wv < bv and not wv <= value <= bv:
-            raise ValueError(f"value {value} is not in range [{wv}, {bv}] (inclusive)")
-        if wv > bv and not wv >= value >= bv:
-            raise ValueError(f"value {value} is not in range [{bv}, {wv}] (inclusive)")
-
-        return values
-
-
-class SampleEvaluationFunction(Protocol, Generic[T_OutputDatum]):
-    @abstractmethod
-    def __call__(self, output_datum: T_OutputDatum) -> SampleMetricValue[T_OutputDatum]:
-        pass
-
 
 # Each test is a (input_datum, evaluation_fn) pair
 UserTestSuiteWithInputs = Sequence[Tuple[str, SampleEvaluationFunction[str]]]
@@ -74,50 +25,13 @@ def __call__(self, output_datum: T_OutputDatum) -> SampleMetricValue[T_OutputDat
 # Each test is a (output_datum, evaluation_fn) pair
 UserTestSuiteOutputsOnly = Sequence[Tuple[str, SampleEvaluationFunction[str]]]
 
-# 
+
+#
 class TestSuiteWithInputsSettings(cu.Record):
     prompt_name: str
     aiconfig_path: str
 
 
-def contains_substring(
-    output_datum: str, substring: str, case_sensitive: bool
-) -> SampleMetricValue[str]:
-    return SampleMetricValue(
-        value=float(
-            check_substring(output_datum, substring, case_sensitive=case_sensitive)
-        ),
-        interpretation=EvaluationMetricMetadata(
-            name="contains_substring",
-            description="1.0 (pass) if contains given substring",
-            best_value=1.0,
-            worst_value=0.0,
-        ),
-    )
-
-
-def substring_match(
-    substring: str, case_sensitive: bool = True
-) -> SampleEvaluationFunction[str]:
-    """Convenience function for running `contains_substring()` on a fixed substring.
-    Can be used directly to construct a user test suite."""
-    return partial(
-        contains_substring, substring=substring, case_sensitive=case_sensitive
-    )
-
-
-def brevity(output_datum: str) -> SampleMetricValue[str]:
-    return SampleMetricValue(
-        value=float(len(output_datum)),
-        interpretation=EvaluationMetricMetadata(
-            name="brevity",
-            description="Absolute text length",
-            best_value=1.0,
-            worst_value=float("inf"),
-        ),
-    )
-
-
 async def run_test_suite_with_inputs(
     test_suite: UserTestSuiteWithInputs,
     settings: TestSuiteWithInputsSettings,
@@ -135,8 +49,6 @@ async def run_test_suite_outputs_only(
     return res.unwrap_or_raise(ValueError)
 
 
-# Implementation
-
 T = TypeVar("T")
 
 
@@ -209,11 +121,13 @@ def eval_res_to_df(
                 value=sample_res.metric_value.value,
                 metric_name=sample_res.metric_value.interpretation.name,
                 metric_description=sample_res.metric_value.interpretation.description,
-                best_value=sample_res.metric_value.interpretation.best_value,
-                worst_value=sample_res.metric_value.interpretation.worst_value,
+                best_possible_value=sample_res.metric_value.interpretation.best_value,
+                worst_possible_value=sample_res.metric_value.interpretation.worst_value,
             )
         )
-    df = pd.DataFrame.from_records(records)  # type: ignore[no-untyped-call]
+    df = pd.DataFrame.from_records(records)  # type: ignore[no-untyped-call]    
+    if len(df) == 0:
+        return df
     for c in ["input", "aiconfig_output", "metric_name", "metric_description"]:
         df[c] = df[c].astype("string").fillna("Missing")  # type: ignore[no-untyped-call]
 
@@ -224,13 +138,12 @@ async def run_aiconfig(
     settings: TestSuiteWithInputsSettings, input_datum: TextInput
 ) -> TextOutput:
     """Helper to run the AIConfig which makes the data to be evaluated."""
-    # TODO: catch exceptions
     prompt_name = settings.prompt_name
     aiconfig_path = settings.aiconfig_path
+    aiconfig = AIConfigRuntime.load(aiconfig_path)  # type: ignore[fixme, no-untyped-call]
 
-    return TextOutput(
-        await run_aiconfig_helper(aiconfig_path, prompt_name, input_datum)
-    )
+    output = await run_aiconfig_helper(aiconfig, prompt_name, input_datum)
+    return output.map(TextOutput).unwrap()
 
 
 async def user_test_suite_with_inputs_to_eval_params_list(
@@ -282,24 +195,16 @@ def user_test_suite_outputs_only_to_eval_params_list(
 
 
 async def run_aiconfig_helper(
-    aiconfig_path: str, prompt_name: str, question: str
-) -> str:
-    runtime = AIConfigRuntime.load(aiconfig_path)  # type: ignore[fixme, no-untyped-call]
-
+    runtime: AIConfigRuntime, prompt_name: str, question: str
+) -> Result[str, str]:
     params = {
         "the_query": question,
     }
 
-    result: Any = await runtime.run(prompt_name, params, run_with_dependencies=True)  # type: ignore[fixme, no-untyped-call]
-    final_output = runtime.get_output_text(prompt_name, result[0])  # type: ignore[fixme, no-untyped-call]
-    return final_output
-
-
-def check_substring(output_datum: str, substring: str, case_sensitive: bool) -> bool:
-    if case_sensitive:
-        return substring in output_datum
-    else:
-        return substring.lower() in output_datum.lower()
+    try:
+        return Ok(await runtime.run_and_get_output_text(prompt_name, params, run_with_dependencies=True))  # type: ignore[fixme, no-untyped-call]
+    except Exception as e:
+        return cu.ErrWithTraceback(e)
 
 
 @dataclass(frozen=True)
diff --git a/python/src/aiconfig/eval/metrics.py b/python/src/aiconfig/eval/metrics.py
new file mode 100644
index 000000000..f61ed8158
--- /dev/null
+++ b/python/src/aiconfig/eval/metrics.py
@@ -0,0 +1,57 @@
+from functools import partial
+
+from aiconfig.eval.common import (
+    EvaluationMetricMetadata,
+    SampleEvaluationFunction,
+    SampleMetricValue,
+)
+
+
+def check_substring(output_datum: str, substring: str, case_sensitive: bool) -> bool:
+    if case_sensitive:
+        return substring in output_datum
+    else:
+        return substring.lower() in output_datum.lower()
+
+
+def contains_substring(
+    output_datum: str, substring: str, case_sensitive: bool
+) -> SampleMetricValue[str]:
+    return SampleMetricValue(
+        value=float(
+            check_substring(output_datum, substring, case_sensitive=case_sensitive)
+        ),
+        interpretation=EvaluationMetricMetadata(
+            name="contains_substring",
+            description="1.0 (pass) if contains given substring",
+            best_value=1.0,
+            worst_value=0.0,
+        ),
+    )
+
+
+def substring_match(
+    substring: str, case_sensitive: bool = True
+) -> SampleEvaluationFunction[str]:
+    """Convenience function for running `contains_substring()` on a fixed substring.
+    Can be used directly to construct a user test suite."""
+    return partial(
+        contains_substring, substring=substring, case_sensitive=case_sensitive
+    )
+
+
+def brevity(output_datum: str) -> SampleMetricValue[str]:
+    def _fn(output_datum: str) -> float:
+        if len(output_datum) == 0:
+            raise ValueError("Brevity is meaningless for empty string.")
+        return float(len(output_datum))
+
+    return SampleMetricValue(
+        value=_fn(output_datum),
+        interpretation=EvaluationMetricMetadata(
+            name="brevity",
+            description="Absolute text length",
+            best_value=1.0,
+            worst_value=float("inf"),
+        ),
+    )
diff --git a/python/src/aiconfig/eval/promptfoo/README.md b/python/src/aiconfig/eval/promptfoo/README.md
index 2b547bb1f..21de182c7 100644
--- a/python/src/aiconfig/eval/promptfoo/README.md
+++ b/python/src/aiconfig/eval/promptfoo/README.md
@@ -1,1 +1,1 @@
-Moved to python/src/aiconfig/eval/README.md.
+Moved to https://github.com/lastmile-ai/aiconfig/blob/main/python/src/aiconfig/eval/README.md
diff --git a/python/src/aiconfig/eval/promptfoo/examples/travel/travel_parametrized.aiconfig.json b/python/src/aiconfig/eval/promptfoo/examples/travel/travel_parametrized.aiconfig.json
deleted file mode 120000
index 32ec5ca0b..000000000
--- a/python/src/aiconfig/eval/promptfoo/examples/travel/travel_parametrized.aiconfig.json
+++ /dev/null
@@ -1,1 +0,0 @@
-../../../custom_eval/examples/travel/travel_parametrized.aiconfig.json
\ No newline at end of file
diff --git a/python/src/aiconfig/eval/promptfoo/run_aiconfig.py b/python/src/aiconfig/eval/promptfoo/run_aiconfig.py
deleted file mode 100644
index 14aca4e9c..000000000
--- a/python/src/aiconfig/eval/promptfoo/run_aiconfig.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import asyncio
-import json
-import os
-import sys
-from typing import Any
-
-import openai
-from dotenv import load_dotenv
-
-from aiconfig.eval.lib import run_aiconfig_helper
-
-
-async def main():
-    settings_path = sys.argv[1]
-    settings = _load_settings(settings_path)
-
-    question = sys.argv[2]
-    prompt_name = settings["prompt_name"]
-    aiconfig_path = settings["aiconfig_path"]
-
-    load_dotenv()
-    openai.api_key = os.getenv("OPENAI_API_KEY")
-    output = await run_aiconfig_helper(aiconfig_path, prompt_name, question)
-    print(output)
-
-
-def _load_settings(settings_path: str) -> dict[str, Any]:
-    with open(settings_path) as f:
-        settings = json.load(f)
-        return settings
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
diff --git a/python/src/aiconfig/scripts/run_aiconfig.py b/python/src/aiconfig/scripts/run_aiconfig.py
new file mode 100644
index 000000000..b047ddf05
--- /dev/null
+++ b/python/src/aiconfig/scripts/run_aiconfig.py
@@ -0,0 +1,69 @@
+import asyncio
+import logging
+import sys
+from typing import IO
+
+from result import Ok, Result
+
+
+from aiconfig.Config import AIConfigRuntime
+from aiconfig.eval.lib import run_aiconfig_helper
+import lastmile_utils.lib.core.api as cu
+
+logging.basicConfig(format=cu.LOGGER_FMT)
+LOGGER = logging.getLogger(__name__)
+
+
+class Settings(cu.Record):
+    prompt_name: str
+    aiconfig_path: str
+
+
+async def main():
+    settings_path = sys.argv[1]
+    res_settings = _load_settings(settings_path)
+
+    question = sys.argv[2]
+    # prompt_name = settings.prompt_name
+    # aiconfig_path = settings.aiconfig_path
+
+    def _load_aiconfig(settings: Settings) -> Result[AIConfigRuntime, str]:
+        try:
+            return Ok(AIConfigRuntime.load(settings.aiconfig_path))  # type: ignore[no-untyped-call]
+        except ValueError as e:
+            return cu.ErrWithTraceback(e)
+
+    # TODO: Need do_async for a different reason: the async expression can't be defined
+    # before the do because it's a function of the stuff unwrapped in the do.
+    settings = res_settings.unwrap()
+    res_aiconfig = _load_aiconfig(settings)
+    aiconfig = res_aiconfig.unwrap()
+    final_value = await run_aiconfig_helper(
+        runtime=aiconfig, prompt_name=settings.prompt_name, question=question
+    )
+
+    print(final_value.unwrap_or_else(str))
+
+    LOGGER.info("Final value: %s", final_value)
+
+
+def _load_settings(settings_path: str) -> Result[Settings, str]:
+    def pydantic_model_validate_from_json_file_handle(
+        f_handle: IO[str],
+    ) -> Result[Settings, str]:
+        def settings_model_validate_json(s: str) -> Result[Settings, str]:
+            try:
+                return Ok(Settings.model_validate_json(s))
+            except ValueError as e:
+                return cu.ErrWithTraceback(e)
+
+        return cu.read_file_from_handle(f_handle).and_then(settings_model_validate_json)
+
+    path_fn = cu.safe_file_io_decorator(
+        pydantic_model_validate_from_json_file_handle, mode="r"
+    )
+    return path_fn(settings_path)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())


Test patch:
diff --git a/python/src/aiconfig/eval/promptfoo/examples/travel/travel_aiconfig_test_suite_settings.json b/python/src/aiconfig/eval/examples/travel/travel_aiconfig_test_suite_settings.json
similarity index 100%
rename from python/src/aiconfig/eval/promptfoo/examples/travel/travel_aiconfig_test_suite_settings.json
rename to python/src/aiconfig/eval/examples/travel/travel_aiconfig_test_suite_settings.json
diff --git a/python/tests/test_eval.py b/python/tests/test_eval.py
new file mode 100644
index 000000000..f523e935b
--- /dev/null
+++ b/python/tests/test_eval.py
@@ -0,0 +1,83 @@
+import hypothesis.strategies as st
+import lastmile_utils.lib.core.api as cu
+import pandas as pd
+import pytest
+from result import Ok
+from aiconfig.eval.api import (
+    TestSuiteWithInputsSettings,
+    brevity,
+    run_test_suite_with_inputs,
+    substring_match,
+)
+from aiconfig.eval.lib import run_test_suite_outputs_only
+from hypothesis import given
+
+
+def test_metrics():
+    assert brevity("hello").value == 5.0
+
+    assert substring_match("lo w")("hello world").value == 1.0
+    assert substring_match("hello", case_sensitive=False)("HELLO world").value == 1.0
+    assert substring_match("hello", case_sensitive=True)("HELLO world").value == 0.0
+
+
+@pytest.mark.asyncio
+async def test_run_with_inputs_sanity_check():
+    """No easy way to mock LLM calls,
+    so just test the imports and sanity check output."""
+    out = await run_test_suite_with_inputs(
+        [],
+        TestSuiteWithInputsSettings(
+            prompt_name="test", aiconfig_path="some/path/we/wont/read"
+        ),
+    )
+    assert isinstance(out, pd.DataFrame)
+    assert out.shape == (0, 0)
+
+
+@pytest.mark.asyncio
+async def test_run_with_outputs_only_basic():
+    test_suite = [
+        ("hello world", brevity),
+        ("hello world", substring_match("hello")),
+    ]
+    out = await run_test_suite_outputs_only(test_suite)
+    exp = pd.DataFrame(
+        {
+            "value": [11.0, 1.0],
+        }
+    )
+    assert out["value"].equals(exp["value"])  # type: ignore
+
+
+@given(st.data())
+@pytest.mark.asyncio
+async def test_run_test_suite_outputs_only(data: st.DataObject):
+    Metrics = [brevity, substring_match("hello")]
+    TestPairs = st.tuples(st.text(min_size=1), st.sampled_from(Metrics))
+    user_test_suite_outputs_only = data.draw(
+        st.lists(
+            TestPairs,
+            min_size=1,
+        )
+    )
+
+    out = await run_test_suite_outputs_only(user_test_suite_outputs_only)
+    assert out.shape[0] == (len(user_test_suite_outputs_only))
+    assert out.columns.tolist() == [
+        "input",
+        "aiconfig_output",
+        "value",
+        "metric_name",
+        "metric_description",
+        "best_possible_value",
+        "worst_possible_value",
+    ]
+    inputs = out["input"].astype(str).tolist()  # type: ignore[no-untyped-call]
+    assert cu.only(inputs) == Ok("Missing")
+
+    out_brevity = out[out["metric_name"] == "brevity"]
+    assert (
+        out_brevity["aiconfig_output"].apply(len)  # type: ignore[no-untyped-call]
+        == out_brevity["value"]  # type: ignore[no-untyped-call]
+    ).all()