Repository: lastmile-ai/aiconfig (language: python)

Issue:
[AIC-py][eval] bundle of small easy fixes pt 2
[AIC-py][eval] bundle of small easy fixes pt 2

- finish wiring up eval fn timeout reasonably to user
- make `common` imports consistent
- fix small user config bug in the example notebook

---
Stack created with [Sapling](https://sapling-scm.com). Best reviewed with [ReviewStack](https://reviewstack.dev/lastmile-ai/aiconfig/pull/547).
* #550
* #549
* #548
* __->__ #547


Discussion / hints:

Fix patch:
diff --git a/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
index e8f97daec..91168e1be 100644
--- a/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
+++ b/python/src/aiconfig/eval/examples/travel/travel_eval.ipynb
@@ -30,14 +30,303 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "lastmile-utils               0.0.9\n"
+      "\u001b[33mWARNING: Ignoring invalid distribution -etuptools (/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0mCollecting lastmile-utils\n",
+      "  Using cached lastmile_utils-0.0.13-py3-none-any.whl.metadata (901 bytes)\n",
+      "Collecting black==23.11.0 (from lastmile-utils)\n",
+      "  Using cached black-23.11.0-cp310-cp310-macosx_11_0_arm64.whl.metadata (66 kB)\n",
+      "Collecting chardet==5.2.0 (from lastmile-utils)\n",
+      "  Using cached chardet-5.2.0-py3-none-any.whl.metadata (3.4 kB)\n",
+      "Collecting flake8==6.1.0 (from lastmile-utils)\n",
+      "  Using cached flake8-6.1.0-py2.py3-none-any.whl.metadata (3.8 kB)\n",
+      "Collecting isort==5.12.0 (from lastmile-utils)\n",
+      "  Using cached isort-5.12.0-py3-none-any.whl (91 kB)\n",
+      "Collecting jsoncomment==0.4.2 (from lastmile-utils)\n",
+      "  Using cached jsoncomment-0.4.2-py3-none-any.whl (6.8 kB)\n",
+      "Collecting pandas==2.1.2 (from lastmile-utils)\n",
+      "  Using cached pandas-2.1.2-cp310-cp310-macosx_11_0_arm64.whl.metadata (18 kB)\n",
+      "Collecting pydantic==2.4.2 (from lastmile-utils)\n",
+      "  Using cached pydantic-2.4.2-py3-none-any.whl.metadata (158 kB)\n",
+      "Collecting pylint==3.0.2 (from lastmile-utils)\n",
+      "  Using cached pylint-3.0.2-py3-none-any.whl.metadata (12 kB)\n",
+      "Collecting pyright==1.1.335 (from lastmile-utils)\n",
+      "  Using cached pyright-1.1.335-py3-none-any.whl.metadata (5.7 kB)\n",
+      "Collecting pytest==7.4.3 (from lastmile-utils)\n",
+      "  Using cached pytest-7.4.3-py3-none-any.whl.metadata (7.9 kB)\n",
+      "Collecting python-dotenv==1.0.0 (from lastmile-utils)\n",
+      "  Using cached python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
+      "Collecting result==0.15.0 (from lastmile-utils)\n",
+      "  Using cached result-0.15.0-py3-none-any.whl.metadata (12 kB)\n",
+      "Collecting autoflake==2.2.1 (from lastmile-utils)\n",
+      "  Using cached autoflake-2.2.1-py3-none-any.whl.metadata (7.3 kB)\n",
+      "Collecting pyflakes>=3.0.0 (from autoflake==2.2.1->lastmile-utils)\n",
+      "  Using cached pyflakes-3.1.0-py2.py3-none-any.whl.metadata (3.5 kB)\n",
+      "Collecting tomli>=2.0.1 (from autoflake==2.2.1->lastmile-utils)\n",
+      "  Using cached tomli-2.0.1-py3-none-any.whl (12 kB)\n",
+      "Collecting click>=8.0.0 (from black==23.11.0->lastmile-utils)\n",
+      "  Using cached click-8.1.7-py3-none-any.whl.metadata (3.0 kB)\n",
+      "Collecting mypy-extensions>=0.4.3 (from black==23.11.0->lastmile-utils)\n",
+      "  Using cached mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
+      "Collecting packaging>=22.0 (from black==23.11.0->lastmile-utils)\n",
+      "  Using cached packaging-23.2-py3-none-any.whl.metadata (3.2 kB)\n",
+      "Collecting pathspec>=0.9.0 (from black==23.11.0->lastmile-utils)\n",
+      "  Using cached pathspec-0.12.1-py3-none-any.whl.metadata (21 kB)\n",
+      "Collecting platformdirs>=2 (from black==23.11.0->lastmile-utils)\n",
+      "  Using cached platformdirs-4.1.0-py3-none-any.whl.metadata (11 kB)\n",
+      "Collecting typing-extensions>=4.0.1 (from black==23.11.0->lastmile-utils)\n",
+      "  Using cached typing_extensions-4.9.0-py3-none-any.whl.metadata (3.0 kB)\n",
+      "Collecting mccabe<0.8.0,>=0.7.0 (from flake8==6.1.0->lastmile-utils)\n",
+      "  Using cached mccabe-0.7.0-py2.py3-none-any.whl (7.3 kB)\n",
+      "Collecting pycodestyle<2.12.0,>=2.11.0 (from flake8==6.1.0->lastmile-utils)\n",
+      "  Using cached pycodestyle-2.11.1-py2.py3-none-any.whl.metadata (4.5 kB)\n",
+      "Collecting json-spec (from jsoncomment==0.4.2->lastmile-utils)\n",
+      "  Using cached json_spec-0.11.0-py3-none-any.whl (41 kB)\n",
+      "Collecting numpy<2,>=1.22.4 (from pandas==2.1.2->lastmile-utils)\n",
+      "  Using cached numpy-1.26.2-cp310-cp310-macosx_11_0_arm64.whl.metadata (61 kB)\n",
+      "Collecting python-dateutil>=2.8.2 (from pandas==2.1.2->lastmile-utils)\n",
+      "  Using cached python_dateutil-2.8.2-py2.py3-none-any.whl (247 kB)\n",
+      "Collecting pytz>=2020.1 (from pandas==2.1.2->lastmile-utils)\n",
+      "  Using cached pytz-2023.3.post1-py2.py3-none-any.whl.metadata (22 kB)\n",
+      "Collecting tzdata>=2022.1 (from pandas==2.1.2->lastmile-utils)\n",
+      "  Using cached tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
+      "Collecting annotated-types>=0.4.0 (from pydantic==2.4.2->lastmile-utils)\n",
+      "  Using cached annotated_types-0.6.0-py3-none-any.whl.metadata (12 kB)\n",
+      "Collecting pydantic-core==2.10.1 (from pydantic==2.4.2->lastmile-utils)\n",
+      "  Using cached pydantic_core-2.10.1-cp310-cp310-macosx_11_0_arm64.whl.metadata (6.5 kB)\n",
+      "Collecting astroid<=3.1.0-dev0,>=3.0.1 (from pylint==3.0.2->lastmile-utils)\n",
+      "  Using cached astroid-3.0.2-py3-none-any.whl.metadata (4.5 kB)\n",
+      "Collecting tomlkit>=0.10.1 (from pylint==3.0.2->lastmile-utils)\n",
+      "  Using cached tomlkit-0.12.3-py3-none-any.whl.metadata (2.7 kB)\n",
+      "Collecting dill>=0.2 (from pylint==3.0.2->lastmile-utils)\n",
+      "  Using cached dill-0.3.7-py3-none-any.whl.metadata (9.9 kB)\n",
+      "Collecting nodeenv>=1.6.0 (from pyright==1.1.335->lastmile-utils)\n",
+      "  Using cached nodeenv-1.8.0-py2.py3-none-any.whl.metadata (21 kB)\n",
+      "Collecting iniconfig (from pytest==7.4.3->lastmile-utils)\n",
+      "  Using cached iniconfig-2.0.0-py3-none-any.whl (5.9 kB)\n",
+      "Collecting pluggy<2.0,>=0.12 (from pytest==7.4.3->lastmile-utils)\n",
+      "  Using cached pluggy-1.3.0-py3-none-any.whl.metadata (4.3 kB)\n",
+      "Collecting exceptiongroup>=1.0.0rc8 (from pytest==7.4.3->lastmile-utils)\n",
+      "  Using cached exceptiongroup-1.2.0-py3-none-any.whl.metadata (6.6 kB)\n",
+      "Collecting setuptools (from nodeenv>=1.6.0->pyright==1.1.335->lastmile-utils)\n",
+      "  Using cached setuptools-69.0.2-py3-none-any.whl.metadata (6.3 kB)\n",
+      "Collecting six>=1.5 (from python-dateutil>=2.8.2->pandas==2.1.2->lastmile-utils)\n",
+      "  Using cached six-1.16.0-py2.py3-none-any.whl (11 kB)\n",
+      "Collecting importlib-metadata<6.0.0,>=5.0.0 (from json-spec->jsoncomment==0.4.2->lastmile-utils)\n",
+      "  Using cached importlib_metadata-5.2.0-py3-none-any.whl (21 kB)\n",
+      "Collecting zipp>=0.5 (from importlib-metadata<6.0.0,>=5.0.0->json-spec->jsoncomment==0.4.2->lastmile-utils)\n",
+      "  Using cached zipp-3.17.0-py3-none-any.whl.metadata (3.7 kB)\n",
+      "Using cached lastmile_utils-0.0.13-py3-none-any.whl (14 kB)\n",
+      "Using cached autoflake-2.2.1-py3-none-any.whl (32 kB)\n",
+      "Using cached black-23.11.0-cp310-cp310-macosx_11_0_arm64.whl (1.4 MB)\n",
+      "Using cached chardet-5.2.0-py3-none-any.whl (199 kB)\n",
+      "Using cached flake8-6.1.0-py2.py3-none-any.whl (58 kB)\n",
+      "Using cached pandas-2.1.2-cp310-cp310-macosx_11_0_arm64.whl (10.9 MB)\n",
+      "Using cached pydantic-2.4.2-py3-none-any.whl (395 kB)\n",
+      "Using cached pylint-3.0.2-py3-none-any.whl (510 kB)\n",
+      "Using cached pyright-1.1.335-py3-none-any.whl (17 kB)\n",
+      "Using cached pytest-7.4.3-py3-none-any.whl (325 kB)\n",
+      "Using cached result-0.15.0-py3-none-any.whl (10 kB)\n",
+      "Using cached pydantic_core-2.10.1-cp310-cp310-macosx_11_0_arm64.whl (1.7 MB)\n",
+      "Using cached annotated_types-0.6.0-py3-none-any.whl (12 kB)\n",
+      "Using cached astroid-3.0.2-py3-none-any.whl (275 kB)\n",
+      "Using cached click-8.1.7-py3-none-any.whl (97 kB)\n",
+      "Using cached dill-0.3.7-py3-none-any.whl (115 kB)\n",
+      "Using cached exceptiongroup-1.2.0-py3-none-any.whl (16 kB)\n",
+      "Using cached nodeenv-1.8.0-py2.py3-none-any.whl (22 kB)\n",
+      "Using cached numpy-1.26.2-cp310-cp310-macosx_11_0_arm64.whl (14.0 MB)\n",
+      "Using cached packaging-23.2-py3-none-any.whl (53 kB)\n",
+      "Using cached pathspec-0.12.1-py3-none-any.whl (31 kB)\n",
+      "Using cached platformdirs-4.1.0-py3-none-any.whl (17 kB)\n",
+      "Using cached pluggy-1.3.0-py3-none-any.whl (18 kB)\n",
+      "Using cached pycodestyle-2.11.1-py2.py3-none-any.whl (31 kB)\n",
+      "Using cached pyflakes-3.1.0-py2.py3-none-any.whl (62 kB)\n",
+      "Using cached pytz-2023.3.post1-py2.py3-none-any.whl (502 kB)\n",
+      "Using cached tomlkit-0.12.3-py3-none-any.whl (37 kB)\n",
+      "Using cached typing_extensions-4.9.0-py3-none-any.whl (32 kB)\n",
+      "Using cached setuptools-69.0.2-py3-none-any.whl (819 kB)\n",
+      "Using cached zipp-3.17.0-py3-none-any.whl (7.4 kB)\n",
+      "\u001b[33mWARNING: Ignoring invalid distribution -etuptools (/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0mInstalling collected packages: pytz, zipp, tzdata, typing-extensions, tomlkit, tomli, six, setuptools, result, python-dotenv, pyflakes, pycodestyle, pluggy, platformdirs, pathspec, packaging, numpy, mypy-extensions, mccabe, isort, iniconfig, exceptiongroup, dill, click, chardet, annotated-types, python-dateutil, pytest, pydantic-core, nodeenv, importlib-metadata, flake8, black, autoflake, astroid, pyright, pylint, pydantic, pandas, json-spec, jsoncomment, lastmile-utils\n",
+      "  Attempting uninstall: pytz\n",
+      "    Found existing installation: pytz 2023.3.post1\n",
+      "    Uninstalling pytz-2023.3.post1:\n",
+      "      Successfully uninstalled pytz-2023.3.post1\n",
+      "  Attempting uninstall: zipp\n",
+      "    Found existing installation: zipp 3.17.0\n",
+      "    Uninstalling zipp-3.17.0:\n",
+      "      Successfully uninstalled zipp-3.17.0\n",
+      "  Attempting uninstall: tzdata\n",
+      "    Found existing installation: tzdata 2023.3\n",
+      "    Uninstalling tzdata-2023.3:\n",
+      "      Successfully uninstalled tzdata-2023.3\n",
+      "  Attempting uninstall: typing-extensions\n",
+      "    Found existing installation: typing_extensions 4.9.0\n",
+      "    Uninstalling typing_extensions-4.9.0:\n",
+      "      Successfully uninstalled typing_extensions-4.9.0\n",
+      "  Attempting uninstall: tomlkit\n",
+      "    Found existing installation: tomlkit 0.12.3\n",
+      "    Uninstalling tomlkit-0.12.3:\n",
+      "      Successfully uninstalled tomlkit-0.12.3\n",
+      "  Attempting uninstall: tomli\n",
+      "    Found existing installation: tomli 2.0.1\n",
+      "    Uninstalling tomli-2.0.1:\n",
+      "      Successfully uninstalled tomli-2.0.1\n",
+      "  Attempting uninstall: six\n",
+      "    Found existing installation: six 1.16.0\n",
+      "    Uninstalling six-1.16.0:\n",
+      "      Successfully uninstalled six-1.16.0\n",
+      "  Attempting uninstall: setuptools\n",
+      "    Found existing installation: setuptools 69.0.2\n",
+      "    Uninstalling setuptools-69.0.2:\n",
+      "      Successfully uninstalled setuptools-69.0.2\n",
+      "  Attempting uninstall: result\n",
+      "    Found existing installation: result 0.15.0\n",
+      "    Uninstalling result-0.15.0:\n",
+      "      Successfully uninstalled result-0.15.0\n",
+      "  Attempting uninstall: python-dotenv\n",
+      "    Found existing installation: python-dotenv 1.0.0\n",
+      "    Uninstalling python-dotenv-1.0.0:\n",
+      "      Successfully uninstalled python-dotenv-1.0.0\n",
+      "  Attempting uninstall: pyflakes\n",
+      "    Found existing installation: pyflakes 3.1.0\n",
+      "    Uninstalling pyflakes-3.1.0:\n",
+      "      Successfully uninstalled pyflakes-3.1.0\n",
+      "  Attempting uninstall: pycodestyle\n",
+      "    Found existing installation: pycodestyle 2.11.1\n",
+      "    Uninstalling pycodestyle-2.11.1:\n",
+      "      Successfully uninstalled pycodestyle-2.11.1\n",
+      "  Attempting uninstall: pluggy\n",
+      "    Found existing installation: pluggy 1.3.0\n",
+      "    Uninstalling pluggy-1.3.0:\n",
+      "      Successfully uninstalled pluggy-1.3.0\n",
+      "  Attempting uninstall: platformdirs\n",
+      "    Found existing installation: platformdirs 4.1.0\n",
+      "    Uninstalling platformdirs-4.1.0:\n",
+      "      Successfully uninstalled platformdirs-4.1.0\n",
+      "  Attempting uninstall: pathspec\n",
+      "    Found existing installation: pathspec 0.12.1\n",
+      "    Uninstalling pathspec-0.12.1:\n",
+      "      Successfully uninstalled pathspec-0.12.1\n",
+      "  Attempting uninstall: packaging\n",
+      "    Found existing installation: packaging 23.2\n",
+      "    Uninstalling packaging-23.2:\n",
+      "      Successfully uninstalled packaging-23.2\n",
+      "  Attempting uninstall: numpy\n",
+      "    Found existing installation: numpy 1.26.2\n",
+      "    Uninstalling numpy-1.26.2:\n",
+      "      Successfully uninstalled numpy-1.26.2\n",
+      "  Attempting uninstall: mypy-extensions\n",
+      "    Found existing installation: mypy-extensions 1.0.0\n",
+      "    Uninstalling mypy-extensions-1.0.0:\n",
+      "      Successfully uninstalled mypy-extensions-1.0.0\n",
+      "  Attempting uninstall: mccabe\n",
+      "    Found existing installation: mccabe 0.7.0\n",
+      "    Uninstalling mccabe-0.7.0:\n",
+      "      Successfully uninstalled mccabe-0.7.0\n",
+      "  Attempting uninstall: isort\n",
+      "    Found existing installation: isort 5.12.0\n",
+      "    Uninstalling isort-5.12.0:\n",
+      "      Successfully uninstalled isort-5.12.0\n",
+      "  Attempting uninstall: iniconfig\n",
+      "    Found existing installation: iniconfig 2.0.0\n",
+      "    Uninstalling iniconfig-2.0.0:\n",
+      "      Successfully uninstalled iniconfig-2.0.0\n",
+      "  Attempting uninstall: exceptiongroup\n",
+      "    Found existing installation: exceptiongroup 1.2.0\n",
+      "    Uninstalling exceptiongroup-1.2.0:\n",
+      "      Successfully uninstalled exceptiongroup-1.2.0\n",
+      "  Attempting uninstall: dill\n",
+      "    Found existing installation: dill 0.3.7\n",
+      "    Uninstalling dill-0.3.7:\n",
+      "      Successfully uninstalled dill-0.3.7\n",
+      "  Attempting uninstall: click\n",
+      "    Found existing installation: click 8.1.7\n",
+      "    Uninstalling click-8.1.7:\n",
+      "      Successfully uninstalled click-8.1.7\n",
+      "  Attempting uninstall: chardet\n",
+      "    Found existing installation: chardet 5.2.0\n",
+      "    Uninstalling chardet-5.2.0:\n",
+      "      Successfully uninstalled chardet-5.2.0\n",
+      "  Attempting uninstall: annotated-types\n",
+      "    Found existing installation: annotated-types 0.6.0\n",
+      "    Uninstalling annotated-types-0.6.0:\n",
+      "      Successfully uninstalled annotated-types-0.6.0\n",
+      "  Attempting uninstall: python-dateutil\n",
+      "    Found existing installation: python-dateutil 2.8.2\n",
+      "    Uninstalling python-dateutil-2.8.2:\n",
+      "      Successfully uninstalled python-dateutil-2.8.2\n",
+      "  Attempting uninstall: pytest\n",
+      "    Found existing installation: pytest 7.4.3\n",
+      "    Uninstalling pytest-7.4.3:\n",
+      "      Successfully uninstalled pytest-7.4.3\n",
+      "  Attempting uninstall: pydantic-core\n",
+      "    Found existing installation: pydantic_core 2.10.1\n",
+      "    Uninstalling pydantic_core-2.10.1:\n",
+      "      Successfully uninstalled pydantic_core-2.10.1\n",
+      "  Attempting uninstall: nodeenv\n",
+      "    Found existing installation: nodeenv 1.8.0\n",
+      "    Uninstalling nodeenv-1.8.0:\n",
+      "      Successfully uninstalled nodeenv-1.8.0\n",
+      "  Attempting uninstall: importlib-metadata\n",
+      "    Found existing installation: importlib-metadata 5.2.0\n",
+      "    Uninstalling importlib-metadata-5.2.0:\n",
+      "      Successfully uninstalled importlib-metadata-5.2.0\n",
+      "  Attempting uninstall: flake8\n",
+      "    Found existing installation: flake8 6.1.0\n",
+      "    Uninstalling flake8-6.1.0:\n",
+      "      Successfully uninstalled flake8-6.1.0\n",
+      "  Attempting uninstall: black\n",
+      "    Found existing installation: black 23.11.0\n",
+      "    Uninstalling black-23.11.0:\n",
+      "      Successfully uninstalled black-23.11.0\n",
+      "  Attempting uninstall: autoflake\n",
+      "    Found existing installation: autoflake 2.2.1\n",
+      "    Uninstalling autoflake-2.2.1:\n",
+      "      Successfully uninstalled autoflake-2.2.1\n",
+      "  Attempting uninstall: astroid\n",
+      "    Found existing installation: astroid 3.0.2\n",
+      "    Uninstalling astroid-3.0.2:\n",
+      "      Successfully uninstalled astroid-3.0.2\n",
+      "  Attempting uninstall: pyright\n",
+      "    Found existing installation: pyright 1.1.335\n",
+      "    Uninstalling pyright-1.1.335:\n",
+      "      Successfully uninstalled pyright-1.1.335\n",
+      "  Attempting uninstall: pylint\n",
+      "    Found existing installation: pylint 3.0.2\n",
+      "    Uninstalling pylint-3.0.2:\n",
+      "      Successfully uninstalled pylint-3.0.2\n",
+      "  Attempting uninstall: pydantic\n",
+      "    Found existing installation: pydantic 2.4.2\n",
+      "    Uninstalling pydantic-2.4.2:\n",
+      "      Successfully uninstalled pydantic-2.4.2\n",
+      "  Attempting uninstall: pandas\n",
+      "    Found existing installation: pandas 2.1.2\n",
+      "    Uninstalling pandas-2.1.2:\n",
+      "      Successfully uninstalled pandas-2.1.2\n",
+      "  Attempting uninstall: json-spec\n",
+      "    Found existing installation: json-spec 0.11.0\n",
+      "    Uninstalling json-spec-0.11.0:\n",
+      "      Successfully uninstalled json-spec-0.11.0\n",
+      "  Attempting uninstall: jsoncomment\n",
+      "    Found existing installation: jsoncomment 0.4.2\n",
+      "    Uninstalling jsoncomment-0.4.2:\n",
+      "      Successfully uninstalled jsoncomment-0.4.2\n",
+      "  Attempting uninstall: lastmile-utils\n",
+      "    Found existing installation: lastmile-utils 0.0.13\n",
+      "    Uninstalling lastmile-utils-0.0.13:\n",
+      "      Successfully uninstalled lastmile-utils-0.0.13\n",
+      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+      "python-aiconfig 1.1.7 requires lastmile-utils==0.0.10, but you have lastmile-utils 0.0.13 which is incompatible.\u001b[0m\u001b[31m\n",
+      "\u001b[0mSuccessfully installed annotated-types-0.6.0 astroid-3.0.2 autoflake-2.2.1 black-23.11.0 chardet-5.2.0 click-8.1.7 dill-0.3.7 exceptiongroup-1.2.0 flake8-6.1.0 importlib-metadata-5.2.0 iniconfig-2.0.0 isort-5.12.0 json-spec-0.11.0 jsoncomment-0.4.2 lastmile-utils-0.0.13 mccabe-0.7.0 mypy-extensions-1.0.0 nodeenv-1.8.0 numpy-1.26.2 packaging-23.2 pandas-2.1.2 pathspec-0.12.1 platformdirs-4.1.0 pluggy-1.3.0 pycodestyle-2.11.1 pydantic-2.4.2 pydantic-core-2.10.1 pyflakes-3.1.0 pylint-3.0.2 pyright-1.1.335 pytest-7.4.3 python-dateutil-2.8.2 python-dotenv-1.0.0 pytz-2023.3.post1 result-0.15.0 setuptools-69.0.2 six-1.16.0 tomli-2.0.1 tomlkit-0.12.3 typing-extensions-4.9.0 tzdata-2023.3 zipp-3.17.0\n",
+      "\u001b[33mWARNING: Ignoring invalid distribution -etuptools (/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0mlastmile-utils               0.0.13\n"
      ]
     }
    ],
@@ -61,7 +350,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
@@ -114,7 +403,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
@@ -176,12 +465,16 @@
     "    test_suite_with_inputs.append((test_input, test_fn1))\n",
     "    # Add substring check function\n",
     "    test_fn2 = metrics.substring_match(substring, case_sensitive=False)\n",
-    "    test_suite_with_inputs.append((test_input, test_fn2))"
+    "    test_suite_with_inputs.append((test_input, test_fn2))\n",
+    "\n",
+    "    # Add a model-graded eval metric that uses GPT 3.5 to return a struct\n",
+    "    test_fn3 = metrics.gpt3_5_text_ratings\n",
+    "    test_suite_with_inputs.append((test_input, test_fn3))"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -193,19 +486,19 @@
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x1696a5a20>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": Infinity,\n",
+      "  \"best_value\": 1,\n",
+      "  \"worst_value\": 9223372036854775807,\n",
       "  \"extra_metadata\": {},\n",
-      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "  \"id\": \"24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3\"\n",
       "}))\n",
       "\n",
       "Test input:\n",
       " different kinds of cuisines \n",
       "Function:\n",
-      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x1036cfac0>, metric_metadata=EvaluationMetricMetadata({\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x1696a6cb0>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
       "  \"description\": \"True (pass) if contains given substring\",\n",
       "  \"best_value\": true,\n",
@@ -218,21 +511,37 @@
       "}))\n",
       "\n",
       "Test input:\n",
+      " different kinds of cuisines \n",
+      "Function:\n",
+      " Metric(evaluation_fn=<function make_structured_llm_metric.<locals>._make_evaluation_fn.<locals>._evaluation_fn at 0x169640430>, metric_metadata=EvaluationMetricMetadata({\n",
+      "  \"name\": \"text_ratings\",\n",
+      "  \"description\": \"Text ratings\",\n",
+      "  \"best_value\": null,\n",
+      "  \"worst_value\": null,\n",
+      "  \"extra_metadata\": {\n",
+      "    \"basemodel_type_name\": \"TextRatingsData\",\n",
+      "    \"eval_llm_name\": \"gpt-3.5-turbo-0613\",\n",
+      "    \"field_descriptions_json\": \"{\\\"conciseness_confidence\\\": \\\"0 to 1.0 rating of confidence in conciseness rating\\\", \\\"conciseness_rating\\\": \\\"1 to 5 rating of conciseness\\\", \\\"conciseness_reasoning\\\": \\\"reasoning behind the conciseness rating\\\"}\"\n",
+      "  },\n",
+      "  \"id\": \"300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7\"\n",
+      "}))\n",
+      "\n",
+      "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x1696a5a20>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": Infinity,\n",
+      "  \"best_value\": 1,\n",
+      "  \"worst_value\": 9223372036854775807,\n",
       "  \"extra_metadata\": {},\n",
-      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "  \"id\": \"24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3\"\n",
       "}))\n",
       "\n",
       "Test input:\n",
       " iconic midtown skyscrapers \n",
       "Function:\n",
-      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x144090e50>, metric_metadata=EvaluationMetricMetadata({\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x13fde56c0>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
       "  \"description\": \"True (pass) if contains given substring\",\n",
       "  \"best_value\": true,\n",
@@ -242,6 +551,22 @@
       "    \"case_sensitive\": false\n",
       "  },\n",
       "  \"id\": \"53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f\"\n",
+      "}))\n",
+      "\n",
+      "Test input:\n",
+      " iconic midtown skyscrapers \n",
+      "Function:\n",
+      " Metric(evaluation_fn=<function make_structured_llm_metric.<locals>._make_evaluation_fn.<locals>._evaluation_fn at 0x169640430>, metric_metadata=EvaluationMetricMetadata({\n",
+      "  \"name\": \"text_ratings\",\n",
+      "  \"description\": \"Text ratings\",\n",
+      "  \"best_value\": null,\n",
+      "  \"worst_value\": null,\n",
+      "  \"extra_metadata\": {\n",
+      "    \"basemodel_type_name\": \"TextRatingsData\",\n",
+      "    \"eval_llm_name\": \"gpt-3.5-turbo-0613\",\n",
+      "    \"field_descriptions_json\": \"{\\\"conciseness_confidence\\\": \\\"0 to 1.0 rating of confidence in conciseness rating\\\", \\\"conciseness_rating\\\": \\\"1 to 5 rating of conciseness\\\", \\\"conciseness_reasoning\\\": \\\"reasoning behind the conciseness rating\\\"}\"\n",
+      "  },\n",
+      "  \"id\": \"300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7\"\n",
       "}))\n"
      ]
     }
@@ -255,7 +580,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -301,22 +626,22 @@
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>1. Explore Chelsea Market's cuisine.\n",
-       "2. Visit Little Italy for authentic fare.\n",
-       "3. Savor Asian delicacies in Flushing, Queens.</td>\n",
-       "      <td>125</td>\n",
-       "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
+       "      <td>1. Explore Chelsea Market delicacies.\n",
+       "2. Embark on Korea Town food tour.\n",
+       "3. Savor Italian cuisine at Bronx's Arthur Avenue.</td>\n",
+       "      <td>123</td>\n",
+       "      <td>24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>inf</td>\n",
+       "      <td>1</td>\n",
+       "      <td>9223372036854775807</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>different kinds of cuisines</td>\n",
-       "      <td>1. Explore Chelsea Market's cuisine.\n",
-       "2. Visit Little Italy for authentic fare.\n",
-       "3. Savor Asian delicacies in Flushing, Queens.</td>\n",
+       "      <td>1. Explore Chelsea Market delicacies.\n",
+       "2. Embark on Korea Town food tour.\n",
+       "3. Savor Italian cuisine at Bronx's Arthur Avenue.</td>\n",
        "      <td>False</td>\n",
        "      <td>0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6</td>\n",
        "      <td>substring_match</td>\n",
@@ -326,23 +651,36 @@
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
+       "      <td>different kinds of cuisines</td>\n",
+       "      <td>1. Explore Chelsea Market delicacies.\n",
+       "2. Embark on Korea Town food tour.\n",
+       "3. Savor Italian cuisine at Bronx's Arthur Avenue.</td>\n",
+       "      <td>CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The text is concise and provides clear information about three different food destinations.\"\\n})</td>\n",
+       "      <td>300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7</td>\n",
+       "      <td>text_ratings</td>\n",
+       "      <td>Text ratings</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Empire State Building Observatory - 86th floor views.\n",
-       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
-       "3. One World Trade Center - One World Observatory visit.</td>\n",
-       "      <td>170</td>\n",
-       "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
+       "      <td>1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \n",
+       "2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\n",
+       "3. Chrysler Building: Admire Art Deco style from the street.</td>\n",
+       "      <td>235</td>\n",
+       "      <td>24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>inf</td>\n",
+       "      <td>1</td>\n",
+       "      <td>9223372036854775807</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>3</th>\n",
+       "      <th>4</th>\n",
        "      <td>iconic midtown skyscrapers</td>\n",
-       "      <td>1. Empire State Building Observatory - 86th floor views.\n",
-       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
-       "3. One World Trade Center - One World Observatory visit.</td>\n",
+       "      <td>1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \n",
+       "2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\n",
+       "3. Chrysler Building: Admire Art Deco style from the street.</td>\n",
        "      <td>True</td>\n",
        "      <td>53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f</td>\n",
        "      <td>substring_match</td>\n",
@@ -350,6 +688,19 @@
        "      <td>True</td>\n",
        "      <td>False</td>\n",
        "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>iconic midtown skyscrapers</td>\n",
+       "      <td>1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \n",
+       "2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\n",
+       "3. Chrysler Building: Admire Art Deco style from the street.</td>\n",
+       "      <td>CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The information is concise and provides a clear description of each attraction.\"\\n})</td>\n",
+       "      <td>300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7</td>\n",
+       "      <td>text_ratings</td>\n",
+       "      <td>Text ratings</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
@@ -358,43 +709,65 @@
        "                         input  \\\n",
        "0  different kinds of cuisines   \n",
        "1  different kinds of cuisines   \n",
-       "2   iconic midtown skyscrapers   \n",
+       "2  different kinds of cuisines   \n",
        "3   iconic midtown skyscrapers   \n",
+       "4   iconic midtown skyscrapers   \n",
+       "5   iconic midtown skyscrapers   \n",
        "\n",
-       "                                                                                                                                                              aiconfig_output  \\\n",
-       "0                                               1. Explore Chelsea Market's cuisine.\n",
-       "2. Visit Little Italy for authentic fare.\n",
-       "3. Savor Asian delicacies in Flushing, Queens.   \n",
-       "1                                               1. Explore Chelsea Market's cuisine.\n",
-       "2. Visit Little Italy for authentic fare.\n",
-       "3. Savor Asian delicacies in Flushing, Queens.   \n",
-       "2  1. Empire State Building Observatory - 86th floor views.\n",
-       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
-       "3. One World Trade Center - One World Observatory visit.   \n",
-       "3  1. Empire State Building Observatory - 86th floor views.\n",
-       "2. Rockefeller Center - Top of the Rock and ice skating.\n",
-       "3. One World Trade Center - One World Observatory visit.   \n",
+       "                                                                                                                                                                                                                               aiconfig_output  \\\n",
+       "0                                                                                                                  1. Explore Chelsea Market delicacies.\n",
+       "2. Embark on Korea Town food tour.\n",
+       "3. Savor Italian cuisine at Bronx's Arthur Avenue.   \n",
+       "1                                                                                                                  1. Explore Chelsea Market delicacies.\n",
+       "2. Embark on Korea Town food tour.\n",
+       "3. Savor Italian cuisine at Bronx's Arthur Avenue.   \n",
+       "2                                                                                                                  1. Explore Chelsea Market delicacies.\n",
+       "2. Embark on Korea Town food tour.\n",
+       "3. Savor Italian cuisine at Bronx's Arthur Avenue.   \n",
+       "3  1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \n",
+       "2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\n",
+       "3. Chrysler Building: Admire Art Deco style from the street.   \n",
+       "4  1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \n",
+       "2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\n",
+       "3. Chrysler Building: Admire Art Deco style from the street.   \n",
+       "5  1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \n",
+       "2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\n",
+       "3. Chrysler Building: Admire Art Deco style from the street.   \n",
        "\n",
-       "   value                                                         metric_id  \\\n",
-       "0    125  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "1  False  0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6   \n",
-       "2    170  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "3   True  53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f   \n",
+       "                                                                                                                                                                                                                           value  \\\n",
+       "0                                                                                                                                                                                                                            123   \n",
+       "1                                                                                                                                                                                                                          False   \n",
+       "2  CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The text is concise and provides clear information about three different food destinations.\"\\n})   \n",
+       "3                                                                                                                                                                                                                            235   \n",
+       "4                                                                                                                                                                                                                           True   \n",
+       "5              CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The information is concise and provides a clear description of each attraction.\"\\n})   \n",
+       "\n",
+       "                                                          metric_id  \\\n",
+       "0  24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3   \n",
+       "1  0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6   \n",
+       "2  300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7   \n",
+       "3  24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3   \n",
+       "4  53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f   \n",
+       "5  300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7   \n",
        "\n",
        "       metric_name                       metric_description  \\\n",
        "0          brevity                     Absolute text length   \n",
        "1  substring_match  True (pass) if contains given substring   \n",
-       "2          brevity                     Absolute text length   \n",
-       "3  substring_match  True (pass) if contains given substring   \n",
+       "2     text_ratings                             Text ratings   \n",
+       "3          brevity                     Absolute text length   \n",
+       "4  substring_match  True (pass) if contains given substring   \n",
+       "5     text_ratings                             Text ratings   \n",
        "\n",
        "  best_possible_value worst_possible_value  \n",
-       "0                 1.0                  inf  \n",
+       "0                   1  9223372036854775807  \n",
        "1                True                False  \n",
-       "2                 1.0                  inf  \n",
-       "3                True                False  "
+       "2                None                 None  \n",
+       "3                   1  9223372036854775807  \n",
+       "4                True                False  \n",
+       "5                None                 None  "
       ]
      },
-     "execution_count": 4,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -413,7 +786,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -447,44 +820,53 @@
        "      <th>metric_name</th>\n",
        "      <th>brevity</th>\n",
        "      <th>substring_match</th>\n",
+       "      <th>text_ratings</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>input</th>\n",
        "      <th>aiconfig_output</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
+       "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>different kinds of cuisines</th>\n",
-       "      <th>1. Explore Chelsea Market's cuisine.\\n2. Visit Little Italy for authentic fare.\\n3. Savor Asian delicacies in Flushing, Queens.</th>\n",
-       "      <td>125</td>\n",
+       "      <th>1. Explore Chelsea Market delicacies.\\n2. Embark on Korea Town food tour.\\n3. Savor Italian cuisine at Bronx's Arthur Avenue.</th>\n",
+       "      <td>123</td>\n",
        "      <td>False</td>\n",
+       "      <td>CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The text is concise and provides clear information about three different food destinations.\"\\n})</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>iconic midtown skyscrapers</th>\n",
-       "      <th>1. Empire State Building Observatory - 86th floor views.\\n2. Rockefeller Center - Top of the Rock and ice skating.\\n3. One World Trade Center - One World Observatory visit.</th>\n",
-       "      <td>170</td>\n",
+       "      <th>1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \\n2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\\n3. Chrysler Building: Admire Art Deco style from the street.</th>\n",
+       "      <td>235</td>\n",
        "      <td>True</td>\n",
+       "      <td>CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The information is concise and provides a clear description of each attraction.\"\\n})</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
-       "metric_name                                                                                                                                                                                              brevity  \\\n",
-       "input                       aiconfig_output                                                                                                                                                                        \n",
-       "different kinds of cuisines 1. Explore Chelsea Market's cuisine.\\n2. Visit Little Italy for authentic fare.\\n3. Savor Asian delicacies in Flushing, Queens.                                                  125   \n",
-       "iconic midtown skyscrapers  1. Empire State Building Observatory - 86th floor views.\\n2. Rockefeller Center - Top of the Rock and ice skating.\\n3. One World Trade Center - One World Observatory visit.     170   \n",
+       "metric_name                                                                                                                                                                                                                                                               brevity  \\\n",
+       "input                       aiconfig_output                                                                                                                                                                                                                                         \n",
+       "different kinds of cuisines 1. Explore Chelsea Market delicacies.\\n2. Embark on Korea Town food tour.\\n3. Savor Italian cuisine at Bronx's Arthur Avenue.                                                                                                                     123   \n",
+       "iconic midtown skyscrapers  1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \\n2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\\n3. Chrysler Building: Admire Art Deco style from the street.     235   \n",
+       "\n",
+       "metric_name                                                                                                                                                                                                                                                               substring_match  \\\n",
+       "input                       aiconfig_output                                                                                                                                                                                                                                                 \n",
+       "different kinds of cuisines 1. Explore Chelsea Market delicacies.\\n2. Embark on Korea Town food tour.\\n3. Savor Italian cuisine at Bronx's Arthur Avenue.                                                                                                                           False   \n",
+       "iconic midtown skyscrapers  1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \\n2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\\n3. Chrysler Building: Admire Art Deco style from the street.            True   \n",
        "\n",
-       "metric_name                                                                                                                                                                                              substring_match  \n",
-       "input                       aiconfig_output                                                                                                                                                                               \n",
-       "different kinds of cuisines 1. Explore Chelsea Market's cuisine.\\n2. Visit Little Italy for authentic fare.\\n3. Savor Asian delicacies in Flushing, Queens.                                                        False  \n",
-       "iconic midtown skyscrapers  1. Empire State Building Observatory - 86th floor views.\\n2. Rockefeller Center - Top of the Rock and ice skating.\\n3. One World Trade Center - One World Observatory visit.            True  "
+       "metric_name                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 text_ratings  \n",
+       "input                       aiconfig_output                                                                                                                                                                                                                                                                                                                                                                                                                                                               \n",
+       "different kinds of cuisines 1. Explore Chelsea Market delicacies.\\n2. Embark on Korea Town food tour.\\n3. Savor Italian cuisine at Bronx's Arthur Avenue.                                                                                                                  CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The text is concise and provides clear information about three different food destinations.\"\\n})  \n",
+       "iconic midtown skyscrapers  1. Empire State Building: Observation deck visit, VR journey through landmarks/history. \\n2. Rockefeller Center: Guided tour, winter - view the Christmas tree and ice skating.\\n3. Chrysler Building: Admire Art Deco style from the street.              CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The information is concise and provides a clear description of each attraction.\"\\n})  "
       ]
      },
-     "execution_count": 5,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -504,7 +886,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
@@ -551,12 +933,16 @@
     "    test_fn2 = metrics.substring_match(substring, case_sensitive=False)\n",
     "    test_suite_outputs_only.append(\n",
     "        (test_output, test_fn2)\n",
-    "    )"
+    "    )\n",
+    "\n",
+    "    # Add a model-graded eval metric that uses GPT 3.5 to return a struct\n",
+    "    test_fn3 = metrics.gpt3_5_text_ratings\n",
+    "    test_suite_outputs_only.append((test_output, test_fn3))    "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -568,19 +954,19 @@
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x1696a5a20>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": Infinity,\n",
+      "  \"best_value\": 1,\n",
+      "  \"worst_value\": 9223372036854775807,\n",
       "  \"extra_metadata\": {},\n",
-      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "  \"id\": \"24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3\"\n",
       "}))\n",
       "\n",
       "Test output:\n",
       " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
       "Function:\n",
-      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x14412c1f0>, metric_metadata=EvaluationMetricMetadata({\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x13f990a60>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
       "  \"description\": \"True (pass) if contains given substring\",\n",
       "  \"best_value\": true,\n",
@@ -593,21 +979,37 @@
       "}))\n",
       "\n",
       "Test output:\n",
+      " Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience \n",
+      "Function:\n",
+      " Metric(evaluation_fn=<function make_structured_llm_metric.<locals>._make_evaluation_fn.<locals>._evaluation_fn at 0x169640430>, metric_metadata=EvaluationMetricMetadata({\n",
+      "  \"name\": \"text_ratings\",\n",
+      "  \"description\": \"Text ratings\",\n",
+      "  \"best_value\": null,\n",
+      "  \"worst_value\": null,\n",
+      "  \"extra_metadata\": {\n",
+      "    \"basemodel_type_name\": \"TextRatingsData\",\n",
+      "    \"eval_llm_name\": \"gpt-3.5-turbo-0613\",\n",
+      "    \"field_descriptions_json\": \"{\\\"conciseness_confidence\\\": \\\"0 to 1.0 rating of confidence in conciseness rating\\\", \\\"conciseness_rating\\\": \\\"1 to 5 rating of conciseness\\\", \\\"conciseness_reasoning\\\": \\\"reasoning behind the conciseness rating\\\"}\"\n",
+      "  },\n",
+      "  \"id\": \"300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7\"\n",
+      "}))\n",
+      "\n",
+      "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " Metric(evaluation_fn=<function _calculate_brevity at 0x14405a320>, metric_metadata=EvaluationMetricMetadata({\n",
+      " Metric(evaluation_fn=<function _calculate_brevity at 0x1696a5a20>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"brevity\",\n",
       "  \"description\": \"Absolute text length\",\n",
-      "  \"best_value\": 1.0,\n",
-      "  \"worst_value\": Infinity,\n",
+      "  \"best_value\": 1,\n",
+      "  \"worst_value\": 9223372036854775807,\n",
       "  \"extra_metadata\": {},\n",
-      "  \"id\": \"1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4\"\n",
+      "  \"id\": \"24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3\"\n",
       "}))\n",
       "\n",
       "Test output:\n",
       " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
       "Function:\n",
-      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x14412dcf0>, metric_metadata=EvaluationMetricMetadata({\n",
+      " Metric(evaluation_fn=<function substring_match.<locals>._fn at 0x13f9917e0>, metric_metadata=EvaluationMetricMetadata({\n",
       "  \"name\": \"substring_match\",\n",
       "  \"description\": \"True (pass) if contains given substring\",\n",
       "  \"best_value\": true,\n",
@@ -617,6 +1019,22 @@
       "    \"case_sensitive\": false\n",
       "  },\n",
       "  \"id\": \"53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f\"\n",
+      "}))\n",
+      "\n",
+      "Test output:\n",
+      " 1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities. \n",
+      "Function:\n",
+      " Metric(evaluation_fn=<function make_structured_llm_metric.<locals>._make_evaluation_fn.<locals>._evaluation_fn at 0x169640430>, metric_metadata=EvaluationMetricMetadata({\n",
+      "  \"name\": \"text_ratings\",\n",
+      "  \"description\": \"Text ratings\",\n",
+      "  \"best_value\": null,\n",
+      "  \"worst_value\": null,\n",
+      "  \"extra_metadata\": {\n",
+      "    \"basemodel_type_name\": \"TextRatingsData\",\n",
+      "    \"eval_llm_name\": \"gpt-3.5-turbo-0613\",\n",
+      "    \"field_descriptions_json\": \"{\\\"conciseness_confidence\\\": \\\"0 to 1.0 rating of confidence in conciseness rating\\\", \\\"conciseness_rating\\\": \\\"1 to 5 rating of conciseness\\\", \\\"conciseness_reasoning\\\": \\\"reasoning behind the conciseness rating\\\"}\"\n",
+      "  },\n",
+      "  \"id\": \"300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7\"\n",
       "}))\n"
      ]
     }
@@ -630,7 +1048,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -678,11 +1096,11 @@
        "      <td>Missing</td>\n",
        "      <td>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</td>\n",
        "      <td>160</td>\n",
-       "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
+       "      <td>24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>inf</td>\n",
+       "      <td>1</td>\n",
+       "      <td>9223372036854775807</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
@@ -698,16 +1116,27 @@
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>Missing</td>\n",
+       "      <td>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</td>\n",
+       "      <td>CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 4,\\n  \"conciseness_confidence\": 0.8,\\n  \"conciseness_reasoning\": \"The text provides a clear and concise description of the itinerary, mentioning the starting point at Chelsea Market, the food tours in Queens, and the conclusion at Smorgasburg.\"\\n})</td>\n",
+       "      <td>300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7</td>\n",
+       "      <td>text_ratings</td>\n",
+       "      <td>Text ratings</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Missing</td>\n",
        "      <td>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</td>\n",
        "      <td>267</td>\n",
-       "      <td>1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4</td>\n",
+       "      <td>24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3</td>\n",
        "      <td>brevity</td>\n",
        "      <td>Absolute text length</td>\n",
-       "      <td>1.0</td>\n",
-       "      <td>inf</td>\n",
+       "      <td>1</td>\n",
+       "      <td>9223372036854775807</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <th>3</th>\n",
+       "      <th>4</th>\n",
        "      <td>Missing</td>\n",
        "      <td>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</td>\n",
        "      <td>True</td>\n",
@@ -717,6 +1146,17 @@
        "      <td>True</td>\n",
        "      <td>False</td>\n",
        "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>Missing</td>\n",
+       "      <td>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</td>\n",
+       "      <td>CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The text provides a concise description of the attractions and activities at each location.\"\\n})</td>\n",
+       "      <td>300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7</td>\n",
+       "      <td>text_ratings</td>\n",
+       "      <td>Text ratings</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
@@ -727,33 +1167,51 @@
        "1  Missing   \n",
        "2  Missing   \n",
        "3  Missing   \n",
+       "4  Missing   \n",
+       "5  Missing   \n",
        "\n",
        "                                                                                                                                                                                                                                                               aiconfig_output  \\\n",
        "0                                                                                                             Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience   \n",
        "1                                                                                                             Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience   \n",
-       "2  1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.   \n",
+       "2                                                                                                             Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience   \n",
        "3  1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.   \n",
+       "4  1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.   \n",
+       "5  1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.   \n",
        "\n",
-       "   value                                                         metric_id  \\\n",
-       "0    160  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "1  False  0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6   \n",
-       "2    267  1370aa8bacbe156352bf5a0d2cbf2b8dd8d54362e3e86809c7a4ae91e52530b4   \n",
-       "3   True  53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f   \n",
+       "                                                                                                                                                                                                                                                                                                                 value  \\\n",
+       "0                                                                                                                                                                                                                                                                                                                  160   \n",
+       "1                                                                                                                                                                                                                                                                                                                False   \n",
+       "2  CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 4,\\n  \"conciseness_confidence\": 0.8,\\n  \"conciseness_reasoning\": \"The text provides a clear and concise description of the itinerary, mentioning the starting point at Chelsea Market, the food tours in Queens, and the conclusion at Smorgasburg.\"\\n})   \n",
+       "3                                                                                                                                                                                                                                                                                                                  267   \n",
+       "4                                                                                                                                                                                                                                                                                                                 True   \n",
+       "5                                                                                        CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The text provides a concise description of the attractions and activities at each location.\"\\n})   \n",
+       "\n",
+       "                                                          metric_id  \\\n",
+       "0  24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3   \n",
+       "1  0c461362f44884023dda5537ce88263ba20d555562bac8abc05bcde0ce1aacf6   \n",
+       "2  300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7   \n",
+       "3  24952ce05ce6dcbd370ccc3b39d410edeab8e1cf420130a83cf9388df6bcfdc3   \n",
+       "4  53e4c7163f49fdc7727286e638ff07bcb570faaa334456775c616c2f4ad3eb3f   \n",
+       "5  300b32bb8a01befd5e729eaf73506bdba01f910c0db0c8f70136dd2e48e298a7   \n",
        "\n",
        "       metric_name                       metric_description  \\\n",
        "0          brevity                     Absolute text length   \n",
        "1  substring_match  True (pass) if contains given substring   \n",
-       "2          brevity                     Absolute text length   \n",
-       "3  substring_match  True (pass) if contains given substring   \n",
+       "2     text_ratings                             Text ratings   \n",
+       "3          brevity                     Absolute text length   \n",
+       "4  substring_match  True (pass) if contains given substring   \n",
+       "5     text_ratings                             Text ratings   \n",
        "\n",
        "  best_possible_value worst_possible_value  \n",
-       "0                 1.0                  inf  \n",
+       "0                   1  9223372036854775807  \n",
        "1                True                False  \n",
-       "2                 1.0                  inf  \n",
-       "3                True                False  "
+       "2                None                 None  \n",
+       "3                   1  9223372036854775807  \n",
+       "4                True                False  \n",
+       "5                None                 None  "
       ]
      },
-     "execution_count": 9,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -769,7 +1227,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -802,11 +1260,13 @@
        "      <th>metric_name</th>\n",
        "      <th>brevity</th>\n",
        "      <th>substring_match</th>\n",
+       "      <th>text_ratings</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>aiconfig_output</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
+       "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -814,11 +1274,13 @@
        "      <th>1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.</th>\n",
        "      <td>267</td>\n",
        "      <td>True</td>\n",
+       "      <td>CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The text provides a concise description of the attractions and activities at each location.\"\\n})</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience</th>\n",
        "      <td>160</td>\n",
        "      <td>False</td>\n",
+       "      <td>CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 4,\\n  \"conciseness_confidence\": 0.8,\\n  \"conciseness_reasoning\": \"The text provides a clear and concise description of the itinerary, mentioning the starting point at Chelsea Market, the food tours in Queens, and the conclusion at Smorgasburg.\"\\n})</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
@@ -830,13 +1292,18 @@
        "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.     267   \n",
        "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                160   \n",
        "\n",
-       "metric_name                                                                                                                                                                                                                                                                 substring_match  \n",
-       "aiconfig_output                                                                                                                                                                                                                                                                              \n",
-       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.            True  \n",
-       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                      False  "
+       "metric_name                                                                                                                                                                                                                                                                 substring_match  \\\n",
+       "aiconfig_output                                                                                                                                                                                                                                                                               \n",
+       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.            True   \n",
+       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                                      False   \n",
+       "\n",
+       "metric_name                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         text_ratings  \n",
+       "aiconfig_output                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   \n",
+       "1. Empire State Building: Observation deck visit, explore exhibits and historical displays. 2. Rockefeller Center: Visit \"Top of the Rock\", ice-skating, NBC Studio tour, shopping and dining. 3. Chrysler Building: Admire exterior and iconic spire, photo opportunities.                                                                                        CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 5,\\n  \"conciseness_confidence\": 0.9,\\n  \"conciseness_reasoning\": \"The text provides a concise description of the attractions and activities at each location.\"\\n})  \n",
+       "Begin at Chelsea Market for diverse food options. Continue to Queens for immersive food tours. Conclude at Smorgasburg for unique outdoor food market experience                                                                                                             CustomMetricPydanticObject(data={\\n  \"conciseness_rating\": 4,\\n  \"conciseness_confidence\": 0.8,\\n  \"conciseness_reasoning\": \"The text provides a clear and concise description of the itinerary, mentioning the starting point at Chelsea Market, the food tours in Queens, and the conclusion at Smorgasburg.\"\\n})  "
       ]
      },
-     "execution_count": 10,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -846,49 +1313,6 @@
     "df_result.set_index([ \"aiconfig_output\", \"metric_name\"])\\\n",
     "        .value.unstack(\"metric_name\")"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/pydantic/_internal/_fields.py:128: UserWarning: Field \"model_parsers\" has conflict with protected namespace \"model_\".\n",
-      "\n",
-      "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
-      "  warnings.warn(\n",
-      "/opt/homebrew/Caskroom/miniconda/base/envs/aiconfig/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "CustomMetricRecord(data={\n",
-       "  \"conciseness_rating\": 5,\n",
-       "  \"conciseness_confidence\": 8.0,\n",
-       "  \"conciseness_reasoning\": \"The text is short and to the point.\"\n",
-       "})"
-      ]
-     },
-     "execution_count": 1,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "import os\n",
-    "import openai\n",
-    "\n",
-    "from aiconfig.eval.api import metrics\n",
-    "\n",
-    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
-    "\n",
-    "await metrics.gpt3_5_text_ratings(\"one two three\")"
-   ]
   }
  ],
  "metadata": {
diff --git a/python/src/aiconfig/eval/lib.py b/python/src/aiconfig/eval/lib.py
index 84c635078..594fb483d 100644
--- a/python/src/aiconfig/eval/lib.py
+++ b/python/src/aiconfig/eval/lib.py
@@ -7,8 +7,8 @@
 import lastmile_utils.lib.core.api as cu
 import pandas as pd
 from aiconfig.Config import AIConfigRuntime
-from aiconfig.eval.common import SampleMetricValue, T_InputDatum, T_MetricValue, T_OutputDatum
 from aiconfig.eval.metrics import Metric
+import aiconfig.eval.common as common
 from result import Err, Ok, Result
 
 logging.basicConfig(format=cu.LOGGER_FMT)
@@ -23,9 +23,19 @@
 UserTestSuiteOutputsOnly = Sequence[Tuple[str, Metric[str, Any]]]
 
 
+@dataclass(frozen=True)
+class TestSuiteGeneralSettings:
+    eval_fn_timeout_s: int = 5
+
+
 class TestSuiteWithInputsSettings(cu.Record):
     prompt_name: str
     aiconfig_path: str
+    general_settings: TestSuiteGeneralSettings = TestSuiteGeneralSettings()
+
+
+class TestSuiteOutputsOnlySettings(cu.Record):
+    general_settings: TestSuiteGeneralSettings = TestSuiteGeneralSettings()
 
 
 async def run_test_suite_with_inputs(
@@ -38,6 +48,7 @@ async def run_test_suite_with_inputs(
             test_suite=test_suite,
             prompt_name=settings.prompt_name,
             aiconfig=aiconfig,
+            general_settings=settings.general_settings,
         )
     )
     return res.unwrap_or_raise(ValueError)
@@ -45,8 +56,9 @@ async def run_test_suite_with_inputs(
 
 async def run_test_suite_outputs_only(
     test_suite: UserTestSuiteOutputsOnly,
+    settings: TestSuiteOutputsOnlySettings = TestSuiteOutputsOnlySettings(),
 ) -> pd.DataFrame:
-    res = await run_test_suite_helper(TestSuiteOutputsOnlySpec(test_suite=test_suite))
+    res = await run_test_suite_helper(TestSuiteOutputsOnlySpec(test_suite=test_suite, general_settings=settings.general_settings))
     return res.unwrap_or_raise(ValueError)
 
 
@@ -65,43 +77,43 @@ class NumericalEvalDataset(cu.Record):
 # TODO:
 # GenericBeforeBaseModelWarning: Classes should inherit from `BaseModel` before generic classes (e.g. `typing.Generic[T]`) for pydantic generics to work properly.
 # But swapping the order breaks
-class SampleEvaluationResult(Generic[T_InputDatum, T_OutputDatum, T_MetricValue], cu.Record):
-    input_datum: T_InputDatum | None
-    output_datum: T_OutputDatum
-    metric_value: SampleMetricValue[T_OutputDatum, T_MetricValue]
+class SampleEvaluationResult(Generic[common.T_InputDatum, common.T_OutputDatum, common.T_MetricValue], cu.Record):
+    input_datum: common.T_InputDatum | None
+    output_datum: common.T_OutputDatum
+    metric_value: common.SampleMetricValue[common.T_OutputDatum, common.T_MetricValue]
 
 
 @dataclass(frozen=True)
-class SampleEvaluationParams(Generic[T_InputDatum, T_OutputDatum, T_MetricValue]):
+class SampleEvaluationParams(Generic[common.T_InputDatum, common.T_OutputDatum, common.T_MetricValue]):
     # input_sample doesn't _need_ to be here, because we already have
     # output_sample ready to input to eval.
     # input_sample is here for documentation/debugging.
-    input_sample: T_InputDatum | None
-    output_sample: T_OutputDatum
-    metric: Metric[T_OutputDatum, T_MetricValue]
+    input_sample: common.T_InputDatum | None
+    output_sample: common.T_OutputDatum
+    metric: Metric[common.T_OutputDatum, common.T_MetricValue]
 
     def __str__(self) -> str:
         return f"\nSampleEvaluationParams:\n\t{self.output_sample=}\n\t{self.metric=}"
 
 
 # TODO: don't use Any.
-DatasetEvaluationResult = Sequence[SampleEvaluationResult[T_InputDatum, T_OutputDatum, Any]]
-DatasetEvaluationParams = Sequence[SampleEvaluationParams[T_InputDatum, T_OutputDatum, Any]]
-MetricList = list[Metric[T_OutputDatum, Any]]
+DatasetEvaluationResult = Sequence[SampleEvaluationResult[common.T_InputDatum, common.T_OutputDatum, Any]]
+DatasetEvaluationParams = Sequence[SampleEvaluationParams[common.T_InputDatum, common.T_OutputDatum, Any]]
+MetricList = list[Metric[common.T_OutputDatum, Any]]
 
 
 async def _evaluate_for_sample(
-    eval_params: SampleEvaluationParams[T_InputDatum, T_OutputDatum, T_MetricValue], timeout: int
-) -> SampleEvaluationResult[T_InputDatum, T_OutputDatum, T_MetricValue]:
+    eval_params: SampleEvaluationParams[common.T_InputDatum, common.T_OutputDatum, common.T_MetricValue], timeout_s: int
+) -> SampleEvaluationResult[common.T_InputDatum, common.T_OutputDatum, common.T_MetricValue]:
     sample, metric = (
         eval_params.output_sample,
         eval_params.metric,
     )
 
-    async def _calculate() -> T_MetricValue:
+    async def _calculate() -> common.T_MetricValue:
         return await metric.evaluation_fn(sample)
 
-    def _ok_with_log(res_: Result[T_MetricValue, str]) -> T_MetricValue | None:
+    def _ok_with_log(res_: Result[common.T_MetricValue, str]) -> common.T_MetricValue | None:
         match res_:
             case Ok(res):
                 return res
@@ -109,11 +121,11 @@ def _ok_with_log(res_: Result[T_MetricValue, str]) -> T_MetricValue | None:
                 LOGGER.error(f"Error evaluating {eval_params=}: {e}")
                 return None
 
-    res_ = await cu.run_thunk_safe(_calculate(), timeout=timeout)
+    res_ = await cu.run_thunk_safe(_calculate(), timeout=timeout_s)
     result = SampleEvaluationResult(
         input_datum=eval_params.input_sample,
         output_datum=sample,
-        metric_value=SampleMetricValue(
+        metric_value=common.SampleMetricValue(
             #
             value=_ok_with_log(res_),
             metric_metadata=metric.metric_metadata,
@@ -123,13 +135,13 @@ def _ok_with_log(res_: Result[T_MetricValue, str]) -> T_MetricValue | None:
 
 
 async def evaluate(
-    evaluation_params_list: DatasetEvaluationParams[T_InputDatum, T_OutputDatum], eval_fn_timeout: int
-) -> Result[DatasetEvaluationResult[T_InputDatum, T_OutputDatum], str]:
-    return Ok(await asyncio.gather(*map(partial(_evaluate_for_sample, timeout=eval_fn_timeout), evaluation_params_list)))
+    evaluation_params_list: DatasetEvaluationParams[common.T_InputDatum, common.T_OutputDatum], eval_fn_timeout_s: int
+) -> Result[DatasetEvaluationResult[common.T_InputDatum, common.T_OutputDatum], str]:
+    return Ok(await asyncio.gather(*map(partial(_evaluate_for_sample, timeout_s=eval_fn_timeout_s), evaluation_params_list)))
 
 
 def eval_res_to_df(
-    eval_res: DatasetEvaluationResult[T_InputDatum, T_OutputDatum],
+    eval_res: DatasetEvaluationResult[common.T_InputDatum, common.T_OutputDatum],
 ) -> pd.DataFrame:
     # TODO: dont use Any
     records: list[dict[str, Any]] = []
@@ -235,11 +247,13 @@ class TestSuiteWithInputsSpec:
     test_suite: UserTestSuiteWithInputs
     prompt_name: str
     aiconfig: AIConfigRuntime
+    general_settings: TestSuiteGeneralSettings
 
 
 @dataclass(frozen=True)
 class TestSuiteOutputsOnlySpec:
     test_suite: UserTestSuiteOutputsOnly
+    general_settings: TestSuiteGeneralSettings
 
 
 TestSuiteSpec = TestSuiteWithInputsSpec | TestSuiteOutputsOnlySpec
@@ -263,7 +277,7 @@ async def _evaluate_with_timeout(
         eval_params_list: DatasetEvaluationParams[TextInput, TextOutput],
     ) -> Result[DatasetEvaluationResult[TextInput, TextOutput], str]:
         # TODO wire up the timeout more and improve default
-        return await evaluate(eval_params_list, eval_fn_timeout=1)
+        return await evaluate(eval_params_list, eval_fn_timeout_s=test_suite_spec.general_settings.eval_fn_timeout_s)
 
     res_evaluated = await eval_params_list.and_then_async(_evaluate_with_timeout)
     res_df_evaluated = res_evaluated.map(eval_res_to_df)
diff --git a/python/src/aiconfig/eval/metrics.py b/python/src/aiconfig/eval/metrics.py
index 640d37874..7cf298c1e 100644
--- a/python/src/aiconfig/eval/metrics.py
+++ b/python/src/aiconfig/eval/metrics.py
@@ -9,28 +9,20 @@
 import nltk
 import pandas as pd
 from aiconfig.eval import common
-from aiconfig.eval.common import (
-    CustomMetricValue,
-    EvaluationFunction,
-    EvaluationMetricMetadata,
-    T_BaseModel,
-    T_Evaluable,
-    T_MetricValue,
-    TextRatingsData,
-)
+
 from aiconfig.eval.openai import OpenAIChatCompletionCreate, default_openai_chat_completion_create, make_fn_completion_text_to_serialized_json
 from nltk.sentiment.vader import SentimentIntensityAnalyzer as NLTKSentimentIntensityAnalyzer
 from result import Err, Ok, Result
 
 
 @dataclass(frozen=True)
-class Metric(Generic[T_Evaluable, T_MetricValue]):
+class Metric(Generic[common.T_Evaluable, common.T_MetricValue]):
     """See metrics.py for examples."""
 
-    evaluation_fn: EvaluationFunction[T_Evaluable, T_MetricValue]
-    metric_metadata: EvaluationMetricMetadata[T_Evaluable, T_MetricValue]
+    evaluation_fn: common.EvaluationFunction[common.T_Evaluable, common.T_MetricValue]
+    metric_metadata: common.EvaluationMetricMetadata[common.T_Evaluable, common.T_MetricValue]
 
-    async def __call__(self, datum: T_Evaluable) -> T_MetricValue:
+    async def __call__(self, datum: common.T_Evaluable) -> common.T_MetricValue:
         """
         For convenience, make a Metric callable.
         Similar to torch Module `forward()`.
@@ -57,7 +49,7 @@ async def _calculate_brevity(datum: str) -> int:
 
 
 @dataclass(frozen=True)
-class TextSentimentScores(CustomMetricValue):
+class TextSentimentScores(common.CustomMetricValue):
     mapping: dict[str, float]
     pos: float
     neg: float
@@ -68,7 +60,7 @@ class TextSentimentScores(CustomMetricValue):
 
 @total_ordering
 @dataclass(frozen=True, eq=False)
-class TextOverallPositiveSentiment(CustomMetricValue):
+class TextOverallPositiveSentiment(common.CustomMetricValue):
     """Compare by total positive sentiment: positive - negative"""
 
     pos: float
@@ -104,14 +96,14 @@ def _get_sentiment_scores(output_datum: str, get_polarity_scores: GetPolaritySco
     return TextSentimentScores(mapping=mapping, **mapping, highest=highest)
 
 
-def make_get_sentiment_scores(get_polarity_scores: GetPolarityScores) -> EvaluationFunction[str, TextSentimentScores]:
+def make_get_sentiment_scores(get_polarity_scores: GetPolarityScores) -> common.EvaluationFunction[str, TextSentimentScores]:
     async def _f(datum: str) -> TextSentimentScores:
         return _get_sentiment_scores(datum, get_polarity_scores)
 
     return _f
 
 
-def make_get_sentiment_class(get_polarity_scores: GetPolarityScores) -> EvaluationFunction[str, str]:
+def make_get_sentiment_class(get_polarity_scores: GetPolarityScores) -> common.EvaluationFunction[str, str]:
     async def _f(datum: str) -> str:
         scores = _get_sentiment_scores(datum, get_polarity_scores)
         return scores.highest
@@ -119,7 +111,7 @@ async def _f(datum: str) -> str:
     return _f
 
 
-def make_get_overall_positive_sentiment(get_polarity_scores: GetPolarityScores) -> EvaluationFunction[str, TextOverallPositiveSentiment]:
+def make_get_overall_positive_sentiment(get_polarity_scores: GetPolarityScores) -> common.EvaluationFunction[str, TextOverallPositiveSentiment]:
     async def _f(datum: str) -> TextOverallPositiveSentiment:
         scores = _get_sentiment_scores(datum, get_polarity_scores)
         return TextOverallPositiveSentiment(pos=scores.pos, neg=scores.neg)
@@ -129,16 +121,16 @@ async def _f(datum: str) -> TextOverallPositiveSentiment:
 
 def make_sentiment_scores_metric(
     get_polarity_scores: GetPolarityScores,
-    make_evaluation_fn: Callable[[GetPolarityScores], EvaluationFunction[str, T_MetricValue]],
+    make_evaluation_fn: Callable[[GetPolarityScores], common.EvaluationFunction[str, common.T_MetricValue]],
     name: str,
     description: str,
-    best_value: T_MetricValue | None = None,
-    worst_value: T_MetricValue | None = None,
-) -> Metric[str, T_MetricValue]:
-    evaluation_fn: EvaluationFunction[str, T_MetricValue] = make_evaluation_fn(get_polarity_scores)
-    out: Metric[str, T_MetricValue] = Metric(
+    best_value: common.T_MetricValue | None = None,
+    worst_value: common.T_MetricValue | None = None,
+) -> Metric[str, common.T_MetricValue]:
+    evaluation_fn: common.EvaluationFunction[str, common.T_MetricValue] = make_evaluation_fn(get_polarity_scores)
+    out: Metric[str, common.T_MetricValue] = Metric(
         evaluation_fn=evaluation_fn,
-        metric_metadata=EvaluationMetricMetadata(
+        metric_metadata=common.EvaluationMetricMetadata(
             #
             name=name,
             description=description,
@@ -153,13 +145,15 @@ def make_sentiment_scores_metric(
 def make_structured_llm_metric(
     chat_completion_create: common.CompletionTextToSerializedJSON,
     eval_llm_name: str,
-    pydantic_basemodel_type: Type[T_BaseModel],
+    pydantic_basemodel_type: Type[common.T_BaseModel],
     metric_name: str,
     metric_description: str,
     field_descriptions: dict[str, str] = {},
-) -> Metric[str, common.CustomMetricPydanticObject[T_BaseModel]]:
-    def _make_evaluation_fn(basemodel_type: Type[T_BaseModel]) -> EvaluationFunction[str, common.CustomMetricPydanticObject[T_BaseModel]]:
-        async def _evaluation_fn(datum: str) -> common.CustomMetricPydanticObject[T_BaseModel]:
+) -> Metric[str, common.CustomMetricPydanticObject[common.T_BaseModel]]:
+    def _make_evaluation_fn(
+        basemodel_type: Type[common.T_BaseModel],
+    ) -> common.EvaluationFunction[str, common.CustomMetricPydanticObject[common.T_BaseModel]]:
+        async def _evaluation_fn(datum: str) -> common.CustomMetricPydanticObject[common.T_BaseModel]:
             resp = common.get_llm_structured_response(
                 input_text=datum,
                 chat_completion_create=chat_completion_create,
@@ -177,7 +171,7 @@ async def _evaluation_fn(datum: str) -> common.CustomMetricPydanticObject[T_Base
 
     return Metric(
         evaluation_fn=_make_evaluation_fn(pydantic_basemodel_type),
-        metric_metadata=EvaluationMetricMetadata(
+        metric_metadata=common.EvaluationMetricMetadata(
             name=metric_name,
             description=metric_description,
             extra_metadata=dict(
@@ -191,12 +185,12 @@ async def _evaluation_fn(datum: str) -> common.CustomMetricPydanticObject[T_Base
 
 def _make_openai_structured_llm_metric_helper(
     eval_llm_name: str,
-    pydantic_basemodel_type: Type[T_BaseModel],
+    pydantic_basemodel_type: Type[common.T_BaseModel],
     metric_name: str,
     metric_description: str,
     field_descriptions: dict[str, str],
     openai_chat_completion_create: OpenAIChatCompletionCreate | None = None,
-) -> Result[Metric[str, common.CustomMetricPydanticObject[T_BaseModel]], str]:
+) -> Result[Metric[str, common.CustomMetricPydanticObject[common.T_BaseModel]], str]:
     schema = pydantic_basemodel_type.model_json_schema()
     properties = schema["properties"]
     required = schema["required"]
@@ -245,12 +239,12 @@ def _with_description(key: str, value: dict[str, str]) -> dict[str, str]:
 
 def make_openai_structured_llm_metric(
     eval_llm_name: str,
-    pydantic_basemodel_type: Type[T_BaseModel],
+    pydantic_basemodel_type: Type[common.T_BaseModel],
     metric_name: str,
     metric_description: str,
     field_descriptions: dict[str, str] = {},
     openai_chat_completion_create: OpenAIChatCompletionCreate | None = None,
-) -> Metric[str, common.CustomMetricPydanticObject[T_BaseModel]]:
+) -> Metric[str, common.CustomMetricPydanticObject[common.T_BaseModel]]:
     res_metric = _make_openai_structured_llm_metric_helper(
         eval_llm_name=eval_llm_name,
         pydantic_basemodel_type=pydantic_basemodel_type,
@@ -278,7 +272,7 @@ async def _fn(datum: str) -> bool:
 
     return Metric(
         evaluation_fn=_fn,
-        metric_metadata=EvaluationMetricMetadata(
+        metric_metadata=common.EvaluationMetricMetadata(
             name="substring_match",
             description="True (pass) if contains given substring",
             best_value=True,
@@ -292,7 +286,7 @@ async def _fn(datum: str) -> bool:
 
 brevity: Metric[str, int] = Metric(
     evaluation_fn=_calculate_brevity,
-    metric_metadata=EvaluationMetricMetadata(
+    metric_metadata=common.EvaluationMetricMetadata(
         name="brevity",
         description="Absolute text length",
         best_value=1,
@@ -303,7 +297,7 @@ async def _fn(datum: str) -> bool:
 
 gpt3_5_text_ratings = make_openai_structured_llm_metric(
     eval_llm_name="gpt-3.5-turbo-0613",
-    pydantic_basemodel_type=TextRatingsData,
+    pydantic_basemodel_type=common.TextRatingsData,
     metric_name="text_ratings",
     metric_description="Text ratings",
     field_descriptions=dict(


Test patch:
diff --git a/python/tests/test_eval.py b/python/tests/test_eval.py
index f7eb0a5a3..fff90b471 100644
--- a/python/tests/test_eval.py
+++ b/python/tests/test_eval.py
@@ -14,7 +14,7 @@
 import pytest
 from aiconfig.eval import common
 from aiconfig.eval.api import TestSuiteWithInputsSettings, metrics, run_test_suite_outputs_only, run_test_suite_with_inputs
-from aiconfig.eval.lib import MetricList, TestSuiteWithInputsSpec, run_test_suite_helper
+from aiconfig.eval.lib import MetricList, TestSuiteGeneralSettings, TestSuiteWithInputsSpec, run_test_suite_helper
 from result import Err, Ok, Result
 
 from . import mocks
@@ -155,9 +155,7 @@ async def test_run_test_suite_with_inputs(data: st.DataObject):
 
     out = await run_test_suite_helper(
         TestSuiteWithInputsSpec(
-            test_suite=user_test_suite_with_inputs,
-            prompt_name="prompt0",
-            aiconfig=mock_aiconfig,
+            test_suite=user_test_suite_with_inputs, prompt_name="prompt0", aiconfig=mock_aiconfig, general_settings=TestSuiteGeneralSettings()
         )
     )
 
@@ -327,7 +325,7 @@ async def test_openai_structured_eval():
         ("one two three", mock_metric),
     ]
     df = await run_test_suite_outputs_only(user_test_suite_outputs_only)
-    metric_data = cast(common.CustomMetricPydanticObject[metrics.TextRatingsData], df.loc[0, "value"]).data
+    metric_data = cast(common.CustomMetricPydanticObject[common.TextRatingsData], df.loc[0, "value"]).data
     assert isinstance(metric_data, common.TextRatingsData)
     metric_json = metric_data.to_dict()
     assert metric_json == {"conciseness_rating": 5, "conciseness_confidence": 0.9, "conciseness_reasoning": "I think it's pretty concise."}