Repository: stanfordnlp/dspy
Pull request #490 (2024-02-28 20:14:50)

Cleaned up functional.py
With new TypedChainOfThought function


Patch:
diff --git a/dspy/functional/__init__.py b/dspy/functional/__init__.py
index 11fb1bc4..3ccced47 100644
--- a/dspy/functional/__init__.py
+++ b/dspy/functional/__init__.py
@@ -1,1 +1,1 @@
-from .functional import cot, predictor, FunctionalModule, TypedPredictor
+from .functional import cot, predictor, FunctionalModule, TypedPredictor, TypedChainOfThought
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py
index bc10def2..6fe34233 100644
--- a/dspy/functional/functional.py
+++ b/dspy/functional/functional.py
@@ -4,21 +4,40 @@ import typing
 from dsp.templates import passages2text
 import json
 
+from dspy.signatures.signature import ensure_signature
+
 
 MAX_RETRIES = 3
 
 
 def predictor(func):
     signature = _func_to_signature(func)
-    return TypedPredictor(signature, chain_of_thought=False, simple_output=True)
+    *_, output_key = signature.output_fields.keys()
+    return _StripOutput(TypedPredictor(signature), output_key)
 
 
 def cot(func):
     signature = _func_to_signature(func)
-    return TypedPredictor(signature, chain_of_thought=True, simple_output=True)
+    *_, output_key = signature.output_fields.keys()
+    return _StripOutput(TypedChainOfThought(signature), output_key)
+
+
+class _StripOutput(dspy.Module):
+    def __init__(self, predictor, output_key):
+        super().__init__()
+        self.predictor = predictor
+        self.output_key = output_key
+    
+    def copy(self):
+        return _StripOutput(self.predictor.copy(), self.output_key)
+
+    def forward(self, **kwargs):
+        prediction = self.predictor(**kwargs)
+        return prediction[self.output_key]
 
 
 class FunctionalModule(dspy.Module):
+    """ To use the @cot and @predictor decorators, your module needs to inheret form this class. """
     def __init__(self):
         super().__init__()
         for name in dir(self):
@@ -27,16 +46,43 @@ class FunctionalModule(dspy.Module):
                 self.__dict__[name] = attr.copy()
 
 
+def TypedChainOfThought(signature, make_example=False):
+    """ Just like TypedPredictor, but adds a ChainOfThought OutputField. """
+    signature = ensure_signature(signature)
+    output_keys = ", ".join(signature.output_fields.keys())
+    return TypedPredictor(signature.prepend(
+        "reasoning",
+        dspy.OutputField(
+            prefix="Reasoning: Let's think step by step in order to",
+            desc="${produce the " + output_keys + "}. We ...",
+        ),
+    ), make_example)
+
+
 class TypedPredictor(dspy.Module):
-    def __init__(self, signature, chain_of_thought=False, simple_output=False):
+    def __init__(self, signature, make_example=False):
         super().__init__()
         self.signature = signature
         self.predictor = dspy.Predict(signature)
-        self.chain_of_thought = chain_of_thought
-        self.simple_output = simple_output
+        self.make_example = make_example
 
     def copy(self):
-        return TypedPredictor(self.signature, self.chain_of_thought, self.simple_output)
+        return TypedPredictor(self.signature, self.make_example)
+
+    @staticmethod
+    def _make_example(type_):
+        # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.
+        return dspy.Predict(
+            dspy.Signature(
+                "json_schema -> json_object",
+                "Make a very succinct json object that validates with the following schema",
+            )
+        )(json_schema=json.dumps(type_.model_json_schema())).json_object
+        # TODO: Another fun idea is to only (but automatically) do this if the output fails.
+        # We could also have a more general "suggest solution" prompt that tries to fix the output
+        # More directly.
+        # TODO: Instead of using a language model to create the example, we can also just use a
+        # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.
 
     def _prepare_signature(self):
         """Add formats and parsers to the signature fields, based on the type
@@ -70,8 +116,11 @@ class TypedPredictor(dspy.Module):
                         + (
                             f". Respond with a single JSON object using the schema "
                             + json.dumps(type_.model_json_schema())
+                            + (". For example: " + self._make_example(type_) if self.make_example else "")
+                        ),
+                        format=lambda x: (
+                            x if isinstance(x, str) else x.model_dump_json()
                         ),
-                        format=lambda x: x if isinstance(x, str) else x.model_dump_json(),
                         parser=lambda x: unwrap(
                             type_.model_validate_json(_unwrap_json(x))
                         ),
@@ -84,19 +133,12 @@ class TypedPredictor(dspy.Module):
                     format = lambda x: x if isinstance(x, str) else x.model_dump_json()
                 signature = signature.with_updated_fields(name, format=format)
 
-        if self.chain_of_thought:
-            output_keys = ", ".join(signature.output_fields.keys())
-            signature = signature.prepend(
-                "reasoning",
-                dspy.OutputField(
-                    prefix="Reasoning: Let's think step by step in order to",
-                    desc="${produce the " + output_keys + "}. We ...",
-                ),
-            )
         return signature
 
     def forward(self, **kwargs):
         modified_kwargs = kwargs.copy()
+        # We have to re-prepare the signature on every forward call, because the base
+        # signature might have been modified by an optimizer or something like that.
         signature = self._prepare_signature()
         for try_i in range(MAX_RETRIES):
             result = self.predictor(**modified_kwargs, new_signature=signature)
@@ -126,9 +168,6 @@ class TypedPredictor(dspy.Module):
                 # If there are no errors, we return the parsed results
                 for name, value in parsed_results.items():
                     setattr(result, name, value)
-                if self.simple_output:
-                    *_, last_output = signature.output_fields.keys()
-                    return result[last_output]
                 return result
         raise ValueError("Too many retries")
 
diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py
index b73c956d..c0a56070 100644
--- a/dspy/signatures/signature.py
+++ b/dspy/signatures/signature.py
@@ -1,6 +1,7 @@
 from copy import deepcopy
 import dsp
 from pydantic import BaseModel, Field, create_model
+from pydantic.fields import FieldInfo
 from typing import Type, Union, Dict, Tuple
 import re
 
@@ -68,9 +69,7 @@ class SignatureMeta(type(BaseModel)):
         return getattr(cls, "__doc__", "")
 
     def with_instructions(cls, instructions: str):
-        return create_model(
-            cls.__name__, __base__=Signature, __doc__=instructions, **cls.fields
-        )
+        return Signature(cls.fields, instructions)
 
     @property
     def fields(cls):
@@ -85,9 +84,7 @@ class SignatureMeta(type(BaseModel)):
             **fields_copy[name].json_schema_extra,
             **kwargs,
         }
-        return create_model(
-            cls.__name__, __base__=Signature, __doc__=cls.instructions, **fields_copy
-        )
+        return Signature(fields_copy, cls.instructions)
 
     @property
     def input_fields(cls):
@@ -113,8 +110,10 @@ class SignatureMeta(type(BaseModel)):
     def insert(cls, index: int, name: str, field, type_: Type = None):
         # It's posisble to set the type as annotation=type in pydantic.Field(...)
         # But this may be annoying for users, so we allow them to pass the type
-        if type_ is not None:
-            field.annotation = type_
+        if type_ is None:
+            type_ = field.annotation 
+        if type_ is None:
+            type_ = str
 
         input_fields = list(cls.input_fields.items())
         output_fields = list(cls.output_fields.items())
@@ -130,14 +129,10 @@ class SignatureMeta(type(BaseModel)):
             index += len(lst) + 1
         if index < 0 or index > len(lst):
             raise ValueError(f"Invalid index: {index}")
-        lst.insert(index, (name, field))
+        lst.insert(index, (name, (type_, field)))
 
         new_fields = dict(input_fields + output_fields)
-        new_signature = create_model(
-            cls.__name__ + "'", __base__=Signature, **new_fields
-        )
-        new_signature.__doc__ = cls.instructions
-        return new_signature
+        return Signature(new_fields, cls.instructions)
 
     def _parse_signature(cls, signature: str) -> Tuple[Type, Field]:
         pattern = r"^\s*[\w\s,]+\s*->\s*[\w\s,]+\s*$"
@@ -157,7 +152,7 @@ class SignatureMeta(type(BaseModel)):
 
     def __call__(
         cls,
-        signature: Union[str, Dict[str, Tuple[type, Field]]],
+        signature: Union[str, Dict[str, Tuple[type, FieldInfo]]],
         instructions: str = None,
     ):
         """
@@ -175,13 +170,35 @@ class SignatureMeta(type(BaseModel)):
             fields = cls._parse_signature(signature)
         else:
             fields = signature
+        
+        # Validate the fields, this is important because we sometimes forget the
+        # slightly unintuitive syntax with tuples of (type, Field)
+        fixed_fields = {}
+        for name, type_field in fields.items():
+            assert isinstance(name, str), f"Field names must be strings, not {type(name)}"
+            if isinstance(type_field, FieldInfo):
+                type_ = type_field.annotation
+                field = type_field
+            else:
+                assert isinstance(type_field, tuple), f"Field values must be tuples, not {type(type_field)}"
+                type_, field = type_field
+            # It might be better to be explicit about the type, but it currently would break
+            # program of thought and teleprompters, so we just silently default to string.
+            if type_ is None:
+                type_ = str
+            assert isinstance(type_, type), f"Field types must be types, not {type(type_)}"
+            assert isinstance(field, FieldInfo), f"Field values must be Field instances, not {type(field)}"
+            fixed_fields[name] = (type_, field)
+        
+        # Fixing the fields shouldn't change the order
+        assert list(fixed_fields.keys()) == list(fields.keys())
 
         # Default prompt when no instructions are provided
         if instructions is None:
             sig = Signature(signature, "")  # Simple way to parse input/output fields
             instructions = _default_instructions(sig)
 
-        signature = create_model("Signature", __base__=Signature, **fields)
+        signature = create_model("Signature", __base__=Signature, **fixed_fields)
         signature.__doc__ = instructions
         return signature
 
diff --git a/examples/functional/functional.ipynb b/examples/functional/functional.ipynb
new file mode 100644
index 00000000..0dcbef5d
--- /dev/null
+++ b/examples/functional/functional.ipynb
@@ -0,0 +1,1215 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: datasets in /opt/homebrew/lib/python3.11/site-packages (2.14.7)\n",
+      "Requirement already satisfied: numpy>=1.17 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (1.26.2)\n",
+      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (12.0.0)\n",
+      "Requirement already satisfied: pyarrow-hotfix in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.6)\n",
+      "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.3.6)\n",
+      "Requirement already satisfied: pandas in /opt/homebrew/lib/python3.11/site-packages (from datasets) (2.1.4)\n",
+      "Requirement already satisfied: requests>=2.19.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (2.31.0)\n",
+      "Requirement already satisfied: tqdm>=4.62.1 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (4.66.1)\n",
+      "Requirement already satisfied: xxhash in /opt/homebrew/lib/python3.11/site-packages (from datasets) (3.2.0)\n",
+      "Requirement already satisfied: multiprocess in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.70.14)\n",
+      "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /opt/homebrew/lib/python3.11/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.10.0)\n",
+      "Requirement already satisfied: aiohttp in /opt/homebrew/lib/python3.11/site-packages (from datasets) (3.9.1)\n",
+      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (0.19.4)\n",
+      "Requirement already satisfied: packaging in /opt/homebrew/lib/python3.11/site-packages (from datasets) (23.2)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from datasets) (6.0.1)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (23.1.0)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (6.0.4)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.9.3)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.4.0)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/homebrew/lib/python3.11/site-packages (from aiohttp->datasets) (1.3.1)\n",
+      "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.13.1)\n",
+      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.8.0)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (3.6)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2.1.0)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.11/site-packages (from requests>=2.19.0->datasets) (2023.11.17)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2.8.2)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n",
+      "Requirement already satisfied: tzdata>=2022.1 in /opt/homebrew/lib/python3.11/site-packages (from pandas->datasets) (2023.3)\n",
+      "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
+      "\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
+      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.11 -m pip install --upgrade pip\u001b[0m\n",
+      "Note: you may need to restart the kernel to use updated packages.\n"
+     ]
+    }
+   ],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "%pip install datasets"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's get the dataset and see what it looks like."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/homebrew/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'task_id': 'HumanEval/0',\n",
+       " 'prompt': 'from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n    \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n    given threshold.\\n    >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n    False\\n    >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n    True\\n    \"\"\"\\n',\n",
+       " 'canonical_solution': '    for idx, elem in enumerate(numbers):\\n        for idx2, elem2 in enumerate(numbers):\\n            if idx != idx2:\\n                distance = abs(elem - elem2)\\n                if distance < threshold:\\n                    return True\\n\\n    return False\\n',\n",
+       " 'test': \"\\n\\nMETADATA = {\\n    'author': 'jt',\\n    'dataset': 'test'\\n}\\n\\n\\ndef check(candidate):\\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\\n    assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\\n\\n\",\n",
+       " 'entry_point': 'has_close_elements'}"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import datasets\n",
+    "ds = datasets.load_dataset(\"openai_humaneval\")\n",
+    "ds['test'][0]\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Before we try to solve the problem, let's just load a language model and make sure everything works."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Prediction(\n",
+      "    answer='Paris'\n",
+      ")\n"
+     ]
+    }
+   ],
+   "source": [
+    "import dspy, dotenv, os\n",
+    "dotenv.load_dotenv(os.path.expanduser(\"~/.env\"))  # load OpenAI API key from .env file\n",
+    "lm = dspy.OpenAI(model=\"gpt-3.5-turbo\", max_tokens=4000)\n",
+    "dspy.settings.configure(lm=lm)\n",
+    "\n",
+    "predictor = dspy.Predict(\"question -> answer\")\n",
+    "print(predictor(question=\"What is the capital of France?\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Next let's write a program that actually outputs code."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Prediction(\n",
+      "    solution=PythonCode(code='def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n    for i in range(len(numbers)):\\n        for j in range(i+1, len(numbers)):\\n            if abs(numbers[i] - numbers[j]) < threshold:\\n                return True\\n    return False')\n",
+      ")\n"
+     ]
+    }
+   ],
+   "source": [
+    "from dspy import InputField, OutputField, Signature\n",
+    "from dspy.functional import TypedPredictor\n",
+    "import pydantic\n",
+    "\n",
+    "# We define a pydantic type that automatically checks if it's argument is valid python code.\n",
+    "class PythonCode(pydantic.BaseModel):\n",
+    "    code: str\n",
+    "\n",
+    "    @pydantic.field_validator('code')\n",
+    "    def check_syntax(cls, v):\n",
+    "        try:\n",
+    "            # Attempt to compile the code snippet\n",
+    "            compile(v, \"<string>\", \"exec\")\n",
+    "        except SyntaxError as e:\n",
+    "            # If a SyntaxError is raised, the code is not syntactically valid\n",
+    "            raise ValueError(f\"Code is not syntactically valid: {e}\")\n",
+    "            \n",
+    "        return v\n",
+    "\n",
+    "# The signature is the main DSpy object. Note that we have types for the input and output fields,\n",
+    "# which was not possible beofore.\n",
+    "class CodeSignature(Signature):\n",
+    "    prompt: str = InputField()\n",
+    "    test: PythonCode = InputField()\n",
+    "    entry_point: str = InputField()\n",
+    "    solution: PythonCode = OutputField()\n",
+    "\n",
+    "predictor = TypedPredictor(CodeSignature, make_example=True)\n",
+    "prediction = predictor(\n",
+    "    prompt=PythonCode(code=ds['test'][0]['prompt']),\n",
+    "    test=PythonCode(code=ds['test'][0]['test']),\n",
+    "    entry_point=ds['test'][0]['entry_point']\n",
+    ")\n",
+    "\n",
+    "print(prediction)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's see what's happening under the hood"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "Make a very succinct json object that validates with the following schema\n",
+      "\n",
+      "---\n",
+      "\n",
+      "Follow the following format.\n",
+      "\n",
+      "Json Schema: ${json_schema}\n",
+      "Json Object: ${json_object}\n",
+      "\n",
+      "---\n",
+      "\n",
+      "Json Schema: {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}\n",
+      "Json Object:\u001b[32m {\"code\": \"print('Hello, World!')\"}\u001b[0m\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "Given the fields `prompt`, `test`, `entry_point`, produce the fields `solution`.\n",
+      "\n",
+      "---\n",
+      "\n",
+      "Follow the following format.\n",
+      "\n",
+      "Prompt: ${prompt}\n",
+      "\n",
+      "Test: ${test}\n",
+      "\n",
+      "Entry Point: ${entry_point}\n",
+      "\n",
+      "Solution: ${solution}. Respond with a single JSON object using the schema {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}. For example: {\"code\": \"print('Hello, World!')\"}\n",
+      "\n",
+      "---\n",
+      "\n",
+      "Prompt: code='from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n    \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n    given threshold.\\n    >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n    False\\n    >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n    True\\n    \"\"\"\\n'\n",
+      "\n",
+      "Test: {\"code\":\"\\n\\nMETADATA = {\\n    'author': 'jt',\\n    'dataset': 'test'\\n}\\n\\n\\ndef check(candidate):\\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\\n    assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\\n\\n\"}\n",
+      "\n",
+      "Entry Point: has_close_elements\n",
+      "\n",
+      "Solution:\u001b[32m {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}\n",
+      "\n",
+      "{\"code\": \"from typing import List\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n    for i in range(len(numbers)):\\n        for j in range(i+1, len(numbers)):\\n            if abs(numbers[i] - numbers[j]) < threshold:\\n                return True\\n    return False\"}\u001b[0m\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "\n",
+      "Given the fields `prompt`, `test`, `entry_point`, produce the fields `solution`.\n",
+      "\n",
+      "---\n",
+      "\n",
+      "Follow the following format.\n",
+      "\n",
+      "Prompt: ${prompt}\n",
+      "\n",
+      "Test: ${test}\n",
+      "\n",
+      "Entry Point: ${entry_point}\n",
+      "\n",
+      "Past Error (solution): An error to avoid in the future\n",
+      "\n",
+      "Solution: ${solution}. Respond with a single JSON object using the schema {\"properties\": {\"code\": {\"title\": \"Code\", \"type\": \"string\"}}, \"required\": [\"code\"], \"title\": \"PythonCode\", \"type\": \"object\"}. For example: {\"code\": \"print('Hello, World!')\"}\n",
+      "\n",
+      "---\n",
+      "\n",
+      "Prompt: code='from typing import List\\n\\n\\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\\n    \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\\n    given threshold.\\n    >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\\n    False\\n    >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\\n    True\\n    \"\"\"\\n'\n",
+      "\n",
+      "Test: {\"code\":\"\\n\\nMETADATA = {\\n    'author': 'jt',\\n    'dataset': 'test'\\n}\\n\\n\\ndef check(candidate):\\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\\n    assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\\n\\n\"}\n",
+      "\n",
+      "Entry Point: has_close_elements\n",
+      "\n",
+      "Past Error (solution): 1 validation error for PythonCode Invalid JSON: trailing characters at line 3 column 1 [type=json_invalid, input_value='{\"properties\": {\"code\": ...ue\\\\n return False\"}', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/json_invalid\n",
+      "\n",
+      "Solution:\u001b[32m {\"code\": \"def has_close_elements(numbers: List[float], threshold: float) -> bool:\\n    for i in range(len(numbers)):\\n        for j in range(i+1, len(numbers)):\\n            if abs(numbers[i] - numbers[j]) < threshold:\\n                return True\\n    return False\"}\u001b[0m\n",
+      "\n",
+      "\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "lm.inspect_history(n=3)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can see `functional` first created an example value {\"code\": \"print('Hello, World!')\"}, which can be useful to boostrap the json generation.\n",
+    "After that it still failed to generate valid json.\n",
+    "It apparently decided to first repeat the schema, and then give the actual code \"as an example\"\n",
+    "The validator caught the error, and gave it as a \"Past Error\", which made the model finally output a valid output."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We need a way to run code. This is actually super tricky to do right in python (see https://stackoverflow.com/questions/3068139/how-can-i-sandbox-python-in-pure-python), so we'll just YOLO and call \"exec\" with globals={}."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "3\n",
+      "None\n",
+      "AssertionError()\n"
+     ]
+    }
+   ],
+   "source": [
+    "from repl import execute_code\n",
+    "print(execute_code(\"print(3)\"))\n",
+    "print(execute_code(\"assert False\"))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's run the evaluator on all the \"canonical solutions\" from HumanEval to check that everything is working."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Score with the original model:\n",
+      "100.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "from dspy import Example\n",
+    "\n",
+    "devset = [Example(\n",
+    "    prompt=PythonCode(code=test['prompt']),\n",
+    "    test=PythonCode(code=test['test']),\n",
+    "    entry_point=test['entry_point'],\n",
+    "    solution=PythonCode(code=test['prompt']+test['canonical_solution']),\n",
+    ").with_inputs('prompt', 'test', 'entry_point') for test in ds['test']]\n",
+    "\n",
+    "trainset = devset[:40]\n",
+    "testset = devset[40:]\n",
+    "\n",
+    "def test_code(timeout=5):\n",
+    "    def metric(example, pred, trace=None):\n",
+    "        if pred.solution.code is None:\n",
+    "            return 0\n",
+    "        error = execute_code(\n",
+    "            \"from typing import List\\n\"\n",
+    "            + f\"{pred.solution.code}\\n\"\n",
+    "            + f\"{example.test.code}\\n\"\n",
+    "            + f\"check({example.entry_point})\",\n",
+    "            timeout=timeout,\n",
+    "        )\n",
+    "        return int(error is None)\n",
+    "    return metric\n",
+    "\n",
+    "metric5s = test_code(timeout=5)\n",
+    "\n",
+    "print(\"Score with the original model:\")\n",
+    "metrix = test_code(timeout=5)\n",
+    "print(100 * sum(metric5s(example, example) for example in testset) / len(testset))\n",
+    "\n",
+    "for example in devset:\n",
+    "    if not metric5s(example, example):\n",
+    "        print(\"Bad example:\")\n",
+    "        code = (\n",
+    "            \"from typing import List\\n\"\n",
+    "            + f\"{example.solution.code}\\n\"\n",
+    "            + f\"{example.test.code}\\n\"\n",
+    "            + f\"check({example.entry_point})\"\n",
+    "        )\n",
+    "        print(code)\n",
+    "        error = execute_code(code)\n",
+    "        print(f\"{error=}\")\n",
+    "        break\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now test our program."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  0%|          | 0/124 [00:00<?, ?it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 0.0 / 3  (0.0):   2%|▏         | 2/124 [00:00<00:01, 77.99it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 24.0 / 37  (64.9):  29%|██▉       | 36/124 [00:00<00:01, 81.56it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 25.0 / 40  (62.5):  31%|███▏      | 39/124 [00:00<00:01, 81.56it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 45.0 / 80  (56.2):  64%|██████▎   | 79/124 [00:00<00:00, 109.09it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 45.0 / 81  (55.6):  65%|██████▍   | 80/124 [00:00<00:00, 109.09it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 64.0 / 119  (53.8):  95%|█████████▌| 118/124 [00:01<00:00, 110.58it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "37\n",
+      "15\n",
+      "8\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 66.0 / 124  (53.2): 100%|██████████| 124/124 [00:01<00:00, 116.06it/s]\n",
+      "/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:143: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
+      "  df = df.applymap(truncate_cell)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 66.0 / 124  (53.2%)\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style type=\"text/css\">\n",
+       "#T_ac8ed th {\n",
+       "  text-align: left;\n",
+       "}\n",
+       "#T_ac8ed td {\n",
+       "  text-align: left;\n",
+       "}\n",
+       "#T_ac8ed_row0_col0, #T_ac8ed_row0_col1, #T_ac8ed_row0_col2, #T_ac8ed_row0_col3, #T_ac8ed_row0_col4, #T_ac8ed_row0_col5, #T_ac8ed_row0_col6, #T_ac8ed_row1_col0, #T_ac8ed_row1_col1, #T_ac8ed_row1_col2, #T_ac8ed_row1_col3, #T_ac8ed_row1_col4, #T_ac8ed_row1_col5, #T_ac8ed_row1_col6, #T_ac8ed_row2_col0, #T_ac8ed_row2_col1, #T_ac8ed_row2_col2, #T_ac8ed_row2_col3, #T_ac8ed_row2_col4, #T_ac8ed_row2_col5, #T_ac8ed_row2_col6, #T_ac8ed_row3_col0, #T_ac8ed_row3_col1, #T_ac8ed_row3_col2, #T_ac8ed_row3_col3, #T_ac8ed_row3_col4, #T_ac8ed_row3_col5, #T_ac8ed_row3_col6, #T_ac8ed_row4_col0, #T_ac8ed_row4_col1, #T_ac8ed_row4_col2, #T_ac8ed_row4_col3, #T_ac8ed_row4_col4, #T_ac8ed_row4_col5, #T_ac8ed_row4_col6 {\n",
+       "  text-align: left;\n",
+       "  white-space: pre-wrap;\n",
+       "  word-wrap: break-word;\n",
+       "  max-width: 400px;\n",
+       "}\n",
+       "</style>\n",
+       "<table id=\"T_ac8ed\">\n",
+       "  <thead>\n",
+       "    <tr>\n",
+       "      <th class=\"blank level0\" >&nbsp;</th>\n",
+       "      <th id=\"T_ac8ed_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",
+       "      <th id=\"T_ac8ed_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",
+       "      <th id=\"T_ac8ed_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",
+       "      <th id=\"T_ac8ed_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",
+       "      <th id=\"T_ac8ed_level0_col4\" class=\"col_heading level0 col4\" >pred_solution</th>\n",
+       "      <th id=\"T_ac8ed_level0_col5\" class=\"col_heading level0 col5\" >metric</th>\n",
+       "      <th id=\"T_ac8ed_level0_col6\" class=\"col_heading level0 col6\" >solution</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th id=\"T_ac8ed_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "      <td id=\"T_ac8ed_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+       "      <td id=\"T_ac8ed_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",
+       "      <td id=\"T_ac8ed_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",
+       "      <td id=\"T_ac8ed_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+       "      <td id=\"T_ac8ed_row0_col4\" class=\"data row0 col4\" >code='def triples_sum_to_zero(l: list):\\n for i in range(len(l)):\\n for j in range(i+1, len(l)):\\n for k in range(j+1, len(l)):\\n if l[i] + l[j] + l[k] == 0:\\n...</td>\n",
+       "      <td id=\"T_ac8ed_row0_col5\" class=\"data row0 col5\" >1.0</td>\n",
+       "      <td id=\"T_ac8ed_row0_col6\" class=\"data row0 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th id=\"T_ac8ed_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+       "      <td id=\"T_ac8ed_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+       "      <td id=\"T_ac8ed_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n    assert candidate(2) == 4\\n    assert candidate(3) == 9\\n    assert candidate(4) == 16\\n    assert candidate(8) == 64\\n    assert candidate(10) == 100\\n\\n'</td>\n",
+       "      <td id=\"T_ac8ed_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",
+       "      <td id=\"T_ac8ed_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+       "      <td id=\"T_ac8ed_row1_col4\" class=\"data row1 col4\" >code='def car_race_collision(n: int):\\n    return n ** 2'</td>\n",
+       "      <td id=\"T_ac8ed_row1_col5\" class=\"data row1 col5\" >1.0</td>\n",
+       "      <td id=\"T_ac8ed_row1_col6\" class=\"data row1 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th id=\"T_ac8ed_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+       "      <td id=\"T_ac8ed_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+       "      <td id=\"T_ac8ed_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",
+       "      <td id=\"T_ac8ed_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",
+       "      <td id=\"T_ac8ed_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+       "      <td id=\"T_ac8ed_row2_col4\" class=\"data row2 col4\" >code='def incr_list(l: list):\\n    return [x + 1 for x in l]'</td>\n",
+       "      <td id=\"T_ac8ed_row2_col5\" class=\"data row2 col5\" >1.0</td>\n",
+       "      <td id=\"T_ac8ed_row2_col6\" class=\"data row2 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th id=\"T_ac8ed_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+       "      <td id=\"T_ac8ed_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+       "      <td id=\"T_ac8ed_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",
+       "      <td id=\"T_ac8ed_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",
+       "      <td id=\"T_ac8ed_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+       "      <td id=\"T_ac8ed_row3_col4\" class=\"data row3 col4\" >code='def pairs_sum_to_zero(l):\\n    return any(-x in l for x in l if x != 0)\\n'</td>\n",
+       "      <td id=\"T_ac8ed_row3_col5\" class=\"data row3 col5\" >1.0</td>\n",
+       "      <td id=\"T_ac8ed_row3_col6\" class=\"data row3 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th id=\"T_ac8ed_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
+       "      <td id=\"T_ac8ed_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+       "      <td id=\"T_ac8ed_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",
+       "      <td id=\"T_ac8ed_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",
+       "      <td id=\"T_ac8ed_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+       "      <td id=\"T_ac8ed_row4_col4\" class=\"data row4 col4\" >code='def change_base(x: int, base: int):\\n    return str(int(str(x), base))'</td>\n",
+       "      <td id=\"T_ac8ed_row4_col5\" class=\"data row4 col5\" >0.0</td>\n",
+       "      <td id=\"T_ac8ed_row4_col6\" class=\"data row4 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x2b70c79d0>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <div style='\n",
+       "                    text-align: center; \n",
+       "                    font-size: 16px; \n",
+       "                    font-weight: bold; \n",
+       "                    color: #555; \n",
+       "                    margin: 10px 0;'>\n",
+       "                    ... 119 more rows not displayed ...\n",
+       "                </div>\n",
+       "                "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "53.23\n"
+     ]
+    }
+   ],
+   "source": [
+    "from dspy.evaluate.evaluate import Evaluate\n",
+    "evaluator = Evaluate(\n",
+    "    devset=testset, num_threads=30,\n",
+    "    display_progress=True,\n",
+    "    display_table=5,\n",
+    "    max_errors=100,\n",
+    ")\n",
+    "res = evaluator(predictor, metric5s)\n",
+    "print(res)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Let's try to optimize it a bit"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Compiling...\n",
+      "Going to sample between 1 and 4 traces per predictor.\n",
+      "Will attempt to train 5 candidate sets.\n",
+      "Error for example in dev set: \t\t Too many retries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 25.0 / 40  (62.5): 100%|██████████| 40/40 [00:01<00:00, 22.16it/s] \n",
+      "/Users/ahle/repos/dspy/dspy/evaluate/evaluate.py:143: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
+      "  df = df.applymap(truncate_cell)\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 25.0 / 40  (62.5%)\n",
+      "Score: 62.5 for set: [0]\n",
+      "New best score: 62.5 for seed -3\n",
+      "Scores so far: [62.5]\n",
+      "Best score: 62.5\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 34.0 / 40  (85.0): 100%|██████████| 40/40 [00:18<00:00,  2.17it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n",
+      "Average Metric: 34.0 / 40  (85.0%)\n",
+      "Score: 85.0 for set: [8]\n",
+      "New best score: 85.0 for seed -2\n",
+      "Scores so far: [62.5, 85.0]\n",
+      "Best score: 85.0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 10%|█         | 4/40 [00:02<00:19,  1.82it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 33 / 40  (82.5): 100%|██████████| 40/40 [01:48<00:00,  2.72s/it] \n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 33 / 40  (82.5%)\n",
+      "Score: 82.5 for set: [8]\n",
+      "Scores so far: [62.5, 85.0, 82.5]\n",
+      "Best score: 85.0\n",
+      "Average of max per entry across top 1 scores: 0.85\n",
+      "Average of max per entry across top 2 scores: 0.975\n",
+      "Average of max per entry across top 3 scores: 1.0\n",
+      "Average of max per entry across top 5 scores: 1.0\n",
+      "Average of max per entry across top 8 scores: 1.0\n",
+      "Average of max per entry across top 9999 scores: 1.0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 10%|█         | 4/40 [00:10<01:37,  2.70s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 35 / 40  (87.5): 100%|██████████| 40/40 [00:12<00:00,  3.15it/s] \n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 35 / 40  (87.5%)\n",
+      "Score: 87.5 for set: [8]\n",
+      "New best score: 87.5 for seed 0\n",
+      "Scores so far: [62.5, 85.0, 82.5, 87.5]\n",
+      "Best score: 87.5\n",
+      "Average of max per entry across top 1 scores: 0.875\n",
+      "Average of max per entry across top 2 scores: 0.975\n",
+      "Average of max per entry across top 3 scores: 1.0\n",
+      "Average of max per entry across top 5 scores: 1.0\n",
+      "Average of max per entry across top 8 scores: 1.0\n",
+      "Average of max per entry across top 9999 scores: 1.0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  5%|▌         | 2/40 [00:03<01:07,  1.79s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Bootstrapped 2 full traces after 3 examples in round 0.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 36 / 40  (90.0): 100%|██████████| 40/40 [00:09<00:00,  4.35it/s] \n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 36 / 40  (90.0%)\n",
+      "Score: 90.0 for set: [8]\n",
+      "New best score: 90.0 for seed 1\n",
+      "Scores so far: [62.5, 85.0, 82.5, 87.5, 90.0]\n",
+      "Best score: 90.0\n",
+      "Average of max per entry across top 1 scores: 0.9\n",
+      "Average of max per entry across top 2 scores: 0.95\n",
+      "Average of max per entry across top 3 scores: 0.975\n",
+      "Average of max per entry across top 5 scores: 1.0\n",
+      "Average of max per entry across top 8 scores: 1.0\n",
+      "Average of max per entry across top 9999 scores: 1.0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  5%|▌         | 2/40 [00:05<01:37,  2.57s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Bootstrapped 1 full traces after 3 examples in round 0.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 34 / 40  (85.0): 100%|██████████| 40/40 [00:12<00:00,  3.31it/s] \n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 34 / 40  (85.0%)\n",
+      "Score: 85.0 for set: [8]\n",
+      "Scores so far: [62.5, 85.0, 82.5, 87.5, 90.0, 85.0]\n",
+      "Best score: 90.0\n",
+      "Average of max per entry across top 1 scores: 0.9\n",
+      "Average of max per entry across top 2 scores: 0.95\n",
+      "Average of max per entry across top 3 scores: 0.975\n",
+      "Average of max per entry across top 5 scores: 1.0\n",
+      "Average of max per entry across top 8 scores: 1.0\n",
+      "Average of max per entry across top 9999 scores: 1.0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  8%|▊         | 3/40 [00:07<01:34,  2.56s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Bootstrapped 2 full traces after 4 examples in round 0.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 32 / 40  (80.0): 100%|██████████| 40/40 [00:18<00:00,  2.16it/s] \n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 32 / 40  (80.0%)\n",
+      "Score: 80.0 for set: [8]\n",
+      "Scores so far: [62.5, 85.0, 82.5, 87.5, 90.0, 85.0, 80.0]\n",
+      "Best score: 90.0\n",
+      "Average of max per entry across top 1 scores: 0.9\n",
+      "Average of max per entry across top 2 scores: 0.95\n",
+      "Average of max per entry across top 3 scores: 0.975\n",
+      "Average of max per entry across top 5 scores: 1.0\n",
+      "Average of max per entry across top 8 scores: 1.0\n",
+      "Average of max per entry across top 9999 scores: 1.0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  5%|▌         | 2/40 [00:03<01:07,  1.76s/it]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Bootstrapped 2 full traces after 3 examples in round 0.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 35 / 40  (87.5): 100%|██████████| 40/40 [00:12<00:00,  3.18it/s] "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 35 / 40  (87.5%)\n",
+      "Score: 87.5 for set: [8]\n",
+      "Scores so far: [62.5, 85.0, 82.5, 87.5, 90.0, 85.0, 80.0, 87.5]\n",
+      "Best score: 90.0\n",
+      "Average of max per entry across top 1 scores: 0.9\n",
+      "Average of max per entry across top 2 scores: 0.95\n",
+      "Average of max per entry across top 3 scores: 0.975\n",
+      "Average of max per entry across top 5 scores: 0.975\n",
+      "Average of max per entry across top 8 scores: 1.0\n",
+      "Average of max per entry across top 9999 scores: 1.0\n",
+      "8 candidate programs found.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from dspy.teleprompt.bootstrap import BootstrapFewShot\n",
+    "from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n",
+    "\n",
+    "print(\"Compiling...\")\n",
+    "compiled = BootstrapFewShotWithRandomSearch(\n",
+    "#compiled = BootstrapFewShot(\n",
+    "    metric=metric5s,\n",
+    "    num_threads=30,\n",
+    "    num_candidate_programs=5,\n",
+    "    max_labeled_demos=8,\n",
+    ").compile(\n",
+    "    predictor,\n",
+    "    trainset=trainset,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Finally evaluate the trained model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Evaluating...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 59.0 / 95  (62.1):  77%|███████▋  | 95/124 [00:16<00:04,  7.14it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 74.0 / 120  (61.7):  97%|█████████▋| 120/124 [00:22<00:01,  2.43it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 75.0 / 123  (61.0):  99%|█████████▉| 123/124 [00:25<00:00,  1.39it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Average Metric: 75.0 / 124  (60.5): 100%|██████████| 124/124 [00:30<00:00,  4.07it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Error for example in dev set: \t\t Too many retries\n",
+      "Average Metric: 75.0 / 124  (60.5%)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<style type=\"text/css\">\n",
+       "#T_1f859 th {\n",
+       "  text-align: left;\n",
+       "}\n",
+       "#T_1f859 td {\n",
+       "  text-align: left;\n",
+       "}\n",
+       "#T_1f859_row0_col0, #T_1f859_row0_col1, #T_1f859_row0_col2, #T_1f859_row0_col3, #T_1f859_row0_col4, #T_1f859_row0_col5, #T_1f859_row0_col6, #T_1f859_row1_col0, #T_1f859_row1_col1, #T_1f859_row1_col2, #T_1f859_row1_col3, #T_1f859_row1_col4, #T_1f859_row1_col5, #T_1f859_row1_col6, #T_1f859_row2_col0, #T_1f859_row2_col1, #T_1f859_row2_col2, #T_1f859_row2_col3, #T_1f859_row2_col4, #T_1f859_row2_col5, #T_1f859_row2_col6, #T_1f859_row3_col0, #T_1f859_row3_col1, #T_1f859_row3_col2, #T_1f859_row3_col3, #T_1f859_row3_col4, #T_1f859_row3_col5, #T_1f859_row3_col6, #T_1f859_row4_col0, #T_1f859_row4_col1, #T_1f859_row4_col2, #T_1f859_row4_col3, #T_1f859_row4_col4, #T_1f859_row4_col5, #T_1f859_row4_col6 {\n",
+       "  text-align: left;\n",
+       "  white-space: pre-wrap;\n",
+       "  word-wrap: break-word;\n",
+       "  max-width: 400px;\n",
+       "}\n",
+       "</style>\n",
+       "<table id=\"T_1f859\">\n",
+       "  <thead>\n",
+       "    <tr>\n",
+       "      <th class=\"blank level0\" >&nbsp;</th>\n",
+       "      <th id=\"T_1f859_level0_col0\" class=\"col_heading level0 col0\" >prompt</th>\n",
+       "      <th id=\"T_1f859_level0_col1\" class=\"col_heading level0 col1\" >test</th>\n",
+       "      <th id=\"T_1f859_level0_col2\" class=\"col_heading level0 col2\" >entry_point</th>\n",
+       "      <th id=\"T_1f859_level0_col3\" class=\"col_heading level0 col3\" >example_solution</th>\n",
+       "      <th id=\"T_1f859_level0_col4\" class=\"col_heading level0 col4\" >pred_solution</th>\n",
+       "      <th id=\"T_1f859_level0_col5\" class=\"col_heading level0 col5\" >metric</th>\n",
+       "      <th id=\"T_1f859_level0_col6\" class=\"col_heading level0 col6\" >solution</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th id=\"T_1f859_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
+       "      <td id=\"T_1f859_row0_col0\" class=\"data row0 col0\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+       "      <td id=\"T_1f859_row0_col1\" class=\"data row0 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, 5, -1]) == False\\n assert candidate([1, 3, -2, 1]) == True\\n...</td>\n",
+       "      <td id=\"T_1f859_row0_col2\" class=\"data row0 col2\" >triples_sum_to_zero</td>\n",
+       "      <td id=\"T_1f859_row0_col3\" class=\"data row0 col3\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+       "      <td id=\"T_1f859_row0_col4\" class=\"data row0 col4\" >code='\\n\\ndef triples_sum_to_zero(l: list):\\n \"\"\"\\n triples_sum_to_zero takes a list of integers as an input.\\n it returns True if there are three distinct elements in the list...</td>\n",
+       "      <td id=\"T_1f859_row0_col5\" class=\"data row0 col5\" >1.0</td>\n",
+       "      <td id=\"T_1f859_row0_col6\" class=\"data row0 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th id=\"T_1f859_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
+       "      <td id=\"T_1f859_row1_col0\" class=\"data row1 col0\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+       "      <td id=\"T_1f859_row1_col1\" class=\"data row1 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n    assert candidate(2) == 4\\n    assert candidate(3) == 9\\n    assert candidate(4) == 16\\n    assert candidate(8) == 64\\n    assert candidate(10) == 100\\n\\n'</td>\n",
+       "      <td id=\"T_1f859_row1_col2\" class=\"data row1 col2\" >car_race_collision</td>\n",
+       "      <td id=\"T_1f859_row1_col3\" class=\"data row1 col3\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+       "      <td id=\"T_1f859_row1_col4\" class=\"data row1 col4\" >code='\\n\\ndef car_race_collision(n: int):\\n \"\"\"\\n Imagine a road that\\'s a perfectly straight infinitely long line.\\n n cars are driving left to right; simultaneously, a different set...</td>\n",
+       "      <td id=\"T_1f859_row1_col5\" class=\"data row1 col5\" >1.0</td>\n",
+       "      <td id=\"T_1f859_row1_col6\" class=\"data row1 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th id=\"T_1f859_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
+       "      <td id=\"T_1f859_row2_col0\" class=\"data row2 col0\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+       "      <td id=\"T_1f859_row2_col1\" class=\"data row2 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([]) == []\\n assert candidate([3, 2, 1]) == [4, 3, 2]\\n assert candidate([5, 2, 5, 2, 3, 3, 9, 0,...</td>\n",
+       "      <td id=\"T_1f859_row2_col2\" class=\"data row2 col2\" >incr_list</td>\n",
+       "      <td id=\"T_1f859_row2_col3\" class=\"data row2 col3\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+       "      <td id=\"T_1f859_row2_col4\" class=\"data row2 col4\" >code='\\n\\ndef incr_list(l: list):\\n \"\"\"Return list with elements incremented by 1.\\n >>> incr_list([1, 2, 3])\\n [2, 3, 4]\\n >>> incr_list([5, 3, 5, 2, 3, 3, 9,...</td>\n",
+       "      <td id=\"T_1f859_row2_col5\" class=\"data row2 col5\" >1.0</td>\n",
+       "      <td id=\"T_1f859_row2_col6\" class=\"data row2 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th id=\"T_1f859_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
+       "      <td id=\"T_1f859_row3_col0\" class=\"data row3 col0\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+       "      <td id=\"T_1f859_row3_col1\" class=\"data row3 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate([1, 3, 5, 0]) == False\\n assert candidate([1, 3, -2, 1]) == False\\n assert candidate([1, 2, 3, 7]) == False\\n...</td>\n",
+       "      <td id=\"T_1f859_row3_col2\" class=\"data row3 col2\" >pairs_sum_to_zero</td>\n",
+       "      <td id=\"T_1f859_row3_col3\" class=\"data row3 col3\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+       "      <td id=\"T_1f859_row3_col4\" class=\"data row3 col4\" >code='\\n\\ndef pairs_sum_to_zero(l):\\n \"\"\"\\n pairs_sum_to_zero takes a list of integers as an input.\\n it returns True if there are two distinct elements in the list that\\n...</td>\n",
+       "      <td id=\"T_1f859_row3_col5\" class=\"data row3 col5\" >1.0</td>\n",
+       "      <td id=\"T_1f859_row3_col6\" class=\"data row3 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th id=\"T_1f859_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
+       "      <td id=\"T_1f859_row4_col0\" class=\"data row4 col0\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+       "      <td id=\"T_1f859_row4_col1\" class=\"data row4 col1\" >code='\\n\\nMETADATA = {}\\n\\n\\ndef check(candidate):\\n assert candidate(8, 3) == \"22\"\\n assert candidate(9, 3) == \"100\"\\n assert candidate(234, 2) == \"11101010\"\\n assert candidate(16, 2) == \"10000\"\\n assert...</td>\n",
+       "      <td id=\"T_1f859_row4_col2\" class=\"data row4 col2\" >change_base</td>\n",
+       "      <td id=\"T_1f859_row4_col3\" class=\"data row4 col3\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+       "      <td id=\"T_1f859_row4_col4\" class=\"data row4 col4\" >code='\\n\\ndef change_base(x: int, base: int):\\n \"\"\"Change numerical base of input number x to base.\\n return string representation after the conversion.\\n base numbers are less than...</td>\n",
+       "      <td id=\"T_1f859_row4_col5\" class=\"data row4 col5\" >1.0</td>\n",
+       "      <td id=\"T_1f859_row4_col6\" class=\"data row4 col6\" >nan</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n"
+      ],
+      "text/plain": [
+       "<pandas.io.formats.style.Styler at 0x2b2fa1410>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "                <div style='\n",
+       "                    text-align: center; \n",
+       "                    font-size: 16px; \n",
+       "                    font-weight: bold; \n",
+       "                    color: #555; \n",
+       "                    margin: 10px 0;'>\n",
+       "                    ... 119 more rows not displayed ...\n",
+       "                </div>\n",
+       "                "
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Compiled HumanEval score: 60.48\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Evaluating...\")\n",
+    "print(\n",
+    "    \"Compiled HumanEval score:\",\n",
+    "    evaluator(compiled, metric=test_code(timeout=100)),\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/examples/functional/repl.py b/examples/functional/repl.py
new file mode 100644
index 00000000..f0fc1801
--- /dev/null
+++ b/examples/functional/repl.py
@@ -0,0 +1,29 @@
+import multiprocessing
+from typing import Optional
+
+
+class PythonREPL:
+    @classmethod
+    def worker(cls, code: str, globals, locals, queue):
+        try:
+            exec(code, globals, locals)
+            queue.put(None)
+        except Exception as e:
+            queue.put(repr(e))
+
+    def run(self, command: str, timeout=5, globals={}, locals=None) -> Optional[str]:
+        queue: multiprocessing.Queue = multiprocessing.Queue()
+        p = multiprocessing.Process(
+            target=self.worker, args=(command, globals, locals, queue)
+        )
+        p.start()
+        p.join(timeout)
+        if p.is_alive():
+            p.terminate()
+            return "TimeoutError"
+        return queue.get_nowait()
+
+
+# We had to move the code above to a class for pickle reasons
+def execute_code(command: str, timeout=5, globals={}, locals=None) -> Optional[str]:
+    return PythonREPL().run(command, timeout, globals, locals)


Test patch:
diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py
index 8adf6cd1..4d77c7e2 100644
--- a/tests/functional/test_functional.py
+++ b/tests/functional/test_functional.py
@@ -9,7 +9,7 @@ import warnings
 import pytest
 
 import dspy
-from dspy.functional import predictor, cot, FunctionalModule, TypedPredictor
+from dspy.functional import predictor, cot, FunctionalModule, TypedPredictor, functional
 from dspy.primitives.example import Example
 from dspy.teleprompt.bootstrap import BootstrapFewShot
 from dspy.utils.dummies import DummyLM
@@ -76,7 +76,7 @@ def test_simple_class():
             description="At least two comments about the answer"
         )
 
-    class QA(dspy.Module):
+    class QA(FunctionalModule):
         @predictor
         def hard_question(self, topic: str) -> str:
             """Think of a hard factual question about a topic. It should be answerable with a number."""
@@ -107,8 +107,13 @@ def test_simple_class():
     dspy.settings.configure(lm=lm)
 
     qa = QA()
+    assert isinstance(qa, FunctionalModule)
+    assert isinstance(qa.answer, functional._StripOutput)
+
     question, answer = qa(topic="Physics")
 
+    print(qa.answer)
+
     assert question == "What is the speed of light?"
     assert answer == expected
 
@@ -168,7 +173,7 @@ def test_named_params():
     named_predictors = list(qa.named_predictors())
     assert len(named_predictors) == 2
     names, _ = zip(*qa.named_predictors())
-    assert set(names) == {"hard_question.predictor", "answer.predictor"}
+    assert set(names) == {"hard_question.predictor.predictor", "answer.predictor.predictor"}
 
 
 def test_bootstrap_effectiveness():
@@ -215,9 +220,10 @@ def test_bootstrap_effectiveness():
     lm.inspect_history(n=2)
 
     # Check that the compiled student has the correct demos
-    assert len(compiled_student.output.predictor.demos) == 1
-    assert compiled_student.output.predictor.demos[0].input == trainset[0].input
-    assert compiled_student.output.predictor.demos[0].output == trainset[0].output
+    demos = compiled_student.predictors()[0].demos
+    assert len(demos) == 1
+    assert demos[0].input == trainset[0].input
+    assert demos[0].output == trainset[0].output
 
     # Test the compiled student's prediction.
     # We are using a DummyLM with follow_examples=True, which means that
diff --git a/tests/predict/test_chain_of_thought_with_hint.py b/tests/predict/test_chain_of_thought_with_hint.py
index b5e62425..c28fb375 100644
--- a/tests/predict/test_chain_of_thought_with_hint.py
+++ b/tests/predict/test_chain_of_thought_with_hint.py
@@ -7,6 +7,7 @@ def test_cot_with_no_hint():
     lm = DummyLM(["find the number after 1", "2"])
     dspy.settings.configure(lm=lm)
     predict = ChainOfThoughtWithHint("question -> answer")
+    # Check output fields have the right order
     assert list(predict.extended_signature2.output_fields.keys()) == [
         "rationale",
         "hint",
diff --git a/tests/pytest.ini b/tests/pytest.ini
new file mode 100644
index 00000000..c24fe5bb
--- /dev/null
+++ b/tests/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+filterwarnings =
+    ignore::DeprecationWarning