Repository: stanfordnlp/dspy
Pull request #509 (2024-03-01 19:05:34)

Type improvements
Cleans up some type stuff (no more pydantic version in error messages),
Removes make_example (now it happens automatically, only if there's actually an error),
Fixes some some issues with generics like List[str]


Patch:
diff --git a/dspy/functional/functional.py b/dspy/functional/functional.py
index c1dcec9a..f349fa2e 100644
--- a/dspy/functional/functional.py
+++ b/dspy/functional/functional.py
@@ -1,5 +1,5 @@
 import inspect, os, openai, dspy, typing, pydantic
-from typing import Annotated
+from typing import Annotated, List, Tuple
 import typing
 from dsp.templates import passages2text
 import json
@@ -27,7 +27,7 @@ class _StripOutput(dspy.Module):
         super().__init__()
         self.predictor = predictor
         self.output_key = output_key
-    
+
     def copy(self):
         return _StripOutput(self.predictor.copy(), self.output_key)
 
@@ -37,7 +37,8 @@ class _StripOutput(dspy.Module):
 
 
 class FunctionalModule(dspy.Module):
-    """ To use the @cot and @predictor decorators, your module needs to inheret form this class. """
+    """To use the @cot and @predictor decorators, your module needs to inheret form this class."""
+
     def __init__(self):
         super().__init__()
         for name in dir(self):
@@ -46,38 +47,45 @@ class FunctionalModule(dspy.Module):
                 self.__dict__[name] = attr.copy()
 
 
-def TypedChainOfThought(signature, make_example=False):
-    """ Just like TypedPredictor, but adds a ChainOfThought OutputField. """
+def TypedChainOfThought(signature):
+    """Just like TypedPredictor, but adds a ChainOfThought OutputField."""
     signature = ensure_signature(signature)
     output_keys = ", ".join(signature.output_fields.keys())
-    return TypedPredictor(signature.prepend(
-        "reasoning",
-        dspy.OutputField(
-            prefix="Reasoning: Let's think step by step in order to",
-            desc="${produce the " + output_keys + "}. We ...",
-        ),
-    ), make_example)
+    return TypedPredictor(
+        signature.prepend(
+            "reasoning",
+            dspy.OutputField(
+                prefix="Reasoning: Let's think step by step in order to",
+                desc="${produce the " + output_keys + "}. We ...",
+            ),
+        )
+    )
 
 
 class TypedPredictor(dspy.Module):
-    def __init__(self, signature, make_example=False):
+    def __init__(self, signature):
         super().__init__()
         self.signature = signature
         self.predictor = dspy.Predict(signature)
-        self.make_example = make_example
 
     def copy(self):
-        return TypedPredictor(self.signature, self.make_example)
+        return TypedPredictor(self.signature)
 
     @staticmethod
     def _make_example(type_):
         # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.
-        return dspy.Predict(
+        json_object = dspy.Predict(
             dspy.Signature(
                 "json_schema -> json_object",
                 "Make a very succinct json object that validates with the following schema",
             )
         )(json_schema=json.dumps(type_.model_json_schema())).json_object
+        # We use the model_validate_json method to make sure the example is valid
+        try:
+            type_.model_validate_json(_unwrap_json(json_object))
+        except (pydantic.ValidationError, ValueError):
+            return ""  # Unable to make an example
+        return json_object
         # TODO: Another fun idea is to only (but automatically) do this if the output fails.
         # We could also have a more general "suggest solution" prompt that tries to fix the output
         # More directly.
@@ -103,29 +111,25 @@ class TypedPredictor(dspy.Module):
                 else:
                     # Anything else we wrap in a pydantic object
                     unwrap = lambda x: x
+                    wrap = lambda x: x
                     if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):
-                        type_ = pydantic.create_model(
-                            "Output", value=(type_, ...), __base__=pydantic.BaseModel
-                        )
+                        type_ = pydantic.create_model("Output", value=(type_, ...), __base__=pydantic.BaseModel)
+                        wrap = lambda x: type_(value=x)
                         unwrap = lambda x: x.value
                     signature = signature.with_updated_fields(
                         name,
                         desc=field.json_schema_extra.get("desc", "")
                         + (
-                            f". Respond with a single JSON object using the schema "
+                            f". Respond with a single JSON object. JSON Schema: "
                             + json.dumps(type_.model_json_schema())
-                            + (". For example: " + self._make_example(type_) if self.make_example else "")
-                        ),
-                        format=lambda x: (
-                            x if isinstance(x, str) else x.model_dump_json()
-                        ),
-                        parser=lambda x: unwrap(
-                            type_.model_validate_json(_unwrap_json(x))
                         ),
+                        format=lambda x: (x if isinstance(x, str) else wrap(x).model_dump_json()),
+                        parser=lambda x: unwrap(type_.model_validate_json(_unwrap_json(x))),
+                        type_=type_,
                     )
             else:  # If input field
                 format = lambda x: x if isinstance(x, str) else str(x)
-                if type_ in (list[str], tuple[str]):
+                if type_ in (List[str], list[str], Tuple[str], tuple[str]):
                     format = passages2text
                 elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):
                     format = lambda x: x if isinstance(x, str) else x.model_dump_json()
@@ -149,16 +153,27 @@ class TypedPredictor(dspy.Module):
                     parser = field.json_schema_extra.get("parser", lambda x: x)
                     parsed_results[name] = parser(value)
                 except (pydantic.ValidationError, ValueError) as e:
-                    errors[name] = e
+                    errors[name] = _format_error(e)
+                    # If we can, we add an example to the error message
+                    current_desc = field.json_schema_extra.get("desc", "")
+                    i = current_desc.find("JSON Schema: ")
+                    if i == -1:
+                        continue  # Only add examples to JSON objects
+                    suffix, current_desc = current_desc[i:], current_desc[:i]
+                    prefix = "You MUST use this format: "
+                    if try_i + 1 < MAX_RETRIES and prefix not in current_desc:
+                        if example := self._make_example(field.annotation):
+                            signature = signature.with_updated_fields(
+                                name, desc=current_desc + "\n" + prefix + example + "\n" + suffix
+                            )
             if errors:
                 # Add new fields for each error
                 for name, error in errors.items():
-                    modified_kwargs[f"error_{name}_{try_i}"] = str(error)
+                    modified_kwargs[f"error_{name}_{try_i}"] = error
                     signature = signature.append(
                         f"error_{name}_{try_i}",
                         dspy.InputField(
-                            prefix=f"Past Error "
-                            + (f"({name}):" if try_i == 0 else f"({name}, {try_i+1}):"),
+                            prefix=f"Past Error " + (f"({name}):" if try_i == 0 else f"({name}, {try_i+1}):"),
                             desc="An error to avoid in the future",
                         ),
                     )
@@ -167,7 +182,20 @@ class TypedPredictor(dspy.Module):
                 for name, value in parsed_results.items():
                     setattr(result, name, value)
                 return result
-        raise ValueError("Too many retries")
+        raise ValueError(
+            "Too many retries trying to get the correct output format. " + "Try simplifying the requirements.", errors
+        )
+
+
+def _format_error(error: Exception):
+    if isinstance(error, pydantic.ValidationError):
+        errors = []
+        for e in error.errors():
+            fields = ", ".join(e["loc"])
+            errors.append(f"{e['msg']}: {fields} (error type: {e['type']})")
+        return "; ".join(errors)
+    else:
+        return str(error)
 
 
 def _func_to_signature(func):
@@ -221,9 +249,7 @@ def main():
     class Answer(pydantic.BaseModel):
         value: float
         certainty: float
-        comments: list[str] = pydantic.Field(
-            description="At least two comments about the answer"
-        )
+        comments: list[str] = pydantic.Field(description="At least two comments about the answer")
 
     class QA(dspy.Module):
         @predictor
@@ -262,16 +288,11 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
     if not dspy.evaluate.answer_passage_match(example, pred):
         return False
 
-    hops = [example.question] + [
-        outputs.query for *_, outputs in trace if "query" in outputs
-    ]
+    hops = [example.question] + [outputs.query for *_, outputs in trace if "query" in outputs]
 
     if max([len(h) for h in hops]) > 100:
         return False
-    if any(
-        dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8)
-        for idx in range(2, len(hops))
-    ):
+    if any(dspy.evaluate.answer_exact_match_str(hops[idx], hops[:idx], frac=0.8) for idx in range(2, len(hops))):
         return False
 
     return True
@@ -279,9 +300,7 @@ def validate_context_and_answer_and_hops(example, pred, trace=None):
 
 def gold_passages_retrieved(example, pred, trace=None):
     gold_titles = set(map(dspy.evaluate.normalize_text, example["gold_titles"]))
-    found_titles = set(
-        map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context])
-    )
+    found_titles = set(map(dspy.evaluate.normalize_text, [c.split(" | ")[0] for c in pred.context]))
 
     return gold_titles.issubset(found_titles)
 
@@ -294,9 +313,7 @@ def hotpot():
     from dspy.teleprompt.bootstrap import BootstrapFewShot
 
     print("Load the dataset.")
-    dataset = HotPotQA(
-        train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0
-    )
+    dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)
     trainset = [x.with_inputs("question") for x in dataset.train]
     devset = [x.with_inputs("question") for x in dataset.dev]
     print("Done")
@@ -333,9 +350,7 @@ def hotpot():
     lm = dspy.OpenAI(model="gpt-3.5-turbo", max_tokens=4000)
     dspy.settings.configure(lm=lm, rm=rm, trace=[])
 
-    evaluate_on_hotpotqa = Evaluate(
-        devset=devset, num_threads=10, display_progress=True, display_table=5
-    )
+    evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=10, display_progress=True, display_table=5)
 
     # uncompiled (i.e., zero-shot) program
     uncompiled_baleen = SimplifiedBaleen()
@@ -345,9 +360,7 @@ def hotpot():
     )
 
     # compiled (i.e., few-shot) program
-    compiled_baleen = BootstrapFewShot(
-        metric=validate_context_and_answer_and_hops
-    ).compile(
+    compiled_baleen = BootstrapFewShot(metric=validate_context_and_answer_and_hops).compile(
         SimplifiedBaleen(),
         teacher=SimplifiedBaleen(passages_per_hop=2),
         trainset=trainset,
diff --git a/dspy/signatures/signature.py b/dspy/signatures/signature.py
index ac9a334c..a5f6b976 100644
--- a/dspy/signatures/signature.py
+++ b/dspy/signatures/signature.py
@@ -77,7 +77,7 @@ class SignatureMeta(type(BaseModel)):
         # Make sure to give input fields before output fields
         return {**cls.input_fields, **cls.output_fields}
 
-    def with_updated_fields(cls, name, **kwargs):
+    def with_updated_fields(cls, name, type_=None, **kwargs):
         """Returns a new Signature type with the field, name, updated
         with fields[name].json_schema_extra[key] = value."""
         fields_copy = deepcopy(cls.fields)
@@ -85,6 +85,8 @@ class SignatureMeta(type(BaseModel)):
             **fields_copy[name].json_schema_extra,
             **kwargs,
         }
+        if type_ is not None:
+            fields_copy[name].annotation = type_
         return Signature(fields_copy, cls.instructions)
 
     @property
diff --git a/examples/functional/functional.ipynb b/examples/functional/functional.ipynb
index 0dcbef5d..cf7088a5 100644
--- a/examples/functional/functional.ipynb
+++ b/examples/functional/functional.ipynb
@@ -175,7 +175,7 @@
     "    entry_point: str = InputField()\n",
     "    solution: PythonCode = OutputField()\n",
     "\n",
-    "predictor = TypedPredictor(CodeSignature, make_example=True)\n",
+    "predictor = TypedPredictor(CodeSignature)\n",
     "prediction = predictor(\n",
     "    prompt=PythonCode(code=ds['test'][0]['prompt']),\n",
     "    test=PythonCode(code=ds['test'][0]['test']),\n",


Test patch:
diff --git a/tests/functional/test_functional.py b/tests/functional/test_functional.py
index 6cb00581..5d0f230b 100644
--- a/tests/functional/test_functional.py
+++ b/tests/functional/test_functional.py
@@ -3,7 +3,6 @@ import textwrap
 import pydantic
 from pydantic import Field, BaseModel, field_validator
 from typing import Annotated
-import warnings
 from typing import List
 
 import pytest
@@ -32,25 +31,8 @@ def test_simple():
 
 def test_list_output():
     @predictor
-    def hard_question(topic: str) -> List[str]:
-        """Think of a hard factual question about a topic."""
-
-    expected = ["What is the speed of light?", "What is the speed of sound?"]
-    lm = DummyLM(
-        ['{"value": ["What is the speed of light?", "What is the speed of sound?"]}']
-    )
-    dspy.settings.configure(lm=lm)
-
-    question = hard_question(topic="Physics")
-    lm.inspect_history(n=2)
-
-    assert question == expected
-
-
-def test_list_output():
-    @predictor
-    def hard_question(topic: str) -> List[str]:
-        """Think of a hard factual question about a topic."""
+    def hard_questions(topics: List[str]) -> List[str]:
+        pass
 
     expected = ["What is the speed of light?", "What is the speed of sound?"]
     lm = DummyLM(
@@ -58,7 +40,7 @@ def test_list_output():
     )
     dspy.settings.configure(lm=lm)
 
-    question = hard_question(topic="Physics")
+    question = hard_questions(topics=["Physics", "Music"])
     lm.inspect_history(n=2)
 
     assert question == expected
@@ -134,6 +116,7 @@ def test_simple_class():
             "What is the speed of light?",
             "Some bad reasoning, 3e8 m/s.",
             "3e8",  # Bad answer 1
+            "{...}",  # Model is asked to create an example
             "Some good reasoning...",
             expected.model_dump_json(),  # Good answer
         ]
@@ -311,6 +294,8 @@ def test_regex():
         [
             # Example with a bad origin code.
             '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',
+            # Example to help the model understand
+            '{...}',
             # Fixed
             '{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',
         ]
@@ -359,7 +344,9 @@ def test_multi_errors():
         [
             # First origin is wrong, then destination, then all is good
             '{"origin": "JF0", "destination": "LAX", "date": "2022-12-25"}',
+            '{...}', # Example to help the model understand
             '{"origin": "JFK", "destination": "LA0", "date": "2022-12-25"}',
+            '{...}', # Example to help the model understand
             '{"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}',
         ]
     )
@@ -368,7 +355,6 @@ def test_multi_errors():
     assert flight_information(email="Some email") == TravelInformation(
         origin="JFK", destination="LAX", date=datetime.date(2022, 12, 25)
     )
-    warnings.warn("This test is dependent on the version of pydantic used.")
     assert lm.get_convo(-1) == textwrap.dedent(
         """\
         Given the fields `email`, produce the fields `flight_information`.
@@ -383,18 +369,17 @@ def test_multi_errors():
 
         Past Error (flight_information, 2): An error to avoid in the future
 
-        Flight Information: ${flight_information}. Respond with a single JSON object using the schema {"properties": {"origin": {"pattern": "^[A-Z]{3}$", "title": "Origin", "type": "string"}, "destination": {"pattern": "^[A-Z]{3}$", "title": "Destination", "type": "string"}, "date": {"format": "date", "title": "Date", "type": "string"}}, "required": ["origin", "destination", "date"], "title": "TravelInformation", "type": "object"}
+        Flight Information: ${flight_information}. Respond with a single JSON object. JSON Schema: {"properties": {"origin": {"pattern": "^[A-Z]{3}$", "title": "Origin", "type": "string"}, "destination": {"pattern": "^[A-Z]{3}$", "title": "Destination", "type": "string"}, "date": {"format": "date", "title": "Date", "type": "string"}}, "required": ["origin", "destination", "date"], "title": "TravelInformation", "type": "object"}
 
         ---
 
         Email: Some email
 
-        Past Error (flight_information): 1 validation error for TravelInformation origin String should match pattern '^[A-Z]{3}$' [type=string_pattern_mismatch, input_value='JF0', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/string_pattern_mismatch
+        Past Error (flight_information): String should match pattern '^[A-Z]{3}$': origin (error type: string_pattern_mismatch)
 
-        Past Error (flight_information, 2): 1 validation error for TravelInformation destination String should match pattern '^[A-Z]{3}$' [type=string_pattern_mismatch, input_value='LA0', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/string_pattern_mismatch
+        Past Error (flight_information, 2): String should match pattern '^[A-Z]{3}$': destination (error type: string_pattern_mismatch)
 
         Flight Information: {"origin": "JFK", "destination": "LAX", "date": "2022-12-25"}"""
-        # Note: Pydantic version is hardcoded in the url here
     )
 
 
@@ -427,7 +412,6 @@ def test_field_validator():
     with pytest.raises(ValueError):
         get_user_details()
 
-    warnings.warn("This test is dependent on the version of pydantic used.")
     assert lm.get_convo(-1) == textwrap.dedent(
         """\
         Given the fields , produce the fields `get_user_details`.
@@ -438,11 +422,28 @@ def test_field_validator():
 
         Past Error (get_user_details): An error to avoid in the future
         Past Error (get_user_details, 2): An error to avoid in the future
-        Get User Details: ${get_user_details}. Respond with a single JSON object using the schema {"properties": {"name": {"title": "Name", "type": "string"}, "age": {"title": "Age", "type": "integer"}}, "required": ["name", "age"], "title": "UserDetails", "type": "object"}
+        Get User Details: ${get_user_details}. Respond with a single JSON object. JSON Schema: {"properties": {"name": {"title": "Name", "type": "string"}, "age": {"title": "Age", "type": "integer"}}, "required": ["name", "age"], "title": "UserDetails", "type": "object"}
 
         ---
 
-        Past Error (get_user_details): 1 validation error for UserDetails name Value error, Name must be in uppercase. [type=value_error, input_value='lower case name', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/value_error
-        Past Error (get_user_details, 2): 1 validation error for UserDetails name Value error, Name must be in uppercase. [type=value_error, input_value='lower case name', input_type=str] For further information visit https://errors.pydantic.dev/2.5/v/value_error
+        Past Error (get_user_details): Value error, Name must be in uppercase.: name (error type: value_error)
+        Past Error (get_user_details, 2): Value error, Name must be in uppercase.: name (error type: value_error)
         Get User Details: {"name": "lower case name", "age": 25}"""
     )
+
+
+def test_annotated_field():
+    # Since we don't currently validate fields on the main signature,
+    # the annotated fields are also not validated.
+    # But at least it should not crash.
+
+    @predictor
+    def test(input: Annotated[str, Field(description="description")]) -> Annotated[float, Field(gt=0, lt=1)]:
+        pass
+
+    lm = DummyLM(["0.5"])
+    dspy.settings.configure(lm=lm)
+
+    output = test(input="input")
+
+    assert output == 0.5