Repository: stanfordnlp/dspy
Pull request #2039 (2025-01-11 07:28:04)

Add arg parsing for dspy.ReAct
Currently we are not parsing the pydantic arg in dspy.ReAct, instead we are just feeding the dict as pydantic arg. This is problematic, see below for a simple reproduce:

```
from pydantic import BaseModel

import dspy

dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini"))


class CalendarEvent(BaseModel):
    name: str
    date: str
    participants: dict[str, str]


def write_invitation_letter(participant_name: str, event_info: CalendarEvent):
    # if isinstance(event_info, dict):
    #     event_info = CalendarEvent.model_validate(event_info)
    if participant_name not in event_info.participants:
        return None
    return f"It's my honor to invite {participant_name} to event {event_info.name} on {event_info.date}"


class InvitationSignature(dspy.Signature):
    participant_name: str = dspy.InputField(desc="The name of the participant to invite")
    event_info: CalendarEvent = dspy.InputField(desc="The information about the event")
    invitation_letter: str = dspy.OutputField(desc="The invitation letter to be sent to the participant")


react = dspy.ReAct(InvitationSignature, tools=[write_invitation_letter])

outputs = react(
    participant_name="Alice",
    event_info=CalendarEvent(
        name="Science Fair",
        date="Friday",
        participants={"Alice": "female", "Bob": "male"},
    ),
)
print(outputs)
```

It doesn't directly error out because we wrap tool calling by try-except block. But if you look into the `outputs.trajectory`, it will tell that errors are thrown. 


Patch:
diff --git a/dspy/adapters/chat_adapter.py b/dspy/adapters/chat_adapter.py
index 3f58fb0e..e51e90d6 100644
--- a/dspy/adapters/chat_adapter.py
+++ b/dspy/adapters/chat_adapter.py
@@ -6,14 +6,14 @@ import re
 import textwrap
 from collections.abc import Mapping
 from itertools import chain
-from typing import Any, Dict, List, Literal, NamedTuple, Union, get_args, get_origin
+from typing import Any, Dict, List, Literal, NamedTuple, Union
 
 import pydantic
 from pydantic import TypeAdapter
 from pydantic.fields import FieldInfo
 
 from dspy.adapters.base import Adapter
-from dspy.adapters.utils import find_enum_member, format_field_value
+from dspy.adapters.utils import find_enum_member, format_field_value, get_annotation_name
 from dspy.signatures.field import OutputField
 from dspy.signatures.signature import Signature, SignatureMeta
 from dspy.signatures.utils import get_dspy_field_type
@@ -267,19 +267,6 @@ def format_turn(signature, values, role, incomplete=False):
     return {"role": role, "content": collapsed_messages}
 
 
-def get_annotation_name(annotation):
-    origin = get_origin(annotation)
-    args = get_args(annotation)
-    if origin is None:
-        if hasattr(annotation, "__name__"):
-            return annotation.__name__
-        else:
-            return str(annotation)
-    else:
-        args_str = ", ".join(get_annotation_name(arg) for arg in args)
-        return f"{get_annotation_name(origin)}[{args_str}]"
-
-
 def enumerate_fields(fields: dict) -> str:
     parts = []
     for idx, (k, v) in enumerate(fields.items()):
@@ -323,7 +310,11 @@ def prepare_instructions(signature: SignatureMeta):
         elif inspect.isclass(field_type) and issubclass(field_type, enum.Enum):
             desc = f"must be one of: {'; '.join(field_type.__members__)}"
         elif hasattr(field_type, "__origin__") and field_type.__origin__ is Literal:
-            desc = f"must be one of: {'; '.join([str(x) for x in field_type.__args__])}"
+            desc = (
+                # Strongly encourage the LM to avoid choosing values that don't appear in the
+                # literal or returning a value of the form 'Literal[<selected_value>]'
+                f"must exactly match (no extra characters) one of: {'; '.join([str(x) for x in field_type.__args__])}"
+            )
         else:
             desc = "must be pareseable according to the following JSON schema: "
             desc += json.dumps(prepare_schema(field_type), ensure_ascii=False)
diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py
index 7eed644f..a2fbcd55 100644
--- a/dspy/adapters/json_adapter.py
+++ b/dspy/adapters/json_adapter.py
@@ -5,7 +5,7 @@ import json
 import logging
 import textwrap
 from copy import deepcopy
-from typing import Any, Dict, KeysView, Literal, NamedTuple, get_args, get_origin
+from typing import Any, Dict, KeysView, Literal, NamedTuple
 
 import json_repair
 import litellm
@@ -15,7 +15,7 @@ from pydantic.fields import FieldInfo
 
 from dspy.adapters.base import Adapter
 from dspy.adapters.image_utils import Image
-from dspy.adapters.utils import find_enum_member, format_field_value, serialize_for_json
+from dspy.adapters.utils import find_enum_member, format_field_value, get_annotation_name, serialize_for_json
 from dspy.signatures.signature import SignatureMeta
 from dspy.signatures.utils import get_dspy_field_type
 
@@ -243,19 +243,6 @@ def format_turn(signature: SignatureMeta, values: Dict[str, Any], role, incomple
     return {"role": role, "content": "\n\n".join(content).strip()}
 
 
-def get_annotation_name(annotation):
-    origin = get_origin(annotation)
-    args = get_args(annotation)
-    if origin is None:
-        if hasattr(annotation, "__name__"):
-            return annotation.__name__
-        else:
-            return str(annotation)
-    else:
-        args_str = ", ".join(get_annotation_name(arg) for arg in args)
-        return f"{get_annotation_name(origin)}[{args_str}]"
-
-
 def enumerate_fields(fields):
     parts = []
     for idx, (k, v) in enumerate(fields.items()):
diff --git a/dspy/adapters/utils.py b/dspy/adapters/utils.py
index 9cddc1fe..f627f074 100644
--- a/dspy/adapters/utils.py
+++ b/dspy/adapters/utils.py
@@ -1,5 +1,5 @@
 import json
-from typing import Any, List, Union
+from typing import Any, List, Literal, Union, get_args, get_origin
 
 from pydantic import TypeAdapter
 from pydantic.fields import FieldInfo
@@ -103,6 +103,26 @@ def find_enum_member(enum, identifier):
     raise ValueError(f"{identifier} is not a valid name or value for the enum {enum.__name__}")
 
 
+def get_annotation_name(annotation):
+    origin = get_origin(annotation)
+    args = get_args(annotation)
+    if origin is None:
+        if hasattr(annotation, "__name__"):
+            return annotation.__name__
+        else:
+            return str(annotation)
+
+    if origin is Literal:
+        args_str = ", ".join(
+            _quoted_string_for_literal_type_annotation(a) if isinstance(a, str) else get_annotation_name(a)
+            for a in args
+        )
+        return f"{get_annotation_name(origin)}[{args_str}]"
+    else:
+        args_str = ", ".join(get_annotation_name(a) for a in args)
+        return f"{get_annotation_name(origin)}[{args_str}]"
+
+
 def _format_input_list_field_value(value: List[Any]) -> str:
     """
     Formats the value of an input field of type List[Any].
@@ -135,3 +155,25 @@ def _format_blob(blob: str) -> str:
 
     modified_blob = blob.replace("\n", "\n    ")
     return f"«««\n    {modified_blob}\n»»»"
+
+
+def _quoted_string_for_literal_type_annotation(s: str) -> str:
+    """
+    Return the specified string quoted for inclusion in a literal type annotation.
+    """
+    has_single = "'" in s
+    has_double = '"' in s
+
+    if has_single and not has_double:
+        # Only single quotes => enclose in double quotes
+        return f'"{s}"'
+    elif has_double and not has_single:
+        # Only double quotes => enclose in single quotes
+        return f"'{s}'"
+    elif has_single and has_double:
+        # Both => enclose in single quotes; escape each single quote with \'
+        escaped = s.replace("'", "\\'")
+        return f"'{escaped}'"
+    else:
+        # Neither => enclose in single quotes
+        return f"'{s}'"
diff --git a/dspy/predict/react.py b/dspy/predict/react.py
index 2690d066..e05fe2fa 100644
--- a/dspy/predict/react.py
+++ b/dspy/predict/react.py
@@ -1,10 +1,9 @@
 import inspect
 from typing import Any, Callable, Literal, get_origin, get_type_hints
 
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 
 import dspy
-from dspy.adapters.json_adapter import get_annotation_name
 from dspy.primitives.program import Module
 from dspy.signatures.signature import ensure_signature
 from dspy.utils.callback import with_callbacks
@@ -16,13 +15,16 @@ class Tool:
         self.func = func
         self.name = name or getattr(func, "__name__", type(func).__name__)
         self.desc = desc or getattr(func, "__doc__", None) or getattr(annotations_func, "__doc__", "")
-        self.args = {
-            k: v.schema()
-            if isinstance((origin := get_origin(v) or v), type) and issubclass(origin, BaseModel)
-            else get_annotation_name(v)
-            for k, v in (args or get_type_hints(annotations_func)).items()
-            if k != "return"
-        }
+        self.args = {}
+        self.arg_types = {}
+        for k, v in (args or get_type_hints(annotations_func)).items():
+            self.arg_types[k] = v
+            if k == "return":
+                continue
+            if isinstance((origin := get_origin(v) or v), type) and issubclass(origin, BaseModel):
+                self.args[k] = v.model_json_schema()
+            else:
+                self.args[k] = TypeAdapter(v).json_schema()
 
     @with_callbacks
     def __call__(self, *args, **kwargs):
@@ -96,7 +98,14 @@ class ReAct(Module):
             trajectory[f"tool_args_{idx}"] = pred.next_tool_args
 
             try:
-                trajectory[f"observation_{idx}"] = self.tools[pred.next_tool_name](**pred.next_tool_args)
+                parsed_tool_args = {}
+                for k, v in pred.next_tool_args.items():
+                    arg_type = self.tools[pred.next_tool_name].arg_types[k]
+                    if isinstance((origin := get_origin(arg_type) or arg_type), type) and issubclass(origin, BaseModel):
+                        parsed_tool_args[k] = arg_type.model_validate(v)
+                    else:
+                        parsed_tool_args[k] = v
+                trajectory[f"observation_{idx}"] = self.tools[pred.next_tool_name](**parsed_tool_args)
             except Exception as e:
                 trajectory[f"observation_{idx}"] = f"Failed to execute: {e}"
 
diff --git a/requirements.txt b/requirements.txt
index 6558a34d..1e9b2b9d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,7 +9,7 @@ httpx
 jinja2
 joblib~=1.3
 json-repair
-litellm[proxy]==1.55.3
+litellm[proxy]==1.57.4
 magicattr~=0.1.6
 openai
 optuna


Test patch:
diff --git a/tests/adapters/test_chat_adapter.py b/tests/adapters/test_chat_adapter.py
new file mode 100644
index 00000000..4866ac01
--- /dev/null
+++ b/tests/adapters/test_chat_adapter.py
@@ -0,0 +1,79 @@
+from typing import Literal
+from unittest import mock
+
+import pytest
+
+import dspy
+
+
+@pytest.mark.parametrize(
+    "input_literal, output_literal, input_value, expected_input_str, expected_output_str",
+    [
+        # Scenario 1: double quotes escaped within strings
+        (
+            Literal["one", "two", 'three"'],
+            Literal["four", "five", 'six"'],
+            "two",
+            "Literal['one', 'two', 'three\"']",
+            "Literal['four', 'five', 'six\"']",
+        ),
+        # Scenario 2: Single quotes inside strings
+        (
+            Literal["she's here", "okay", "test"],
+            Literal["done", "maybe'soon", "later"],
+            "she's here",
+            "Literal[\"she's here\", 'okay', 'test']",
+            "Literal['done', \"maybe'soon\", 'later']",
+        ),
+        # Scenario 3: Strings containing both single and double quotes
+        (
+            Literal["both\"and'", "another"],
+            Literal["yet\"another'", "plain"],
+            "another",
+            "Literal['both\"and\\'', 'another']",
+            "Literal['yet\"another\\'', 'plain']",
+        ),
+        # Scenario 4: No quotes at all (check the default)
+        (
+            Literal["foo", "bar"],
+            Literal["baz", "qux"],
+            "foo",
+            "Literal['foo', 'bar']",
+            "Literal['baz', 'qux']",
+        ),
+        # Scenario 5: Mixed types
+        (
+            Literal[1, "bar"],
+            Literal[True, 3, "foo"],
+            "bar",
+            "Literal[1, 'bar']",
+            "Literal[True, 3, 'foo']",
+        ),
+    ],
+)
+def test_chat_adapter_quotes_literals_as_expected(
+    input_literal, output_literal, input_value, expected_input_str, expected_output_str
+):
+    """
+    This test verifies that when we declare Literal fields with various mixes
+    of single/double quotes, the generated content string includes those
+    Literals exactly as we want them to appear (like IPython does).
+    """
+
+    class TestSignature(dspy.Signature):
+        input_text: input_literal = dspy.InputField()
+        output_text: output_literal = dspy.OutputField()
+
+    program = dspy.Predict(TestSignature)
+
+    dspy.configure(lm=dspy.LM(model="openai/gpt4o"), adapter=dspy.ChatAdapter())
+
+    with mock.patch("litellm.completion") as mock_completion:
+        program(input_text=input_value)
+
+    mock_completion.assert_called_once()
+    _, call_kwargs = mock_completion.call_args
+    content = call_kwargs["messages"][0]["content"]
+
+    assert expected_input_str in content
+    assert expected_output_str in content
diff --git a/tests/test_json_adapter.py b/tests/adapters/test_json_adapter.py
similarity index 100%
rename from tests/test_json_adapter.py
rename to tests/adapters/test_json_adapter.py
diff --git a/tests/predict/test_react.py b/tests/predict/test_react.py
index 4c6a150d..2d99cda1 100644
--- a/tests/predict/test_react.py
+++ b/tests/predict/test_react.py
@@ -3,6 +3,7 @@ from dataclasses import dataclass
 import dspy
 from dspy.utils.dummies import DummyLM, dummy_rm
 from dspy.predict import react
+from pydantic import BaseModel
 
 
 # def test_example_no_tools():
@@ -124,15 +125,17 @@ from dspy.predict import react
 #     assert react.react[0].signature.instructions is not None
 #     assert react.react[0].signature.instructions.startswith("You are going to generate output based on input.")
 
+
 def test_tool_from_function():
     def foo(a: int, b: int) -> int:
         """Add two numbers."""
         return a + b
-    
+
     tool = react.Tool(foo)
     assert tool.name == "foo"
     assert tool.desc == "Add two numbers."
-    assert tool.args == {"a": "int", "b": "int"}
+    assert tool.args == {"a": {"type": "integer"}, "b": {"type": "integer"}}
+
 
 def test_tool_from_class():
     class Foo:
@@ -146,4 +149,82 @@ def test_tool_from_class():
     tool = react.Tool(Foo("123").foo)
     assert tool.name == "foo"
     assert tool.desc == "Add two numbers."
-    assert tool.args == {"a": "int", "b": "int"}
+    assert tool.args == {"a": {"type": "integer"}, "b": {"type": "integer"}}
+
+
+def test_tool_calling_with_pydantic_args():
+    class CalendarEvent(BaseModel):
+        name: str
+        date: str
+        participants: dict[str, str]
+
+    def write_invitation_letter(participant_name: str, event_info: CalendarEvent):
+        if participant_name not in event_info.participants:
+            return None
+        return f"It's my honor to invite {participant_name} to event {event_info.name} on {event_info.date}"
+
+    class InvitationSignature(dspy.Signature):
+        participant_name: str = dspy.InputField(desc="The name of the participant to invite")
+        event_info: CalendarEvent = dspy.InputField(desc="The information about the event")
+        invitation_letter: str = dspy.OutputField(desc="The invitation letter to be sent to the participant")
+
+    react = dspy.ReAct(InvitationSignature, tools=[write_invitation_letter])
+
+    lm = DummyLM(
+        [
+            {
+                "next_thought": "I need to write an invitation letter for Alice to the Science Fair event.",
+                "next_tool_name": "write_invitation_letter",
+                "next_tool_args": {
+                    "participant_name": "Alice",
+                    "event_info": {
+                        "name": "Science Fair",
+                        "date": "Friday",
+                        "participants": {"Alice": "female", "Bob": "male"},
+                    },
+                },
+            },
+            {
+                "next_thought": (
+                    "I have successfully written the invitation letter for Alice to the Science Fair. Now "
+                    "I can finish the task."
+                ),
+                "next_tool_name": "finish",
+                "next_tool_args": {},
+            },
+            {
+                "reasoning": "This is a very rigorous reasoning process, trust me bro!",
+                "invitation_letter": "It's my honor to invite Alice to the Science Fair event on Friday.",
+            },
+        ]
+    )
+    dspy.settings.configure(lm=lm)
+
+    outputs = react(
+        participant_name="Alice",
+        event_info=CalendarEvent(
+            name="Science Fair",
+            date="Friday",
+            participants={"Alice": "female", "Bob": "male"},
+        ),
+    )
+    assert outputs.invitation_letter == "It's my honor to invite Alice to the Science Fair event on Friday."
+
+    expected_trajectory = {
+        "thought_0": "I need to write an invitation letter for Alice to the Science Fair event.",
+        "tool_name_0": "write_invitation_letter",
+        "tool_args_0": {
+            "participant_name": "Alice",
+            "event_info": {
+                "name": "Science Fair",
+                "date": "Friday",
+                "participants": {"Alice": "female", "Bob": "male"},
+            },
+        },
+        "observation_0": "It's my honor to invite Alice to event Science Fair on Friday",
+        "thought_1": "I have successfully written the invitation letter for Alice to the Science Fair. Now I can finish the task.",
+        "tool_name_1": "finish",
+        "tool_args_1": {},
+        "observation_1": "Completed.",
+    }
+    assert outputs.trajectory == expected_trajectory
diff --git a/tests/reliability/test_pydantic_models.py b/tests/reliability/test_pydantic_models.py
index bb699154..89ddd747 100644
--- a/tests/reliability/test_pydantic_models.py
+++ b/tests/reliability/test_pydantic_models.py
@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import List
+from typing import Any, List, Literal
 
 import pydantic
 import pytest
@@ -103,3 +103,85 @@ def test_entity_extraction_with_multiple_primitive_outputs():
             " The categories should be separated by the character '|'."
         ),
     )
+
+
+@pytest.mark.parametrize("module", [dspy.Predict, dspy.ChainOfThought])
+def test_tool_calling_with_literals(module):
+    next_tool_names = [
+        "get_docs",
+        "finish",
+        "search_policy",
+        "notify_manager",
+        "calculate_accrual",
+        "combine_leave",
+        "review_seniority_rules",
+        "fetch_calendar",
+        "verify_compensation",
+        "check_carryover_policy",
+    ]
+
+    class ToolCalling(dspy.Signature):
+        """
+        Given the fields question, produce the fields response.
+        You will be given question and your goal is to finish with response.
+        To do this, you will interleave Thought, Tool Name, and Tool Args, and receive a resulting Observation.
+        """
+
+        question: str = dspy.InputField()
+        trajectory: str = dspy.InputField()
+        next_thought: str = dspy.OutputField()
+        next_tool_name: Literal[
+            "get_docs",
+            "finish",
+            "search_policy",
+            "notify_manager",
+            "calculate_accrual",
+            "combine_leave",
+            "review_seniority_rules",
+            "fetch_calendar",
+            "verify_compensation",
+            "check_carryover_policy",
+        ] = dspy.OutputField()
+        next_tool_args: dict[str, Any] = dspy.OutputField()
+        response_status: Literal["success", "error", "pending"] = dspy.OutputField()
+        user_intent: Literal["informational", "transactional", "exploratory"] = dspy.OutputField()
+
+    program = dspy.Predict(ToolCalling)
+    prediction = program(
+        question=(
+            "Tell me more about the company's internal policy for paid time off (PTO), "
+            "including as many details as possible. I want to know how PTO is accrued—are "
+            "there fixed rates, and do they vary by employee seniority or length of service? "
+            "Are there specific rules about carrying unused PTO into the next calendar year, "
+            "or is it a 'use it or lose it' system? Additionally, if an employee plans to take "
+            "extended leave for a vacation or personal reasons, what is the process for submitting "
+            "a request, and how far in advance should they notify their manager? Is there any overlap "
+            "or interaction between PTO and other forms of leave, such as sick leave or parental leave? "
+            "For example, can PTO be combined with those leave types to create a longer absence, or are "
+            "they strictly separate? I’d also like to know if there are any restrictions on when PTO can "
+            "be used, such as during critical business periods or holidays. Finally, what is the official "
+            "policy if an employee resigns or is terminated—are they compensated for unused PTO days, and if "
+            "so, at what rate?"
+        ),
+        trajectory=(
+            "["
+            "{'thought': 'Start by understanding PTO accrual rules.', 'tool_name': 'search_policy', 'tool_args': {'topic': 'PTO accrual rates'}}, "
+            "{'thought': 'Clarify whether PTO accrual rates vary by seniority.', 'tool_name': 'review_seniority_rules', 'tool_args': {}}, "
+            "{'thought': 'Identify carryover rules for unused PTO.', 'tool_name': 'check_carryover_policy', 'tool_args': {'year': 'current year'}}, "
+            "{'thought': 'Determine policies on extended leave requests.', 'tool_name': 'search_policy', 'tool_args': {'topic': 'PTO leave request process'}}, "
+            "{'thought': 'Check the notification requirements for extended PTO.', 'tool_name': 'notify_manager', 'tool_args': {'type': 'extended leave'}}, "
+            "{'thought': 'Investigate overlap between PTO and sick leave.', 'tool_name': 'combine_leave', 'tool_args': {'types': ['PTO', 'sick leave']}}, "
+            "{'thought': 'Explore how PTO interacts with parental leave.', 'tool_name': 'combine_leave', 'tool_args': {'types': ['PTO', 'parental leave']}}, "
+            "{'thought': 'Fetch the company calendar to determine critical business periods.', 'tool_name': 'fetch_calendar', 'tool_args': {'year': 'current year'}}, "
+            "{'thought': 'Verify restrictions on PTO usage during holidays.', 'tool_name': 'search_policy', 'tool_args': {'topic': 'holiday restrictions on PTO'}}, "
+            "{'thought': 'Confirm whether unused PTO is compensated upon termination.', 'tool_name': 'verify_compensation', 'tool_args': {'scenario': 'termination'}}, "
+            "{'thought': 'Check if PTO is compensated differently upon resignation.', 'tool_name': 'verify_compensation', 'tool_args': {'scenario': 'resignation'}}, "
+            "{'thought': 'Review if accrual caps limit PTO earnings.', 'tool_name': 'calculate_accrual', 'tool_args': {'cap': True}}, "
+            "{'thought': 'Investigate whether senior employees receive additional PTO benefits.', 'tool_name': 'review_seniority_rules', 'tool_args': {'seniority_level': 'high'}}, "
+            "{'thought': 'Assess policy transparency in PTO documentation.', 'tool_name': 'search_policy', 'tool_args': {'topic': 'PTO documentation clarity'}}, "
+            "{'thought': 'Explore how leave types can be optimized together.', 'tool_name': 'combine_leave', 'tool_args': {'types': ['PTO', 'other leave']}}, "
+            "{'thought': 'Check historical trends in PTO policy changes.', 'tool_name': 'get_docs', 'tool_args': {'document': 'PTO history'}}"
+            "]"
+        ),
+    )
+    assert prediction.next_tool_name in next_tool_names