Repository: stanfordnlp/dspy

Issue:
[Bug] mlflow.dspy.util warning with structured output - failed to save dspy module state (object is not JSON serializable)
### What happened?

@TomeHirata 

As requested in [MLFLow Issue 16939](https://github.com/mlflow/mlflow/issues/16939) 

> Can you file an issue on https://github.com/stanfordnlp/dspy/issues with your motivation of using `pydantic.HttpUrl` as part of Signature and tag me?

Apologies for the confusion. The actual signature we're using looks more like this:

```
class MyCustomType(BaseModel):
    # various fields ...
    url: HttpUrl
    

class MySignature(dspy.Signature):
    text: str = dspy.InputField()
    output: MyCustomType = dspy.OutputField()
```

I just used the `HttpUrl` directly in the signature in the linked issue to minimise the reproducible example.

Please let me know if you need any more information.

### Steps to reproduce

Both of these result in the same warning being emitted.

More realistic code:
```
import dspy
import mlflow

from dspy.teleprompt import BootstrapFewShot
from pydantic import HttpUrl, BaseModel

dspy.configure(lm=dspy.LM("gemini/gemini-2.5-flash"))

mlflow.set_experiment("dspy-example")
mlflow.set_tracking_uri("http://localhost:5000")
mlflow.dspy.autolog(
    log_compiles=True,
    log_traces_from_compile=True,
)


class Output(BaseModel):
    url: HttpUrl


def metric(example, pred, trace):
    return 0


def main():
    with mlflow.start_run():
        # Using HttpUrl directly for brevity, although this was first encountered
        # while using a `BaseModel` with an `HttpUrl` field.
        model = dspy.Predict("question: str -> answer: Output")

        trainset = [
            dspy.Example(
                question="Give me a random email address.",
                answer=Output(url="https://www.example.com"),
            ).with_inputs("question"),
        ]

        optimiser: BootstrapFewShot = BootstrapFewShot(
            metric=metric,
            max_bootstrapped_demos=1,
            max_labeled_demos=1,
            max_rounds=1,
        )
        optimised = optimiser.compile(model, trainset=trainset)

        # Confirm log_model works fine
        mlflow.dspy.log_model(optimised, "optimised_model")


if __name__ == "__main__":
    main()
```


Original example:
```
import dspy
import mlflow

from dspy.teleprompt import BootstrapFewShot
from pydantic import HttpUrl

dspy.configure(lm=dspy.LM("gemini/gemini-2.5-flash"))

mlflow.set_experiment("dspy-example")
mlflow.set_tracking_uri("http://localhost:5000")
mlflow.dspy.autolog(
    log_compiles=True,
    log_traces_from_compile=True,
)


def metric(example, pred, trace):
    return 0


def main():
    with mlflow.start_run():
        # Using HttpUrl directly for brevity, although this was first encountered
        # while using a `BaseModel` with an `HttpUrl` field.
        model = dspy.Predict("question: str -> answer: HttpUrl")

        trainset = [
            dspy.Example(
                question="Give me a random URL",
                answer=HttpUrl("https://example.com"),
            ).with_inputs("question"),
        ]

        optimiser: BootstrapFewShot = BootstrapFewShot(
            metric=metric,
            max_bootstrapped_demos=1,
            max_labeled_demos=1,
            max_rounds=1,
        )
        mlflow.doctor()
        optimised = optimiser.compile(model, trainset=trainset)

        # Confirm log_model works fine
        mlflow.dspy.log_model(optimised, "optimised_model")


if __name__ == "__main__":
    main()
```

### DSPy version

2.6.27


Discussion:
Thanks for relaying the issue here, let me file a fix for this.



Patch:
diff --git a/dspy/clients/cache.py b/dspy/clients/cache.py
index 7ba9b0e573..7a9885f1bc 100644
--- a/dspy/clients/cache.py
+++ b/dspy/clients/cache.py
@@ -75,7 +75,7 @@ def transform_value(value):
             if isinstance(value, type) and issubclass(value, pydantic.BaseModel):
                 return value.model_json_schema()
             elif isinstance(value, pydantic.BaseModel):
-                return value.model_dump()
+                return value.model_dump(mode="json")
             elif callable(value):
                 # Try to get the source code of the callable if available
                 import inspect
diff --git a/dspy/predict/predict.py b/dspy/predict/predict.py
index a875237150..59939b1efa 100644
--- a/dspy/predict/predict.py
+++ b/dspy/predict/predict.py
@@ -203,8 +203,9 @@ def serialize_object(obj):
     Supports Pydantic models, lists, dicts, and primitive types.
     """
     if isinstance(obj, BaseModel):
-        # Use model_dump to convert the model into a JSON-serializable dict
-        return obj.model_dump()
+        # Use model_dump with mode="json" to ensure all fields (including HttpUrl, datetime, etc.)
+        # are converted to JSON-serializable types (strings)
+        return obj.model_dump(mode="json")
     elif isinstance(obj, list):
         return [serialize_object(item) for item in obj]
     elif isinstance(obj, tuple):


Test patch:
diff --git a/tests/predict/test_predict.py b/tests/predict/test_predict.py
index e60cd14e91..37b9c87fcf 100644
--- a/tests/predict/test_predict.py
+++ b/tests/predict/test_predict.py
@@ -10,9 +10,11 @@
 import pytest
 import ujson
 from litellm import ModelResponse
+from pydantic import BaseModel, HttpUrl
 
 import dspy
 from dspy import Predict, Signature
+from dspy.predict.predict import serialize_object
 from dspy.utils.dummies import DummyLM
 
 
@@ -684,3 +686,46 @@ def test_predicted_outputs_piped_from_predict_to_lm_call():
         program(question="Why did a chicken cross the kitchen?", prediction="To get to the other side!")
 
     assert "prediction" not in mock_completion.call_args[1]
+
+
+def test_dump_state_pydantic_non_primitive_types():
+    class WebsiteInfo(BaseModel):
+        name: str
+        url: HttpUrl
+        description: str | None = None
+        created_at: datetime
+
+    class TestSignature(dspy.Signature):
+        website_info: WebsiteInfo = dspy.InputField()
+        summary: str = dspy.OutputField()
+
+    website_info = WebsiteInfo(
+        name="Example",
+        url="https://www.example.com",
+        description="Test website",
+        created_at=datetime(2021, 1, 1, 12, 0, 0),
+    )
+
+    serialized = serialize_object(website_info)
+
+    assert serialized["url"] == "https://www.example.com/"
+    assert serialized["created_at"] == "2021-01-01T12:00:00"
+
+    json_str = ujson.dumps(serialized)
+    reloaded = ujson.loads(json_str)
+    assert reloaded == serialized
+
+    predictor = Predict(TestSignature)
+    demo = {
+        "website_info": website_info,
+        "summary": "This is a test website."
+    }
+    predictor.demos = [demo]
+
+    state = predictor.dump_state()
+    json_str = ujson.dumps(state)
+    reloaded_state = ujson.loads(json_str)
+
+    demo_data = reloaded_state["demos"][0]
+    assert demo_data["website_info"]["url"] == "https://www.example.com/"
+    assert demo_data["website_info"]["created_at"] == "2021-01-01T12:00:00"