Repository: stanfordnlp/dspy (python)

Issue:
[Bug] AzureOpenAI ContentPolicyViolation raising RuntimeError
### What happened?

When a ContentPolicyViolation from AzureOpenAI is triggered, the JSON-Adapter is currently raising a RuntimeError instead of propagating the proper error from litellm, because of a blanket Exception statement in the adapter.

Full Traceback:
```
Traceback (most recent call last):
File "/home/site/wwwroot/validation_script/script.py", line 876, in category_processor
enriched_event = await find_category(
^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/validation_script/script.py", line 949, in find_category
extrakt = await ev_loop.run_in_executor(None, prompt_args["dspy_program"], prompt_input)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/python/3.12/lib/python3.12/concurrent/futures/thread.py", line 58, in run
result = self.fn(*self.args, **self.kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/utils/callback.py", line 326, in sync_wrapper
return fn(instance, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/primitives/program.py", line 60, in __call__
return self.forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/utils/dspy_utils.py", line 154, in forward
response = self.classifier(text=text, lm=llm)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/utils/callback.py", line 326, in sync_wrapper
return fn(instance, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/primitives/program.py", line 60, in __call__
return self.forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/predict/chain_of_thought.py", line 38, in forward
return self.predict(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/predict/predict.py", line 85, in __call__
return super().__call__(**kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/utils/callback.py", line 326, in sync_wrapper
return fn(instance, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/primitives/program.py", line 60, in __call__
return self.forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/predict/predict.py", line 157, in forward
completions = adapter(lm, lm_kwargs=config, signature=signature, demos=demos, inputs=kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/adapters/chat_adapter.py", line 51, in __call__
return JSONAdapter()(lm, lm_kwargs, signature, demos, inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/site/wwwroot/.python_packages/lib/site-packages/dspy/adapters/json_adapter.py", line 75, in __call__
raise RuntimeError(
RuntimeError: Both structured output format and JSON mode failed. Please choose a model that supports `response_format` argument. Original error: litellm.BadRequestError: litellm.ContentPolicyViolationError: litellm.ContentPolicyViolationError: AzureException - The response was filtered due to the prompt triggering Azure OpenAI's content management policy. Please modify your prompt and retry. To learn more about our content filtering policies please read our documentation: https://go.microsoft.com/fwlink/?linkid=2198766
```

I believe it would be better to simply propagate the ContentViolationError from litelllm in these cases and let it be handled through application logic, as in our case e.g. we would like to flag cases like these and remove them from further processing.

### Steps to reproduce

Simply run any harmful prompt through AzureOpenAI. In our case we are using the following general structure for the module:

```python
    class GeneralModule(dspy.Module):

        def __init__(self):
            super().__init__()
            self.classifier = dspy.ChainOfThought(signature, **predictor_config)

        def forward(self, text, retry_count: int = 0):
            try:
                response = self.classifier(text=text, lm=llm)
            except AttributeError as e:
                raise ValueError("empty response from Azure OpenAI") from e
            except RateLimitError as e:
                # pylint: disable-next=no-member
                if (retry_after := int(e.litellm_response_headers.get("Retry-After", 60))) > 0:
                    logger.warning("Rate limited. Retrying in %d seconds.", retry_after)
                    time.sleep(retry_after)

                if retry_count < 5:
                    return self.forward(text, retry_count + 1)

                raise ValueError("Rate limit exceeded") from e
            except ContentPolicyViolationError as e:
                raise ValueError(f"Content Policy Violation: {e.message}") from e

            return response
```

But the exception handler for ContentPolicyViolation would never trigger since its intercept in the adapter

### DSPy version

2.6.27

Pull request:
Allow exception to pass through in JSONAdapter
resolve #8430 

We always capture the exception to turn it into a `RuntimeError`, but the wrapper provides less value than vagueness in reality. We are just letting the original exception to go through. 

Patch:
diff --git a/docs/docs/index.md b/docs/docs/index.md
index fac5194b..4c5ff98c 100644
--- a/docs/docs/index.md
+++ b/docs/docs/index.md
@@ -31,16 +31,16 @@ Instead of wrangling prompts or training jobs, DSPy (Declarative Self-improving
 
         ```python linenums="1"
         import dspy
-        lm = dspy.LM('openai/gpt-4o-mini', api_key='YOUR_OPENAI_API_KEY')
+        lm = dspy.LM("openai/gpt-4o-mini", api_key="YOUR_OPENAI_API_KEY")
         dspy.configure(lm=lm)
         ```
 
     === "Anthropic"
-        You can authenticate by setting the ANTHROPIC_API_KEY env variable or passing `api_key` below.
+        You can authenticate by setting the `ANTHROPIC_API_KEY` env variable or passing `api_key` below.
 
         ```python linenums="1"
         import dspy
-        lm = dspy.LM('anthropic/claude-3-opus-20240229', api_key='YOUR_ANTHROPIC_API_KEY')
+        lm = dspy.LM("anthropic/claude-3-opus-20240229", api_key="YOUR_ANTHROPIC_API_KEY")
         dspy.configure(lm=lm)
         ```
 
@@ -50,7 +50,7 @@ Instead of wrangling prompts or training jobs, DSPy (Declarative Self-improving
         ```python linenums="1"
         import dspy
         lm = dspy.LM(
-            'databricks/databricks-llama-4-maverick',
+            "databricks/databricks-llama-4-maverick",
             api_key="YOUR_DATABRICKS_ACCESS_TOKEN",
             api_base="YOUR_DATABRICKS_WORKSPACE_URL",  # e.g.: https://dbc-64bf4923-e39e.cloud.databricks.com/serving-endpoints
         )
@@ -58,11 +58,11 @@ Instead of wrangling prompts or training jobs, DSPy (Declarative Self-improving
         ```
 
     === "Gemini"
-        You can authenticate by setting the GEMINI_API_KEY env variable or passing `api_key` below.
+        You can authenticate by setting the `GEMINI_API_KEY` env variable or passing `api_key` below.
 
         ```python linenums="1"
         import dspy
-        lm = dspy.LM('gemini/gemini-2.5-flash', api_key='YOUR_GEMINI_API_KEY')
+        lm = dspy.LM("gemini/gemini-2.5-flash", api_key="YOUR_GEMINI_API_KEY")
         dspy.configure(lm=lm)
         ```
 
@@ -78,7 +78,7 @@ Instead of wrangling prompts or training jobs, DSPy (Declarative Self-improving
 
         ```python linenums="1"
         import dspy
-        lm = dspy.LM('ollama_chat/llama3.2', api_base='http://localhost:11434', api_key='')
+        lm = dspy.LM("ollama_chat/llama3.2", api_base="http://localhost:11434", api_key="")
         dspy.configure(lm=lm)
         ```
 
@@ -99,7 +99,7 @@ Instead of wrangling prompts or training jobs, DSPy (Declarative Self-improving
           ```python linenums="1"
           lm = dspy.LM("openai/meta-llama/Llama-3.1-8B-Instruct",
                        api_base="http://localhost:7501/v1",  # ensure this points to your port
-                       api_key="local", model_type='chat')
+                       api_key="local", model_type="chat")
           dspy.configure(lm=lm)
           ```
 
@@ -118,7 +118,7 @@ Instead of wrangling prompts or training jobs, DSPy (Declarative Self-improving
 
         ```python linenums="1"
         import dspy
-        lm = dspy.LM('openai/your-model-name', api_key='PROVIDER_API_KEY', api_base='YOUR_PROVIDER_URL')
+        lm = dspy.LM("openai/your-model-name", api_key="PROVIDER_API_KEY", api_base="YOUR_PROVIDER_URL")
         dspy.configure(lm=lm)
         ```
 
@@ -161,10 +161,10 @@ DSPy shifts your focus from tinkering with prompt strings to **programming with
 
         ```python linenums="1"       
         def search_wikipedia(query: str) -> list[str]:
-            results = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')(query, k=3)
-            return [x['text'] for x in results]
+            results = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")(query, k=3)
+            return [x["text"] for x in results]
         
-        rag = dspy.ChainOfThought('context, question -> response')
+        rag = dspy.ChainOfThought("context, question -> response")
 
         question = "What's the name of the castle that David Gregory inherited?"
         rag(context=search_wikipedia(question), question=question)
@@ -187,7 +187,7 @@ DSPy shifts your focus from tinkering with prompt strings to **programming with
             """Classify sentiment of a given sentence."""
             
             sentence: str = dspy.InputField()
-            sentiment: Literal['positive', 'negative', 'neutral'] = dspy.OutputField()
+            sentiment: Literal["positive", "negative", "neutral"] = dspy.OutputField()
             confidence: float = dspy.OutputField()
 
         classify = dspy.Predict(Classify)
@@ -239,8 +239,8 @@ DSPy shifts your focus from tinkering with prompt strings to **programming with
             return dspy.PythonInterpreter({}).execute(expression)
 
         def search_wikipedia(query: str):
-            results = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')(query, k=3)
-            return [x['text'] for x in results]
+            results = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")(query, k=3)
+            return [x["text"] for x in results]
 
         react = dspy.ReAct("question -> answer: float", tools=[evaluate_math, search_wikipedia])
 
@@ -311,14 +311,14 @@ DSPy shifts your focus from tinkering with prompt strings to **programming with
 
 ??? "Using DSPy in practice: from quick scripting to building sophisticated systems."
 
-    Standard prompts conflate interface (“what should the LM do?”) with implementation (“how do we tell it to do that?”). DSPy isolates the former as _signatures_ so we can infer the latter or learn it from data — in the context of a bigger program.
+    Standard prompts conflate interface ("what should the LM do?") with implementation ("how do we tell it to do that?"). DSPy isolates the former as _signatures_ so we can infer the latter or learn it from data — in the context of a bigger program.
     
     Even before you start using optimizers, DSPy's modules allow you to script effective LM systems as ergonomic, portable _code_. Across many tasks and LMs, we maintain _signature test suites_ that assess the reliability of the built-in DSPy adapters. Adapters are the components that map signatures to prompts prior to optimization. If you find a task where a simple prompt consistently outperforms idiomatic DSPy for your LM, consider that a bug and [file an issue](https://github.com/stanfordnlp/dspy/issues). We'll use this to improve the built-in adapters.
 
 
 ## 2) **Optimizers** tune the prompts and weights of your AI modules.
 
-DSPy provides you with the tools to compile high-level code with natural language annotations into the low-level computations, prompts, or weight updates that align your LM with your program’s structure and metrics. If you change your code or your metrics, you can simply re-compile accordingly.
+DSPy provides you with the tools to compile high-level code with natural language annotations into the low-level computations, prompts, or weight updates that align your LM with your program's structure and metrics. If you change your code or your metrics, you can simply re-compile accordingly.
 
 Given a few tens or hundreds of representative _inputs_ of your task and a _metric_ that can measure the quality of your system's outputs, you can use a DSPy optimizer. Different optimizers in DSPy work by **synthesizing good few-shot examples** for every module, like `dspy.BootstrapRS`,<sup>[1](https://arxiv.org/abs/2310.03714)</sup> **proposing and intelligently exploring better natural-language instructions** for every prompt, like `dspy.MIPROv2`,<sup>[2](https://arxiv.org/abs/2406.11695)</sup> and **building datasets for your modules and using them to finetune the LM weights** in your system, like `dspy.BootstrapFinetune`.<sup>[3](https://arxiv.org/abs/2407.10930)</sup>
 
@@ -342,11 +342,11 @@ Given a few tens or hundreds of representative _inputs_ of your task and a _metr
         import dspy
         from dspy.datasets import HotPotQA
 
-        dspy.configure(lm=dspy.LM('openai/gpt-4o-mini'))
+        dspy.configure(lm=dspy.LM("openai/gpt-4o-mini"))
 
         def search_wikipedia(query: str) -> list[str]:
-            results = dspy.ColBERTv2(url='http://20.102.90.50:2017/wiki17_abstracts')(query, k=3)
-            return [x['text'] for x in results]
+            results = dspy.ColBERTv2(url="http://20.102.90.50:2017/wiki17_abstracts")(query, k=3)
+            return [x["text"] for x in results]
 
         trainset = [x.with_inputs('question') for x in HotPotQA(train_seed=2024, train_size=500).train]
         react = dspy.ReAct("question -> answer", tools=[search_wikipedia])
@@ -364,7 +364,7 @@ Given a few tens or hundreds of representative _inputs_ of your task and a _metr
         class RAG(dspy.Module):
             def __init__(self, num_docs=5):
                 self.num_docs = num_docs
-                self.respond = dspy.ChainOfThought('context, question -> response')
+                self.respond = dspy.ChainOfThought("context, question -> response")
 
             def forward(self, question):
                 context = search(question, k=self.num_docs)   # defined in tutorial linked below
@@ -403,10 +403,10 @@ Given a few tens or hundreds of representative _inputs_ of your task and a _metr
 
         ```python linenums="1"
         import dspy
-        dspy.configure(lm=dspy.LM('openai/gpt-4o-mini-2024-07-18'))
+        dspy.configure(lm=dspy.LM("openai/gpt-4o-mini-2024-07-18"))
 
         # Define the DSPy module for classification. It will use the hint at training time, if available.
-        signature = dspy.Signature("text, hint -> label").with_updated_fields('label', type_=Literal[tuple(CLASSES)])
+        signature = dspy.Signature("text, hint -> label").with_updated_fields("label", type_=Literal[tuple(CLASSES)])
         classify = dspy.ChainOfThought(signature)
 
         # Optimize via BootstrapFinetune.
diff --git a/dspy/adapters/json_adapter.py b/dspy/adapters/json_adapter.py
index 5e599d29..9b14691c 100644
--- a/dspy/adapters/json_adapter.py
+++ b/dspy/adapters/json_adapter.py
@@ -22,11 +22,6 @@ from dspy.utils.exceptions import AdapterParseError
 
 logger = logging.getLogger(__name__)
 
-ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE = (
-    "Both structured output format and JSON mode failed. Please choose a model that supports "
-    "`response_format` argument. Original error: {}"
-)
-
 
 def _has_open_ended_mapping(signature: SignatureMeta) -> bool:
     """
@@ -72,13 +67,8 @@ class JSONAdapter(ChatAdapter):
             return super().__call__(lm, lm_kwargs, signature, demos, inputs)
         except Exception:
             logger.warning("Failed to use structured output format, falling back to JSON mode.")
-            try:
-                lm_kwargs["response_format"] = {"type": "json_object"}
-                return super().__call__(lm, lm_kwargs, signature, demos, inputs)
-            except AdapterParseError as e:
-                raise e
-            except Exception as e:
-                raise RuntimeError(ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE.format(e)) from e
+            lm_kwargs["response_format"] = {"type": "json_object"}
+            return super().__call__(lm, lm_kwargs, signature, demos, inputs)
 
     async def acall(
         self,
@@ -98,13 +88,8 @@ class JSONAdapter(ChatAdapter):
             return await super().acall(lm, lm_kwargs, signature, demos, inputs)
         except Exception:
             logger.warning("Failed to use structured output format, falling back to JSON mode.")
-            try:
-                lm_kwargs["response_format"] = {"type": "json_object"}
-                return await super().acall(lm, lm_kwargs, signature, demos, inputs)
-            except AdapterParseError as e:
-                raise e
-            except Exception as e:
-                raise RuntimeError(ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE.format(e)) from e
+            lm_kwargs["response_format"] = {"type": "json_object"}
+            return await super().acall(lm, lm_kwargs, signature, demos, inputs)
 
     def _call_preprocess(
         self,
diff --git a/dspy/clients/lm.py b/dspy/clients/lm.py
index e9dfc901..228d78b6 100644
--- a/dspy/clients/lm.py
+++ b/dspy/clients/lm.py
@@ -269,6 +269,9 @@ def _get_stream_completion_fn(
     stream = cast(MemoryObjectSendStream, stream)
     caller_predict_id = id(caller_predict) if caller_predict else None
 
+    if dspy.settings.track_usage:
+        request["stream_options"] = {"include_usage": True}
+
     async def stream_completion(request: Dict[str, Any], cache_kwargs: Dict[str, Any]):
         response = await litellm.acompletion(
             cache=cache_kwargs,


Test patch:
diff --git a/tests/adapters/test_json_adapter.py b/tests/adapters/test_json_adapter.py
index f688af43..eb90a754 100644
--- a/tests/adapters/test_json_adapter.py
+++ b/tests/adapters/test_json_adapter.py
@@ -5,7 +5,6 @@ import pytest
 from litellm.utils import Choices, Message, ModelResponse
 
 import dspy
-from dspy.adapters.json_adapter import ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE
 
 
 def test_json_adapter_passes_structured_output_when_supported_by_model():
@@ -616,12 +615,18 @@ def test_error_message_on_json_adapter_failure():
     dspy.configure(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter())
 
     with mock.patch("litellm.completion") as mock_completion:
-        mock_completion.side_effect = RuntimeError("Failed!")
+        mock_completion.side_effect = RuntimeError("RuntimeError!")
 
         with pytest.raises(RuntimeError) as error:
             program(question="Dummy question!")
 
-        assert ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE[:50] in str(error.value)
+        assert "RuntimeError!" in str(error.value)
+
+        mock_completion.side_effect = ValueError("ValueError!")
+        with pytest.raises(ValueError) as error:
+            program(question="Dummy question!")
+
+        assert "ValueError!" in str(error.value)
 
 
 @pytest.mark.asyncio
@@ -633,10 +638,15 @@ async def test_error_message_on_json_adapter_failure_async():
     program = dspy.Predict(TestSignature)
 
     with mock.patch("litellm.acompletion") as mock_acompletion:
-        mock_acompletion.side_effect = RuntimeError("Failed!")
-
         with dspy.context(lm=dspy.LM(model="openai/gpt-4o-mini", cache=False), adapter=dspy.JSONAdapter()):
+            mock_acompletion.side_effect = RuntimeError("RuntimeError!")
             with pytest.raises(RuntimeError) as error:
                 await program.acall(question="Dummy question!")
 
-        assert ERROR_MESSAGE_ON_JSON_ADAPTER_FAILURE[:50] in str(error.value)
+            assert "RuntimeError!" in str(error.value)
+
+            mock_acompletion.side_effect = ValueError("ValueError!")
+            with pytest.raises(ValueError) as error:
+                await program.acall(question="Dummy question!")
+
+            assert "ValueError!" in str(error.value)