Repository: stanfordnlp/dspy

Issue:
[Bug] StreamListener doesn't parse Litellm ModelResponseStream correctly
### What happened?

I'm trying out the new `StreamListener` support and write some unit test for our integration. I see an issue when `litellm` has the following streaming chunks
```
mock_llm_stream = [
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="[[ ## answer ## ]]\n"))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="How "))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="are "))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="you "))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="doing?"))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="\n\n[[ ## completed ## ]]"))]),
]
```
the final prediction will become "Howare you doing?" where the space after "How" is stripped by 
`concat_message = "".join(self.field_start_queue).strip()` on line 102 in `dspy.streaming.streaming_listener`


### Steps to reproduce

mock litellm.accompletion with an async gen that `yield from`
```
mock_llm_stream = [
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="[[ ## answer ## ]]\n"))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="How "))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="are "))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="you "))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="doing?"))]),
    ModelResponseStream(choices=[StreamingChoices(delta=Delta(content="\n\n[[ ## completed ## ]]"))]),
]
```
use `dspy.Predict("question->answer") as module, and stream the module with `StreamListener(signature_field_name="answer")`

### DSPy version

2.6.26


Discussion:
@YGao119 I can reproduce the issue, and made a fix in #8360.  

But I am curious are you seeing this error from real production? Most of the models are using leading space, which is a standard since byte-pair encoder, so although I can reproduce this error from a unit test, I cannot reproduce it from a real case. 



Patch:
diff --git a/dspy/streaming/streaming_listener.py b/dspy/streaming/streaming_listener.py
index c07ca9f65..46c53ea1f 100644
--- a/dspy/streaming/streaming_listener.py
+++ b/dspy/streaming/streaming_listener.py
@@ -99,7 +99,7 @@ def receive(self, chunk: ModelResponseStream):
             # We keep appending the tokens to the queue until we have a full identifier or the concanated
             # tokens no longer match our expected identifier.
             self.field_start_queue.append(chunk_message)
-            concat_message = "".join(self.field_start_queue).strip()
+            concat_message = "".join(self.field_start_queue)
 
             if start_identifier in concat_message:
                 # We have a full identifier, we can start the stream.
@@ -113,8 +113,9 @@ def receive(self, chunk: ModelResponseStream):
                     # because there could be a few splitters between ':' and '"', e.g., '"name": "value"'.
                     chunk_message = chunk_message[1:]
 
-            elif self._buffered_message_end_with_start_identifier(concat_message, start_identifier):
-                # If the buffered message ends with part of the start_identifier, we can start the stream.
+            elif self._buffered_message_end_with_start_identifier(concat_message.strip(), start_identifier):
+                # If the buffered message ends with part of the start_identifier, we keep looking for the
+                # start_identifier from the token stream.
                 return
             else:
                 # Doesn't match the expected identifier, reset the queue.


Test patch:
diff --git a/tests/streaming/test_streaming.py b/tests/streaming/test_streaming.py
index 6019964c1..0486bc814 100644
--- a/tests/streaming/test_streaming.py
+++ b/tests/streaming/test_streaming.py
@@ -229,6 +229,36 @@ def __call__(self, x: str, **kwargs):
     assert all_chunks[-1].signature_field_name == "judgement"
 
 
+@pytest.mark.anyio
+async def test_streaming_handles_space_correctly():
+    dspy.settings.configure(lm=dspy.LM("openai/gpt-4o-mini", cache=False), adapter=dspy.ChatAdapter())
+    my_program = dspy.Predict("question->answer")
+    program = dspy.streamify(
+        my_program, stream_listeners=[dspy.streaming.StreamListener(signature_field_name="answer")]
+    )
+
+    async def gpt_4o_mini_stream(*args, **kwargs):
+        yield ModelResponseStream(
+            model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="[[ ## answer ## ]]\n"))]
+        )
+        yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="How "))])
+        yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="are "))])
+        yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="you "))])
+        yield ModelResponseStream(model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="doing?"))])
+        yield ModelResponseStream(
+            model="gpt-4o-mini", choices=[StreamingChoices(delta=Delta(content="\n\n[[ ## completed ## ]]"))]
+        )
+
+    with mock.patch("litellm.acompletion", side_effect=gpt_4o_mini_stream):
+        output = program(question="What is the capital of France?")
+        all_chunks = []
+        async for value in output:
+            if isinstance(value, dspy.streaming.StreamResponse):
+                all_chunks.append(value)
+
+    assert all_chunks[0].chunk == "How are you doing?"
+
+
 @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not found in environment variables")
 def test_sync_streaming():
     class MyProgram(dspy.Module):