Repository: tensorzero/tensorzero (language: rust)

Issue:
Add an example that uses streaming inference



Discussion / hints:

Fix patch:
diff --git a/examples/guides/streaming/config/tensorzero.toml b/examples/guides/streaming/config/tensorzero.toml
new file mode 100644
index 000000000..389781887
--- /dev/null
+++ b/examples/guides/streaming/config/tensorzero.toml
@@ -0,0 +1,17 @@
+[models.gpt_4o_mini]
+routing = ["openai"]
+
+[models.gpt_4o_mini.providers.openai]
+type = "openai"
+model_name = "gpt-4o-mini"
+
+[functions.chatbot]
+type = "chat"
+
+[functions.chatbot.variants.openai]
+type = "chat_completion"
+model = "gpt_4o_mini"
+
+# Disable observability to keep this example minimal (not recommended in production)
+[gateway]
+disable_observability = true
diff --git a/examples/guides/streaming/docker-compose.yml b/examples/guides/streaming/docker-compose.yml
new file mode 100644
index 000000000..8688339ae
--- /dev/null
+++ b/examples/guides/streaming/docker-compose.yml
@@ -0,0 +1,13 @@
+# This is a simplified example for learning purposes. Do not use this in production.
+# For production-ready deployments, see: https://www.tensorzero.com/docs/gateway/deployment
+
+services:
+  gateway:
+    image: tensorzero/gateway
+    volumes:
+      # Mount our tensorzero.toml file into the container
+      - ./config:/app/config:ro
+    environment:
+      - OPENAI_API_KEY=${OPENAI_API_KEY:?Environment variable OPENAI_API_KEY must be set.}
+    ports:
+      - "3000:3000"
diff --git a/examples/guides/streaming/run.py b/examples/guides/streaming/run.py
new file mode 100644
index 000000000..0b3c3c3e4
--- /dev/null
+++ b/examples/guides/streaming/run.py
@@ -0,0 +1,32 @@
+import asyncio
+import os
+
+from tensorzero import AsyncTensorZeroGateway
+
+
+async def main(gateway_url: str):
+    async with AsyncTensorZeroGateway(gateway_url) as client:
+        stream = await client.inference(
+            function_name="chatbot",
+            input={
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": "Share an extensive list of fun facts about Japan.",
+                    },
+                ],
+            },
+            stream=True,
+        )
+
+        async for chunk in stream:
+            if len(chunk.content) > 0:
+                print(chunk.content[0].text, end="")
+
+
+if __name__ == "__main__":
+    gateway_url = os.getenv("TENSORZERO_GATEWAY_URL")
+    if not gateway_url:
+        gateway_url = "http://localhost:3000"
+
+    asyncio.run(main(gateway_url))


Test patch: