Stream a response from LangChain's OpenAI with python fastapi
Labels: name: bug, name: question

### System Info

langchain == 0.0.205
python == 3.10.11

### Who can help?

@hwchase17 

### Information

- [ ] The official example notebooks/scripts
- [X] My own modified scripts

### Related Components

- [X] LLMs/Chat Models
- [ ] Embedding Models
- [ ] Prompts / Prompt Templates / Prompt Selectors
- [ ] Output Parsers
- [ ] Document Loaders
- [ ] Vector Stores / Retrievers
- [ ] Memory
- [ ] Agents / Agent Executors
- [ ] Tools / Toolkits
- [ ] Chains
- [ ] Callbacks/Tracing
- [X] Async

### Reproduction

```
from fastapi import FastAPI, Depends, Request, Response
from typing import Any, Dict, List, Generator
import asyncio
from langchain.llms import OpenAI
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
from langchain.schema import LLMResult, HumanMessage, SystemMessage
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import StreamingResponse
from html import escape


app = FastAPI()

# 添加 CORS 中间件
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_methods=["*"],
    allow_headers=["*"],
    allow_credentials=True,
)

q = asyncio.Queue()
stop_item = "###finish###"


class StreamingStdOutCallbackHandlerYield(StreamingStdOutCallbackHandler):
    async def on_llm_start(
        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
    ) -> None:
        """Run when LLM starts running."""
        # Clear the queue at the start
        while not q.empty():
            await q.get()

    async def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
        """Run on new LLM token. Only available when streaming is enabled."""
        await q.put(token)

    async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
        """Run when LLM ends running."""
        await q.put(stop_item)


async def generate_llm_stream(prompt: str) -> Generator[str, None, None]:
    llm = OpenAI(temperature=0.5, streaming=True, callbacks=[StreamingStdOutCallbackHandlerYield()])
    result = await llm.agenerate([prompt])
    
    while True:
        item = await q.get()
        if item == stop_item:
            break
        yield item


@app.get("/generate-song", status_code=200)
async def generate_song() -> Response:
    prompt = "Write me a song about sparkling water."

    async def event_stream() -> Generator[str, None, None]:
        async for item in generate_llm_stream(prompt):
            escaped_chunk = escape(item).replace("\n", "<br>").replace(" ", "&nbsp;")
            yield f"data:{escaped_chunk}\n\n"

    return StreamingResponse(event_stream(), media_type="text/event-stream")


if __name__ == "__main__":
    import uvicorn
    uvicorn.run("easy:app", host="0.0.0.0", port=8000, reload=True)
```

### Expected behavior

```Retrying langchain.llms.openai.acompletion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
Retrying langchain.llms.openai.acompletion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
Retrying langchain.llms.openai.acompletion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
Retrying langchain.llms.openai.acompletion_with_retry.<locals>._completion_with_retry in 8.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
Retrying langchain.llms.openai.acompletion_with_retry.<locals>._completion_with_retry in 10.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
ERROR:    Exception in ASGI application
Traceback (most recent call last):
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\aiohttp\connector.py", line 980, in _wrap_create_connection
    return await self._loop.create_connection(*args, **kwargs)  # type: ignore[return-value]  # noqa
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\asyncio\base_events.py", line 1076, in create_connection
    raise exceptions[0]
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\asyncio\base_events.py", line 1060, in create_connection
    sock = await self._connect_sock(
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\asyncio\base_events.py", line 969, in _connect_sock
    await self.sock_connect(sock, address)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\asyncio\selector_events.py", line 501, in sock_connect
    return await fut
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\asyncio\selector_events.py", line 541, in _sock_connect_cb
    raise OSError(err, f'Connect call failed {address}')
TimeoutError: [Errno 10060] Connect call failed ('108.160.166.253', 443)

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\openai\api_requestor.py", line 592, in arequest_raw
    result = await session.request(**request_kwargs)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\aiohttp\client.py", line 536, in _request
    conn = await self._connector.connect(
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\aiohttp\connector.py", line 540, in connect
    proto = await self._create_connection(req, traces, timeout)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\aiohttp\connector.py", line 901, in _create_connection
    _, proto = await self._create_direct_connection(req, traces, timeout)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\aiohttp\connector.py", line 1206, in _create_direct_connection
    raise last_exc
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\aiohttp\connector.py", line 1175, in _create_direct_connection
    transp, proto = await self._wrap_create_connection(
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\aiohttp\connector.py", line 988, in _wrap_create_connection
    raise client_error(req.connection_key, exc) from exc
aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host api.openai.com:443 ssl:default [Connect call failed ('108.160.166.253', 443)]

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\uvicorn\protocols\http\httptools_impl.py", line 435, in run_asgi
    result = await app(  # type: ignore[func-returns-value]
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\uvicorn\middleware\proxy_headers.py", line 78, in __call__
    return await self.app(scope, receive, send)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\fastapi\applications.py", line 282, in __call__
    await super().__call__(scope, receive, send)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\applications.py", line 122, in __call__
    await self.middleware_stack(scope, receive, send)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\middleware\errors.py", line 184, in __call__
    raise exc
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\middleware\errors.py", line 162, in __call__
    await self.app(scope, receive, _send)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\middleware\cors.py", line 91, in __call__
    await self.simple_response(scope, receive, send, request_headers=headers)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\middleware\cors.py", line 146, in simple_response
    await self.app(scope, receive, send)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\middleware\exceptions.py", line 79, in __call__
    raise exc
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\middleware\exceptions.py", line 68, in __call__
    await self.app(scope, receive, sender)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\fastapi\middleware\asyncexitstack.py", line 20, in __call__
    raise e
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\fastapi\middleware\asyncexitstack.py", line 17, in __call__
    await self.app(scope, receive, send)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\routing.py", line 718, in __call__
    await route.handle(scope, receive, send)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\routing.py", line 276, in handle
    await self.app(scope, receive, send)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\routing.py", line 69, in app
    await response(scope, receive, send)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\responses.py", line 270, in __call__
    async with anyio.create_task_group() as task_group:
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\anyio\_backends\_asyncio.py", line 662, in __aexit__
    raise exceptions[0]
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\responses.py", line 273, in wrap
    await func()
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\starlette\responses.py", line 262, in stream_response
    async for chunk in self.body_iterator:
  File "C:\AI\openai\easy.py", line 61, in event_stream
    async for item in generate_llm_stream(prompt):
  File "C:\AI\openai\easy.py", line 47, in generate_llm_stream
    result = await llm.agenerate([prompt])
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\langchain\llms\base.py", line 287, in agenerate
    raise e
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\langchain\llms\base.py", line 279, in agenerate
    await self._agenerate(
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\langchain\llms\openai.py", line 355, in _agenerate
    async for stream_resp in await acompletion_with_retry(
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\langchain\llms\openai.py", line 120, in acompletion_with_retry
    return await _completion_with_retry(**kwargs)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\tenacity\_asyncio.py", line 88, in async_wrapped
    return await fn(*args, **kwargs)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\tenacity\_asyncio.py", line 47, in __call__
    do = self.iter(retry_state=retry_state)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\tenacity\__init__.py", line 325, in iter
    raise retry_exc.reraise()
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\tenacity\__init__.py", line 158, in reraise
    raise self.last_attempt.result()
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\concurrent\futures\_base.py", line 451, in result
    return self.__get_result()
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\concurrent\futures\_base.py", line 403, in __get_result
    raise self._exception
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\tenacity\_asyncio.py", line 50, in __call__
    result = await fn(*args, **kwargs)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\langchain\llms\openai.py", line 118, in _completion_with_retry
    return await llm.client.acreate(**kwargs)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\openai\api_resources\completion.py", line 45, in acreate
    return await super().acreate(*args, **kwargs)
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\openai\api_resources\abstract\engine_api_resource.py", line 217, in acreate
    response, _, api_key = await requestor.arequest(
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\openai\api_requestor.py", line 304, in arequest
    result = await self.arequest_raw(
  File "C:\Users\jhrsya\Anaconda3\envs\langchain\lib\site-packages\openai\api_requestor.py", line 609, in arequest_raw
    raise error.APIConnectionError("Error communicating with OpenAI") from e
openai.error.APIConnectionError: Error communicating with OpenAI```