Async callback has issues
Labels: name: auto:bug

### System Info

Langchain Version: 0.0.224
Platform: Mac
Python Version: 3.10.9

### Who can help?

_No response_

### Information

- [ ] The official example notebooks/scripts
- [ ] My own modified scripts

### Related Components

- [ ] LLMs/Chat Models
- [ ] Embedding Models
- [ ] Prompts / Prompt Templates / Prompt Selectors
- [ ] Output Parsers
- [ ] Document Loaders
- [ ] Vector Stores / Retrievers
- [ ] Memory
- [ ] Agents / Agent Executors
- [ ] Tools / Toolkits
- [ ] Chains
- [x] Callbacks/Tracing
- [x] Async

### Reproduction

Copy/Paste of example snippet from official documentation:
https://python.langchain.com/docs/modules/callbacks/how_to/async_callbacks

import asyncio
from typing import Any, Dict, List

from langchain.chat_models import ChatOpenAI
from langchain.schema import LLMResult, HumanMessage
from langchain.callbacks.base import AsyncCallbackHandler, BaseCallbackHandler
from dotenv import load_dotenv

load_dotenv()

class MyCustomSyncHandler(BaseCallbackHandler):
    def on_llm_new_token(self, token: str, **kwargs) -> None:
        print(f"Sync handler being called in a `thread_pool_executor`: token: {token}")


class MyCustomAsyncHandler(AsyncCallbackHandler):
    """Async callback handler that can be used to handle callbacks from langchain."""

    async def on_llm_start(
        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
    ) -> None:
        """Run when chain starts running."""
        print("zzzz....")
        await asyncio.sleep(0.3)
        class_name = serialized["name"]
        print("Hi! I just woke up. Your llm is starting")

    async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
        """Run when chain ends running."""
        print("zzzz....")
        await asyncio.sleep(0.3)
        print("Hi! I just woke up. Your llm is ending")


# To enable streaming, we pass in `streaming=True` to the ChatModel constructor
# Additionally, we pass in a list with our custom handler
chat = ChatOpenAI(
    max_tokens=25,
    streaming=True,
    callbacks=[MyCustomSyncHandler(), MyCustomAsyncHandler()],
)

async def main():
    await chat.agenerate([[HumanMessage(content="Tell me a joke")]])

if __name__ == '__main__':
    asyncio.run(main())

#####################################################
Output:

zzzz....
Error in MyCustomAsyncHandler.on_llm_start callback: 'name'
Retrying langchain.chat_models.openai.acompletion_with_retry.<locals>._completion_with_retry in 1.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
Retrying langchain.chat_models.openai.acompletion_with_retry.<locals>._completion_with_retry in 2.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
Retrying langchain.chat_models.openai.acompletion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
Retrying langchain.chat_models.openai.acompletion_with_retry.<locals>._completion_with_retry in 8.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
Retrying langchain.chat_models.openai.acompletion_with_retry.<locals>._completion_with_retry in 16.0 seconds as it raised APIConnectionError: Error communicating with OpenAI.
Traceback (most recent call last):
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/aiohttp/connector.py", line 980, in _wrap_create_connection
    return await self._loop.create_connection(*args, **kwargs)  # type: ignore[return-value]  # noqa
  File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 1097, in create_connection
    transport, protocol = await self._create_connection_transport(
  File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 1127, in _create_connection_transport
    await waiter
  File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/sslproto.py", line 534, in data_received
    ssldata, appdata = self._sslpipe.feed_ssldata(data)
  File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/sslproto.py", line 188, in feed_ssldata
    self._sslobj.do_handshake()
  File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/ssl.py", line 975, in do_handshake
    self._sslobj.do_handshake()
ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:997)

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/openai/api_requestor.py", line 592, in arequest_raw
    result = await session.request(**request_kwargs)
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/aiohttp/client.py", line 536, in _request
    conn = await self._connector.connect(
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/aiohttp/connector.py", line 540, in connect
    proto = await self._create_connection(req, traces, timeout)
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/aiohttp/connector.py", line 901, in _create_connection
    _, proto = await self._create_direct_connection(req, traces, timeout)
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/aiohttp/connector.py", line 1206, in _create_direct_connection
    raise last_exc
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/aiohttp/connector.py", line 1175, in _create_direct_connection
    transp, proto = await self._wrap_create_connection(
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/aiohttp/connector.py", line 982, in _wrap_create_connection
    raise ClientConnectorCertificateError(req.connection_key, exc) from exc
aiohttp.client_exceptions.ClientConnectorCertificateError: Cannot connect to host api.openai.com:443 ssl:True [SSLCertVerificationError: (1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:997)')]

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/pnsvk/my-apps/p_y/llm-apps/async_callbacks.py", line 49, in <module>
    asyncio.run(main())
  File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/Users/pnsvk/my-apps/p_y/llm-apps/async_callbacks.py", line 45, in main
    res = await chat.agenerate([[HumanMessage(content="Tell me a joke")]])
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/langchain/chat_models/base.py", line 191, in agenerate
    raise exceptions[0]
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/langchain/chat_models/base.py", line 292, in _agenerate_with_cache
    return await self._agenerate(
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/langchain/chat_models/openai.py", line 408, in _agenerate
    async for stream_resp in await acompletion_with_retry(
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/langchain/chat_models/openai.py", line 95, in acompletion_with_retry
    return await _completion_with_retry(**kwargs)
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/tenacity/_asyncio.py", line 86, in async_wrapped
    return await fn(*args, **kwargs)
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/tenacity/_asyncio.py", line 48, in __call__
    do = self.iter(retry_state=retry_state)
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/tenacity/__init__.py", line 362, in iter
    raise retry_exc.reraise()
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/tenacity/__init__.py", line 195, in reraise
    raise self.last_attempt.result()
  File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/concurrent/futures/_base.py", line 451, in result
    return self.__get_result()
  File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
    raise self._exception
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/tenacity/_asyncio.py", line 51, in __call__
    result = await fn(*args, **kwargs)
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/langchain/chat_models/openai.py", line 93, in _completion_with_retry
    return await llm.client.acreate(**kwargs)
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/openai/api_resources/chat_completion.py", line 45, in acreate
    return await super().acreate(*args, **kwargs)
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/openai/api_resources/abstract/engine_api_resource.py", line 217, in acreate
    response, _, api_key = await requestor.arequest(
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/openai/api_requestor.py", line 304, in arequest
    result = await self.arequest_raw(
  File "/Users/pnsvk/my-venv-3.10/lib/python3.10/site-packages/openai/api_requestor.py", line 609, in arequest_raw
    raise error.APIConnectionError("Error communicating with OpenAI") from e
openai.error.APIConnectionError: Error communicating with OpenAI


### Expected behavior

As mentioned in the official documentation:

    zzzz....
    Hi! I just woke up. Your llm is starting
    Sync handler being called in a `thread_pool_executor`: token: 
    Sync handler being called in a `thread_pool_executor`: token: Why
    Sync handler being called in a `thread_pool_executor`: token:  don
    Sync handler being called in a `thread_pool_executor`: token: 't
    Sync handler being called in a `thread_pool_executor`: token:  scientists
    Sync handler being called in a `thread_pool_executor`: token:  trust
    Sync handler being called in a `thread_pool_executor`: token:  atoms
    Sync handler being called in a `thread_pool_executor`: token: ?
    Sync handler being called in a `thread_pool_executor`: token:  
    
    
    Sync handler being called in a `thread_pool_executor`: token: Because
    Sync handler being called in a `thread_pool_executor`: token:  they
    Sync handler being called in a `thread_pool_executor`: token:  make
    Sync handler being called in a `thread_pool_executor`: token:  up
    Sync handler being called in a `thread_pool_executor`: token:  everything
    Sync handler being called in a `thread_pool_executor`: token: .
    Sync handler being called in a `thread_pool_executor`: token: 
    zzzz....
    Hi! I just woke up. Your llm is ending





    LLMResult(generations=[[ChatGeneration(text="Why don't scientists trust atoms? \n\nBecause they make up everything.", generation_info=None, message=AIMessage(content="Why don't scientists trust atoms? \n\nBecause they make up everything.", additional_kwargs={}, example=False))]], llm_output={'token_usage': {}, 'model_name': 'gpt-3.5-turbo'})