Issue: RetrievalQA -> ConversationalChatAgent -> AgentExecutor gives no response if document-related

### Issue you'd like to raise.

Hi everybody,

I'm working with an LLM setup inspired from @pelyhe 's implementation #4573 .
It uses a RetrievalQA that queries a persistent embedded ChromaDB, then feeds it into a ConversationalChatAgent and then an AgentExecutor.

Currently, this setup works for only basic situations which definitely have nothing to do with documents. Once I ask it something document relevant, it gives an empty response. I have a nagging suspicion I've simply wired things up incorrectly, but it's not clear how to fix it.
```
@st.cache_resource
def load_agent():
    vectorstore = Chroma(persist_directory=CHROMA_DIR)

    basic_prompt_template = """If the context is not relevant, 
    please answer the question by using your own knowledge about the topic.
    
    ###Context:
    {context}
    ###Human:
    {question}
    ###Assistant:
    """

    prompt = PromptTemplate(
        template=basic_prompt_template, input_variables=["context", "question"]
    )

    system_msg = "You are a helpful assistant."

    chain_type_kwargs = {"prompt": prompt}
    # Time to initialize the LLM, as late as possible so everything not requiring the LLM instance to fail fast
    llm = GPT4All(
        model=MODEL,
        verbose=True,
    )

    # Initialise QA chain for document-relevant queries
    qa = RetrievalQA.from_chain_type(
        llm=llm,
        chain_type="stuff",
        retriever=vectorstore.as_retriever(),
        chain_type_kwargs=chain_type_kwargs,
    )

    tools = [
        Tool(
            name="Document tool",
            func=qa.run,
            description="useful for when you need to answer questions from documents.",
        ),
    ]

    agent = ConversationalChatAgent.from_llm_and_tools(
        llm=llm, tools=tools, system_message=system_msg, verbose=True
    )

    return AgentExecutor.from_agent_and_tools(
        agent=agent,
        tools=tools,
        verbose=True,
        memory=ConversationBufferMemory(
            memory_key="chat_history", return_messages=True
        ),
    )


agent = load_agent()

###########################
# Streamlit UI operation. #
###########################
if "generated" not in st.session_state:
    st.session_state["generated"] = []

if "past" not in st.session_state:
    st.session_state["past"] = []


def get_text():
    input_text = st.text_input(label="", key="question")
    return input_text


user_input = get_text()

if user_input:
    try:
        output = agent.run(input=user_input)
    except ValueError as e:
        output = str(e)
        if not output.startswith("Could not parse LLM output: "):
            raise Exception(output)
        output = output.removeprefix("Could not parse LLM output: ").removesuffix("`")
    st.session_state.past.append(user_input)
    st.session_state.generated.append(output)

if st.session_state["generated"]:
    for i in range(len(st.session_state["generated"]) - 1, -1, -1):
        message(st.session_state["generated"][i], key=str(i))
        message(st.session_state["past"][i], is_user=True, key=str(i) + "_user")
```

### Suggestion:

_No response_