Response taking way to long
Labels: name: auto:question, name: auto:improvement, name: area: models, name: integration: chroma

Im building an embedded chatbot using langchain and openai its working fine but the issue is that the responses takes around 15-25 seconds and i tried to use the time library to know which line is taking this much
`import os
import sys

from langchain.chains import ConversationalRetrievalChain
from langchain.chat_models import ChatOpenAI
from langchain.document_loaders import TextLoader
from langchain.embeddings import OpenAIEmbeddings
from langchain.indexes import VectorstoreIndexCreator
from langchain.indexes.vectorstore import VectorStoreIndexWrapper
from langchain.vectorstores import Chroma
from cachetools import TTLCache
import time

import constants

os.environ["OPENAI_API_KEY"] = constants.APIKEY
cache = TTLCache(maxsize=100, ttl=3600)  # Example: Cache up to 100 items for 1 hour

PERSIST = False

template_prompt = "If the user greets you, greet back. If there is a link in the response return it as a clickable link as if it is an a tag '<a>'. If you don't know the answer, you can say, 'I don't have the information you need, I recommend contacting our support team for assistance.' Here is the user prompt: 'On the Hawsabah platform"


def initialize_chatbot():
    query = None
    if len(sys.argv) > 1:
        query = sys.argv[1]

    if PERSIST and os.path.exists("persist"):
        print("Reusing index...\n")
        vectorstore = Chroma(persist_directory="persist", embedding_function=OpenAIEmbeddings())
        index = VectorStoreIndexWrapper(vectorstore=vectorstore)
    else:
        loader = TextLoader("data/data.txt")
        if PERSIST:
            index = VectorstoreIndexCreator(vectorstore_kwargs={"persist_directory":"persist"}).from_loaders([loader])
        else:
            index = VectorstoreIndexCreator().from_loaders([loader])

    chat_chain = ConversationalRetrievalChain.from_llm(
        llm=ChatOpenAI(model="gpt-3.5-turbo"),
        retriever=index.vectorstore.as_retriever(search_kwargs={"k": 1}),
    )

    chat_history = []

    return chat_chain, chat_history

MAX_CONVERSATION_HISTORY = 3  # Set the maximum number of interactions to keep in the buffer

def chatbot_response(user_prompt, chat_chain, chat_history):
    # Check if the response is cached
    cached_response = cache.get(user_prompt)
    if cached_response:
        return cached_response

    # Check if the user's query is a greeting or unrelated
    is_greeting = check_for_greeting(user_prompt)

    # Conditionally clear the conversation history
    if is_greeting:
        chat_history.clear()

    query_with_template = f"{template_prompt} {user_prompt}'"
    s = time.time() 
    result = chat_chain({"question": query_with_template, "chat_history": chat_history})
    e = time.time()

    # Append the new interaction and limit the conversation buffer to the last MAX_CONVERSATION_HISTORY interactions
    chat_history.append((user_prompt, result['answer']))
    if len(chat_history) > MAX_CONVERSATION_HISTORY:
        chat_history.pop(0)  # Remove the oldest interaction

    response = result['answer']

    # Cache the response for future use
    cache[user_prompt] = response

    print("Time taken by chatbot_response:", (e - s) * 1000, "ms")

    return response`
the line result = chat_chain({"question": query_with_template, "chat_history": chat_history}) was the one taking this long i tried to figure out how to fix this but i couldnt i also tried to implement word streaming to help make it look faster but it only worked for the davinci model. Is there a way or method to make responses faster?