Update main.py

main.py

@@ -70,7 +70,7 @@ def limit_tokens(input_string, token_limit=6000):
 def calculate_tokens(msgs):
     return sum(len(encoding.encode(str(m))) for m in msgs)
 
-def chat_with_llama_stream(messages, model="gpt-3.5-turbo", max_llm_history=4, max_output_tokens=2500):
+async def chat_with_llama_stream(messages, model="gpt-3.5-turbo", max_llm_history=4, max_output_tokens=2500):
     while calculate_tokens(messages) > (8000 - max_output_tokens):
         if len(messages) > max_llm_history:
             messages = [messages[0]] + messages[-max_llm_history:]
@@ -80,7 +80,7 @@ def chat_with_llama_stream(messages, model="gpt-3.5-turbo", max_llm_history=4, m
                 raise ValueError("Unable to reduce message length below token limit")
 
     try:
-        response = or_client.chat.completions.create(
+        response = await or_client.chat.completions.create(
             model=model,
             messages=messages,
             max_tokens=max_output_tokens,
@@ -88,7 +88,7 @@ def chat_with_llama_stream(messages, model="gpt-3.5-turbo", max_llm_history=4, m
         )
         
         full_response = ""
-        for chunk in response:
+        async for chunk in response:
             if chunk.choices[0].delta.content is not None:
                 content = chunk.choices[0].delta.content
                 full_response += content
@@ -100,6 +100,7 @@ def chat_with_llama_stream(messages, model="gpt-3.5-turbo", max_llm_history=4, m
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error in model response: {str(e)}")
 
+
 async def verify_api_key(api_key: str = Security(api_key_header)):
     if api_key != API_KEY:
         raise HTTPException(status_code=403, detail="Could not validate credentials")