Update app.py

app.py

@@ -23,8 +23,8 @@ if 'memory' not in st.session_state:
     st.session_state['memory'] = ConversationBufferMemory(return_messages=True)
 
 if 'chain' not in st.session_state:
-    st.session_state['chain'] = custom_chain_with_history(llm=CustomLLM(repo_id="mistralai/Mixtral-8x7B-Instruct-v0.1", model_type='text-generation', api_token=API_TOKEN, stop=["\n<|","<|"]), memory=st.session_state.memory)
-
+    # st.session_state['chain'] = custom_chain_with_history(llm=CustomLLM(repo_id="mistralai/Mixtral-8x7B-Instruct-v0.1", model_type='text-generation', api_token=API_TOKEN, stop=["\n<|","<|"]), memory=st.session_state.memory)
+    st.session_state['chain'] = custom_chain_with_history(llm=InferenceClient("https://api-inference.huggingface.co/models/mistralai/Mixtral-8x7B-Instruct-v0.1", headers = {"Authorization": f"Bearer {API_TOKEN}"}, stream=True, max_new_tokens=512, temperature=0.01), memory=st.session_state.memory)
 st.title("LMD Chatbot V2 Sample")
 st.subheader("Knowledge-base from web scrapping and FAQ")
 
@@ -44,12 +44,19 @@ if prompt := st.chat_input("Ask me anything.."):
     # Add user message to chat history
     st.session_state.messages.append({"role": "User", "content": prompt})
     
-    response = st.session_state.chain.invoke(prompt).split("\n<|")[0]
+    # full_response = st.session_state.chain.invoke(prompt).split("\n<|")[0]
 
     # Display assistant response in chat message container
     with st.chat_message("assistant"):
-        st.markdown(response)
-    st.session_state.memory.save_context({"question":prompt}, {"output":prompt})
+        message_placeholder = st.empty()
+        full_response = ""
+        for chunk in st.session_state.chain.stream(prompt):
+            full_response += chunk + " "
+            message_placeholder.markdown(full_response + " ")
+            if full_response[-4:] == "\n<|":
+                break
+        # st.markdown(full_response)
+    st.session_state.memory.save_context({"question":prompt}, {"output":full_response})
     st.session_state.memory.chat_memory.messages = st.session_state.memory.chat_memory.messages[-15:]
     # Add assistant response to chat history
     st.session_state.messages.append({"role": "assistant", "content": response})