Update app.py

app.py

@@ -15,11 +15,14 @@ with st.sidebar:
     st.image('img/logo.png', use_column_width=True)
     st.title("Nanbeige Chatbot💬")
     st.markdown("""
-    <p>Created by Nanbeige Lab
+    <p>Created by Nanbeige Lab 
     </p>""", unsafe_allow_html=True)
 
     temperature = st.sidebar.slider('temperature', min_value=0.01, max_value=1.0, value=0.3, step=0.01)
     top_p = st.sidebar.slider('top_p', min_value=0.01, max_value=1.0, value=0.9, step=0.01)
+    # st.text_input("System prompt: ",
+    #               key='sys_prompt',
+    #               placeholder="If not provided, the default system prompt will be used.")
 
     st.sidebar.button('Clear Chat History', on_click=clear_chat_history)
 
@@ -40,9 +43,11 @@ def generate_response():
     payload = json.dumps({
         'model': 'NBG-plus',
         'messages': messages,
-        'max_tokens': "4096",
-        'temperature': 0.3,
-        'top_p': 0.9,
+        'max_tokens': 4096,
+        'temperature': 0.7,
+        'stream': True,
+        'output_accumulate': True,
+        'top_p': 1,
     })
 
     headers = {
@@ -50,14 +55,19 @@ def generate_response():
         'Content-Type': 'application/json'
     }
 
-    r = requests.request("POST", st.secrets["secret_url"], headers=headers, data=payload)
-    return r.json()['reply']
+    return requests.request("POST", st.secrets["secret_url"], headers=headers, data=payload, stream=True)
 
 
-def stream_response(text):
-    for word in text:
-        yield word
-        time.sleep(0.02)
+def stream_response(resp):
+    last_length = 0
+    for line in resp.iter_lines():
+        line = line.decode()
+        if len(line) == 0:
+            continue
+        r = json.loads(line[6:])
+        content = r['modelServerData']['choices'][0]['delta']['content']
+        yield content[last_length:]
+        last_length = len(content)
 
 
 if prompt := st.chat_input():
@@ -68,8 +78,8 @@ if prompt := st.chat_input():
 if st.session_state.messages[-1]["role"] != "assistant":
     with st.chat_message("assistant"):
         with st.spinner("Thinking..."):
-            response = generate_response()
-        st.write_stream(stream_response(response))
+            stream_resp = generate_response()
+        resp = st.write_stream(stream_response(stream_resp))
 
-    message = {"role": "assistant", "content": response}
+    message = {"role": "assistant", "content": resp}
     st.session_state.messages.append(message)