Update app.py

app.py

@@ -15,32 +15,36 @@ def respond(
     temperature,
     top_p,
 ):
-    # Flatten the history and system message into a list of strings
-    messages = [system_message]  # Start with the system message
+    # Initialize an empty list to hold tokenized messages
+    tokenized_messages = []
 
+    # Tokenize the system message
+    tokenized_messages.append(llm.tokenizer.encode(system_message))
+
+    # Tokenize the history messages
     for val in history:
         if val[0]:
-            messages.append(val[0])  # Append user message
+            tokenized_messages.append(llm.tokenizer.encode(val[0]))  # User message
         if val[1]:
-            messages.append(val[1])  # Append assistant message
+            tokenized_messages.append(llm.tokenizer.encode(val[1]))  # Assistant message
 
-    messages.append(message)  # Add the current user message
+    # Tokenize the current user message
+    tokenized_messages.append(llm.tokenizer.encode(message))
 
     response = ""
 
-    # Use llm.create_completion instead of llm.chat_completion
-    for message in llm.create_completion(
-        messages,
+    # Use llm.create_completion with tokenized messages
+    for token in llm.create_completion(
+        tokenized_messages,
         max_tokens=max_tokens,
         stream=True,
         temperature=temperature,
         top_p=top_p,
     ):
-        token = message['choices'][0]['delta']['content']
-        response += token
+        token_content = token['choices'][0]['delta']['content']
+        response += token_content
         yield response
 
-
 # Gradio demo setup
 demo = gr.ChatInterface(
     respond,