Update app.py

app.py

@@ -7,6 +7,9 @@ llm = Llama.from_pretrained(
     filename="Marco-o1-Q4_K_M.gguf",
 )
 
+# Access the tokenizer from the Llama model
+tokenizer = llm.get_tokenizer()
+
 def respond(
     message,
     history: list[tuple[str, str]],
@@ -19,48 +22,18 @@ def respond(
     tokenized_messages = []
 
     # Tokenize the system message
-    tokenized_messages.append(llm.tokenizer.encode(system_message))
+    tokenized_messages.append(tokenizer.encode(system_message))
 
     # Tokenize the history messages
     for val in history:
         if val[0]:
-            tokenized_messages.append(llm.tokenizer.encode(val[0]))  # User message
+            tokenized_messages.append(tokenizer.encode(val[0]))  # User message
         if val[1]:
-            tokenized_messages.append(llm.tokenizer.encode(val[1]))  # Assistant message
+            tokenized_messages.append(tokenizer.encode(val[1]))  # Assistant message
 
     # Tokenize the current user message
-    tokenized_messages.append(llm.tokenizer.encode(message))
+    tokenized_messages.append(tokenizer.encode(message))
 
     response = ""
 
-    # Use llm.create_completion with tokenized messages
-    for token in llm.create_completion(
-        tokenized_messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token_content = token['choices'][0]['delta']['content']
-        response += token_content
-        yield response
-
-# Gradio demo setup
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="", label="System message"),
-        gr.Slider(minimum=1, maximum=8192, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
-
-if __name__ == "__main__":
-    demo.launch()
+    # Use llm.create_completion with tokenized message