greedy decoding of the output

app.py

@@ -1,6 +1,6 @@
 import gradio as gr
 import spaces
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
 
 model = AutoModelForCausalLM.from_pretrained("BatsResearch/bonito-v1")
 tokenizer = AutoTokenizer.from_pretrained("BatsResearch/bonito-v1")
@@ -26,10 +26,19 @@ def respond(
         temperature=temperature,
         top_p=top_p,
         do_sample=True,
+        stop_strings=["<|pipe|>"],
+        tokenizer=tokenizer,
+    )
+    output_with_greedy_response = model.generate(
+        output,
+        max_new_tokens=max_tokens,
+        temperature=0.0,
+        top_p=1.0,
+        do_sample=False,
     )
     pred_start = int(input_ids.shape[-1])
 
-    response = tokenizer.decode(output[0][pred_start:], skip_special_tokens=True)
+    response = tokenizer.decode(output_with_greedy_response[0][pred_start:], skip_special_tokens=True)
 
     # check if <|pipe|> is in the response
     if "<|pipe|>" in response: