Update app.py

app.py

@@ -49,10 +49,10 @@ model, tokenizer = FastLanguageModel.from_pretrained(
     load_in_4bit = True,
 )
 model.load_adapter(peft_model_adapter_id, token=token)
-# terminators = [
-#     tokenizer.eos_token_id,
-#     tokenizer.convert_tokens_to_ids("<|eot_id|>")
-# ]
+terminators = [
+    tokenizer.eos_token_id,
+    tokenizer.convert_tokens_to_ids("<|eot_id|>")
+]
 FastLanguageModel.for_inference(model) # Enable native 2x faster inference
 
 from transformers import pipeline, TextIteratorStreamer
@@ -79,7 +79,7 @@ def run_model_on_text(text):
 
   streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
 
-  generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024, do_sample=True, repetition_penalty=1.2,)
+  generation_kwargs = dict(inputs, streamer=streamer,eos_token_id=terminators, max_new_tokens=1024, do_sample=True, repetition_penalty=1.2,)
   thread = Thread(target=model.generate, kwargs=generation_kwargs)
   thread.start()