Update app.py

app.py

@@ -4,10 +4,10 @@ import gradio as gr
 # Load model directly
 from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
 
-pipe = pipeline("text-generation", model="shivanikerai/TinyLlama-1.1B-Chat-v1.0-seo-optimised-title-suggestion-v1.0")
+# pipe = pipeline("text-generation", model="shivanikerai/TinyLlama-1.1B-Chat-v1.0-seo-optimised-title-suggestion-v1.0")
 
-# tokenizer = AutoTokenizer.from_pretrained("shivanikerai/TinyLlama-1.1B-Chat-v1.0-seo-optimised-title-suggestion-v1.0")
-# model = AutoModelForCausalLM.from_pretrained("shivanikerai/TinyLlama-1.1B-Chat-v1.0-seo-optimised-title-suggestion-v1.0")
+tokenizer = AutoTokenizer.from_pretrained("shivanikerai/TinyLlama-1.1B-Chat-v1.0-seo-optimised-title-suggestion-v1.0")
+model = AutoModelForCausalLM.from_pretrained("shivanikerai/TinyLlama-1.1B-Chat-v1.0-seo-optimised-title-suggestion-v1.0")
 def generate_title_suggestions(keywords, product_info):
     # Define the roles and markers
     B_SYS, E_SYS = "<<SYS>>", "<</SYS>>"
@@ -21,29 +21,29 @@ def generate_title_suggestions(keywords, product_info):
 
     # print("Prompt:")
     # print(prompt)
-    predictions = pipeline(prompt)
-    output=((predictions[0]['generated_text']).split(B_out)[-1]).strip()
-    return (output)
-    # encoding = tokenizer(prompt, return_tensors="pt").to("cuda:0")
-    # output = model.generate(input_ids=encoding.input_ids,
-    #                         attention_mask=encoding.attention_mask,
-    #                         max_new_tokens=1024,
-    #                         do_sample=True,
-    #                         temperature=0.01,
-    #                         eos_token_id=tokenizer.eos_token_id,
-    #                         top_k=0)
+    # predictions = pipeline(prompt)
+    # output=((predictions[0]['generated_text']).split(B_out)[-1]).strip()
+    # return (output)
+    encoding = tokenizer(prompt, return_tensors="pt").to("cuda:0")
+    output = model.generate(input_ids=encoding.input_ids,
+                            attention_mask=encoding.attention_mask,
+                            max_new_tokens=1024,
+                            do_sample=True,
+                            temperature=0.01,
+                            eos_token_id=tokenizer.eos_token_id,
+                            top_k=0)
 
     # print()
 
     # Subtract the length of input_ids from output to get only the model's response
-    # output_text = tokenizer.decode(output[0, len(encoding.input_ids[0]):], skip_special_tokens=False)
-    # output_text = re.sub('\n+', '\n', output_text)  # remove excessive newline characters
+    output_text = tokenizer.decode(output[0, len(encoding.input_ids[0]):], skip_special_tokens=False)
+    output_text = re.sub('\n+', '\n', output_text)  # remove excessive newline characters
 
     # print("Generated Assistant Response:")
-    # print(output_text)
+    return (output_text)
     gr.Interface(
     generate_title_suggestions,
     inputs=['text','text'],
     outputs='text',
-    title="Title Suggestion",
+    title="SEO Optimised Title Suggestion",
 ).launch()