app update

app.py

@@ -6,28 +6,22 @@ model_path = "finetuned_phi2"
 model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True)
 tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
 
-# gen = pipeline('text-generation', model=model, tokenizer=tokenizer)
-# result = gen(prompt)
-# print(result[0]['generated_text'])
 
 
-def generate(prompt, history, temperature=0.3, max_new_tokens=512, top_p=0.95, repetition_penalty=1.0,):
-    temperature = float(temperature)
-    if temperature < 1e-2:
-        temperature = 1e-2
-    top_p = float(top_p)
-
-    generate_kwargs = dict(temperature=temperature,
-                            max_new_tokens=max_new_tokens,
-                            top_p=top_p,
-                            repetition_penalty=repetition_penalty,
-                            do_sample=True,
-                            seed=42)
+def generate(question):
+    system_message = "You are a question answering chatbot. Provide a clear and detailed explanation"
+    prompt = f"[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n {question} [/INST]" # replace the command here with something relevant to your task
+    
+    num_new_tokens = 500  # change to the number of new tokens you want to generate
+    # Count the number of tokens in the prompt
+    num_prompt_tokens = len(tokenizer(prompt)['input_ids'])
+    # Calculate the maximum length for the generation
+    max_length = num_prompt_tokens + num_new_tokens
+    
+    gen = pipeline('text-generation', model=model, tokenizer=tokenizer, max_length=max_length)
+    result = gen(prompt)
+    return (result[0]['generated_text'].replace(prompt, ''))
     
-    #output = gen(prompt, **generate_kwargs)
-    gen = pipeline('text-generation', model=model, tokenizer=tokenizer)
-    output = gen(prompt)
-    return output[0]['generated_text']
 
 bbchatbot = gr.Chatbot(
     avatar_images=["logo/user logo.png", "logo/bot logo.png"], bubble_full_width=False, show_label=False, show_copy_button=True, likeable=True,)