Update app.py

app.py

@@ -24,7 +24,6 @@ Do not give any additional explanations or use any tags or headers, only return
 """
 
 test_input_string = """
-###Question:
 <Title:>Vinorelbine in Treating Patients With Advanced Solid Tumors That Have Not Responded to Treatment and Liver Dysfunction <BriefSummary:>RATIONALE: Drugs used in chemotherapy, such as vinorelbine, work in different ways to stop the growth of tumor cells, either by killing the cells or by stopping them from dividing.
 PURPOSE: This pilot trial is studying the side effects and best dose of vinorelbine in treating patients with advanced solid tumors that have not responded to treatment and liver dysfunction. <EligibilityCriteria:>DISEASE CHARACTERISTICS:
 * Histologically confirmed advanced solid tumor
@@ -65,22 +64,46 @@ def load_model(model_name):
     model.to(device)
     tokenizer = AutoTokenizer.from_pretrained(model_name)
 
+alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
+
+### Instruction:
+{}
+
+### Input:
+{}
+
+### Response:
+{}"""
+
 @spaces.GPU
 def generate_response(system_instruction, user_input):
-    # Format the prompt using the messages structure
-    messages = [
-        {"role": "system", "content": system_instruction},
-        {"role": "user", "content": user_input},
-    ]
-    encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt").to(device)
-    model_inputs = encodeds.to(device)
-
-    # Generate model response
-    with torch.no_grad():
-        generated_ids = model.generate(model_inputs, max_new_tokens=1000, do_sample=True)
-    # Find everything after the <|assistant|> tag
+    # # Format the prompt using the messages structure
+    # messages = [
+    #     {"role": "system", "content": system_instruction},
+    #     {"role": "user", "content": user_input},
+    # ]
+    # encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt").to(device)
+    # model_inputs = encodeds.to(device)
+
+    inputs = tokenizer([
+        alpaca_prompt.format(
+            test_instruction_string, # instruction
+            test_input_string, # input
+            "", # output - leave this blank for generation!
+        )
+    ], return_tensors = "pt").to("cuda")
+
+    outputs = model.generate(**inputs, max_new_tokens = 1000, use_cache = True)
     decoded_output = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
-    assistant_response = decoded_output.split("<|assistant|>")[-1].strip()
+    assistant_response = decoded_output
+    # tokenizer.batch_decode(outputs)
+
+    # # Generate model response
+    # with torch.no_grad():
+    #     generated_ids = model.generate(model_inputs, max_new_tokens=1000, do_sample=True)
+    # # Find everything after the <|assistant|> tag
+    # decoded_output = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    # assistant_response = decoded_output.split("<|assistant|>")[-1].strip()
 
     return assistant_response