Update app.py

app.py

@@ -85,41 +85,41 @@ def generate(
     top_k: int = 50,
     repetition_penalty: float = 1.2,
 ) -> Iterator[str]:
-    # print_gpu()
-
-    # conversation = []
-    # if system_prompt:
-    #     conversation.append({"role": "system", "content": system_prompt})
-    # for user, assistant in chat_history:
-    #     conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
-    # conversation.append({"role": "user", "content": message})
-
-    # input_ids = tokenizer.apply_chat_template(conversation, tokenize=False,add_generation_prompt=True)
-    # input_ids = tokenizer([input_ids],return_tensors="pt").to(model.device)
-
-    # streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
-    # generate_kwargs = dict(
-    #     input_ids=input_ids.input_ids,
-    #     streamer=streamer,
-    #     max_new_tokens=max_new_tokens,
-    #     do_sample=True,
-    #     top_p=top_p,
-    #     top_k=top_k,
-    #     temperature=temperature,
-    #     repetition_penalty=repetition_penalty,
-    # ) 
-    # t = Thread(target=model.generate, kwargs=generate_kwargs)
-    # t.start()
-    # #dictionary update sequence element #0 has length 19; 2 is required
-
-    # outputs = []
-    # for text in streamer:
-    #     outputs.append(text)
-    #     yield "".join(outputs)
-
-    # #outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
-    # print(outputs)
-    #yield outputs
+    print_gpu()
+
+    conversation = []
+    if system_prompt:
+        conversation.append({"role": "system", "content": system_prompt})
+    for user, assistant in chat_history:
+        conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
+    conversation.append({"role": "user", "content": message})
+
+    input_ids = tokenizer.apply_chat_template(conversation, tokenize=False,add_generation_prompt=True)
+    input_ids = tokenizer([input_ids],return_tensors="pt").to(model.device)
+
+    streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        input_ids=input_ids.input_ids,
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        top_p=top_p,
+        top_k=top_k,
+        temperature=temperature,
+        repetition_penalty=repetition_penalty,
+    ) 
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+    #dictionary update sequence element #0 has length 19; 2 is required
+
+    outputs = []
+    for text in streamer:
+        outputs.append(text)
+        yield "".join(outputs)
+
+    #outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    print(outputs)
+    yield outputs
 
 
 chat_interface = gr.ChatInterface(