new

app.py

@@ -1,7 +1,30 @@
 import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+def vicuna(input):
+    tokenizer = AutoTokenizer.from_pretrained("path/to/stable-vicuna-13b-applied")
+    model = AutoModelForCausalLM.from_pretrained("path/to/stable-vicuna-13b-applied")
+    model.half().cuda()
+
+    # prompt = """\
+    # ### Human: Write a Python script for text classification using Transformers and PyTorch
+    # ### Assistant:\
+    # """
+    prompt=input
+    inputs = tokenizer(prompt, return_tensors='pt').to('cuda')
+    tokens = model.generate(
+    **inputs,
+    max_new_tokens=256,
+    do_sample=True,
+    temperature=1.0,
+    top_p=1.0,
+    )
+    print(tokenizer.decode(tokens[0], skip_special_tokens=True))
+
+
 
 def greet(name):
     return "Hello " + name + "!!"
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
+iface = gr.Interface(fn=vicuna, inputs="text", outputs="text")
 iface.launch()