Update app.py

app.py

@@ -30,6 +30,14 @@ routes.get_types = get_types
 
 # App code
 
+model_name = "daekeun-ml/Llama-2-ko-instruct-13B"
+
+#daekeun-ml/Llama-2-ko-instruct-13B
+#quantumaikr/llama-2-70b-fb16-korean
+
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoDistributedModelForCausalLM.from_pretrained(model_name)
+
 def chat(id, npc, prompt):
 
     # get_coin endpoint
@@ -43,6 +51,11 @@ def chat(id, npc, prompt):
         return "no coin"
     
     # model inference
+
+    prom = ""
+    inputs = tokenizer(prom, return_tensors="pt")["input_ids"]
+    outputs = model.generate(inputs, max_new_tokens=100)
+    print(tokenizer.decode(outputs[0]))
     
     output = "AI 응답입니다."