Update app.py

app.py

@@ -3,8 +3,9 @@ import gradio as gr
 import inspect
 from gradio import routes
 from typing import List, Type
-from petals import AutoDistributedModelForCausalLM
+import torch
 from transformers import AutoTokenizer
+from petals import AutoDistributedModelForCausalLM
 import requests, os, re, asyncio, json, time
 
 loop = asyncio.get_event_loop()
@@ -38,7 +39,7 @@ model_name = "quantumaikr/llama-2-70b-fb16-korean"
 #quantumaikr/llama-2-70b-fb16-korean
 tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
 
-#model = None
+model = None
 model = AutoDistributedModelForCausalLM.from_pretrained(model_name)
 
 history = {
@@ -262,12 +263,14 @@ def chat(id, npc, text):
 [대화기록]{history[npc][id]}
 """
 
-        inputs = tokenizer("아", return_tensors="pt")["input_ids"]
-        outputs = model.generate(inputs, do_sample=True, temperature=0.6, top_p=0.75, max_new_tokens=2)
-        output = tokenizer.decode(outputs[0])[len(prom)+3:-1].split("<")[0].split("###")[0].replace(". ", ".\n")
+        #inputs = tokenizer("아", return_tensors="pt")["input_ids"]
+        #outputs = model.generate(inputs, do_sample=True, temperature=0.6, top_p=0.75, max_new_tokens=2)
+        #output = tokenizer.decode(outputs[0])[len(prom)+3:-1].split("<")[0].split("###")[0].replace(". ", ".\n")
         #output = cleanText(output)
         #print(tokenizer.decode(outputs[0]))
         #output = f"{npc}의 응답입니다."
+        
+        
         response = client.chat.completions.create(
         model=os.environ['MODEL'],
         messages=[