Update app.py

app.py

@@ -1,20 +1,22 @@
 import os
-os.system('pip install dashscope')
+os.system('pip install --upgrade pip')
+os.system('pip install gradio transformers torch')
+
 import gradio as gr
-from http import HTTPStatus
-import dashscope
-from dashscope import Generation
-from dashscope.api_entities.dashscope_response import Role
 from typing import List, Optional, Tuple, Dict
-from urllib.error import HTTPError
-default_system = 'You are JOSIE which is an acronym for "Just an Outstandingly Smart Intelligent Entity", a private and super-intelligent AI assistant, created by Gökdeniz Gülmez.'
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
+import torch
 
-YOUR_API_TOKEN = os.getenv('YOUR_API_TOKEN')
-dashscope.api_key = YOUR_API_TOKEN
+default_system = 'You are JOSIE which is an acronym for "Just an Outstandingly Smart Intelligent Entity", a private and super-intelligent AI assistant, created by Gökdeniz Gülmez.'
 
 History = List[Tuple[str, str]]
 Messages = List[Dict[str, str]]
 
+# Load model and tokenizer
+model_name = 'Isaak-Carter/J.O.S.I.E.3-Beta12-7B-slerp'
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name)
+
 def clear_session() -> History:
     return '', []
 
@@ -24,55 +26,43 @@ def modify_system_session(system: str) -> str:
     return system, system, []
 
 def history_to_messages(history: History, system: str) -> Messages:
-    messages = [{'role': Role.SYSTEM, 'content': system}]
+    messages = [{'role': 'system', 'content': system}]
     for h in history:
-        messages.append({'role': Role.USER, 'content': h[0]})
-        messages.append({'role': Role.ASSISTANT, 'content': h[1]})
+        messages.append({'role': 'user', 'content': h[0]})
+        messages.append({'role': 'assistant', 'content': h[1]})
     return messages
 
-
 def messages_to_history(messages: Messages) -> Tuple[str, History]:
-    assert messages[0]['role'] == Role.SYSTEM
+    assert messages[0]['role'] == 'system'
     system = messages[0]['content']
     history = []
     for q, r in zip(messages[1::2], messages[2::2]):
         history.append([q['content'], r['content']])
     return system, history
 
+def generate_response(messages: Messages) -> str:
+    prompt = "\n".join([f"{msg['role']}: {msg['content']}" for msg in messages])
+    inputs = tokenizer(prompt, return_tensors='pt')
+    outputs = model.generate(inputs['input_ids'], max_length=512, pad_token_id=tokenizer.eos_token_id)
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return response.split('assistant:')[-1].strip()
 
-def model_chat(query: Optional[str], history: Optional[History], system: str
-) -> Tuple[str, str, History]:
+def model_chat(query: Optional[str], history: Optional[History], system: str) -> Tuple[str, str, History]:
     if query is None:
         query = ''
     if history is None:
         history = []
     messages = history_to_messages(history, system)
-    messages.append({'role': Role.USER, 'content': query})
-    gen = Generation.call(
-        model='J.O.S.I.E.3-Beta12-7B-slerp',
-        messages=messages,
-        result_format='message',
-        stream=True
-    )
-    for response in gen:
-        if response.status_code == HTTPStatus.OK:
-            role = response.output.choices[0].message.role
-            response = response.output.choices[0].message.content
-            system, history = messages_to_history(messages + [{'role': role, 'content': response}])
-            yield '', history, system
-        else:
-            raise ValueError('Request id: %s, Status code: %s, error code: %s, error message: %s' % (
-                response.request_id, response.status_code,
-                response.code, response.message
-            ))
-
+    messages.append({'role': 'user', 'content': query})
+    response = generate_response(messages)
+    messages.append({'role': 'assistant', 'content': response})
+    system, history = messages_to_history(messages)
+    return '', history, system
 
 with gr.Blocks() as demo:
     gr.Markdown("""<center><font size=8>J.O.S.I.E.3-Beta12 Preview👾</center>""")
 
     with gr.Row():
-        # with gr.Column(scale=3):
-        #     system_input = gr.Textbox(value=default_system, lines=1, label='System')
         with gr.Column(scale=1):
             modify_system = gr.Button("🛠️ Set system prompt and clear history", scale=2)
         system_state = gr.Textbox(value=default_system, visible=False)
@@ -81,18 +71,18 @@ with gr.Blocks() as demo:
 
     with gr.Row():
         clear_history = gr.Button("🧹 Clear history")
-        sumbit = gr.Button("🚀 Send")
+        submit = gr.Button("🚀 Send")
 
-    sumbit.click(model_chat,
+    submit.click(model_chat,
                  inputs=[textbox, chatbot, system_state],
-                 outputs=[textbox, chatbot, system_input],
-                 concurrency_limit = 5)
+                 outputs=[textbox, chatbot, system_state],
+                 concurrency_limit=5)
     clear_history.click(fn=clear_session,
                         inputs=[],
                         outputs=[textbox, chatbot])
     modify_system.click(fn=modify_system_session,
-                        inputs=[system_input],
-                        outputs=[system_state, system_input, chatbot])
+                        inputs=[system_state],
+                        outputs=[system_state, system_state, chatbot])
 
 demo.queue(api_open=False)
 demo.launch(max_threads=5)