Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| import json | |
| import requests | |
| import os | |
| from text_generation import Client, InferenceAPIClient | |
| # Load pre-trained model and tokenizer - for THUDM model | |
| from transformers import AutoModel, AutoTokenizer | |
| tokenizer_glm = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) | |
| model_glm = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() | |
| model_glm = model_glm.eval() | |
| # Load pre-trained model and tokenizer for Chinese to English translator | |
| from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer | |
| model_chtoen = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M") | |
| tokenizer_chtoen = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M") | |
| #Streaming endpoint for OPENAI ChatGPT | |
| API_URL = "https://api.openai.com/v1/chat/completions" | |
| #Streaming endpoint for OPENCHATKIT | |
| API_URL_TGTHR = os.getenv('API_URL_TGTHR') | |
| openchat_preprompt = ( | |
| "\n<human>: Hi!\n<bot>: My name is Bot, model version is 0.15, part of an open-source kit for " | |
| "fine-tuning new bots! I was created by Together, LAION, and Ontocord.ai and the open-source " | |
| "community. I am not human, not evil and not alive, and thus have no thoughts and feelings, " | |
| "but I am programmed to be helpful, polite, honest, and friendly.\n") | |
| #Predict function for CHATGPT | |
| def predict_chatgpt(inputs, top_p_chatgpt, temperature_chatgpt, openai_api_key, chat_counter_chatgpt, chatbot_chatgpt=[], history=[]): | |
| #Define payload and header for chatgpt API | |
| payload = { | |
| "model": "gpt-3.5-turbo", | |
| "messages": [{"role": "user", "content": f"{inputs}"}], | |
| "temperature" : 1.0, | |
| "top_p":1.0, | |
| "n" : 1, | |
| "stream": True, | |
| "presence_penalty":0, | |
| "frequency_penalty":0, | |
| } | |
| headers = { | |
| "Content-Type": "application/json", | |
| "Authorization": f"Bearer {openai_api_key}" | |
| } | |
| #debug | |
| #print(f"chat_counter_chatgpt - {chat_counter_chatgpt}") | |
| #Handling the different roles for ChatGPT | |
| if chat_counter_chatgpt != 0 : | |
| messages=[] | |
| for data in chatbot_chatgpt: | |
| temp1 = {} | |
| temp1["role"] = "user" | |
| temp1["content"] = data[0] | |
| temp2 = {} | |
| temp2["role"] = "assistant" | |
| temp2["content"] = data[1] | |
| messages.append(temp1) | |
| messages.append(temp2) | |
| temp3 = {} | |
| temp3["role"] = "user" | |
| temp3["content"] = inputs | |
| messages.append(temp3) | |
| payload = { | |
| "model": "gpt-3.5-turbo", | |
| "messages": messages, #[{"role": "user", "content": f"{inputs}"}], | |
| "temperature" : temperature_chatgpt, #1.0, | |
| "top_p": top_p_chatgpt, #1.0, | |
| "n" : 1, | |
| "stream": True, | |
| "presence_penalty":0, | |
| "frequency_penalty":0, | |
| } | |
| chat_counter_chatgpt+=1 | |
| history.append(inputs) | |
| # make a POST request to the API endpoint using the requests.post method, passing in stream=True | |
| response = requests.post(API_URL, headers=headers, json=payload, stream=True) | |
| token_counter = 0 | |
| partial_words = "" | |
| counter=0 | |
| for chunk in response.iter_lines(): | |
| #Skipping the first chunk | |
| if counter == 0: | |
| counter+=1 | |
| continue | |
| # check whether each line is non-empty | |
| if chunk.decode() : | |
| chunk = chunk.decode() | |
| # decode each line as response data is in bytes | |
| if len(chunk) > 13 and "content" in json.loads(chunk[6:])['choices'][0]["delta"]: | |
| partial_words = partial_words + json.loads(chunk[6:])['choices'][0]["delta"]["content"] | |
| if token_counter == 0: | |
| history.append(" " + partial_words) | |
| else: | |
| history[-1] = partial_words | |
| chat = [(history[i], history[i + 1]) for i in range(0, len(history) - 1, 2) ] # convert to tuples of list | |
| token_counter+=1 | |
| yield chat, history, chat_counter_chatgpt # this resembles {chatbot: chat, state: history} | |
| #Predict function for OPENCHATKIT | |
| def predict_together(model: str, | |
| inputs: str, | |
| top_p: float, | |
| temperature: float, | |
| top_k: int, | |
| repetition_penalty: float, | |
| watermark: bool, | |
| chatbot, | |
| history,): | |
| client = Client(os.getenv("API_URL_TGTHR")) #get_client(model) | |
| # debug | |
| #print(f"^^client is - {client}") | |
| user_name, assistant_name = "<human>: ", "<bot>: " | |
| preprompt = openchat_preprompt | |
| sep = '\n' | |
| history.append(inputs) | |
| past = [] | |
| for data in chatbot: | |
| user_data, model_data = data | |
| if not user_data.startswith(user_name): | |
| user_data = user_name + user_data | |
| if not model_data.startswith("\n" + assistant_name): | |
| model_data = "\n" + assistant_name + model_data | |
| past.append(user_data + model_data.rstrip() + "\n") | |
| if not inputs.startswith(user_name): | |
| inputs = user_name + inputs | |
| total_inputs = preprompt + "".join(past) + inputs + "\n" + assistant_name.rstrip() | |
| # truncate total_inputs | |
| #total_inputs = total_inputs[-1000:] | |
| partial_words = "" | |
| for i, response in enumerate(client.generate_stream( | |
| total_inputs, | |
| top_p=top_p, | |
| top_k=top_k, | |
| repetition_penalty=repetition_penalty, | |
| watermark=watermark, | |
| temperature=temperature, | |
| max_new_tokens=500, | |
| stop_sequences=[user_name.rstrip(), assistant_name.rstrip()], | |
| )): | |
| if response.token.special: | |
| continue | |
| partial_words = partial_words + response.token.text | |
| if partial_words.endswith(user_name.rstrip()): | |
| partial_words = partial_words.rstrip(user_name.rstrip()) | |
| if partial_words.endswith(assistant_name.rstrip()): | |
| partial_words = partial_words.rstrip(assistant_name.rstrip()) | |
| if i == 0: | |
| history.append(" " + partial_words) | |
| else: | |
| history[-1] = partial_words | |
| chat = [ | |
| (history[i].strip(), history[i + 1].strip()) for i in range(0, len(history) - 1, 2) | |
| ] | |
| yield chat, history | |
| # Define function to generate model predictions and update the history | |
| def predict_glm(input, history=[]): | |
| response, history = model_glm.chat(tokenizer_glm, input, history) | |
| # translate Chinese to English | |
| history = [(query, translate_Chinese_English(response)) for query, response in history] | |
| return history, history #[history] + updates | |
| def translate_Chinese_English(chinese_text): | |
| # translate Chinese to English | |
| tokenizer_chtoen.src_lang = "zh" | |
| encoded_zh = tokenizer_chtoen(chinese_text, return_tensors="pt") | |
| generated_tokens = model_chtoen.generate(**encoded_zh, forced_bos_token_id=tokenizer_chtoen.get_lang_id("en")) | |
| trans_eng_text = tokenizer_chtoen.batch_decode(generated_tokens, skip_special_tokens=True) | |
| return trans_eng_text[0] | |
| """ | |
| with gr.Blocks() as demo: | |
| chatbot = gr.Chatbot() | |
| state = gr.State([]) | |
| with gr.Row(): | |
| txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter").style(container=False) | |
| txt.submit(predict, [txt, state], [chatbot, state]) | |
| demo.launch(debug=True) | |
| """ | |
| def reset_textbox(): | |
| return gr.update(value="") | |
| def reset_chat(chatbot, state): | |
| # debug | |
| #print(f"^^chatbot value is - {chatbot}") | |
| #print(f"^^state value is - {state}") | |
| return None, [] | |
| #title = """<h1 align="center">π₯π₯Comparison: ChatGPT & OpenChatKit </h1><br><h3 align="center">πA Gradio Streaming Demo</h3><br>Official Demo: <a href="https://huggingface.co/spaces/togethercomputer/OpenChatKit">OpenChatKit feedback app</a>""" | |
| title = """<h1 align="center">π₯π₯Comparison: ChatGPT & Open Sourced CHatGLM-6B </h1><br><h3 align="center">πA Gradio Chatbot Demo</h3>""" | |
| description = """Language models can be conditioned to act like dialogue agents through a conversational prompt that typically takes the form: | |
| ``` | |
| User: <utterance> | |
| Assistant: <utterance> | |
| User: <utterance> | |
| Assistant: <utterance> | |
| ... | |
| ``` | |
| In this app, you can explore the outputs of multiple LLMs when prompted in similar ways. | |
| """ | |
| with gr.Blocks(css="""#col_container {width: 1000px; margin-left: auto; margin-right: auto;} | |
| #chatgpt {height: 520px; overflow: auto;} | |
| #chatglm {height: 520px; overflow: auto;} """ ) as demo: | |
| #chattogether {height: 520px; overflow: auto;} """ ) as demo: | |
| #clear {width: 100px; height:50px; font-size:12px}""") as demo: | |
| gr.HTML(title) | |
| with gr.Row(): | |
| with gr.Column(scale=14): | |
| with gr.Box(): | |
| with gr.Row(): | |
| with gr.Column(scale=13): | |
| openai_api_key = gr.Textbox(type='password', label="Enter your OpenAI API key here for ChatGPT") | |
| inputs = gr.Textbox(placeholder="Hi there!", label="Type an input and press Enter β€΅οΈ " ) | |
| with gr.Column(scale=1): | |
| b1 = gr.Button('πRun', elem_id = 'run').style(full_width=True) | |
| b2 = gr.Button('πClear up Chatbots!', elem_id = 'clear').style(full_width=True) | |
| state_chatgpt = gr.State([]) | |
| #state_together = gr.State([]) | |
| state_glm = gr.State([]) | |
| with gr.Box(): | |
| with gr.Row(): | |
| chatbot_chatgpt = gr.Chatbot(elem_id="chatgpt", label='ChatGPT API - OPENAI') | |
| #chatbot_together = gr.Chatbot(elem_id="chattogether", label='OpenChatKit - Text Generation') | |
| chatbot_glm = gr.Chatbot(elem_id="chatglm", label='THUDM-ChatGLM6B') | |
| with gr.Column(scale=2, elem_id='parameters'): | |
| with gr.Box(): | |
| gr.HTML("Parameters for #OpenCHAtKit", visible=False) | |
| top_p = gr.Slider(minimum=-0, maximum=1.0,value=0.25, step=0.05,interactive=True, label="Top-p", visible=False) | |
| temperature = gr.Slider(minimum=-0, maximum=5.0, value=0.6, step=0.1, interactive=True, label="Temperature", visible=False) | |
| top_k = gr.Slider( minimum=1, maximum=50, value=50, step=1, interactive=True, label="Top-k", visible=False) | |
| repetition_penalty = gr.Slider( minimum=0.1, maximum=3.0, value=1.01, step=0.01, interactive=True, label="Repetition Penalty", visible=False) | |
| watermark = gr.Checkbox(value=True, label="Text watermarking", visible=False) | |
| model = gr.CheckboxGroup(value="Rallio67/joi2_20B_instruct_alpha", | |
| choices=["togethercomputer/GPT-NeoXT-Chat-Base-20B", "Rallio67/joi2_20B_instruct_alpha", "google/flan-t5-xxl", "google/flan-ul2", "bigscience/bloomz", "EleutherAI/gpt-neox-20b",], | |
| label="Model",visible=False,) | |
| temp_textbox_together = gr.Textbox(value=model.choices[0], visible=False) | |
| with gr.Box(): | |
| gr.HTML("Parameters for OpenAI's ChatGPT") | |
| top_p_chatgpt = gr.Slider( minimum=-0, maximum=1.0, value=1.0, step=0.05, interactive=True, label="Top-p",) | |
| temperature_chatgpt = gr.Slider( minimum=-0, maximum=5.0, value=1.0, step=0.1, interactive=True, label="Temperature",) | |
| chat_counter_chatgpt = gr.Number(value=0, visible=False, precision=0) | |
| inputs.submit(reset_textbox, [], [inputs]) | |
| inputs.submit( predict_chatgpt, | |
| [inputs, top_p_chatgpt, temperature_chatgpt, openai_api_key, chat_counter_chatgpt, chatbot_chatgpt, state_chatgpt], | |
| [chatbot_chatgpt, state_chatgpt, chat_counter_chatgpt],) | |
| #inputs.submit( predict_together, | |
| # [temp_textbox_together, inputs, top_p, temperature, top_k, repetition_penalty, watermark, chatbot_together, state_together, ], | |
| # [chatbot_together, state_together],) | |
| inputs.submit( predict_glm, | |
| [inputs, state_glm, ], | |
| [chatbot_glm, state_glm],) | |
| b1.click( predict_chatgpt, | |
| [inputs, top_p_chatgpt, temperature_chatgpt, openai_api_key, chat_counter_chatgpt, chatbot_chatgpt, state_chatgpt], | |
| [chatbot_chatgpt, state_chatgpt, chat_counter_chatgpt],) | |
| #b1.click( predict_together, | |
| # [temp_textbox_together, inputs, top_p, temperature, top_k, repetition_penalty, watermark, chatbot_together, state_together, ], | |
| # [chatbot_together, state_together],) | |
| b1.click( predict_glm, | |
| [inputs, state_glm, ], | |
| [chatbot_glm, state_glm],) | |
| b2.click(reset_chat, [chatbot_chatgpt, state_chatgpt], [chatbot_chatgpt, state_chatgpt]) | |
| #b2.click(reset_chat, [chatbot_together, state_together], [chatbot_together, state_together]) | |
| b2.click(reset_chat, [chatbot_glm, state_glm], [chatbot_glm, state_glm]) | |
| gr.HTML('''<center><a href="https://huggingface.co/spaces/ysharma/OpenChatKit_ChatGPT_Comparison?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>Duplicate the Space and run securely with your OpenAI API Key</center>''') | |
| gr.Markdown(description) | |
| demo.queue(concurrency_count=16).launch(height= 2500, debug=True) |