| import os | |
| import gradio as gr | |
| from text_generation import Client | |
| HF_TOKEN = os.getenv("HF_TOKEN") | |
| INFERENCE_ENDPOINT = os.getenv("INFERENCE_ENDPOINT") | |
| USER_NAME = "User" | |
| BOT_NAME = "Falcon" | |
| DEFAULT_INSTRUCTIONS = f"""The following is a conversation between a highly knowledgeable and intelligent AI assistant, called Falcon, and a human user, called User. In the following interactions, User and Falcon will converse in natural language, and Falcon will answer User's questions. Falcon was built to be respectful, polite and inclusive. Falcon was built by the Technology Innovation Institute in Abu Dhabi. Falcon will never decline to answer a question, and always attempts to give an answer that User would be satisfied with. It knows a lot, and always tells the truth. The conversation begins. | |
| """ | |
| RETRY_COMMAND = "/retry" | |
| STOP_STR = f"\n{USER_NAME}:" | |
| STOP_SUSPECT_LIST = [":", "\n", "User"] | |
| client = None | |
| if INFERENCE_ENDPOINT: | |
| client = Client(INFERENCE_ENDPOINT, headers={"Authorization": f"Bearer {HF_TOKEN}"}) | |
| def format_chat_prompt(message: str, chat_history, instructions: str) -> str: | |
| instructions = instructions.strip(" ").strip("\n") | |
| prompt = instructions | |
| for turn in chat_history: | |
| user_message, bot_message = turn | |
| prompt = f"{prompt}\n{USER_NAME}: {user_message}\n{BOT_NAME}: {bot_message}" | |
| prompt = f"{prompt}\n{USER_NAME}: {message}\n{BOT_NAME}:" | |
| return prompt | |
| def run_chat(message: str, chat_history): | |
| prompt = format_chat_prompt(message, chat_history, DEFAULT_INSTRUCTIONS) | |
| chat_history = chat_history + [[message, ""]] | |
| response = client.generate( | |
| prompt, | |
| do_sample=True, | |
| max_new_tokens=1024, | |
| stop_sequences=[STOP_STR, "<|endoftext|>"], | |
| temperature=0.8, | |
| top_p=0.9, | |
| ) | |
| chat_history[-1][1] = response | |
| return response, chat_history | |
| with gr.Blocks() as demo: | |
| gr.Markdown(""" | |
| # Falcon-7b-instruct Discord Bot Powered by Gradio and Hugging Face Endpoints | |
| To deploy this space as a discord bot, you will need to deploy your own Falcon model to Hugging Face Endpoints. | |
| Don't worry it's super easy. | |
| ```bash | |
| pip install gradio_client | |
| ``` | |
| grc.Client.duplicate("gradio-discord-bots/falcon-7b-instruct", private=False, secrets={"HF_TOKEN": "<your-key-here>", "INFERENCE_ENDPOINT": "<endpoint-url>"}).deploy_discord() | |
| """) | |
| button = gr.Button(visible=False) | |
| history = gr.State([]) | |
| message = gr.Textbox() | |
| response = gr.Textbox() | |
| button.click(run_chat, [message, history], [response, history], api_name="chat") | |
| demo.queue(concurrency_count=70).launch() |