File size: 3,001 Bytes
41f8286 e34e07c 41f8286 e34e07c 41f8286 1890c65 e34e07c 41f8286 e34e07c 41f8286 e34e07c 94f6ac7 e34e07c 94f6ac7 e34e07c 9728299 94f6ac7 e34e07c 41f8286 e34e07c 94f6ac7 41f8286 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
import gradio as gr
import torch
import os
from model import get_input_token_length, run
DEFAULT_SYSTEM_PROMPT = """\
You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.\
"""
MAX_MAX_NEW_TOKENS = 2048
DEFAULT_MAX_NEW_TOKENS = 1024
MAX_INPUT_TOKEN_LENGTH = 4000
LICENSE = """
<p/>
---
As a derivate work of [Llama-2-13b-chat](https://huggingface.co/meta-llama/Llama-2-13b-chat) by Meta,
this demo is governed by the original [license](https://huggingface.co/spaces/huggingface-projects/llama-2-13b-chat/blob/main/LICENSE.txt) and [acceptable use policy](https://huggingface.co/spaces/huggingface-projects/llama-2-13b-chat/blob/main/USE_POLICY.md).
"""
is_spaces = True if "SPACE_ID" in os.environ else False
if is_spaces :
is_shared_ui = True if "gradio-discord-bots/llama-2-13b-chat-transformers" in os.environ['SPACE_ID'] else False
else:
is_shared_ui = False
is_gpu_associated = torch.cuda.is_available()
def generate(
message: str,
history: list[tuple[str, str]],
system_prompt=DEFAULT_SYSTEM_PROMPT,
max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
temperature=1.0,
top_p=0.95,
top_k=50,
) -> tuple[str, list[tuple[str, str]]]:
if is_shared_ui:
raise ValueError("Cannot use demo running in shared_ui. Must duplicate your own space.")
if max_new_tokens > MAX_MAX_NEW_TOKENS:
raise ValueError
input_token_length = get_input_token_length(message, history, system_prompt)
if input_token_length > MAX_INPUT_TOKEN_LENGTH:
response = f'The accumulated input is too long ({input_token_length} > {MAX_INPUT_TOKEN_LENGTH}). Please create a new thread.'
else:
response = run(message, history, system_prompt, max_new_tokens, temperature, top_p, top_k)
return response
interface = gr.ChatInterface(generate)
with gr.Blocks() as demo:
gr.Markdown(
"""
# Llama-2-13b-chat-hf Discord Bot Powered by Gradio and Hugging Face Transformers
### First install the `gradio_client`
```bash
pip install gradio_client
```
### Then deploy to discord in one line! ⚡️
```python
secrets = {"HUGGING_FACE_HUB_TOKEN": "<your-key-here>",}
client = grc.Client.duplicate("gradio-discord-bots/llama-2-13b-chat-transformers", secrets=secrets, hardware="a10g-small", sleep_timeout=2880)
client.deploy_discord(api_names=["chat"], hf_token="<your-key-here>")
```
"""
)
gr.Markdown(LICENSE)
with gr.Row(visible=False):
interface.render()
demo.queue(max_size=20).launch()
|