File size: 5,421 Bytes
93ed498 e2ec341 f4903ba 0d02f18 addc716 93ed498 b43bcfd 17749ab 93ed498 16e49b4 4ad0753 e18bb87 4ad0753 17749ab 58a3a72 7b32bf2 16e49b4 58a3a72 16e49b4 93ed498 6390b56 06cc7e1 b43bcfd 6390b56 ec7e05a 84d8cd6 d1b5796 760415f d1b5796 54d537a 7c3e16e 54d537a 29e7041 3ae4a47 b4565b6 d491d34 cac98b2 7387712 760415f 1d433d0 7387712 ccb1159 fdd9d54 0888c95 0eaea57 c535860 29e7041 d972151 b4565b6 b5411ca 195b309 d491d34 4ad0753 3ae4a47 2d9088a 0d02f18 5554587 4ad0753 195b309 f9e7094 93ed498 c7296f8 586b115 2cb2161 5554587 2cb2161 b4565b6 82c113d dbfa4f4 129a84f 82c113d 96f65f7 a3c3c74 82c113d ab6fbd7 b5411ca 2cb2161 d1b5796 84d8cd6 195b309 84d8cd6 ec7e05a 84d8cd6 6390b56 93ed498 6390b56 93ed498 6390b56 93ed498 6390b56 93ed498 6390b56 93ed498 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 |
import gradio as gr
import spaces
import torch
from torch.cuda.amp import autocast
import subprocess
from huggingface_hub import InferenceClient
import os
import psutil
"""
For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
"""
from accelerate import init_empty_weights, infer_auto_device_map, load_checkpoint_and_dispatch
from accelerate import Accelerator
subprocess.run(
"pip install psutil",
shell=True,
)
import bitsandbytes as bnb # Import bitsandbytes for 8-bit quantization
from datetime import datetime
subprocess.run(
"pip install flash-attn --no-build-isolation",
env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
shell=True,
)
client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
# pip install 'git+https://github.com/huggingface/transformers.git'
token=os.getenv('token')
print('token = ',token)
from transformers import AutoModelForCausalLM, AutoTokenizer
# model_id = "mistralai/Mistral-7B-v0.3"
# model_id = "openchat/openchat-3.6-8b-20240522"
model_id = "meta-llama/Meta-Llama-3-8B-Instruct"
tokenizer = AutoTokenizer.from_pretrained(
# model_id
model_id
, token= token,)
accelerator = Accelerator()
model = AutoModelForCausalLM.from_pretrained(model_id, token= token,
# torch_dtype= torch.uint8,
torch_dtype=torch.bfloat16,
# load_in_4bit=True,
# # # torch_dtype=torch.fl,
attn_implementation="flash_attention_2",
low_cpu_mem_usage=True,
# device_map='cuda',
# device_map=accelerator.device_map,
)
#
model = accelerator.prepare(model)
# device_map = infer_auto_device_map(model, max_memory={0: "79GB", "cpu":"65GB" })
# Load the model with the inferred device map
# model = load_checkpoint_and_dispatch(model, model_id, device_map=device_map, no_split_module_classes=["GPTJBlock"])
# model.half()
import json
def str_to_json(str_obj):
json_obj = json.loads(str_obj)
return json_obj
@spaces.GPU(duration=170)
def respond(
message,
history: list[tuple[str, str]],
system_message,
max_tokens,
temperature,
top_p,
):
# yield 'retuend'
# model.to(accelerator.device)
messages = []
json_obj = str_to_json(message)
print(json_obj)
messages= json_obj
input_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(accelerator.device)
input_ids2 = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, return_tensors="pt") #.to('cuda')
print(f"Converted input_ids dtype: {input_ids.dtype}")
input_str= str(input_ids2)
print('input str = ', input_str)
with torch.no_grad():
gen_tokens = model.generate(
input_ids,
max_new_tokens=max_tokens,
# do_sample=True,
temperature=temperature,
)
gen_text = tokenizer.decode(gen_tokens[0])
print(gen_text)
gen_text= gen_text.replace(input_str,'')
gen_text= gen_text.replace('<|eot_id|>','')
yield gen_text
# messages = [
# # {"role": "user", "content": "What is your favourite condiment?"},
# # {"role": "assistant", "content": "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!"},
# # {"role": "user", "content": "Do you have mayonnaise recipes?"}
# ]
# inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to("cuda")
# outputs = model.generate(inputs, max_new_tokens=2000)
# gen_text=tokenizer.decode(outputs[0], skip_special_tokens=True)
# print(gen_text)
# yield gen_text
# for val in history:
# if val[0]:
# messages.append({"role": "user", "content": val[0]})
# if val[1]:
# messages.append({"role": "assistant", "content": val[1]})
# messages.append({"role": "user", "content": message})
# response = ""
# for message in client.chat_completion(
# messages,
# max_tokens=max_tokens,
# stream=True,
# temperature=temperature,
# top_p=top_p,
# ):
# token = message.choices[0].delta.content
# response += token
# yield response
"""
For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
"""
demo = gr.ChatInterface(
respond,
additional_inputs=[
gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
gr.Slider(
minimum=0.1,
maximum=1.0,
value=0.95,
step=0.05,
label="Top-p (nucleus sampling)",
),
],
)
if __name__ == "__main__":
demo.launch() |