File size: 3,310 Bytes
f0dc90a 558a253 f0dc90a 558a253 f0dc90a 558a253 f0dc90a dbde5f6 f0dc90a 558a253 72e002c 8eaa299 558a253 9190a90 f0dc90a 72e002c f0dc90a 558a253 f0dc90a a0013c1 255e744 ba0a926 255e744 ba0a926 255e744 d3d1451 a0013c1 255e744 a0013c1 255e744 f0dc90a 255e744 f0dc90a 90afce1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
import gradio as gr
from gpt4all import GPT4All
from huggingface_hub import hf_hub_download
title = "安风"
description = """
Infer service
"""
model_path = "tastypear/CausalLM-7B-DPO-alpha-GGUF"
model_name = "causallm_7b-dpo-alpha.Q4_K_M.gguf"
hf_hub_download(repo_id="tastypear/CausalLM-7B-DPO-alpha-GGUF", filename=model_name, local_dir=model_path, local_dir_use_symlinks=True)
print("Start the model init process")
model = model = GPT4All(model_name, model_path, allow_download = True, device="cpu")
print("Finish the model init process")
model.config["promptTemplate"] = '''<|im_start|>user
{0}<|im_end|>
<|im_start|>assistant
'''
model.config["systemPrompt"] = "You are a helpful assistant named 安风."
model._is_chat_session_activated = True
max_new_tokens = 2048
def generater(message, history, temperature, top_p, top_k):
prompt = ""
for user_message, assistant_message in history:
prompt += model.config["promptTemplate"].format(user_message)
prompt += assistant_message + "<|im_end|>"
prompt += model.config["promptTemplate"].format(message)
outputs = []
for token in model.generate(prompt=prompt, temp=temperature, top_k = top_k, top_p = top_p, max_tokens = max_new_tokens, streaming=True):
outputs.append(token)
yield "".join(outputs)
def vote(data: gr.LikeData):
if data.liked:
return
else:
return
chatbot = gr.Chatbot(avatar_images=('resourse/user-icon.png', 'resourse/chatbot-icon.png'),bubble_full_width = False)
additional_inputs=[
gr.Slider(
label="temperature",
value=0.5,
minimum=0.0,
maximum=2.0,
step=0.05,
interactive=True,
info="Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.",
),
gr.Slider(
label="top_p",
value=1.0,
minimum=0.0,
maximum=1.0,
step=0.01,
interactive=True,
info="0.1 means only the tokens comprising the top 10% probability mass are considered. Suggest set to 1 and use temperature. 1 means 100% and will disable it",
),
gr.Slider(
label="top_k",
value=40,
minimum=0,
maximum=1000,
step=1,
interactive=True,
info="limits candidate tokens to a fixed number after sorting by probability. Setting it higher than the vocabulary size deactivates this limit.",
)
]
iface = gr.ChatInterface(
fn = generater,
title=title,
description = description,
additional_inputs=additional_inputs,
examples=[
["Can you tell me how the Namib Desert Beetle inspires water collection methods?"],
["I'm working on a project related to sustainable architecture. How can biomimicry guide my design process?"],
["Can you explain the concept of biomimicry and its importance in today’s world?"],
["I need some ideas for a biomimicry project in my biology class. Can you suggest some organisms to study?"],
["How does the structure of a lotus leaf help in creating self-cleaning surfaces?"]
]
)
with gr.Blocks(css="resourse/style/custom.css") as demo:
chatbot.like(vote, None, None)
iface.render()
if __name__ == "__main__":
demo.queue().launch()
|