Spaces:
Sleeping
Sleeping
File size: 3,679 Bytes
093d71e 519a55d 669d949 519a55d b293ad4 54929aa b293ad4 0d516fd b293ad4 669d949 b293ad4 669d949 b293ad4 54929aa b293ad4 54929aa b293ad4 54929aa b293ad4 54929aa 519a55d 6fc5da1 519a55d 6fc5da1 519a55d 093d71e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 |
import gradio as gr
import os
import requests
import json
sbc_host_url = os.environ['URL']
# def get_completion(prompt:str, messages:str = '', n_predict=128):
# system = "### System: You are a helpful assistant helps to brainstorm ideas.\n"
# prompt_templated = f'{system} {messages}\n ### HUMAN:\n{prompt} \n ### ASSISTANT:'
# headers = {
# "Content-Type": "application/json"
# }
# data = {
# "prompt": prompt_templated,
# "n_predict": n_predict,
# "stop": ["### HUMAN:", "### ASSISTANT:", "HUMAN"],
# "stream": "True"
# }
# try:
# response = requests.post(sbc_host_url, headers=headers, data=json.dumps(data))
# if response.status_code == 200:
# return response.json()['content']
# else:
# response.raise_for_status()
# except:
# raise gr.Warning("Apologies for the inconvenience! Our model is currently self-hosted and unavailable at the moment.")
# def chatty(prompt, messages):
# # print(prompt)
# # print(f'messages: {messages}')
# past_messages = ''
# if len(messages) > 0:
# for idx, message in enumerate(messages):
# print(f'idx: {idx}, message: {message}')
# past_messages += f'\n### HUMAN: {message[0]}'
# past_messages += f'\n### ASSISTANT: {message[1]}'
# # past_messages = messages[0][0]
# # print(f'past_messages: {past_messages}')
# messages = get_completion(prompt, past_messages)
# return messages.split('### ASSISTANT:')[-1]
# stream
def chatty(prompt, messages, n_predict=128):
# print(prompt)
# print(f'messages: {messages}')
past_messages = ''
if len(messages) > 0:
for idx, message in enumerate(messages):
# print(f'idx: {idx}, message: {message}')
past_messages += f'\n### HUMAN: {message[0]}'
past_messages += f'\n### ASSISTANT: {message[1]}'
system = "### System: You help to brainstorm ideas.\n"
prompt_templated = f'{system} {messages}\n ### HUMAN:\n{prompt} \n ### ASSISTANT:'
headers = {
"Content-Type": "application/json"
}
data = {
"prompt": prompt_templated,
"n_predict": n_predict,
"stop": ["### HUMAN:", "### ASSISTANT:", "HUMAN"],
"stream": True
}
result = ""
try:
response = requests.post(sbc_host_url, headers=headers, data=json.dumps(data), stream=True)
if response.status_code == 200:
for line in response.iter_lines():
if line:
try:
result += json.loads(line.decode('utf-8').replace('data: ', ''))['content']
except:
# LMStudio response has empty token
pass
yield result
else:
response.raise_for_status()
except requests.exceptions.RequestException as e:
raise gr.Warning("Apologies for the inconvenience! Our model is currently self-hosted and unavailable at the moment.")
with gr.Blocks() as demo:
gr.Image("sbc.jpg")
gr.ChatInterface(
fn=chatty,
title="DevQuasar/llama3_8b_chat_brainstorm-GGUF on Orange Pi5 plus with llama.cpp",
description="Brainstorm facilitates idea exploration through interaction with a Language Model (LLM). Rather than providing direct answers, the model engages in a dialogue with users, offering probing questions aimed at fostering deeper contemplation and consideration of various facets of their ideas."
)
if __name__ == "__main__":
demo.launch() |