Spaces:
Q4234
/
Runtime error

a2 / app.py
Q4234's picture
Update app.py
d301d7f
raw
history blame
1.82 kB
import gradio as gr
import ctransformers
class Z(object):
def __init__(self):
self.llm = None
def init(self):
pass
def greet(self, txt0, paramTemp):
prompt0 = txt0
# for Wizard-Vicuna-13B
#prompt00 = f'''USER: {prompt0}
#ASSISTANT:'''
# for starcoder
prompt00 = f'''{prompt0}'''
prompt00 = f'''Below is an instruction that describes a task. Write a response that appropriately completes the request.
### Instruction:
{prompt0}
### Response:'''
response0 = llm(prompt00, max_new_tokens=198, temperature=paramTemp) # 0.5, 0.3
return f'{response0}'
from ctransformers import AutoModelForCausalLM
# wizzard vicuna
# see https://github.com/melodysdreamj/WizardVicunaLM
#llm = AutoModelForCausalLM.from_pretrained('TheBloke/Wizard-Vicuna-13B-Uncensored-GGML', model_file='Wizard-Vicuna-13B-Uncensored.ggmlv3.q4_0.bin', model_type='llama')
#llm = AutoModelForCausalLM.from_pretrained('mverrilli/dolly-v2-12b-ggml', model_file='ggml-model-q5_0.bin', model_type='dolly-v2')
#llm = AutoModelForCausalLM.from_pretrained('mverrilli/dolly-v2-7b-ggml', model_file='ggml-model-q5_0.bin', model_type='dolly-v2')
# non-RLHF model
# 4 may 2023
# site https://huggingface.co/bigcode/starcoder
modelInfo = {'path':'NeoDim/starcoder-GGML', 'subPath':'starcoder-ggml-q8_0.bin', 'promptType':'raw', 'modelType':'starcoder'}
llm = AutoModelForCausalLM.from_pretrained(modelInfo['path'], model_file=modelInfo['subPath'], model_type=modelInfo['modelType'])
z = Z()
z.llm = llm
z.modelInfo = modelInfo
z.init()
def greet(prompt, temperature):
global z
return z.greet(prompt, temperature)
iface = gr.Interface(fn=greet, inputs=["text", gr.Slider(0.0, 1.0, value=0.41)], outputs="text")
iface.launch()