|
import gradio as gr |
|
|
|
import ctransformers |
|
|
|
class Z(object): |
|
def __init__(self): |
|
self.llm = None |
|
|
|
def init(self): |
|
pass |
|
|
|
def greet(self, txt0): |
|
prompt0 = txt0 |
|
|
|
|
|
prompt00 = f'''USER: {prompt0} |
|
ASSISTANT:''' |
|
|
|
prompt00 = f'''Below is an instruction that describes a task. Write a response that appropriately completes the request. |
|
|
|
### Instruction: |
|
{prompt0} |
|
|
|
### Response:''' |
|
|
|
response0 = llm(prompt00, max_new_tokens=128, temperature=0.5) |
|
|
|
return f'{response0}' |
|
|
|
from ctransformers import AutoModelForCausalLM |
|
|
|
|
|
|
|
|
|
|
|
llm = AutoModelForCausalLM.from_pretrained('mverrilli/dolly-v2-12b-ggml', model_file='ggml-model-q5_0.bin', model_type='dolly') |
|
|
|
|
|
|
|
z = Z() |
|
z.llm = llm |
|
z.init() |
|
|
|
def greet(arg0): |
|
global z |
|
return z.greet(arg0) |
|
|
|
iface = gr.Interface(fn=greet, inputs="text", outputs="text") |
|
iface.launch() |