app.py

import gradio as gr

import ctransformers

class Z(object):
    def __init__(self):
        self.llm = None

    def init(self):
        pass

    def run0(self, txt0, paramTemp):
        prompt0 = txt0

        # for Wizard-Vicuna-13B
        prompt00 = f'''USER: {prompt0}
ASSISTANT:'''

        # for TheBloke/Wizard-Vicuna-13B-Uncensored-GGML
        prompt00 = f'''Below is an instruction that describes a task. Write a response that appropriately completes the request.

### Instruction:
{prompt0}

### Response:'''

        # raw
        prompt00 = prompt0
        
        response0 = llm(prompt00, max_new_tokens=198, temperature=paramTemp) # 0.5, 0.3
        
        return f'{response0}'

from ctransformers import AutoModelForCausalLM

# experiment
#llm = AutoModelForCausalLM.from_pretrained('mverrilli/dolly-v2-12b-ggml', model_file='ggml-model-q5_0.bin', model_type='dolly-v2')

# experiment
#llm = AutoModelForCausalLM.from_pretrained('mverrilli/dolly-v2-7b-ggml', model_file='ggml-model-q5_0.bin', model_type='dolly-v2')


# wizzard vicuna
# see https://github.com/melodysdreamj/WizardVicunaLM
#llm = AutoModelForCausalLM.from_pretrained('TheBloke/Wizard-Vicuna-13B-Uncensored-GGML', model_file='Wizard-Vicuna-13B-Uncensored.ggmlv3.q4_0.bin', model_type='llama')


modelInfo = {'path2':'TheBloke/hippogriff-30b-chat-GGML:hippogriff-30b.ggmlv3.q4_1.bin', 'promptType':'raw', 'modelType':'llama'}

print('[D] load LMt...')

llm = AutoModelForCausalLM.from_pretrained(modelInfo['path2'].split(':')[0], model_file=modelInfo['path2'].split(':')[1], model_type=modelInfo['modelType'])

print('[D] ...done')


z = Z()
z.llm = llm
z.init()

def run0(prompt, temperature):
    global z
    return z.run0(prompt, temperature)

iface = gr.Interface(fn=run0, inputs=["text", gr.Slider(0.0, 1.0, value=0.41)], outputs="text")
iface.launch()