File size: 1,436 Bytes
b9e24ef
ff11a2a
 
 
302a52d
 
ff11a2a
 
7a17c0a
8fdc643
a330329
ff11a2a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7a17c0a
ff11a2a
 
 
 
 
 
781821b
 
 
 
ff11a2a
 
781821b
0a15ada
 
ff11a2a
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
import os
import gradio as gr
import ctransformers

configObj = ctransformers.Config(stop=["\n", 'User'], context_length=2048)
config = ctransformers.AutoConfig(config=configObj, model_type='llama')
config.config.stop = ["\n"]

# path_to_llm = os.path.abspath("llama-2-7b-chat.ggmlv3.q4_1.bin")

llm = ctransformers.AutoModelForCausalLM.from_pretrained("TheBloke/Llama-2-7b-Chat-GGUF", model_file="llama-2-7b-chat.Q4_K_M.gguf", config=config)

def complete(prompt, stop=["User", "Assistant"]):
  tokens = llm.tokenize(prompt)
  output = ''
  for token in llm.generate(tokens):
    result = llm.detokenize(token)
    output += result
    for word in stop:
      if word in output:
        print('\n')
        return output
    print(result, end='',flush=True)

  print('\n')
  return output

title = "llama2-7b-chat-ggml"
description = "This space is an attempt to run the GGUF 4 bit quantized version of 'llama2-7b-chat' on a CPU"

example_1 = "Write a 7 line poem on AI"
example_2 = "Tell me a joke"

examples = [example_1, example_2]

def generate_response(user_input):
    prompt = f'User: {user_input}\nAssistant: '
    response = complete(prompt)
    return response

UI = gr.Interface(
    fn=generate_response,
    inputs=gr.Textbox(label="User Query", placeholder="Ask your queries here...."),
    outputs=gr.Textbox(label="Assistant Response"),
    title=title,
    description=description,
    examples=examples
)

UI.launch()