Kush1 commited on
Commit
8e9efce
1 Parent(s): 01886a5

Changes in app.py

Browse files
Files changed (2) hide show
  1. app.py +32 -141
  2. requirements.txt +1 -9
app.py CHANGED
@@ -1,145 +1,36 @@
1
- from threading import Thread
2
- from typing import Iterator
3
-
4
  import gradio as gr
5
- import spaces
6
- import torch
7
- from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
8
-
9
- DEFAULT_SYSTEM_PROMPT = "You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."
10
- MAX_MAX_NEW_TOKENS = 2048
11
- DEFAULT_MAX_NEW_TOKENS = 1024
12
- MAX_INPUT_TOKEN_LENGTH = 4096
13
-
14
- DESCRIPTION = """\
15
- # Llama-2 7B Chat
16
- This Space demonstrates model [Llama-2-7b-chat](https://huggingface.co/meta-llama/Llama-2-7b-chat) by Meta, a Llama 2 model with 7B parameters fine-tuned for chat instructions. Feel free to play with it, or duplicate to run generations without a queue! If you want to run your own service, you can also [deploy the model on Inference Endpoints](https://huggingface.co/inference-endpoints).
17
- 🔎 For more details about the Llama 2 family of models and how to use them with `transformers`, take a look [at our blog post](https://huggingface.co/blog/llama2).
18
- 🔨 Looking for an even more powerful model? Check out the [13B version](https://huggingface.co/spaces/huggingface-projects/llama-2-13b-chat) or the large [70B model demo](https://huggingface.co/spaces/ysharma/Explore_llamav2_with_TGI).
19
- """
20
-
21
- LICENSE = """
22
- <p/>
23
- ---
24
- As a derivate work of [Llama-2-7b-chat](https://huggingface.co/meta-llama/Llama-2-7b-chat) by Meta,
25
- this demo is governed by the original [license](https://huggingface.co/spaces/huggingface-projects/llama-2-7b-chat/blob/main/LICENSE.txt) and [acceptable use policy](https://huggingface.co/spaces/huggingface-projects/llama-2-7b-chat/blob/main/USE_POLICY.md).
26
- """
27
-
28
- if not torch.cuda.is_available():
29
- DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
30
-
31
-
32
- if torch.cuda.is_available():
33
- model_id = "meta-llama/Llama-2-7b-chat-hf"
34
- model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16, device_map="auto")
35
- tokenizer = AutoTokenizer.from_pretrained(model_id)
36
- tokenizer.use_default_system_prompt = False
37
-
38
-
39
- @spaces.GPU
40
- def generate(
41
- tokenizer,
42
- message: str,
43
- chat_history: list[tuple[str, str]],
44
- system_prompt: str,
45
- max_new_tokens: int = 1024,
46
- temperature: float = 0.6,
47
- top_p: float = 0.9,
48
- top_k: int = 50,
49
- repetition_penalty: float = 1.2,
50
- ) -> Iterator[str]:
51
- conversation = []
52
- if system_prompt:
53
- conversation.append({"role": "system", "content": system_prompt})
54
- for user, assistant in chat_history:
55
- conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
56
- conversation.append({"role": "user", "content": message})
57
-
58
- chat = tokenizer.apply_chat_template(conversation, tokenize=False)
59
- inputs = tokenizer(chat, return_tensors="pt", add_special_tokens=False).to("cuda")
60
- if len(inputs) > MAX_INPUT_TOKEN_LENGTH:
61
- inputs = inputs[-MAX_INPUT_TOKEN_LENGTH:]
62
- gr.Warning("Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
63
-
64
- streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
65
- generate_kwargs = dict(
66
- tokenizer,
67
- inputs,
68
- streamer=streamer,
69
- max_new_tokens=max_new_tokens,
70
- do_sample=True,
71
- top_p=top_p,
72
- top_k=top_k,
73
- temperature=temperature,
74
- num_beams=1,
75
- repetition_penalty=repetition_penalty,
76
  )
77
- t = Thread(target=model.generate, kwargs=generate_kwargs)
78
- t.start()
79
-
80
- outputs = []
81
- for text in streamer:
82
- outputs.append(text)
83
- yield "".join(outputs)
84
 
 
85
 
86
- chat_interface = gr.ChatInterface(
87
- fn=generate,
88
- additional_inputs=[
89
- tokenizer,
90
- gr.Textbox(label="System prompt", value=DEFAULT_SYSTEM_PROMPT, lines=6),
91
- gr.Slider(
92
- label="Max new tokens",
93
- minimum=1,
94
- maximum=MAX_MAX_NEW_TOKENS,
95
- step=1,
96
- value=DEFAULT_MAX_NEW_TOKENS,
97
- ),
98
- gr.Slider(
99
- label="Temperature",
100
- minimum=0.1,
101
- maximum=4.0,
102
- step=0.1,
103
- value=0.6,
104
- ),
105
- gr.Slider(
106
- label="Top-p (nucleus sampling)",
107
- minimum=0.05,
108
- maximum=1.0,
109
- step=0.05,
110
- value=0.9,
111
- ),
112
- gr.Slider(
113
- label="Top-k",
114
- minimum=1,
115
- maximum=1000,
116
- step=1,
117
- value=50,
118
- ),
119
- gr.Slider(
120
- label="Repetition penalty",
121
- minimum=1.0,
122
- maximum=2.0,
123
- step=0.05,
124
- value=1.2,
125
- ),
126
- ],
127
- stop_btn=None,
128
- examples=[
129
- ["Hello there! How are you doing?"],
130
- ["Can you explain briefly to me what is the Python programming language?"],
131
- ["Explain the plot of Cinderella in a sentence."],
132
- ["How many hours does it take a man to eat a Helicopter?"],
133
- ["Write a 100-word article on 'Benefits of Open-Source in AI research'"],
134
- ],
135
- )
136
-
137
- with gr.Blocks(css="style.css") as demo:
138
- gr.Markdown(DESCRIPTION)
139
- gr.DuplicateButton(value="Duplicate Space for private use", elem_id="duplicate-button")
140
- chat_interface.render()
141
- gr.Markdown(LICENSE)
142
-
143
- if __name__ == "__main__":
144
- demo.queue(max_size=20).launch()
145
-
 
 
 
 
1
  import gradio as gr
2
+ import time
3
+ from ctransformers import AutoModelForCausalLM
4
+
5
+ def load_llm():
6
+ llm = AutoModelForCausalLM.from_pretrained("codellama-13b-instruct.Q4_K_M.gguf",
7
+ model_type='llama',
8
+ max_new_tokens = 1096,
9
+ repetition_penalty = 1.13,
10
+ temperature = 0.1
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  )
12
+ return llm
 
 
 
 
 
 
13
 
14
+ def llm_function(message, chat_history):
15
 
16
+ llm = load_llm()
17
+ response = llm(
18
+ message
19
+ )
20
+ output_texts = response
21
+ return output_texts
22
+
23
+ title = "CodeLlama 13B GGUF Demo"
24
+
25
+ examples = [
26
+ 'Write a python code to connect with a SQL database and list down all the tables.',
27
+ 'Write the python code to train a linear regression model using Scikit Learn.',
28
+ 'Explain the concepts of Functional Programming.',
29
+ 'Can you explain the benefits of Python programming language?'
30
+ ]
31
+
32
+ gr.ChatInterface(
33
+ fn=llm_function,
34
+ title=title,
35
+ examples=examples
36
+ ).launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
requirements.txt CHANGED
@@ -1,9 +1 @@
1
- accelerate==0.23.0
2
- bitsandbytes==0.41.1
3
- gradio==3.47.1
4
- protobuf==3.20.3
5
- scipy==1.11.2
6
- sentencepiece==0.1.99
7
- spaces==0.16.1
8
- torch==2.0.0
9
- transformers==4.34.0
 
1
+ ctransformers == 0.2.24