Tonic commited on
Commit
b6dc5a5
1 Parent(s): 9bc49ef

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +18 -13
app.py CHANGED
@@ -5,17 +5,19 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
5
  import torch
6
  import gradio as gr
7
  import sentencepiece
 
8
 
9
- title = "Welcome to Tonic's 🐋🐳Orca-2-13B (in 8bit)!"
10
- description = "You can use [🐋🐳microsoft/Orca-2-13b](https://huggingface.co/microsoft/Orca-2-13b) via API using Gradio by scrolling down and clicking Use 'Via API' or privately by [cloning this space on huggingface](https://huggingface.co/spaces/Tonic1/TonicsOrca2?duplicate=true) . [Join my active builders' server on discord](https://discord.gg/VqTxc76K3u). Let's build together! Big thanks to the HuggingFace Organisation for the Community Grant."
11
 
 
12
  device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
13
- model_name = "microsoft/Orca-2-13b"
14
- tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False)
15
- model = transformers.AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)
16
 
17
- class OrcaChatBot:
18
- def __init__(self, model, tokenizer, system_message="You are Orca, an AI language model created by Microsoft. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior."):
19
  self.model = model
20
  self.tokenizer = tokenizer
21
  self.system_message = system_message
@@ -24,16 +26,19 @@ class OrcaChatBot:
24
  self.system_message = new_system_message
25
 
26
  def format_prompt(self, user_message):
27
- prompt = f"<|im_start|>assistant\n{self.system_message}<|im_end|>\n<|im_start|>\nuser\n{user_message}<|im_end|>\nassistant\n"
28
  return prompt
29
 
30
  def predict(self, user_message, temperature=0.4, max_new_tokens=70, top_p=0.99, repetition_penalty=1.9):
31
  prompt = self.format_prompt(user_message)
32
  inputs = self.tokenizer(prompt, return_tensors='pt', add_special_tokens=False)
33
  input_ids = inputs["input_ids"].to(self.model.device)
 
 
34
 
35
  output_ids = self.model.generate(
36
  input_ids,
 
37
  max_length=input_ids.shape[1] + max_new_tokens,
38
  temperature=temperature,
39
  top_p=top_p,
@@ -45,11 +50,11 @@ class OrcaChatBot:
45
  return response
46
 
47
  def gradio_predict(user_message, system_message, max_new_tokens, temperature, top_p, repetition_penalty):
48
- Orca_bot.set_system_message(system_message)
49
- response = Orca_bot.predict(user_message, temperature, max_new_tokens, top_p, repetition_penalty)
50
  return response
51
 
52
- Orca_bot = OrcaChatBot(model, tokenizer)
53
 
54
  iface = gr.Interface(
55
  fn=gradio_predict,
@@ -58,8 +63,8 @@ iface = gr.Interface(
58
  inputs=[
59
  gr.Textbox(label="Your Message", type="text", lines=3),
60
  gr.Textbox(label="Introduce a Character Here or Set a Scene (system prompt)", type="text", lines=2),
61
- gr.Slider(label="Max new tokens", value=550, minimum=360, maximum=600, step=1),
62
- gr.Slider(label="Temperature", value=0.1, minimum=0.05, maximum=1.0, step=0.05),
63
  gr.Slider(label="Top-p (nucleus sampling)", value=0.90, minimum=0.01, maximum=0.99, step=0.05),
64
  gr.Slider(label="Repetition penalty", value=1.9, minimum=1.0, maximum=2.0, step=0.05)
65
  ],
 
5
  import torch
6
  import gradio as gr
7
  import sentencepiece
8
+ from tokenization_xgen import XgenTokenizer
9
 
10
+ title = "Welcome to 🙋🏻‍♂️Tonic's🌷Tulu Chat!"
11
+ description = "[allenai/tulu-2-dpo-7b](https://huggingface.co/allenai/tulu-2-dpo-7b) and larger Tulu-2 models are Instruct Llama Finetunes using the [mistralai/Mistral-7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) recipe. You can use [allenai/tulu-2-13b](https://huggingface.co/allenai/tulu-2-13b) here via API using Gradio by scrolling down and clicking Use 'Via API' or privately by [cloning this space on huggingface](https://huggingface.co/spaces/Tonic1/TuluDemo?duplicate=true) See also the large model here : [allenai/tulu-2-dpo-70b](https://huggingface.co/allenai/tulu-2-dpo-70b) . [Join my active builders' server on discord](https://discord.gg/VqTxc76K3u). Let's build together!."
12
 
13
+ os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:50'
14
  device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
15
+ model_name = "allenai/tulu-2-dpo-13b"
16
+ tokenizer = AutoTokenizer.from_pretrained("allenai/tulu-2-dpo-13b")
17
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.bfloat16, device_map="auto")
18
 
19
+ class TuluChatBot:
20
+ def __init__(self, model, tokenizer, system_message="You are 🌷Tulu, an AI language model created by Tonic-AI. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior."):
21
  self.model = model
22
  self.tokenizer = tokenizer
23
  self.system_message = system_message
 
26
  self.system_message = new_system_message
27
 
28
  def format_prompt(self, user_message):
29
+ prompt = f"<|assistant|>\n {self.system_message}\n\n <|user|>{user_message}\n\n<|assistant|>\n"
30
  return prompt
31
 
32
  def predict(self, user_message, temperature=0.4, max_new_tokens=70, top_p=0.99, repetition_penalty=1.9):
33
  prompt = self.format_prompt(user_message)
34
  inputs = self.tokenizer(prompt, return_tensors='pt', add_special_tokens=False)
35
  input_ids = inputs["input_ids"].to(self.model.device)
36
+ attention_mask = inputs["attention_mask"].to(self.model.device)
37
+
38
 
39
  output_ids = self.model.generate(
40
  input_ids,
41
+ attention_mask=attention_mask,
42
  max_length=input_ids.shape[1] + max_new_tokens,
43
  temperature=temperature,
44
  top_p=top_p,
 
50
  return response
51
 
52
  def gradio_predict(user_message, system_message, max_new_tokens, temperature, top_p, repetition_penalty):
53
+ Tulu_bot.set_system_message(system_message)
54
+ response = Tulu_bot.predict(user_message, temperature, max_new_tokens, top_p, repetition_penalty)
55
  return response
56
 
57
+ Tulu_bot = TuluChatBot(model, tokenizer)
58
 
59
  iface = gr.Interface(
60
  fn=gradio_predict,
 
63
  inputs=[
64
  gr.Textbox(label="Your Message", type="text", lines=3),
65
  gr.Textbox(label="Introduce a Character Here or Set a Scene (system prompt)", type="text", lines=2),
66
+ gr.Slider(label="Max new tokens", value=1269, minimum=550, maximum=3200, step=1),
67
+ gr.Slider(label="Temperature", value=1.2, minimum=0.05, maximum=4.0, step=0.05),
68
  gr.Slider(label="Top-p (nucleus sampling)", value=0.90, minimum=0.01, maximum=0.99, step=0.05),
69
  gr.Slider(label="Repetition penalty", value=1.9, minimum=1.0, maximum=2.0, step=0.05)
70
  ],